作者: 谢瑞阳 10225101483 徐翔宇 10225101535
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

387 line
12 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. //
  5. // We recover the contents of the descriptor from the other files we find.
  6. // (1) Any log files are first converted to tables
  7. // (2) We scan every table to compute
  8. // (a) smallest/largest for the table
  9. // (b) largest sequence number in the table
  10. // (3) We generate descriptor contents:
  11. // - log number is set to zero
  12. // - next-file-number is set to 1 + largest file number we found
  13. // - last-sequence-number is set to largest sequence# found across
  14. // all tables (see 2c)
  15. // - compaction pointers are cleared
  16. // - every table file is added at level 0
  17. //
  18. // Possible optimization 1:
  19. // (a) Compute total size and use to pick appropriate max-level M
  20. // (b) Sort tables by largest sequence# in the table
  21. // (c) For each table: if it overlaps earlier table, place in level-0,
  22. // else place in level-M.
  23. // Possible optimization 2:
  24. // Store per-table metadata (smallest, largest, largest-seq#, ...)
  25. // in the table's meta section to speed up ScanTable.
  26. #include "db/builder.h"
  27. #include "db/db_impl.h"
  28. #include "db/dbformat.h"
  29. #include "db/filename.h"
  30. #include "db/log_reader.h"
  31. #include "db/log_writer.h"
  32. #include "db/memtable.h"
  33. #include "db/table_cache.h"
  34. #include "db/version_edit.h"
  35. #include "db/write_batch_internal.h"
  36. #include "leveldb/comparator.h"
  37. #include "leveldb/db.h"
  38. #include "leveldb/env.h"
  39. namespace leveldb {
  40. namespace {
  41. class Repairer {
  42. public:
  43. Repairer(const std::string& dbname, const Options& options)
  44. : dbname_(dbname),
  45. env_(options.env),
  46. icmp_(options.comparator),
  47. options_(SanitizeOptions(dbname, &icmp_, options)),
  48. owns_info_log_(options_.info_log != options.info_log),
  49. owns_cache_(options_.block_cache != options.block_cache),
  50. next_file_number_(1) {
  51. // TableCache can be small since we expect each table to be opened once.
  52. table_cache_ = new TableCache(dbname_, &options_, 10);
  53. }
  54. ~Repairer() {
  55. delete table_cache_;
  56. if (owns_info_log_) {
  57. delete options_.info_log;
  58. }
  59. if (owns_cache_) {
  60. delete options_.block_cache;
  61. }
  62. }
  63. Status Run() {
  64. Status status = FindFiles();
  65. if (status.ok()) {
  66. ConvertLogFilesToTables();
  67. ExtractMetaData();
  68. status = WriteDescriptor();
  69. }
  70. if (status.ok()) {
  71. unsigned long long bytes = 0;
  72. for (size_t i = 0; i < tables_.size(); i++) {
  73. bytes += tables_[i].meta.file_size;
  74. }
  75. Log(options_.info_log,
  76. "**** Repaired leveldb %s; "
  77. "recovered %d files; %llu bytes. "
  78. "Some data may have been lost. "
  79. "****",
  80. dbname_.c_str(),
  81. static_cast<int>(tables_.size()),
  82. bytes);
  83. }
  84. return status;
  85. }
  86. private:
  87. struct TableInfo {
  88. FileMetaData meta;
  89. SequenceNumber max_sequence;
  90. };
  91. std::string const dbname_;
  92. Env* const env_;
  93. InternalKeyComparator const icmp_;
  94. Options const options_;
  95. bool owns_info_log_;
  96. bool owns_cache_;
  97. TableCache* table_cache_;
  98. VersionEdit edit_;
  99. std::vector<std::string> manifests_;
  100. std::vector<uint64_t> table_numbers_;
  101. std::vector<uint64_t> logs_;
  102. std::vector<TableInfo> tables_;
  103. uint64_t next_file_number_;
  104. Status FindFiles() {
  105. std::vector<std::string> filenames;
  106. Status status = env_->GetChildren(dbname_, &filenames);
  107. if (!status.ok()) {
  108. return status;
  109. }
  110. if (filenames.empty()) {
  111. return Status::IOError(dbname_, "repair found no files");
  112. }
  113. uint64_t number;
  114. FileType type;
  115. for (size_t i = 0; i < filenames.size(); i++) {
  116. if (ParseFileName(filenames[i], &number, &type)) {
  117. if (type == kDescriptorFile) {
  118. manifests_.push_back(filenames[i]);
  119. } else {
  120. if (number + 1 > next_file_number_) {
  121. next_file_number_ = number + 1;
  122. }
  123. if (type == kLogFile) {
  124. logs_.push_back(number);
  125. } else if (type == kTableFile) {
  126. table_numbers_.push_back(number);
  127. } else {
  128. // Ignore other files
  129. }
  130. }
  131. }
  132. }
  133. return status;
  134. }
  135. void ConvertLogFilesToTables() {
  136. for (size_t i = 0; i < logs_.size(); i++) {
  137. std::string logname = LogFileName(dbname_, logs_[i]);
  138. Status status = ConvertLogToTable(logs_[i]);
  139. if (!status.ok()) {
  140. Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
  141. (unsigned long long) logs_[i],
  142. status.ToString().c_str());
  143. }
  144. ArchiveFile(logname);
  145. }
  146. }
  147. Status ConvertLogToTable(uint64_t log) {
  148. struct LogReporter : public log::Reader::Reporter {
  149. Env* env;
  150. Logger* info_log;
  151. uint64_t lognum;
  152. virtual void Corruption(size_t bytes, const Status& s) {
  153. // We print error messages for corruption, but continue repairing.
  154. Log(info_log, "Log #%llu: dropping %d bytes; %s",
  155. (unsigned long long) lognum,
  156. static_cast<int>(bytes),
  157. s.ToString().c_str());
  158. }
  159. };
  160. // Open the log file
  161. std::string logname = LogFileName(dbname_, log);
  162. SequentialFile* lfile;
  163. Status status = env_->NewSequentialFile(logname, &lfile);
  164. if (!status.ok()) {
  165. return status;
  166. }
  167. // Create the log reader.
  168. LogReporter reporter;
  169. reporter.env = env_;
  170. reporter.info_log = options_.info_log;
  171. reporter.lognum = log;
  172. // We intentially make log::Reader do checksumming so that
  173. // corruptions cause entire commits to be skipped instead of
  174. // propagating bad information (like overly large sequence
  175. // numbers).
  176. log::Reader reader(lfile, &reporter, false/*do not checksum*/,
  177. 0/*initial_offset*/);
  178. // Read all the records and add to a memtable
  179. std::string scratch;
  180. Slice record;
  181. WriteBatch batch;
  182. MemTable* mem = new MemTable(icmp_);
  183. mem->Ref();
  184. int counter = 0;
  185. while (reader.ReadRecord(&record, &scratch)) {
  186. if (record.size() < 12) {
  187. reporter.Corruption(
  188. record.size(), Status::Corruption("log record too small"));
  189. continue;
  190. }
  191. WriteBatchInternal::SetContents(&batch, record);
  192. status = WriteBatchInternal::InsertInto(&batch, mem);
  193. if (status.ok()) {
  194. counter += WriteBatchInternal::Count(&batch);
  195. } else {
  196. Log(options_.info_log, "Log #%llu: ignoring %s",
  197. (unsigned long long) log,
  198. status.ToString().c_str());
  199. status = Status::OK(); // Keep going with rest of file
  200. }
  201. }
  202. delete lfile;
  203. // Do not record a version edit for this conversion to a Table
  204. // since ExtractMetaData() will also generate edits.
  205. FileMetaData meta;
  206. meta.number = next_file_number_++;
  207. Iterator* iter = mem->NewIterator();
  208. status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
  209. delete iter;
  210. mem->Unref();
  211. mem = NULL;
  212. if (status.ok()) {
  213. if (meta.file_size > 0) {
  214. table_numbers_.push_back(meta.number);
  215. }
  216. }
  217. Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
  218. (unsigned long long) log,
  219. counter,
  220. (unsigned long long) meta.number,
  221. status.ToString().c_str());
  222. return status;
  223. }
  224. void ExtractMetaData() {
  225. std::vector<TableInfo> kept;
  226. for (size_t i = 0; i < table_numbers_.size(); i++) {
  227. TableInfo t;
  228. t.meta.number = table_numbers_[i];
  229. Status status = ScanTable(&t);
  230. if (!status.ok()) {
  231. std::string fname = TableFileName(dbname_, table_numbers_[i]);
  232. Log(options_.info_log, "Table #%llu: ignoring %s",
  233. (unsigned long long) table_numbers_[i],
  234. status.ToString().c_str());
  235. ArchiveFile(fname);
  236. } else {
  237. tables_.push_back(t);
  238. }
  239. }
  240. }
  241. Status ScanTable(TableInfo* t) {
  242. std::string fname = TableFileName(dbname_, t->meta.number);
  243. int counter = 0;
  244. Status status = env_->GetFileSize(fname, &t->meta.file_size);
  245. if (status.ok()) {
  246. Iterator* iter = table_cache_->NewIterator(
  247. ReadOptions(), t->meta.number, t->meta.file_size);
  248. bool empty = true;
  249. ParsedInternalKey parsed;
  250. t->max_sequence = 0;
  251. for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
  252. Slice key = iter->key();
  253. if (!ParseInternalKey(key, &parsed)) {
  254. Log(options_.info_log, "Table #%llu: unparsable key %s",
  255. (unsigned long long) t->meta.number,
  256. EscapeString(key).c_str());
  257. continue;
  258. }
  259. counter++;
  260. if (empty) {
  261. empty = false;
  262. t->meta.smallest.DecodeFrom(key);
  263. }
  264. t->meta.largest.DecodeFrom(key);
  265. if (parsed.sequence > t->max_sequence) {
  266. t->max_sequence = parsed.sequence;
  267. }
  268. }
  269. if (!iter->status().ok()) {
  270. status = iter->status();
  271. }
  272. delete iter;
  273. }
  274. Log(options_.info_log, "Table #%llu: %d entries %s",
  275. (unsigned long long) t->meta.number,
  276. counter,
  277. status.ToString().c_str());
  278. return status;
  279. }
  280. Status WriteDescriptor() {
  281. std::string tmp = TempFileName(dbname_, 1);
  282. WritableFile* file;
  283. Status status = env_->NewWritableFile(tmp, &file);
  284. if (!status.ok()) {
  285. return status;
  286. }
  287. SequenceNumber max_sequence = 0;
  288. for (size_t i = 0; i < tables_.size(); i++) {
  289. if (max_sequence < tables_[i].max_sequence) {
  290. max_sequence = tables_[i].max_sequence;
  291. }
  292. }
  293. edit_.SetComparatorName(icmp_.user_comparator()->Name());
  294. edit_.SetLogNumber(0);
  295. edit_.SetNextFile(next_file_number_);
  296. edit_.SetLastSequence(max_sequence);
  297. for (size_t i = 0; i < tables_.size(); i++) {
  298. // TODO(opt): separate out into multiple levels
  299. const TableInfo& t = tables_[i];
  300. edit_.AddFile(0, t.meta.number, t.meta.file_size,
  301. t.meta.smallest, t.meta.largest);
  302. }
  303. //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
  304. {
  305. log::Writer log(file);
  306. std::string record;
  307. edit_.EncodeTo(&record);
  308. status = log.AddRecord(record);
  309. }
  310. if (status.ok()) {
  311. status = file->Close();
  312. }
  313. delete file;
  314. file = NULL;
  315. if (!status.ok()) {
  316. env_->DeleteFile(tmp);
  317. } else {
  318. // Discard older manifests
  319. for (size_t i = 0; i < manifests_.size(); i++) {
  320. ArchiveFile(dbname_ + "/" + manifests_[i]);
  321. }
  322. // Install new manifest
  323. status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1));
  324. if (status.ok()) {
  325. status = SetCurrentFile(env_, dbname_, 1);
  326. } else {
  327. env_->DeleteFile(tmp);
  328. }
  329. }
  330. return status;
  331. }
  332. void ArchiveFile(const std::string& fname) {
  333. // Move into another directory. E.g., for
  334. // dir/foo
  335. // rename to
  336. // dir/lost/foo
  337. const char* slash = strrchr(fname.c_str(), '/');
  338. std::string new_dir;
  339. if (slash != NULL) {
  340. new_dir.assign(fname.data(), slash - fname.data());
  341. }
  342. new_dir.append("/lost");
  343. env_->CreateDir(new_dir); // Ignore error
  344. std::string new_file = new_dir;
  345. new_file.append("/");
  346. new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
  347. Status s = env_->RenameFile(fname, new_file);
  348. Log(options_.info_log, "Archiving %s: %s\n",
  349. fname.c_str(), s.ToString().c_str());
  350. }
  351. };
  352. } // namespace
  353. Status RepairDB(const std::string& dbname, const Options& options) {
  354. Repairer repairer(dbname, options);
  355. return repairer.Run();
  356. }
  357. } // namespace leveldb