作者: 谢瑞阳 10225101483 徐翔宇 10225101535

461 行
13 KiB

Release 1.18 Changes are: * Update version number to 1.18 * Replace the basic fprintf call with a call to fwrite in order to work around the apparent compiler optimization/rewrite failure that we are seeing with the new toolchain/iOS SDKs provided with Xcode6 and iOS8. * Fix ALL the header guards. * Createed a README.md with the LevelDB project description. * A new CONTRIBUTING file. * Don't implicitly convert uint64_t to size_t or int. Either preserve it as uint64_t, or explicitly cast. This fixes MSVC warnings about possible value truncation when compiling this code in Chromium. * Added a DumpFile() library function that encapsulates the guts of the "leveldbutil dump" command. This will allow clients to dump data to their log files instead of stdout. It will also allow clients to supply their own environment. * leveldb: Remove unused function 'ConsumeChar'. * leveldbutil: Remove unused member variables from WriteBatchItemPrinter. * OpenBSD, NetBSD and DragonflyBSD have _LITTLE_ENDIAN, so define PLATFORM_IS_LITTLE_ENDIAN like on FreeBSD. This fixes: * issue #143 * issue #198 * issue #249 * Switch from <cstdatomic> to <atomic>. The former never made it into the standard and doesn't exist in modern gcc versions at all. The later contains everything that leveldb was using from the former. This problem was noticed when porting to Portable Native Client where no memory barrier is defined. The fact that <cstdatomic> is missing normally goes unnoticed since memory barriers are defined for most architectures. * Make Hash() treat its input as unsigned. Before this change LevelDB files from platforms with different signedness of char were not compatible. This change fixes: issue #243 * Verify checksums of index/meta/filter blocks when paranoid_checks set. * Invoke all tools for iOS with xcrun. (This was causing problems with the new XCode 5.1.1 image on pulse.) * include <sys/stat.h> only once, and fix the following linter warning: "Found C system header after C++ system header" * When encountering a corrupted table file, return Status::Corruption instead of Status::InvalidArgument. * Support cygwin as build platform, patch is from https://code.google.com/p/leveldb/issues/detail?id=188 * Fix typo, merge patch from https://code.google.com/p/leveldb/issues/detail?id=159 * Fix typos and comments, and address the following two issues: * issue #166 * issue #241 * Add missing db synchronize after "fillseq" in the benchmark. * Removed unused variable in SeekRandom: value (issue #201)
10年前
  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. //
  5. // We recover the contents of the descriptor from the other files we find.
  6. // (1) Any log files are first converted to tables
  7. // (2) We scan every table to compute
  8. // (a) smallest/largest for the table
  9. // (b) largest sequence number in the table
  10. // (3) We generate descriptor contents:
  11. // - log number is set to zero
  12. // - next-file-number is set to 1 + largest file number we found
  13. // - last-sequence-number is set to largest sequence# found across
  14. // all tables (see 2c)
  15. // - compaction pointers are cleared
  16. // - every table file is added at level 0
  17. //
  18. // Possible optimization 1:
  19. // (a) Compute total size and use to pick appropriate max-level M
  20. // (b) Sort tables by largest sequence# in the table
  21. // (c) For each table: if it overlaps earlier table, place in level-0,
  22. // else place in level-M.
  23. // Possible optimization 2:
  24. // Store per-table metadata (smallest, largest, largest-seq#, ...)
  25. // in the table's meta section to speed up ScanTable.
  26. #include "db/builder.h"
  27. #include "db/db_impl.h"
  28. #include "db/dbformat.h"
  29. #include "db/filename.h"
  30. #include "db/log_reader.h"
  31. #include "db/log_writer.h"
  32. #include "db/memtable.h"
  33. #include "db/table_cache.h"
  34. #include "db/version_edit.h"
  35. #include "db/write_batch_internal.h"
  36. #include "leveldb/comparator.h"
  37. #include "leveldb/db.h"
  38. #include "leveldb/env.h"
  39. namespace leveldb {
  40. namespace {
  41. class Repairer {
  42. public:
  43. Repairer(const std::string& dbname, const Options& options)
  44. : dbname_(dbname),
  45. env_(options.env),
  46. icmp_(options.comparator),
  47. ipolicy_(options.filter_policy),
  48. options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
  49. owns_info_log_(options_.info_log != options.info_log),
  50. owns_cache_(options_.block_cache != options.block_cache),
  51. next_file_number_(1) {
  52. // TableCache can be small since we expect each table to be opened once.
  53. table_cache_ = new TableCache(dbname_, options_, 10);
  54. }
  55. ~Repairer() {
  56. delete table_cache_;
  57. if (owns_info_log_) {
  58. delete options_.info_log;
  59. }
  60. if (owns_cache_) {
  61. delete options_.block_cache;
  62. }
  63. }
  64. Status Run() {
  65. Status status = FindFiles();
  66. if (status.ok()) {
  67. ConvertLogFilesToTables();
  68. ExtractMetaData();
  69. status = WriteDescriptor();
  70. }
  71. if (status.ok()) {
  72. unsigned long long bytes = 0;
  73. for (size_t i = 0; i < tables_.size(); i++) {
  74. bytes += tables_[i].meta.file_size;
  75. }
  76. Log(options_.info_log,
  77. "**** Repaired leveldb %s; "
  78. "recovered %d files; %llu bytes. "
  79. "Some data may have been lost. "
  80. "****",
  81. dbname_.c_str(),
  82. static_cast<int>(tables_.size()),
  83. bytes);
  84. }
  85. return status;
  86. }
  87. private:
  88. struct TableInfo {
  89. FileMetaData meta;
  90. SequenceNumber max_sequence;
  91. };
  92. std::string const dbname_;
  93. Env* const env_;
  94. InternalKeyComparator const icmp_;
  95. InternalFilterPolicy const ipolicy_;
  96. Options const options_;
  97. bool owns_info_log_;
  98. bool owns_cache_;
  99. TableCache* table_cache_;
  100. VersionEdit edit_;
  101. std::vector<std::string> manifests_;
  102. std::vector<uint64_t> table_numbers_;
  103. std::vector<uint64_t> logs_;
  104. std::vector<TableInfo> tables_;
  105. uint64_t next_file_number_;
  106. Status FindFiles() {
  107. std::vector<std::string> filenames;
  108. Status status = env_->GetChildren(dbname_, &filenames);
  109. if (!status.ok()) {
  110. return status;
  111. }
  112. if (filenames.empty()) {
  113. return Status::IOError(dbname_, "repair found no files");
  114. }
  115. uint64_t number;
  116. FileType type;
  117. for (size_t i = 0; i < filenames.size(); i++) {
  118. if (ParseFileName(filenames[i], &number, &type)) {
  119. if (type == kDescriptorFile) {
  120. manifests_.push_back(filenames[i]);
  121. } else {
  122. if (number + 1 > next_file_number_) {
  123. next_file_number_ = number + 1;
  124. }
  125. if (type == kLogFile) {
  126. logs_.push_back(number);
  127. } else if (type == kTableFile) {
  128. table_numbers_.push_back(number);
  129. } else {
  130. // Ignore other files
  131. }
  132. }
  133. }
  134. }
  135. return status;
  136. }
  137. void ConvertLogFilesToTables() {
  138. for (size_t i = 0; i < logs_.size(); i++) {
  139. std::string logname = LogFileName(dbname_, logs_[i]);
  140. Status status = ConvertLogToTable(logs_[i]);
  141. if (!status.ok()) {
  142. Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
  143. (unsigned long long) logs_[i],
  144. status.ToString().c_str());
  145. }
  146. ArchiveFile(logname);
  147. }
  148. }
  149. Status ConvertLogToTable(uint64_t log) {
  150. struct LogReporter : public log::Reader::Reporter {
  151. Env* env;
  152. Logger* info_log;
  153. uint64_t lognum;
  154. virtual void Corruption(size_t bytes, const Status& s) {
  155. // We print error messages for corruption, but continue repairing.
  156. Log(info_log, "Log #%llu: dropping %d bytes; %s",
  157. (unsigned long long) lognum,
  158. static_cast<int>(bytes),
  159. s.ToString().c_str());
  160. }
  161. };
  162. // Open the log file
  163. std::string logname = LogFileName(dbname_, log);
  164. SequentialFile* lfile;
  165. Status status = env_->NewSequentialFile(logname, &lfile);
  166. if (!status.ok()) {
  167. return status;
  168. }
  169. // Create the log reader.
  170. LogReporter reporter;
  171. reporter.env = env_;
  172. reporter.info_log = options_.info_log;
  173. reporter.lognum = log;
  174. // We intentionally make log::Reader do checksumming so that
  175. // corruptions cause entire commits to be skipped instead of
  176. // propagating bad information (like overly large sequence
  177. // numbers).
  178. log::Reader reader(lfile, &reporter, false/*do not checksum*/,
  179. 0/*initial_offset*/);
  180. // Read all the records and add to a memtable
  181. std::string scratch;
  182. Slice record;
  183. WriteBatch batch;
  184. MemTable* mem = new MemTable(icmp_);
  185. mem->Ref();
  186. int counter = 0;
  187. while (reader.ReadRecord(&record, &scratch)) {
  188. if (record.size() < 12) {
  189. reporter.Corruption(
  190. record.size(), Status::Corruption("log record too small"));
  191. continue;
  192. }
  193. WriteBatchInternal::SetContents(&batch, record);
  194. status = WriteBatchInternal::InsertInto(&batch, mem);
  195. if (status.ok()) {
  196. counter += WriteBatchInternal::Count(&batch);
  197. } else {
  198. Log(options_.info_log, "Log #%llu: ignoring %s",
  199. (unsigned long long) log,
  200. status.ToString().c_str());
  201. status = Status::OK(); // Keep going with rest of file
  202. }
  203. }
  204. delete lfile;
  205. // Do not record a version edit for this conversion to a Table
  206. // since ExtractMetaData() will also generate edits.
  207. FileMetaData meta;
  208. meta.number = next_file_number_++;
  209. Iterator* iter = mem->NewIterator();
  210. status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
  211. delete iter;
  212. mem->Unref();
  213. mem = nullptr;
  214. if (status.ok()) {
  215. if (meta.file_size > 0) {
  216. table_numbers_.push_back(meta.number);
  217. }
  218. }
  219. Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
  220. (unsigned long long) log,
  221. counter,
  222. (unsigned long long) meta.number,
  223. status.ToString().c_str());
  224. return status;
  225. }
  226. void ExtractMetaData() {
  227. for (size_t i = 0; i < table_numbers_.size(); i++) {
  228. ScanTable(table_numbers_[i]);
  229. }
  230. }
  231. Iterator* NewTableIterator(const FileMetaData& meta) {
  232. // Same as compaction iterators: if paranoid_checks are on, turn
  233. // on checksum verification.
  234. ReadOptions r;
  235. r.verify_checksums = options_.paranoid_checks;
  236. return table_cache_->NewIterator(r, meta.number, meta.file_size);
  237. }
  238. void ScanTable(uint64_t number) {
  239. TableInfo t;
  240. t.meta.number = number;
  241. std::string fname = TableFileName(dbname_, number);
  242. Status status = env_->GetFileSize(fname, &t.meta.file_size);
  243. if (!status.ok()) {
  244. // Try alternate file name.
  245. fname = SSTTableFileName(dbname_, number);
  246. Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
  247. if (s2.ok()) {
  248. status = Status::OK();
  249. }
  250. }
  251. if (!status.ok()) {
  252. ArchiveFile(TableFileName(dbname_, number));
  253. ArchiveFile(SSTTableFileName(dbname_, number));
  254. Log(options_.info_log, "Table #%llu: dropped: %s",
  255. (unsigned long long) t.meta.number,
  256. status.ToString().c_str());
  257. return;
  258. }
  259. // Extract metadata by scanning through table.
  260. int counter = 0;
  261. Iterator* iter = NewTableIterator(t.meta);
  262. bool empty = true;
  263. ParsedInternalKey parsed;
  264. t.max_sequence = 0;
  265. for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
  266. Slice key = iter->key();
  267. if (!ParseInternalKey(key, &parsed)) {
  268. Log(options_.info_log, "Table #%llu: unparsable key %s",
  269. (unsigned long long) t.meta.number,
  270. EscapeString(key).c_str());
  271. continue;
  272. }
  273. counter++;
  274. if (empty) {
  275. empty = false;
  276. t.meta.smallest.DecodeFrom(key);
  277. }
  278. t.meta.largest.DecodeFrom(key);
  279. if (parsed.sequence > t.max_sequence) {
  280. t.max_sequence = parsed.sequence;
  281. }
  282. }
  283. if (!iter->status().ok()) {
  284. status = iter->status();
  285. }
  286. delete iter;
  287. Log(options_.info_log, "Table #%llu: %d entries %s",
  288. (unsigned long long) t.meta.number,
  289. counter,
  290. status.ToString().c_str());
  291. if (status.ok()) {
  292. tables_.push_back(t);
  293. } else {
  294. RepairTable(fname, t); // RepairTable archives input file.
  295. }
  296. }
  297. void RepairTable(const std::string& src, TableInfo t) {
  298. // We will copy src contents to a new table and then rename the
  299. // new table over the source.
  300. // Create builder.
  301. std::string copy = TableFileName(dbname_, next_file_number_++);
  302. WritableFile* file;
  303. Status s = env_->NewWritableFile(copy, &file);
  304. if (!s.ok()) {
  305. return;
  306. }
  307. TableBuilder* builder = new TableBuilder(options_, file);
  308. // Copy data.
  309. Iterator* iter = NewTableIterator(t.meta);
  310. int counter = 0;
  311. for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
  312. builder->Add(iter->key(), iter->value());
  313. counter++;
  314. }
  315. delete iter;
  316. ArchiveFile(src);
  317. if (counter == 0) {
  318. builder->Abandon(); // Nothing to save
  319. } else {
  320. s = builder->Finish();
  321. if (s.ok()) {
  322. t.meta.file_size = builder->FileSize();
  323. }
  324. }
  325. delete builder;
  326. builder = nullptr;
  327. if (s.ok()) {
  328. s = file->Close();
  329. }
  330. delete file;
  331. file = nullptr;
  332. if (counter > 0 && s.ok()) {
  333. std::string orig = TableFileName(dbname_, t.meta.number);
  334. s = env_->RenameFile(copy, orig);
  335. if (s.ok()) {
  336. Log(options_.info_log, "Table #%llu: %d entries repaired",
  337. (unsigned long long) t.meta.number, counter);
  338. tables_.push_back(t);
  339. }
  340. }
  341. if (!s.ok()) {
  342. env_->DeleteFile(copy);
  343. }
  344. }
  345. Status WriteDescriptor() {
  346. std::string tmp = TempFileName(dbname_, 1);
  347. WritableFile* file;
  348. Status status = env_->NewWritableFile(tmp, &file);
  349. if (!status.ok()) {
  350. return status;
  351. }
  352. SequenceNumber max_sequence = 0;
  353. for (size_t i = 0; i < tables_.size(); i++) {
  354. if (max_sequence < tables_[i].max_sequence) {
  355. max_sequence = tables_[i].max_sequence;
  356. }
  357. }
  358. edit_.SetComparatorName(icmp_.user_comparator()->Name());
  359. edit_.SetLogNumber(0);
  360. edit_.SetNextFile(next_file_number_);
  361. edit_.SetLastSequence(max_sequence);
  362. for (size_t i = 0; i < tables_.size(); i++) {
  363. // TODO(opt): separate out into multiple levels
  364. const TableInfo& t = tables_[i];
  365. edit_.AddFile(0, t.meta.number, t.meta.file_size,
  366. t.meta.smallest, t.meta.largest);
  367. }
  368. //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
  369. {
  370. log::Writer log(file);
  371. std::string record;
  372. edit_.EncodeTo(&record);
  373. status = log.AddRecord(record);
  374. }
  375. if (status.ok()) {
  376. status = file->Close();
  377. }
  378. delete file;
  379. file = nullptr;
  380. if (!status.ok()) {
  381. env_->DeleteFile(tmp);
  382. } else {
  383. // Discard older manifests
  384. for (size_t i = 0; i < manifests_.size(); i++) {
  385. ArchiveFile(dbname_ + "/" + manifests_[i]);
  386. }
  387. // Install new manifest
  388. status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1));
  389. if (status.ok()) {
  390. status = SetCurrentFile(env_, dbname_, 1);
  391. } else {
  392. env_->DeleteFile(tmp);
  393. }
  394. }
  395. return status;
  396. }
  397. void ArchiveFile(const std::string& fname) {
  398. // Move into another directory. E.g., for
  399. // dir/foo
  400. // rename to
  401. // dir/lost/foo
  402. const char* slash = strrchr(fname.c_str(), '/');
  403. std::string new_dir;
  404. if (slash != nullptr) {
  405. new_dir.assign(fname.data(), slash - fname.data());
  406. }
  407. new_dir.append("/lost");
  408. env_->CreateDir(new_dir); // Ignore error
  409. std::string new_file = new_dir;
  410. new_file.append("/");
  411. new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
  412. Status s = env_->RenameFile(fname, new_file);
  413. Log(options_.info_log, "Archiving %s: %s\n",
  414. fname.c_str(), s.ToString().c_str());
  415. }
  416. };
  417. } // namespace
  418. Status RepairDB(const std::string& dbname, const Options& options) {
  419. Repairer repairer(dbname, options);
  420. return repairer.Run();
  421. }
  422. } // namespace leveldb