1257 Zeilen
39 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "db/version_set.h"
  5. #include <algorithm>
  6. #include <stdio.h>
  7. #include "db/filename.h"
  8. #include "db/log_reader.h"
  9. #include "db/log_writer.h"
  10. #include "db/memtable.h"
  11. #include "db/table_cache.h"
  12. #include "leveldb/env.h"
  13. #include "leveldb/table_builder.h"
  14. #include "table/merger.h"
  15. #include "table/two_level_iterator.h"
  16. #include "util/coding.h"
  17. #include "util/logging.h"
  18. namespace leveldb {
  19. static const int kTargetFileSize = 2 * 1048576;
  20. // Maximum bytes of overlaps in grandparent (i.e., level+2) before we
  21. // stop building a single file in a level->level+1 compaction.
  22. static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize;
  23. static double MaxBytesForLevel(int level) {
  24. // Note: the result for level zero is not really used since we set
  25. // the level-0 compaction threshold based on number of files.
  26. double result = 10 * 1048576.0; // Result for both level-0 and level-1
  27. while (level > 1) {
  28. result *= 10;
  29. level--;
  30. }
  31. return result;
  32. }
  33. static uint64_t MaxFileSizeForLevel(int level) {
  34. return kTargetFileSize; // We could vary per level to reduce number of files?
  35. }
  36. namespace {
  37. std::string IntSetToString(const std::set<uint64_t>& s) {
  38. std::string result = "{";
  39. for (std::set<uint64_t>::const_iterator it = s.begin();
  40. it != s.end();
  41. ++it) {
  42. result += (result.size() > 1) ? "," : "";
  43. result += NumberToString(*it);
  44. }
  45. result += "}";
  46. return result;
  47. }
  48. }
  49. Version::~Version() {
  50. assert(refs_ == 0);
  51. // Remove from linked list
  52. prev_->next_ = next_;
  53. next_->prev_ = prev_;
  54. // Drop references to files
  55. for (int level = 0; level < config::kNumLevels; level++) {
  56. for (size_t i = 0; i < files_[level].size(); i++) {
  57. FileMetaData* f = files_[level][i];
  58. assert(f->refs > 0);
  59. f->refs--;
  60. if (f->refs <= 0) {
  61. delete f;
  62. }
  63. }
  64. }
  65. }
  66. int FindFile(const InternalKeyComparator& icmp,
  67. const std::vector<FileMetaData*>& files,
  68. const Slice& key) {
  69. uint32_t left = 0;
  70. uint32_t right = files.size();
  71. while (left < right) {
  72. uint32_t mid = (left + right) / 2;
  73. const FileMetaData* f = files[mid];
  74. if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) {
  75. // Key at "mid.largest" is < "target". Therefore all
  76. // files at or before "mid" are uninteresting.
  77. left = mid + 1;
  78. } else {
  79. // Key at "mid.largest" is >= "target". Therefore all files
  80. // after "mid" are uninteresting.
  81. right = mid;
  82. }
  83. }
  84. return right;
  85. }
  86. bool SomeFileOverlapsRange(
  87. const InternalKeyComparator& icmp,
  88. const std::vector<FileMetaData*>& files,
  89. const InternalKey& smallest,
  90. const InternalKey& largest) {
  91. const int index = FindFile(icmp, files, smallest.Encode());
  92. return ((index < files.size()) &&
  93. icmp.Compare(largest, files[index]->smallest) >= 0);
  94. }
  95. // An internal iterator. For a given version/level pair, yields
  96. // information about the files in the level. For a given entry, key()
  97. // is the largest key that occurs in the file, and value() is an
  98. // 16-byte value containing the file number and file size, both
  99. // encoded using EncodeFixed64.
  100. class Version::LevelFileNumIterator : public Iterator {
  101. public:
  102. LevelFileNumIterator(const InternalKeyComparator& icmp,
  103. const std::vector<FileMetaData*>* flist)
  104. : icmp_(icmp),
  105. flist_(flist),
  106. index_(flist->size()) { // Marks as invalid
  107. }
  108. virtual bool Valid() const {
  109. return index_ < flist_->size();
  110. }
  111. virtual void Seek(const Slice& target) {
  112. index_ = FindFile(icmp_, *flist_, target);
  113. }
  114. virtual void SeekToFirst() { index_ = 0; }
  115. virtual void SeekToLast() {
  116. index_ = flist_->empty() ? 0 : flist_->size() - 1;
  117. }
  118. virtual void Next() {
  119. assert(Valid());
  120. index_++;
  121. }
  122. virtual void Prev() {
  123. assert(Valid());
  124. if (index_ == 0) {
  125. index_ = flist_->size(); // Marks as invalid
  126. } else {
  127. index_--;
  128. }
  129. }
  130. Slice key() const {
  131. assert(Valid());
  132. return (*flist_)[index_]->largest.Encode();
  133. }
  134. Slice value() const {
  135. assert(Valid());
  136. EncodeFixed64(value_buf_, (*flist_)[index_]->number);
  137. EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size);
  138. return Slice(value_buf_, sizeof(value_buf_));
  139. }
  140. virtual Status status() const { return Status::OK(); }
  141. private:
  142. const InternalKeyComparator icmp_;
  143. const std::vector<FileMetaData*>* const flist_;
  144. uint32_t index_;
  145. // Backing store for value(). Holds the file number and size.
  146. mutable char value_buf_[16];
  147. };
  148. static Iterator* GetFileIterator(void* arg,
  149. const ReadOptions& options,
  150. const Slice& file_value) {
  151. TableCache* cache = reinterpret_cast<TableCache*>(arg);
  152. if (file_value.size() != 16) {
  153. return NewErrorIterator(
  154. Status::Corruption("FileReader invoked with unexpected value"));
  155. } else {
  156. return cache->NewIterator(options,
  157. DecodeFixed64(file_value.data()),
  158. DecodeFixed64(file_value.data() + 8));
  159. }
  160. }
  161. Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
  162. int level) const {
  163. return NewTwoLevelIterator(
  164. new LevelFileNumIterator(vset_->icmp_, &files_[level]),
  165. &GetFileIterator, vset_->table_cache_, options);
  166. }
  167. void Version::AddIterators(const ReadOptions& options,
  168. std::vector<Iterator*>* iters) {
  169. // Merge all level zero files together since they may overlap
  170. for (size_t i = 0; i < files_[0].size(); i++) {
  171. iters->push_back(
  172. vset_->table_cache_->NewIterator(
  173. options, files_[0][i]->number, files_[0][i]->file_size));
  174. }
  175. // For levels > 0, we can use a concatenating iterator that sequentially
  176. // walks through the non-overlapping files in the level, opening them
  177. // lazily.
  178. for (int level = 1; level < config::kNumLevels; level++) {
  179. if (!files_[level].empty()) {
  180. iters->push_back(NewConcatenatingIterator(options, level));
  181. }
  182. }
  183. }
  184. // If "*iter" points at a value or deletion for user_key, store
  185. // either the value, or a NotFound error and return true.
  186. // Else return false.
  187. static bool GetValue(Iterator* iter, const Slice& user_key,
  188. std::string* value,
  189. Status* s) {
  190. if (!iter->Valid()) {
  191. return false;
  192. }
  193. ParsedInternalKey parsed_key;
  194. if (!ParseInternalKey(iter->key(), &parsed_key)) {
  195. *s = Status::Corruption("corrupted key for ", user_key);
  196. return true;
  197. }
  198. if (parsed_key.user_key != user_key) {
  199. return false;
  200. }
  201. switch (parsed_key.type) {
  202. case kTypeDeletion:
  203. *s = Status::NotFound(Slice()); // Use an empty error message for speed
  204. break;
  205. case kTypeValue: {
  206. Slice v = iter->value();
  207. value->assign(v.data(), v.size());
  208. break;
  209. }
  210. }
  211. return true;
  212. }
  213. static bool NewestFirst(FileMetaData* a, FileMetaData* b) {
  214. return a->number > b->number;
  215. }
  216. Status Version::Get(const ReadOptions& options,
  217. const LookupKey& k,
  218. std::string* value,
  219. GetStats* stats) {
  220. Slice ikey = k.internal_key();
  221. Slice user_key = k.user_key();
  222. const Comparator* ucmp = vset_->icmp_.user_comparator();
  223. Status s;
  224. stats->seek_file = NULL;
  225. stats->seek_file_level = -1;
  226. FileMetaData* last_file_read = NULL;
  227. // We can search level-by-level since entries never hop across
  228. // levels. Therefore we are guaranteed that if we find data
  229. // in an smaller level, later levels are irrelevant.
  230. std::vector<FileMetaData*> tmp;
  231. FileMetaData* tmp2;
  232. for (int level = 0; level < config::kNumLevels; level++) {
  233. size_t num_files = files_[level].size();
  234. if (num_files == 0) continue;
  235. // Get the list of files to search in this level
  236. FileMetaData* const* files = &files_[level][0];
  237. if (level == 0) {
  238. // Level-0 files may overlap each other. Find all files that
  239. // overlap user_key and process them in order from newest to oldest.
  240. tmp.reserve(num_files);
  241. for (int i = 0; i < num_files; i++) {
  242. FileMetaData* f = files[i];
  243. if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
  244. ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
  245. tmp.push_back(f);
  246. }
  247. }
  248. if (tmp.empty()) continue;
  249. std::sort(tmp.begin(), tmp.end(), NewestFirst);
  250. files = &tmp[0];
  251. num_files = tmp.size();
  252. } else {
  253. // Binary search to find earliest index whose largest key >= ikey.
  254. uint32_t index = FindFile(vset_->icmp_, files_[level], ikey);
  255. if (index >= num_files) {
  256. files = NULL;
  257. num_files = 0;
  258. } else {
  259. tmp2 = files[index];
  260. if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
  261. // All of "tmp2" is past any data for user_key
  262. files = NULL;
  263. num_files = 0;
  264. } else {
  265. files = &tmp2;
  266. num_files = 1;
  267. }
  268. }
  269. }
  270. for (int i = 0; i < num_files; ++i) {
  271. if (last_file_read != NULL && stats->seek_file == NULL) {
  272. // We have had more than one seek for this read. Charge the 1st file.
  273. stats->seek_file = last_file_read;
  274. stats->seek_file_level = (i == 0 ? level - 1 : level);
  275. }
  276. FileMetaData* f = files[i];
  277. last_file_read = f;
  278. Iterator* iter = vset_->table_cache_->NewIterator(
  279. options,
  280. f->number,
  281. f->file_size);
  282. iter->Seek(ikey);
  283. const bool done = GetValue(iter, user_key, value, &s);
  284. if (!iter->status().ok()) {
  285. s = iter->status();
  286. delete iter;
  287. return s;
  288. } else {
  289. delete iter;
  290. if (done) {
  291. return s;
  292. }
  293. }
  294. }
  295. }
  296. return Status::NotFound(Slice()); // Use an empty error message for speed
  297. }
  298. bool Version::UpdateStats(const GetStats& stats) {
  299. FileMetaData* f = stats.seek_file;
  300. if (f != NULL) {
  301. f->allowed_seeks--;
  302. if (f->allowed_seeks <= 0 && file_to_compact_ == NULL) {
  303. file_to_compact_ = f;
  304. file_to_compact_level_ = stats.seek_file_level;
  305. return true;
  306. }
  307. }
  308. return false;
  309. }
  310. void Version::Ref() {
  311. ++refs_;
  312. }
  313. void Version::Unref() {
  314. assert(this != &vset_->dummy_versions_);
  315. assert(refs_ >= 1);
  316. --refs_;
  317. if (refs_ == 0) {
  318. delete this;
  319. }
  320. }
  321. bool Version::OverlapInLevel(int level,
  322. const InternalKey& smallest,
  323. const InternalKey& largest) {
  324. return SomeFileOverlapsRange(vset_->icmp_, files_[level], smallest, largest);
  325. }
  326. std::string Version::DebugString() const {
  327. std::string r;
  328. for (int level = 0; level < config::kNumLevels; level++) {
  329. // E.g.,
  330. // --- level 1 ---
  331. // 17:123['a' .. 'd']
  332. // 20:43['e' .. 'g']
  333. r.append("--- level ");
  334. AppendNumberTo(&r, level);
  335. r.append(" ---\n");
  336. const std::vector<FileMetaData*>& files = files_[level];
  337. for (size_t i = 0; i < files.size(); i++) {
  338. r.push_back(' ');
  339. AppendNumberTo(&r, files[i]->number);
  340. r.push_back(':');
  341. AppendNumberTo(&r, files[i]->file_size);
  342. r.append("['");
  343. AppendEscapedStringTo(&r, files[i]->smallest.Encode());
  344. r.append("' .. '");
  345. AppendEscapedStringTo(&r, files[i]->largest.Encode());
  346. r.append("']\n");
  347. }
  348. }
  349. return r;
  350. }
  351. // A helper class so we can efficiently apply a whole sequence
  352. // of edits to a particular state without creating intermediate
  353. // Versions that contain full copies of the intermediate state.
  354. class VersionSet::Builder {
  355. private:
  356. // Helper to sort by v->files_[file_number].smallest
  357. struct BySmallestKey {
  358. const InternalKeyComparator* internal_comparator;
  359. bool operator()(FileMetaData* f1, FileMetaData* f2) const {
  360. int r = internal_comparator->Compare(f1->smallest, f2->smallest);
  361. if (r != 0) {
  362. return (r < 0);
  363. } else {
  364. // Break ties by file number
  365. return (f1->number < f2->number);
  366. }
  367. }
  368. };
  369. typedef std::set<FileMetaData*, BySmallestKey> FileSet;
  370. struct LevelState {
  371. std::set<uint64_t> deleted_files;
  372. FileSet* added_files;
  373. };
  374. VersionSet* vset_;
  375. Version* base_;
  376. LevelState levels_[config::kNumLevels];
  377. public:
  378. // Initialize a builder with the files from *base and other info from *vset
  379. Builder(VersionSet* vset, Version* base)
  380. : vset_(vset),
  381. base_(base) {
  382. base_->Ref();
  383. BySmallestKey cmp;
  384. cmp.internal_comparator = &vset_->icmp_;
  385. for (int level = 0; level < config::kNumLevels; level++) {
  386. levels_[level].added_files = new FileSet(cmp);
  387. }
  388. }
  389. ~Builder() {
  390. for (int level = 0; level < config::kNumLevels; level++) {
  391. std::vector<FileMetaData*> to_unref(levels_[level].added_files->begin(),
  392. levels_[level].added_files->end());
  393. delete levels_[level].added_files;
  394. for (int i = 0; i < to_unref.size(); i++) {
  395. FileMetaData* f = to_unref[i];
  396. f->refs--;
  397. if (f->refs <= 0) {
  398. delete f;
  399. }
  400. }
  401. }
  402. base_->Unref();
  403. }
  404. // Apply all of the edits in *edit to the current state.
  405. void Apply(VersionEdit* edit) {
  406. // Update compaction pointers
  407. for (size_t i = 0; i < edit->compact_pointers_.size(); i++) {
  408. const int level = edit->compact_pointers_[i].first;
  409. vset_->compact_pointer_[level] =
  410. edit->compact_pointers_[i].second.Encode().ToString();
  411. }
  412. // Delete files
  413. const VersionEdit::DeletedFileSet& del = edit->deleted_files_;
  414. for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin();
  415. iter != del.end();
  416. ++iter) {
  417. const int level = iter->first;
  418. const uint64_t number = iter->second;
  419. levels_[level].deleted_files.insert(number);
  420. }
  421. // Add new files
  422. for (size_t i = 0; i < edit->new_files_.size(); i++) {
  423. const int level = edit->new_files_[i].first;
  424. FileMetaData* f = new FileMetaData(edit->new_files_[i].second);
  425. f->refs = 1;
  426. // We arrange to automatically compact this file after
  427. // a certain number of seeks. Let's assume:
  428. // (1) One seek costs 10ms
  429. // (2) Writing or reading 1MB costs 10ms (100MB/s)
  430. // (3) A compaction of 1MB does 25MB of IO:
  431. // 1MB read from this level
  432. // 10-12MB read from next level (boundaries may be misaligned)
  433. // 10-12MB written to next level
  434. // This implies that 25 seeks cost the same as the compaction
  435. // of 1MB of data. I.e., one seek costs approximately the
  436. // same as the compaction of 40KB of data. We are a little
  437. // conservative and allow approximately one seek for every 16KB
  438. // of data before triggering a compaction.
  439. f->allowed_seeks = (f->file_size / 16384);
  440. if (f->allowed_seeks < 100) f->allowed_seeks = 100;
  441. levels_[level].deleted_files.erase(f->number);
  442. levels_[level].added_files->insert(f);
  443. }
  444. }
  445. // Save the current state in *v.
  446. void SaveTo(Version* v) {
  447. BySmallestKey cmp;
  448. cmp.internal_comparator = &vset_->icmp_;
  449. for (int level = 0; level < config::kNumLevels; level++) {
  450. // Merge the set of added files with the set of pre-existing files.
  451. // Drop any deleted files. Store the result in *v.
  452. const std::vector<FileMetaData*>& base_files = base_->files_[level];
  453. std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin();
  454. std::vector<FileMetaData*>::const_iterator base_end = base_files.end();
  455. const FileSet* added = levels_[level].added_files;
  456. v->files_[level].reserve(base_files.size() + added->size());
  457. for (FileSet::const_iterator added_iter = added->begin();
  458. added_iter != added->end();
  459. ++added_iter) {
  460. // Add all smaller files listed in base_
  461. for (std::vector<FileMetaData*>::const_iterator bpos
  462. = std::upper_bound(base_iter, base_end, *added_iter, cmp);
  463. base_iter != bpos;
  464. ++base_iter) {
  465. MaybeAddFile(v, level, *base_iter);
  466. }
  467. MaybeAddFile(v, level, *added_iter);
  468. }
  469. // Add remaining base files
  470. for (; base_iter != base_end; ++base_iter) {
  471. MaybeAddFile(v, level, *base_iter);
  472. }
  473. #ifndef NDEBUG
  474. // Make sure there is no overlap in levels > 0
  475. if (level > 0) {
  476. for (int i = 1; i < v->files_[level].size(); i++) {
  477. const InternalKey& prev_end = v->files_[level][i-1]->largest;
  478. const InternalKey& this_begin = v->files_[level][i]->smallest;
  479. if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) {
  480. fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
  481. EscapeString(prev_end.Encode()).c_str(),
  482. EscapeString(this_begin.Encode()).c_str());
  483. abort();
  484. }
  485. }
  486. }
  487. #endif
  488. }
  489. }
  490. void MaybeAddFile(Version* v, int level, FileMetaData* f) {
  491. if (levels_[level].deleted_files.count(f->number) > 0) {
  492. // File is deleted: do nothing
  493. } else {
  494. std::vector<FileMetaData*>* files = &v->files_[level];
  495. if (level > 0 && !files->empty()) {
  496. // Must not overlap
  497. assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest,
  498. f->smallest) < 0);
  499. }
  500. f->refs++;
  501. files->push_back(f);
  502. }
  503. }
  504. };
  505. VersionSet::VersionSet(const std::string& dbname,
  506. const Options* options,
  507. TableCache* table_cache,
  508. const InternalKeyComparator* cmp)
  509. : env_(options->env),
  510. dbname_(dbname),
  511. options_(options),
  512. table_cache_(table_cache),
  513. icmp_(*cmp),
  514. next_file_number_(2),
  515. manifest_file_number_(0), // Filled by Recover()
  516. last_sequence_(0),
  517. log_number_(0),
  518. prev_log_number_(0),
  519. descriptor_file_(NULL),
  520. descriptor_log_(NULL),
  521. dummy_versions_(this),
  522. current_(NULL) {
  523. AppendVersion(new Version(this));
  524. }
  525. VersionSet::~VersionSet() {
  526. current_->Unref();
  527. assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty
  528. delete descriptor_log_;
  529. delete descriptor_file_;
  530. }
  531. void VersionSet::AppendVersion(Version* v) {
  532. // Make "v" current
  533. assert(v->refs_ == 0);
  534. assert(v != current_);
  535. if (current_ != NULL) {
  536. current_->Unref();
  537. }
  538. current_ = v;
  539. v->Ref();
  540. // Append to linked list
  541. v->prev_ = dummy_versions_.prev_;
  542. v->next_ = &dummy_versions_;
  543. v->prev_->next_ = v;
  544. v->next_->prev_ = v;
  545. }
  546. Status VersionSet::LogAndApply(VersionEdit* edit) {
  547. if (edit->has_log_number_) {
  548. assert(edit->log_number_ >= log_number_);
  549. assert(edit->log_number_ < next_file_number_);
  550. } else {
  551. edit->SetLogNumber(log_number_);
  552. }
  553. if (!edit->has_prev_log_number_) {
  554. edit->SetPrevLogNumber(prev_log_number_);
  555. }
  556. edit->SetNextFile(next_file_number_);
  557. edit->SetLastSequence(last_sequence_);
  558. Version* v = new Version(this);
  559. {
  560. Builder builder(this, current_);
  561. builder.Apply(edit);
  562. builder.SaveTo(v);
  563. }
  564. Finalize(v);
  565. // Initialize new descriptor log file if necessary by creating
  566. // a temporary file that contains a snapshot of the current version.
  567. std::string new_manifest_file;
  568. Status s;
  569. if (descriptor_log_ == NULL) {
  570. assert(descriptor_file_ == NULL);
  571. new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_);
  572. edit->SetNextFile(next_file_number_);
  573. s = env_->NewWritableFile(new_manifest_file, &descriptor_file_);
  574. if (s.ok()) {
  575. descriptor_log_ = new log::Writer(descriptor_file_);
  576. s = WriteSnapshot(descriptor_log_);
  577. }
  578. }
  579. // Write new record to MANIFEST log
  580. if (s.ok()) {
  581. std::string record;
  582. edit->EncodeTo(&record);
  583. s = descriptor_log_->AddRecord(record);
  584. if (s.ok()) {
  585. s = descriptor_file_->Sync();
  586. }
  587. }
  588. // If we just created a new descriptor file, install it by writing a
  589. // new CURRENT file that points to it.
  590. if (s.ok() && !new_manifest_file.empty()) {
  591. s = SetCurrentFile(env_, dbname_, manifest_file_number_);
  592. }
  593. // Install the new version
  594. if (s.ok()) {
  595. AppendVersion(v);
  596. log_number_ = edit->log_number_;
  597. prev_log_number_ = edit->prev_log_number_;
  598. } else {
  599. delete v;
  600. if (!new_manifest_file.empty()) {
  601. delete descriptor_log_;
  602. delete descriptor_file_;
  603. descriptor_log_ = NULL;
  604. descriptor_file_ = NULL;
  605. env_->DeleteFile(new_manifest_file);
  606. }
  607. }
  608. return s;
  609. }
  610. Status VersionSet::Recover() {
  611. struct LogReporter : public log::Reader::Reporter {
  612. Status* status;
  613. virtual void Corruption(size_t bytes, const Status& s) {
  614. if (this->status->ok()) *this->status = s;
  615. }
  616. };
  617. // Read "CURRENT" file, which contains a pointer to the current manifest file
  618. std::string current;
  619. Status s = ReadFileToString(env_, CurrentFileName(dbname_), &current);
  620. if (!s.ok()) {
  621. return s;
  622. }
  623. if (current.empty() || current[current.size()-1] != '\n') {
  624. return Status::Corruption("CURRENT file does not end with newline");
  625. }
  626. current.resize(current.size() - 1);
  627. std::string dscname = dbname_ + "/" + current;
  628. SequentialFile* file;
  629. s = env_->NewSequentialFile(dscname, &file);
  630. if (!s.ok()) {
  631. return s;
  632. }
  633. bool have_log_number = false;
  634. bool have_prev_log_number = false;
  635. bool have_next_file = false;
  636. bool have_last_sequence = false;
  637. uint64_t next_file = 0;
  638. uint64_t last_sequence = 0;
  639. uint64_t log_number = 0;
  640. uint64_t prev_log_number = 0;
  641. Builder builder(this, current_);
  642. {
  643. LogReporter reporter;
  644. reporter.status = &s;
  645. log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/);
  646. Slice record;
  647. std::string scratch;
  648. while (reader.ReadRecord(&record, &scratch) && s.ok()) {
  649. VersionEdit edit;
  650. s = edit.DecodeFrom(record);
  651. if (s.ok()) {
  652. if (edit.has_comparator_ &&
  653. edit.comparator_ != icmp_.user_comparator()->Name()) {
  654. s = Status::InvalidArgument(
  655. edit.comparator_ + "does not match existing comparator ",
  656. icmp_.user_comparator()->Name());
  657. }
  658. }
  659. if (s.ok()) {
  660. builder.Apply(&edit);
  661. }
  662. if (edit.has_log_number_) {
  663. log_number = edit.log_number_;
  664. have_log_number = true;
  665. }
  666. if (edit.has_prev_log_number_) {
  667. prev_log_number = edit.prev_log_number_;
  668. have_prev_log_number = true;
  669. }
  670. if (edit.has_next_file_number_) {
  671. next_file = edit.next_file_number_;
  672. have_next_file = true;
  673. }
  674. if (edit.has_last_sequence_) {
  675. last_sequence = edit.last_sequence_;
  676. have_last_sequence = true;
  677. }
  678. }
  679. }
  680. delete file;
  681. file = NULL;
  682. if (s.ok()) {
  683. if (!have_next_file) {
  684. s = Status::Corruption("no meta-nextfile entry in descriptor");
  685. } else if (!have_log_number) {
  686. s = Status::Corruption("no meta-lognumber entry in descriptor");
  687. } else if (!have_last_sequence) {
  688. s = Status::Corruption("no last-sequence-number entry in descriptor");
  689. }
  690. if (!have_prev_log_number) {
  691. prev_log_number = 0;
  692. }
  693. }
  694. if (s.ok()) {
  695. Version* v = new Version(this);
  696. builder.SaveTo(v);
  697. // Install recovered version
  698. Finalize(v);
  699. AppendVersion(v);
  700. manifest_file_number_ = next_file;
  701. next_file_number_ = next_file + 1;
  702. last_sequence_ = last_sequence;
  703. log_number_ = log_number;
  704. prev_log_number_ = prev_log_number;
  705. }
  706. return s;
  707. }
  708. static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
  709. int64_t sum = 0;
  710. for (size_t i = 0; i < files.size(); i++) {
  711. sum += files[i]->file_size;
  712. }
  713. return sum;
  714. }
  715. void VersionSet::Finalize(Version* v) {
  716. // Precomputed best level for next compaction
  717. int best_level = -1;
  718. double best_score = -1;
  719. for (int level = 0; level < config::kNumLevels-1; level++) {
  720. double score;
  721. if (level == 0) {
  722. // We treat level-0 specially by bounding the number of files
  723. // instead of number of bytes for two reasons:
  724. //
  725. // (1) With larger write-buffer sizes, it is nice not to do too
  726. // many level-0 compactions.
  727. //
  728. // (2) The files in level-0 are merged on every read and
  729. // therefore we wish to avoid too many files when the individual
  730. // file size is small (perhaps because of a small write-buffer
  731. // setting, or very high compression ratios, or lots of
  732. // overwrites/deletions).
  733. score = v->files_[level].size() /
  734. static_cast<double>(config::kL0_CompactionTrigger);
  735. } else {
  736. // Compute the ratio of current size to size limit.
  737. const uint64_t level_bytes = TotalFileSize(v->files_[level]);
  738. score = static_cast<double>(level_bytes) / MaxBytesForLevel(level);
  739. }
  740. if (score > best_score) {
  741. best_level = level;
  742. best_score = score;
  743. }
  744. }
  745. v->compaction_level_ = best_level;
  746. v->compaction_score_ = best_score;
  747. }
  748. Status VersionSet::WriteSnapshot(log::Writer* log) {
  749. // TODO: Break up into multiple records to reduce memory usage on recovery?
  750. // Save metadata
  751. VersionEdit edit;
  752. edit.SetComparatorName(icmp_.user_comparator()->Name());
  753. // Save compaction pointers
  754. for (int level = 0; level < config::kNumLevels; level++) {
  755. if (!compact_pointer_[level].empty()) {
  756. InternalKey key;
  757. key.DecodeFrom(compact_pointer_[level]);
  758. edit.SetCompactPointer(level, key);
  759. }
  760. }
  761. // Save files
  762. for (int level = 0; level < config::kNumLevels; level++) {
  763. const std::vector<FileMetaData*>& files = current_->files_[level];
  764. for (size_t i = 0; i < files.size(); i++) {
  765. const FileMetaData* f = files[i];
  766. edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest);
  767. }
  768. }
  769. std::string record;
  770. edit.EncodeTo(&record);
  771. return log->AddRecord(record);
  772. }
  773. int VersionSet::NumLevelFiles(int level) const {
  774. assert(level >= 0);
  775. assert(level < config::kNumLevels);
  776. return current_->files_[level].size();
  777. }
  778. const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
  779. // Update code if kNumLevels changes
  780. assert(config::kNumLevels == 7);
  781. snprintf(scratch->buffer, sizeof(scratch->buffer),
  782. "files[ %d %d %d %d %d %d %d ]",
  783. int(current_->files_[0].size()),
  784. int(current_->files_[1].size()),
  785. int(current_->files_[2].size()),
  786. int(current_->files_[3].size()),
  787. int(current_->files_[4].size()),
  788. int(current_->files_[5].size()),
  789. int(current_->files_[6].size()));
  790. return scratch->buffer;
  791. }
  792. uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
  793. uint64_t result = 0;
  794. for (int level = 0; level < config::kNumLevels; level++) {
  795. const std::vector<FileMetaData*>& files = v->files_[level];
  796. for (size_t i = 0; i < files.size(); i++) {
  797. if (icmp_.Compare(files[i]->largest, ikey) <= 0) {
  798. // Entire file is before "ikey", so just add the file size
  799. result += files[i]->file_size;
  800. } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) {
  801. // Entire file is after "ikey", so ignore
  802. if (level > 0) {
  803. // Files other than level 0 are sorted by meta->smallest, so
  804. // no further files in this level will contain data for
  805. // "ikey".
  806. break;
  807. }
  808. } else {
  809. // "ikey" falls in the range for this table. Add the
  810. // approximate offset of "ikey" within the table.
  811. Table* tableptr;
  812. Iterator* iter = table_cache_->NewIterator(
  813. ReadOptions(), files[i]->number, files[i]->file_size, &tableptr);
  814. if (tableptr != NULL) {
  815. result += tableptr->ApproximateOffsetOf(ikey.Encode());
  816. }
  817. delete iter;
  818. }
  819. }
  820. }
  821. return result;
  822. }
  823. void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
  824. for (Version* v = dummy_versions_.next_;
  825. v != &dummy_versions_;
  826. v = v->next_) {
  827. for (int level = 0; level < config::kNumLevels; level++) {
  828. const std::vector<FileMetaData*>& files = v->files_[level];
  829. for (size_t i = 0; i < files.size(); i++) {
  830. live->insert(files[i]->number);
  831. }
  832. }
  833. }
  834. }
  835. int64_t VersionSet::NumLevelBytes(int level) const {
  836. assert(level >= 0);
  837. assert(level < config::kNumLevels);
  838. return TotalFileSize(current_->files_[level]);
  839. }
  840. int64_t VersionSet::MaxNextLevelOverlappingBytes() {
  841. int64_t result = 0;
  842. std::vector<FileMetaData*> overlaps;
  843. for (int level = 1; level < config::kNumLevels - 1; level++) {
  844. for (size_t i = 0; i < current_->files_[level].size(); i++) {
  845. const FileMetaData* f = current_->files_[level][i];
  846. GetOverlappingInputs(level+1, f->smallest, f->largest, &overlaps);
  847. const int64_t sum = TotalFileSize(overlaps);
  848. if (sum > result) {
  849. result = sum;
  850. }
  851. }
  852. }
  853. return result;
  854. }
  855. // Store in "*inputs" all files in "level" that overlap [begin,end]
  856. void VersionSet::GetOverlappingInputs(
  857. int level,
  858. const InternalKey& begin,
  859. const InternalKey& end,
  860. std::vector<FileMetaData*>* inputs) {
  861. inputs->clear();
  862. Slice user_begin = begin.user_key();
  863. Slice user_end = end.user_key();
  864. const Comparator* user_cmp = icmp_.user_comparator();
  865. for (size_t i = 0; i < current_->files_[level].size(); i++) {
  866. FileMetaData* f = current_->files_[level][i];
  867. if (user_cmp->Compare(f->largest.user_key(), user_begin) < 0 ||
  868. user_cmp->Compare(f->smallest.user_key(), user_end) > 0) {
  869. // Either completely before or after range; skip it
  870. } else {
  871. inputs->push_back(f);
  872. }
  873. }
  874. }
  875. // Stores the minimal range that covers all entries in inputs in
  876. // *smallest, *largest.
  877. // REQUIRES: inputs is not empty
  878. void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
  879. InternalKey* smallest,
  880. InternalKey* largest) {
  881. assert(!inputs.empty());
  882. smallest->Clear();
  883. largest->Clear();
  884. for (size_t i = 0; i < inputs.size(); i++) {
  885. FileMetaData* f = inputs[i];
  886. if (i == 0) {
  887. *smallest = f->smallest;
  888. *largest = f->largest;
  889. } else {
  890. if (icmp_.Compare(f->smallest, *smallest) < 0) {
  891. *smallest = f->smallest;
  892. }
  893. if (icmp_.Compare(f->largest, *largest) > 0) {
  894. *largest = f->largest;
  895. }
  896. }
  897. }
  898. }
  899. // Stores the minimal range that covers all entries in inputs1 and inputs2
  900. // in *smallest, *largest.
  901. // REQUIRES: inputs is not empty
  902. void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
  903. const std::vector<FileMetaData*>& inputs2,
  904. InternalKey* smallest,
  905. InternalKey* largest) {
  906. std::vector<FileMetaData*> all = inputs1;
  907. all.insert(all.end(), inputs2.begin(), inputs2.end());
  908. GetRange(all, smallest, largest);
  909. }
  910. Iterator* VersionSet::MakeInputIterator(Compaction* c) {
  911. ReadOptions options;
  912. options.verify_checksums = options_->paranoid_checks;
  913. options.fill_cache = false;
  914. // Level-0 files have to be merged together. For other levels,
  915. // we will make a concatenating iterator per level.
  916. // TODO(opt): use concatenating iterator for level-0 if there is no overlap
  917. const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2);
  918. Iterator** list = new Iterator*[space];
  919. int num = 0;
  920. for (int which = 0; which < 2; which++) {
  921. if (!c->inputs_[which].empty()) {
  922. if (c->level() + which == 0) {
  923. const std::vector<FileMetaData*>& files = c->inputs_[which];
  924. for (size_t i = 0; i < files.size(); i++) {
  925. list[num++] = table_cache_->NewIterator(
  926. options, files[i]->number, files[i]->file_size);
  927. }
  928. } else {
  929. // Create concatenating iterator for the files from this level
  930. list[num++] = NewTwoLevelIterator(
  931. new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]),
  932. &GetFileIterator, table_cache_, options);
  933. }
  934. }
  935. }
  936. assert(num <= space);
  937. Iterator* result = NewMergingIterator(&icmp_, list, num);
  938. delete[] list;
  939. return result;
  940. }
  941. Compaction* VersionSet::PickCompaction() {
  942. Compaction* c;
  943. int level;
  944. // We prefer compactions triggered by too much data in a level over
  945. // the compactions triggered by seeks.
  946. const bool size_compaction = (current_->compaction_score_ >= 1);
  947. const bool seek_compaction = (current_->file_to_compact_ != NULL);
  948. if (size_compaction) {
  949. level = current_->compaction_level_;
  950. assert(level >= 0);
  951. assert(level+1 < config::kNumLevels);
  952. c = new Compaction(level);
  953. // Pick the first file that comes after compact_pointer_[level]
  954. for (size_t i = 0; i < current_->files_[level].size(); i++) {
  955. FileMetaData* f = current_->files_[level][i];
  956. if (compact_pointer_[level].empty() ||
  957. icmp_.Compare(f->largest.Encode(), compact_pointer_[level]) > 0) {
  958. c->inputs_[0].push_back(f);
  959. break;
  960. }
  961. }
  962. if (c->inputs_[0].empty()) {
  963. // Wrap-around to the beginning of the key space
  964. c->inputs_[0].push_back(current_->files_[level][0]);
  965. }
  966. } else if (seek_compaction) {
  967. level = current_->file_to_compact_level_;
  968. c = new Compaction(level);
  969. c->inputs_[0].push_back(current_->file_to_compact_);
  970. } else {
  971. return NULL;
  972. }
  973. c->input_version_ = current_;
  974. c->input_version_->Ref();
  975. // Files in level 0 may overlap each other, so pick up all overlapping ones
  976. if (level == 0) {
  977. InternalKey smallest, largest;
  978. GetRange(c->inputs_[0], &smallest, &largest);
  979. // Note that the next call will discard the file we placed in
  980. // c->inputs_[0] earlier and replace it with an overlapping set
  981. // which will include the picked file.
  982. GetOverlappingInputs(0, smallest, largest, &c->inputs_[0]);
  983. assert(!c->inputs_[0].empty());
  984. }
  985. SetupOtherInputs(c);
  986. return c;
  987. }
  988. void VersionSet::SetupOtherInputs(Compaction* c) {
  989. const int level = c->level();
  990. InternalKey smallest, largest;
  991. GetRange(c->inputs_[0], &smallest, &largest);
  992. GetOverlappingInputs(level+1, smallest, largest, &c->inputs_[1]);
  993. // Get entire range covered by compaction
  994. InternalKey all_start, all_limit;
  995. GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
  996. // See if we can grow the number of inputs in "level" without
  997. // changing the number of "level+1" files we pick up.
  998. if (!c->inputs_[1].empty()) {
  999. std::vector<FileMetaData*> expanded0;
  1000. GetOverlappingInputs(level, all_start, all_limit, &expanded0);
  1001. if (expanded0.size() > c->inputs_[0].size()) {
  1002. InternalKey new_start, new_limit;
  1003. GetRange(expanded0, &new_start, &new_limit);
  1004. std::vector<FileMetaData*> expanded1;
  1005. GetOverlappingInputs(level+1, new_start, new_limit, &expanded1);
  1006. if (expanded1.size() == c->inputs_[1].size()) {
  1007. Log(env_, options_->info_log,
  1008. "Expanding@%d %d+%d to %d+%d\n",
  1009. level,
  1010. int(c->inputs_[0].size()),
  1011. int(c->inputs_[1].size()),
  1012. int(expanded0.size()),
  1013. int(expanded1.size()));
  1014. smallest = new_start;
  1015. largest = new_limit;
  1016. c->inputs_[0] = expanded0;
  1017. c->inputs_[1] = expanded1;
  1018. GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
  1019. }
  1020. }
  1021. }
  1022. // Compute the set of grandparent files that overlap this compaction
  1023. // (parent == level+1; grandparent == level+2)
  1024. if (level + 2 < config::kNumLevels) {
  1025. GetOverlappingInputs(level + 2, all_start, all_limit, &c->grandparents_);
  1026. }
  1027. if (false) {
  1028. Log(env_, options_->info_log, "Compacting %d '%s' .. '%s'",
  1029. level,
  1030. EscapeString(smallest.Encode()).c_str(),
  1031. EscapeString(largest.Encode()).c_str());
  1032. }
  1033. // Update the place where we will do the next compaction for this level.
  1034. // We update this immediately instead of waiting for the VersionEdit
  1035. // to be applied so that if the compaction fails, we will try a different
  1036. // key range next time.
  1037. compact_pointer_[level] = largest.Encode().ToString();
  1038. c->edit_.SetCompactPointer(level, largest);
  1039. }
  1040. Compaction* VersionSet::CompactRange(
  1041. int level,
  1042. const InternalKey& begin,
  1043. const InternalKey& end) {
  1044. std::vector<FileMetaData*> inputs;
  1045. GetOverlappingInputs(level, begin, end, &inputs);
  1046. if (inputs.empty()) {
  1047. return NULL;
  1048. }
  1049. Compaction* c = new Compaction(level);
  1050. c->input_version_ = current_;
  1051. c->input_version_->Ref();
  1052. c->inputs_[0] = inputs;
  1053. SetupOtherInputs(c);
  1054. return c;
  1055. }
  1056. Compaction::Compaction(int level)
  1057. : level_(level),
  1058. max_output_file_size_(MaxFileSizeForLevel(level)),
  1059. input_version_(NULL),
  1060. grandparent_index_(0),
  1061. seen_key_(false),
  1062. overlapped_bytes_(0) {
  1063. for (int i = 0; i < config::kNumLevels; i++) {
  1064. level_ptrs_[i] = 0;
  1065. }
  1066. }
  1067. Compaction::~Compaction() {
  1068. if (input_version_ != NULL) {
  1069. input_version_->Unref();
  1070. }
  1071. }
  1072. bool Compaction::IsTrivialMove() const {
  1073. // Avoid a move if there is lots of overlapping grandparent data.
  1074. // Otherwise, the move could create a parent file that will require
  1075. // a very expensive merge later on.
  1076. return (num_input_files(0) == 1 &&
  1077. num_input_files(1) == 0 &&
  1078. TotalFileSize(grandparents_) <= kMaxGrandParentOverlapBytes);
  1079. }
  1080. void Compaction::AddInputDeletions(VersionEdit* edit) {
  1081. for (int which = 0; which < 2; which++) {
  1082. for (size_t i = 0; i < inputs_[which].size(); i++) {
  1083. edit->DeleteFile(level_ + which, inputs_[which][i]->number);
  1084. }
  1085. }
  1086. }
  1087. bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
  1088. // Maybe use binary search to find right entry instead of linear search?
  1089. const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
  1090. for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) {
  1091. const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
  1092. for (; level_ptrs_[lvl] < files.size(); ) {
  1093. FileMetaData* f = files[level_ptrs_[lvl]];
  1094. if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
  1095. // We've advanced far enough
  1096. if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
  1097. // Key falls in this file's range, so definitely not base level
  1098. return false;
  1099. }
  1100. break;
  1101. }
  1102. level_ptrs_[lvl]++;
  1103. }
  1104. }
  1105. return true;
  1106. }
  1107. bool Compaction::ShouldStopBefore(const Slice& internal_key) {
  1108. // Scan to find earliest grandparent file that contains key.
  1109. const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
  1110. while (grandparent_index_ < grandparents_.size() &&
  1111. icmp->Compare(internal_key,
  1112. grandparents_[grandparent_index_]->largest.Encode()) > 0) {
  1113. if (seen_key_) {
  1114. overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
  1115. }
  1116. grandparent_index_++;
  1117. }
  1118. seen_key_ = true;
  1119. if (overlapped_bytes_ > kMaxGrandParentOverlapBytes) {
  1120. // Too much overlap for current output; start new output
  1121. overlapped_bytes_ = 0;
  1122. return true;
  1123. } else {
  1124. return false;
  1125. }
  1126. }
  1127. void Compaction::ReleaseInputs() {
  1128. if (input_version_ != NULL) {
  1129. input_version_->Unref();
  1130. input_version_ = NULL;
  1131. }
  1132. }
  1133. }