作者: 谢瑞阳 10225101483 徐翔宇 10225101535
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

1060 rader
32 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "db/version_set.h"
  5. #include <algorithm>
  6. #include <stdio.h>
  7. #include "db/filename.h"
  8. #include "db/log_reader.h"
  9. #include "db/log_writer.h"
  10. #include "db/memtable.h"
  11. #include "db/table_cache.h"
  12. #include "leveldb/env.h"
  13. #include "leveldb/table_builder.h"
  14. #include "table/merger.h"
  15. #include "table/two_level_iterator.h"
  16. #include "util/coding.h"
  17. #include "util/logging.h"
  18. namespace leveldb {
  19. static const int kTargetFileSize = 2 * 1048576;
  20. // Maximum bytes of overlaps in grandparent (i.e., level+2) before we
  21. // stop building a single file in a level->level+1 compaction.
  22. static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize;
  23. static double MaxBytesForLevel(int level) {
  24. // Note: the result for level zero is not really used since we set
  25. // the level-0 compaction threshold based on number of files.
  26. double result = 10 * 1048576.0; // Result for both level-0 and level-1
  27. while (level > 1) {
  28. result *= 10;
  29. level--;
  30. }
  31. return result;
  32. }
  33. static uint64_t MaxFileSizeForLevel(int level) {
  34. return kTargetFileSize; // We could vary per level to reduce number of files?
  35. }
  36. namespace {
  37. std::string IntSetToString(const std::set<uint64_t>& s) {
  38. std::string result = "{";
  39. for (std::set<uint64_t>::const_iterator it = s.begin();
  40. it != s.end();
  41. ++it) {
  42. result += (result.size() > 1) ? "," : "";
  43. result += NumberToString(*it);
  44. }
  45. result += "}";
  46. return result;
  47. }
  48. }
  49. Version::~Version() {
  50. assert(refs_ == 0);
  51. // Remove from linked list
  52. prev_->next_ = next_;
  53. next_->prev_ = prev_;
  54. // Drop references to files
  55. for (int level = 0; level < config::kNumLevels; level++) {
  56. for (size_t i = 0; i < files_[level].size(); i++) {
  57. FileMetaData* f = files_[level][i];
  58. assert(f->refs > 0);
  59. f->refs--;
  60. if (f->refs <= 0) {
  61. delete f;
  62. }
  63. }
  64. }
  65. }
  66. // An internal iterator. For a given version/level pair, yields
  67. // information about the files in the level. For a given entry, key()
  68. // is the largest key that occurs in the file, and value() is an
  69. // 16-byte value containing the file number and file size, both
  70. // encoded using EncodeFixed64.
  71. class Version::LevelFileNumIterator : public Iterator {
  72. public:
  73. LevelFileNumIterator(const InternalKeyComparator& icmp,
  74. const std::vector<FileMetaData*>* flist)
  75. : icmp_(icmp),
  76. flist_(flist),
  77. index_(flist->size()) { // Marks as invalid
  78. }
  79. virtual bool Valid() const {
  80. return index_ < flist_->size();
  81. }
  82. virtual void Seek(const Slice& target) {
  83. uint32_t left = 0;
  84. uint32_t right = flist_->size() - 1;
  85. while (left < right) {
  86. uint32_t mid = (left + right) / 2;
  87. int cmp = icmp_.Compare((*flist_)[mid]->largest.Encode(), target);
  88. if (cmp < 0) {
  89. // Key at "mid.largest" is < than "target". Therefore all
  90. // files at or before "mid" are uninteresting.
  91. left = mid + 1;
  92. } else {
  93. // Key at "mid.largest" is >= "target". Therefore all files
  94. // after "mid" are uninteresting.
  95. right = mid;
  96. }
  97. }
  98. index_ = left;
  99. }
  100. virtual void SeekToFirst() { index_ = 0; }
  101. virtual void SeekToLast() {
  102. index_ = flist_->empty() ? 0 : flist_->size() - 1;
  103. }
  104. virtual void Next() {
  105. assert(Valid());
  106. index_++;
  107. }
  108. virtual void Prev() {
  109. assert(Valid());
  110. if (index_ == 0) {
  111. index_ = flist_->size(); // Marks as invalid
  112. } else {
  113. index_--;
  114. }
  115. }
  116. Slice key() const {
  117. assert(Valid());
  118. return (*flist_)[index_]->largest.Encode();
  119. }
  120. Slice value() const {
  121. assert(Valid());
  122. EncodeFixed64(value_buf_, (*flist_)[index_]->number);
  123. EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size);
  124. return Slice(value_buf_, sizeof(value_buf_));
  125. }
  126. virtual Status status() const { return Status::OK(); }
  127. private:
  128. const InternalKeyComparator icmp_;
  129. const std::vector<FileMetaData*>* const flist_;
  130. uint32_t index_;
  131. // Backing store for value(). Holds the file number and size.
  132. mutable char value_buf_[16];
  133. };
  134. static Iterator* GetFileIterator(void* arg,
  135. const ReadOptions& options,
  136. const Slice& file_value) {
  137. TableCache* cache = reinterpret_cast<TableCache*>(arg);
  138. if (file_value.size() != 16) {
  139. return NewErrorIterator(
  140. Status::Corruption("FileReader invoked with unexpected value"));
  141. } else {
  142. return cache->NewIterator(options,
  143. DecodeFixed64(file_value.data()),
  144. DecodeFixed64(file_value.data() + 8));
  145. }
  146. }
  147. Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
  148. int level) const {
  149. return NewTwoLevelIterator(
  150. new LevelFileNumIterator(vset_->icmp_, &files_[level]),
  151. &GetFileIterator, vset_->table_cache_, options);
  152. }
  153. void Version::AddIterators(const ReadOptions& options,
  154. std::vector<Iterator*>* iters) {
  155. // Merge all level zero files together since they may overlap
  156. for (size_t i = 0; i < files_[0].size(); i++) {
  157. iters->push_back(
  158. vset_->table_cache_->NewIterator(
  159. options, files_[0][i]->number, files_[0][i]->file_size));
  160. }
  161. // For levels > 0, we can use a concatenating iterator that sequentially
  162. // walks through the non-overlapping files in the level, opening them
  163. // lazily.
  164. for (int level = 1; level < config::kNumLevels; level++) {
  165. if (!files_[level].empty()) {
  166. iters->push_back(NewConcatenatingIterator(options, level));
  167. }
  168. }
  169. }
  170. void Version::Ref() {
  171. ++refs_;
  172. }
  173. void Version::Unref() {
  174. assert(this != &vset_->dummy_versions_);
  175. assert(refs_ >= 1);
  176. --refs_;
  177. if (refs_ == 0) {
  178. delete this;
  179. }
  180. }
  181. std::string Version::DebugString() const {
  182. std::string r;
  183. for (int level = 0; level < config::kNumLevels; level++) {
  184. // E.g., level 1: 17:123['a' .. 'd'] 20:43['e' .. 'g']
  185. r.append("level ");
  186. AppendNumberTo(&r, level);
  187. r.push_back(':');
  188. const std::vector<FileMetaData*>& files = files_[level];
  189. for (size_t i = 0; i < files.size(); i++) {
  190. r.push_back(' ');
  191. AppendNumberTo(&r, files[i]->number);
  192. r.push_back(':');
  193. AppendNumberTo(&r, files[i]->file_size);
  194. r.append("['");
  195. AppendEscapedStringTo(&r, files[i]->smallest.Encode());
  196. r.append("' .. '");
  197. AppendEscapedStringTo(&r, files[i]->largest.Encode());
  198. r.append("']");
  199. }
  200. r.push_back('\n');
  201. }
  202. return r;
  203. }
  204. // A helper class so we can efficiently apply a whole sequence
  205. // of edits to a particular state without creating intermediate
  206. // Versions that contain full copies of the intermediate state.
  207. class VersionSet::Builder {
  208. private:
  209. // Helper to sort by v->files_[file_number].smallest
  210. struct BySmallestKey {
  211. const InternalKeyComparator* internal_comparator;
  212. bool operator()(FileMetaData* f1, FileMetaData* f2) const {
  213. int r = internal_comparator->Compare(f1->smallest, f2->smallest);
  214. if (r != 0) {
  215. return (r < 0);
  216. } else {
  217. // Break ties by file number
  218. return (f1->number < f2->number);
  219. }
  220. }
  221. };
  222. typedef std::set<FileMetaData*, BySmallestKey> FileSet;
  223. struct LevelState {
  224. std::set<uint64_t> deleted_files;
  225. FileSet* added_files;
  226. };
  227. VersionSet* vset_;
  228. Version* base_;
  229. LevelState levels_[config::kNumLevels];
  230. public:
  231. // Initialize a builder with the files from *base and other info from *vset
  232. Builder(VersionSet* vset, Version* base)
  233. : vset_(vset),
  234. base_(base) {
  235. base_->Ref();
  236. BySmallestKey cmp;
  237. cmp.internal_comparator = &vset_->icmp_;
  238. for (int level = 0; level < config::kNumLevels; level++) {
  239. levels_[level].added_files = new FileSet(cmp);
  240. }
  241. }
  242. ~Builder() {
  243. for (int level = 0; level < config::kNumLevels; level++) {
  244. std::vector<FileMetaData*> to_unref(levels_[level].added_files->begin(),
  245. levels_[level].added_files->end());
  246. delete levels_[level].added_files;
  247. for (int i = 0; i < to_unref.size(); i++) {
  248. FileMetaData* f = to_unref[i];
  249. f->refs--;
  250. if (f->refs <= 0) {
  251. delete f;
  252. }
  253. }
  254. }
  255. base_->Unref();
  256. }
  257. // Apply all of the edits in *edit to the current state.
  258. void Apply(VersionEdit* edit) {
  259. // Update compaction pointers
  260. for (size_t i = 0; i < edit->compact_pointers_.size(); i++) {
  261. const int level = edit->compact_pointers_[i].first;
  262. vset_->compact_pointer_[level] =
  263. edit->compact_pointers_[i].second.Encode().ToString();
  264. }
  265. // Delete files
  266. const VersionEdit::DeletedFileSet& del = edit->deleted_files_;
  267. for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin();
  268. iter != del.end();
  269. ++iter) {
  270. const int level = iter->first;
  271. const uint64_t number = iter->second;
  272. levels_[level].deleted_files.insert(number);
  273. }
  274. // Add new files
  275. for (size_t i = 0; i < edit->new_files_.size(); i++) {
  276. const int level = edit->new_files_[i].first;
  277. FileMetaData* f = new FileMetaData(edit->new_files_[i].second);
  278. f->refs = 1;
  279. levels_[level].deleted_files.erase(f->number);
  280. levels_[level].added_files->insert(f);
  281. }
  282. }
  283. // Save the current state in *v.
  284. void SaveTo(Version* v) {
  285. BySmallestKey cmp;
  286. cmp.internal_comparator = &vset_->icmp_;
  287. for (int level = 0; level < config::kNumLevels; level++) {
  288. // Merge the set of added files with the set of pre-existing files.
  289. // Drop any deleted files. Store the result in *v.
  290. const std::vector<FileMetaData*>& base_files = base_->files_[level];
  291. std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin();
  292. std::vector<FileMetaData*>::const_iterator base_end = base_files.end();
  293. const FileSet* added = levels_[level].added_files;
  294. v->files_[level].reserve(base_files.size() + added->size());
  295. for (FileSet::const_iterator added_iter = added->begin();
  296. added_iter != added->end();
  297. ++added_iter) {
  298. // Add all smaller files listed in base_
  299. for (std::vector<FileMetaData*>::const_iterator bpos
  300. = std::upper_bound(base_iter, base_end, *added_iter, cmp);
  301. base_iter != bpos;
  302. ++base_iter) {
  303. MaybeAddFile(v, level, *base_iter);
  304. }
  305. MaybeAddFile(v, level, *added_iter);
  306. }
  307. // Add remaining base files
  308. for (; base_iter != base_end; ++base_iter) {
  309. MaybeAddFile(v, level, *base_iter);
  310. }
  311. #ifndef NDEBUG
  312. // Make sure there is no overlap in levels > 0
  313. if (level > 0) {
  314. for (int i = 1; i < v->files_[level].size(); i++) {
  315. const InternalKey& prev_end = v->files_[level][i-1]->largest;
  316. const InternalKey& this_begin = v->files_[level][i]->smallest;
  317. if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) {
  318. fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
  319. EscapeString(prev_end.Encode()).c_str(),
  320. EscapeString(this_begin.Encode()).c_str());
  321. abort();
  322. }
  323. }
  324. }
  325. #endif
  326. }
  327. }
  328. void MaybeAddFile(Version* v, int level, FileMetaData* f) {
  329. if (levels_[level].deleted_files.count(f->number) > 0) {
  330. // File is deleted: do nothing
  331. } else {
  332. f->refs++;
  333. v->files_[level].push_back(f);
  334. }
  335. }
  336. };
  337. VersionSet::VersionSet(const std::string& dbname,
  338. const Options* options,
  339. TableCache* table_cache,
  340. const InternalKeyComparator* cmp)
  341. : env_(options->env),
  342. dbname_(dbname),
  343. options_(options),
  344. table_cache_(table_cache),
  345. icmp_(*cmp),
  346. next_file_number_(2),
  347. manifest_file_number_(0), // Filled by Recover()
  348. last_sequence_(0),
  349. log_number_(0),
  350. prev_log_number_(0),
  351. descriptor_file_(NULL),
  352. descriptor_log_(NULL),
  353. dummy_versions_(this),
  354. current_(NULL) {
  355. AppendVersion(new Version(this));
  356. }
  357. VersionSet::~VersionSet() {
  358. current_->Unref();
  359. assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty
  360. delete descriptor_log_;
  361. delete descriptor_file_;
  362. }
  363. void VersionSet::AppendVersion(Version* v) {
  364. // Make "v" current
  365. assert(v->refs_ == 0);
  366. assert(v != current_);
  367. if (current_ != NULL) {
  368. current_->Unref();
  369. }
  370. current_ = v;
  371. v->Ref();
  372. // Append to linked list
  373. v->prev_ = dummy_versions_.prev_;
  374. v->next_ = &dummy_versions_;
  375. v->prev_->next_ = v;
  376. v->next_->prev_ = v;
  377. }
  378. Status VersionSet::LogAndApply(VersionEdit* edit) {
  379. if (edit->has_log_number_) {
  380. assert(edit->log_number_ >= log_number_);
  381. assert(edit->log_number_ < next_file_number_);
  382. } else {
  383. edit->SetLogNumber(log_number_);
  384. }
  385. if (!edit->has_prev_log_number_) {
  386. edit->SetPrevLogNumber(prev_log_number_);
  387. }
  388. edit->SetNextFile(next_file_number_);
  389. edit->SetLastSequence(last_sequence_);
  390. Version* v = new Version(this);
  391. {
  392. Builder builder(this, current_);
  393. builder.Apply(edit);
  394. builder.SaveTo(v);
  395. }
  396. Finalize(v);
  397. // Initialize new descriptor log file if necessary by creating
  398. // a temporary file that contains a snapshot of the current version.
  399. std::string new_manifest_file;
  400. Status s;
  401. if (descriptor_log_ == NULL) {
  402. assert(descriptor_file_ == NULL);
  403. new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_);
  404. edit->SetNextFile(next_file_number_);
  405. s = env_->NewWritableFile(new_manifest_file, &descriptor_file_);
  406. if (s.ok()) {
  407. descriptor_log_ = new log::Writer(descriptor_file_);
  408. s = WriteSnapshot(descriptor_log_);
  409. }
  410. }
  411. // Write new record to MANIFEST log
  412. if (s.ok()) {
  413. std::string record;
  414. edit->EncodeTo(&record);
  415. s = descriptor_log_->AddRecord(record);
  416. if (s.ok()) {
  417. s = descriptor_file_->Sync();
  418. }
  419. }
  420. // If we just created a new descriptor file, install it by writing a
  421. // new CURRENT file that points to it.
  422. if (s.ok() && !new_manifest_file.empty()) {
  423. s = SetCurrentFile(env_, dbname_, manifest_file_number_);
  424. }
  425. // Install the new version
  426. if (s.ok()) {
  427. AppendVersion(v);
  428. log_number_ = edit->log_number_;
  429. prev_log_number_ = edit->prev_log_number_;
  430. } else {
  431. delete v;
  432. if (!new_manifest_file.empty()) {
  433. delete descriptor_log_;
  434. delete descriptor_file_;
  435. descriptor_log_ = NULL;
  436. descriptor_file_ = NULL;
  437. env_->DeleteFile(new_manifest_file);
  438. }
  439. }
  440. return s;
  441. }
  442. Status VersionSet::Recover() {
  443. struct LogReporter : public log::Reader::Reporter {
  444. Status* status;
  445. virtual void Corruption(size_t bytes, const Status& s) {
  446. if (this->status->ok()) *this->status = s;
  447. }
  448. };
  449. // Read "CURRENT" file, which contains a pointer to the current manifest file
  450. std::string current;
  451. Status s = ReadFileToString(env_, CurrentFileName(dbname_), &current);
  452. if (!s.ok()) {
  453. return s;
  454. }
  455. if (current.empty() || current[current.size()-1] != '\n') {
  456. return Status::Corruption("CURRENT file does not end with newline");
  457. }
  458. current.resize(current.size() - 1);
  459. std::string dscname = dbname_ + "/" + current;
  460. SequentialFile* file;
  461. s = env_->NewSequentialFile(dscname, &file);
  462. if (!s.ok()) {
  463. return s;
  464. }
  465. bool have_log_number = false;
  466. bool have_prev_log_number = false;
  467. bool have_next_file = false;
  468. bool have_last_sequence = false;
  469. uint64_t next_file = 0;
  470. uint64_t last_sequence = 0;
  471. uint64_t log_number = 0;
  472. uint64_t prev_log_number = 0;
  473. Builder builder(this, current_);
  474. {
  475. LogReporter reporter;
  476. reporter.status = &s;
  477. log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/);
  478. Slice record;
  479. std::string scratch;
  480. while (reader.ReadRecord(&record, &scratch) && s.ok()) {
  481. VersionEdit edit;
  482. s = edit.DecodeFrom(record);
  483. if (s.ok()) {
  484. if (edit.has_comparator_ &&
  485. edit.comparator_ != icmp_.user_comparator()->Name()) {
  486. s = Status::InvalidArgument(
  487. edit.comparator_ + "does not match existing comparator ",
  488. icmp_.user_comparator()->Name());
  489. }
  490. }
  491. if (s.ok()) {
  492. builder.Apply(&edit);
  493. }
  494. if (edit.has_log_number_) {
  495. log_number = edit.log_number_;
  496. have_log_number = true;
  497. }
  498. if (edit.has_prev_log_number_) {
  499. prev_log_number = edit.prev_log_number_;
  500. have_prev_log_number = true;
  501. }
  502. if (edit.has_next_file_number_) {
  503. next_file = edit.next_file_number_;
  504. have_next_file = true;
  505. }
  506. if (edit.has_last_sequence_) {
  507. last_sequence = edit.last_sequence_;
  508. have_last_sequence = true;
  509. }
  510. }
  511. }
  512. delete file;
  513. file = NULL;
  514. if (s.ok()) {
  515. if (!have_next_file) {
  516. s = Status::Corruption("no meta-nextfile entry in descriptor");
  517. } else if (!have_log_number) {
  518. s = Status::Corruption("no meta-lognumber entry in descriptor");
  519. } else if (!have_last_sequence) {
  520. s = Status::Corruption("no last-sequence-number entry in descriptor");
  521. }
  522. if (!have_prev_log_number) {
  523. prev_log_number = 0;
  524. }
  525. }
  526. if (s.ok()) {
  527. Version* v = new Version(this);
  528. builder.SaveTo(v);
  529. // Install recovered version
  530. Finalize(v);
  531. AppendVersion(v);
  532. manifest_file_number_ = next_file;
  533. next_file_number_ = next_file + 1;
  534. last_sequence_ = last_sequence;
  535. log_number_ = log_number;
  536. prev_log_number_ = prev_log_number;
  537. }
  538. return s;
  539. }
  540. static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
  541. int64_t sum = 0;
  542. for (size_t i = 0; i < files.size(); i++) {
  543. sum += files[i]->file_size;
  544. }
  545. return sum;
  546. }
  547. void VersionSet::Finalize(Version* v) {
  548. // Precomputed best level for next compaction
  549. int best_level = -1;
  550. double best_score = -1;
  551. for (int level = 0; level < config::kNumLevels-1; level++) {
  552. double score;
  553. if (level == 0) {
  554. // We treat level-0 specially by bounding the number of files
  555. // instead of number of bytes for two reasons:
  556. //
  557. // (1) With larger write-buffer sizes, it is nice not to do too
  558. // many level-0 compactions.
  559. //
  560. // (2) The files in level-0 are merged on every read and
  561. // therefore we wish to avoid too many files when the individual
  562. // file size is small (perhaps because of a small write-buffer
  563. // setting, or very high compression ratios, or lots of
  564. // overwrites/deletions).
  565. score = v->files_[level].size() /
  566. static_cast<double>(config::kL0_CompactionTrigger);
  567. } else {
  568. // Compute the ratio of current size to size limit.
  569. const uint64_t level_bytes = TotalFileSize(v->files_[level]);
  570. score = static_cast<double>(level_bytes) / MaxBytesForLevel(level);
  571. }
  572. if (score > best_score) {
  573. best_level = level;
  574. best_score = score;
  575. }
  576. }
  577. v->compaction_level_ = best_level;
  578. v->compaction_score_ = best_score;
  579. }
  580. Status VersionSet::WriteSnapshot(log::Writer* log) {
  581. // TODO: Break up into multiple records to reduce memory usage on recovery?
  582. // Save metadata
  583. VersionEdit edit;
  584. edit.SetComparatorName(icmp_.user_comparator()->Name());
  585. // Save compaction pointers
  586. for (int level = 0; level < config::kNumLevels; level++) {
  587. if (!compact_pointer_[level].empty()) {
  588. InternalKey key;
  589. key.DecodeFrom(compact_pointer_[level]);
  590. edit.SetCompactPointer(level, key);
  591. }
  592. }
  593. // Save files
  594. for (int level = 0; level < config::kNumLevels; level++) {
  595. const std::vector<FileMetaData*>& files = current_->files_[level];
  596. for (size_t i = 0; i < files.size(); i++) {
  597. const FileMetaData* f = files[i];
  598. edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest);
  599. }
  600. }
  601. std::string record;
  602. edit.EncodeTo(&record);
  603. return log->AddRecord(record);
  604. }
  605. int VersionSet::NumLevelFiles(int level) const {
  606. assert(level >= 0);
  607. assert(level < config::kNumLevels);
  608. return current_->files_[level].size();
  609. }
  610. const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
  611. // Update code if kNumLevels changes
  612. assert(config::kNumLevels == 7);
  613. snprintf(scratch->buffer, sizeof(scratch->buffer),
  614. "files[ %d %d %d %d %d %d %d ]",
  615. int(current_->files_[0].size()),
  616. int(current_->files_[1].size()),
  617. int(current_->files_[2].size()),
  618. int(current_->files_[3].size()),
  619. int(current_->files_[4].size()),
  620. int(current_->files_[5].size()),
  621. int(current_->files_[6].size()));
  622. return scratch->buffer;
  623. }
  624. uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
  625. uint64_t result = 0;
  626. for (int level = 0; level < config::kNumLevels; level++) {
  627. const std::vector<FileMetaData*>& files = v->files_[level];
  628. for (size_t i = 0; i < files.size(); i++) {
  629. if (icmp_.Compare(files[i]->largest, ikey) <= 0) {
  630. // Entire file is before "ikey", so just add the file size
  631. result += files[i]->file_size;
  632. } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) {
  633. // Entire file is after "ikey", so ignore
  634. if (level > 0) {
  635. // Files other than level 0 are sorted by meta->smallest, so
  636. // no further files in this level will contain data for
  637. // "ikey".
  638. break;
  639. }
  640. } else {
  641. // "ikey" falls in the range for this table. Add the
  642. // approximate offset of "ikey" within the table.
  643. Table* tableptr;
  644. Iterator* iter = table_cache_->NewIterator(
  645. ReadOptions(), files[i]->number, files[i]->file_size, &tableptr);
  646. if (tableptr != NULL) {
  647. result += tableptr->ApproximateOffsetOf(ikey.Encode());
  648. }
  649. delete iter;
  650. }
  651. }
  652. }
  653. return result;
  654. }
  655. void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
  656. for (Version* v = dummy_versions_.next_;
  657. v != &dummy_versions_;
  658. v = v->next_) {
  659. for (int level = 0; level < config::kNumLevels; level++) {
  660. const std::vector<FileMetaData*>& files = v->files_[level];
  661. for (size_t i = 0; i < files.size(); i++) {
  662. live->insert(files[i]->number);
  663. }
  664. }
  665. }
  666. }
  667. int64_t VersionSet::NumLevelBytes(int level) const {
  668. assert(level >= 0);
  669. assert(level < config::kNumLevels);
  670. return TotalFileSize(current_->files_[level]);
  671. }
  672. int64_t VersionSet::MaxNextLevelOverlappingBytes() {
  673. int64_t result = 0;
  674. std::vector<FileMetaData*> overlaps;
  675. for (int level = 0; level < config::kNumLevels - 1; level++) {
  676. for (size_t i = 0; i < current_->files_[level].size(); i++) {
  677. const FileMetaData* f = current_->files_[level][i];
  678. GetOverlappingInputs(level+1, f->smallest, f->largest, &overlaps);
  679. const int64_t sum = TotalFileSize(overlaps);
  680. if (sum > result) {
  681. result = sum;
  682. }
  683. }
  684. }
  685. return result;
  686. }
  687. // Store in "*inputs" all files in "level" that overlap [begin,end]
  688. void VersionSet::GetOverlappingInputs(
  689. int level,
  690. const InternalKey& begin,
  691. const InternalKey& end,
  692. std::vector<FileMetaData*>* inputs) {
  693. inputs->clear();
  694. Slice user_begin = begin.user_key();
  695. Slice user_end = end.user_key();
  696. const Comparator* user_cmp = icmp_.user_comparator();
  697. for (size_t i = 0; i < current_->files_[level].size(); i++) {
  698. FileMetaData* f = current_->files_[level][i];
  699. if (user_cmp->Compare(f->largest.user_key(), user_begin) < 0 ||
  700. user_cmp->Compare(f->smallest.user_key(), user_end) > 0) {
  701. // Either completely before or after range; skip it
  702. } else {
  703. inputs->push_back(f);
  704. }
  705. }
  706. }
  707. // Stores the minimal range that covers all entries in inputs in
  708. // *smallest, *largest.
  709. // REQUIRES: inputs is not empty
  710. void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
  711. InternalKey* smallest,
  712. InternalKey* largest) {
  713. assert(!inputs.empty());
  714. smallest->Clear();
  715. largest->Clear();
  716. for (size_t i = 0; i < inputs.size(); i++) {
  717. FileMetaData* f = inputs[i];
  718. if (i == 0) {
  719. *smallest = f->smallest;
  720. *largest = f->largest;
  721. } else {
  722. if (icmp_.Compare(f->smallest, *smallest) < 0) {
  723. *smallest = f->smallest;
  724. }
  725. if (icmp_.Compare(f->largest, *largest) > 0) {
  726. *largest = f->largest;
  727. }
  728. }
  729. }
  730. }
  731. // Stores the minimal range that covers all entries in inputs1 and inputs2
  732. // in *smallest, *largest.
  733. // REQUIRES: inputs is not empty
  734. void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
  735. const std::vector<FileMetaData*>& inputs2,
  736. InternalKey* smallest,
  737. InternalKey* largest) {
  738. std::vector<FileMetaData*> all = inputs1;
  739. all.insert(all.end(), inputs2.begin(), inputs2.end());
  740. GetRange(all, smallest, largest);
  741. }
  742. Iterator* VersionSet::MakeInputIterator(Compaction* c) {
  743. ReadOptions options;
  744. options.verify_checksums = options_->paranoid_checks;
  745. options.fill_cache = false;
  746. // Level-0 files have to be merged together. For other levels,
  747. // we will make a concatenating iterator per level.
  748. // TODO(opt): use concatenating iterator for level-0 if there is no overlap
  749. const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2);
  750. Iterator** list = new Iterator*[space];
  751. int num = 0;
  752. for (int which = 0; which < 2; which++) {
  753. if (!c->inputs_[which].empty()) {
  754. if (c->level() + which == 0) {
  755. const std::vector<FileMetaData*>& files = c->inputs_[which];
  756. for (size_t i = 0; i < files.size(); i++) {
  757. list[num++] = table_cache_->NewIterator(
  758. options, files[i]->number, files[i]->file_size);
  759. }
  760. } else {
  761. // Create concatenating iterator for the files from this level
  762. list[num++] = NewTwoLevelIterator(
  763. new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]),
  764. &GetFileIterator, table_cache_, options);
  765. }
  766. }
  767. }
  768. assert(num <= space);
  769. Iterator* result = NewMergingIterator(&icmp_, list, num);
  770. delete[] list;
  771. return result;
  772. }
  773. Compaction* VersionSet::PickCompaction() {
  774. if (!NeedsCompaction()) {
  775. return NULL;
  776. }
  777. const int level = current_->compaction_level_;
  778. assert(level >= 0);
  779. assert(level+1 < config::kNumLevels);
  780. Compaction* c = new Compaction(level);
  781. c->input_version_ = current_;
  782. c->input_version_->Ref();
  783. // Pick the first file that comes after compact_pointer_[level]
  784. for (size_t i = 0; i < current_->files_[level].size(); i++) {
  785. FileMetaData* f = current_->files_[level][i];
  786. if (compact_pointer_[level].empty() ||
  787. icmp_.Compare(f->largest.Encode(), compact_pointer_[level]) > 0) {
  788. c->inputs_[0].push_back(f);
  789. break;
  790. }
  791. }
  792. if (c->inputs_[0].empty()) {
  793. // Wrap-around to the beginning of the key space
  794. c->inputs_[0].push_back(current_->files_[level][0]);
  795. }
  796. // Files in level 0 may overlap each other, so pick up all overlapping ones
  797. if (level == 0) {
  798. InternalKey smallest, largest;
  799. GetRange(c->inputs_[0], &smallest, &largest);
  800. // Note that the next call will discard the file we placed in
  801. // c->inputs_[0] earlier and replace it with an overlapping set
  802. // which will include the picked file.
  803. GetOverlappingInputs(0, smallest, largest, &c->inputs_[0]);
  804. assert(!c->inputs_[0].empty());
  805. }
  806. SetupOtherInputs(c);
  807. return c;
  808. }
  809. void VersionSet::SetupOtherInputs(Compaction* c) {
  810. const int level = c->level();
  811. InternalKey smallest, largest;
  812. GetRange(c->inputs_[0], &smallest, &largest);
  813. GetOverlappingInputs(level+1, smallest, largest, &c->inputs_[1]);
  814. // Get entire range covered by compaction
  815. InternalKey all_start, all_limit;
  816. GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
  817. // See if we can grow the number of inputs in "level" without
  818. // changing the number of "level+1" files we pick up.
  819. if (!c->inputs_[1].empty()) {
  820. std::vector<FileMetaData*> expanded0;
  821. GetOverlappingInputs(level, all_start, all_limit, &expanded0);
  822. if (expanded0.size() > c->inputs_[0].size()) {
  823. InternalKey new_start, new_limit;
  824. GetRange(expanded0, &new_start, &new_limit);
  825. std::vector<FileMetaData*> expanded1;
  826. GetOverlappingInputs(level+1, new_start, new_limit, &expanded1);
  827. if (expanded1.size() == c->inputs_[1].size()) {
  828. Log(env_, options_->info_log,
  829. "Expanding@%d %d+%d to %d+%d\n",
  830. level,
  831. int(c->inputs_[0].size()),
  832. int(c->inputs_[1].size()),
  833. int(expanded0.size()),
  834. int(expanded1.size()));
  835. smallest = new_start;
  836. largest = new_limit;
  837. c->inputs_[0] = expanded0;
  838. c->inputs_[1] = expanded1;
  839. GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
  840. }
  841. }
  842. }
  843. // Compute the set of grandparent files that overlap this compaction
  844. // (parent == level+1; grandparent == level+2)
  845. if (level + 2 < config::kNumLevels) {
  846. GetOverlappingInputs(level + 2, all_start, all_limit, &c->grandparents_);
  847. }
  848. if (false) {
  849. Log(env_, options_->info_log, "Compacting %d '%s' .. '%s'",
  850. level,
  851. EscapeString(smallest.Encode()).c_str(),
  852. EscapeString(largest.Encode()).c_str());
  853. }
  854. // Update the place where we will do the next compaction for this level.
  855. // We update this immediately instead of waiting for the VersionEdit
  856. // to be applied so that if the compaction fails, we will try a different
  857. // key range next time.
  858. compact_pointer_[level] = largest.Encode().ToString();
  859. c->edit_.SetCompactPointer(level, largest);
  860. }
  861. Compaction* VersionSet::CompactRange(
  862. int level,
  863. const InternalKey& begin,
  864. const InternalKey& end) {
  865. std::vector<FileMetaData*> inputs;
  866. GetOverlappingInputs(level, begin, end, &inputs);
  867. if (inputs.empty()) {
  868. return NULL;
  869. }
  870. Compaction* c = new Compaction(level);
  871. c->input_version_ = current_;
  872. c->input_version_->Ref();
  873. c->inputs_[0] = inputs;
  874. SetupOtherInputs(c);
  875. return c;
  876. }
  877. Compaction::Compaction(int level)
  878. : level_(level),
  879. max_output_file_size_(MaxFileSizeForLevel(level)),
  880. input_version_(NULL),
  881. grandparent_index_(0),
  882. seen_key_(false),
  883. overlapped_bytes_(0) {
  884. for (int i = 0; i < config::kNumLevels; i++) {
  885. level_ptrs_[i] = 0;
  886. }
  887. }
  888. Compaction::~Compaction() {
  889. if (input_version_ != NULL) {
  890. input_version_->Unref();
  891. }
  892. }
  893. bool Compaction::IsTrivialMove() const {
  894. // Avoid a move if there is lots of overlapping grandparent data.
  895. // Otherwise, the move could create a parent file that will require
  896. // a very expensive merge later on.
  897. return (num_input_files(0) == 1 &&
  898. num_input_files(1) == 0 &&
  899. TotalFileSize(grandparents_) <= kMaxGrandParentOverlapBytes);
  900. }
  901. void Compaction::AddInputDeletions(VersionEdit* edit) {
  902. for (int which = 0; which < 2; which++) {
  903. for (size_t i = 0; i < inputs_[which].size(); i++) {
  904. edit->DeleteFile(level_ + which, inputs_[which][i]->number);
  905. }
  906. }
  907. }
  908. bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
  909. // Maybe use binary search to find right entry instead of linear search?
  910. const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
  911. for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) {
  912. const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
  913. for (; level_ptrs_[lvl] < files.size(); ) {
  914. FileMetaData* f = files[level_ptrs_[lvl]];
  915. if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
  916. // We've advanced far enough
  917. if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
  918. // Key falls in this file's range, so definitely not base level
  919. return false;
  920. }
  921. break;
  922. }
  923. level_ptrs_[lvl]++;
  924. }
  925. }
  926. return true;
  927. }
  928. bool Compaction::ShouldStopBefore(const Slice& internal_key) {
  929. // Scan to find earliest grandparent file that contains key.
  930. const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
  931. while (grandparent_index_ < grandparents_.size() &&
  932. icmp->Compare(internal_key,
  933. grandparents_[grandparent_index_]->largest.Encode()) > 0) {
  934. if (seen_key_) {
  935. overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
  936. }
  937. grandparent_index_++;
  938. }
  939. seen_key_ = true;
  940. if (overlapped_bytes_ > kMaxGrandParentOverlapBytes) {
  941. // Too much overlap for current output; start new output
  942. overlapped_bytes_ = 0;
  943. return true;
  944. } else {
  945. return false;
  946. }
  947. }
  948. void Compaction::ReleaseInputs() {
  949. if (input_version_ != NULL) {
  950. input_version_->Unref();
  951. input_version_ = NULL;
  952. }
  953. }
  954. }