作者: 韩晨旭 10225101440 李畅 10225102463

1484 rindas
46 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "db/version_set.h"
  5. #include <algorithm>
  6. #include <stdio.h>
  7. #include "db/filename.h"
  8. #include "db/log_reader.h"
  9. #include "db/log_writer.h"
  10. #include "db/memtable.h"
  11. #include "db/table_cache.h"
  12. #include "leveldb/env.h"
  13. #include "leveldb/table_builder.h"
  14. #include "table/merger.h"
  15. #include "table/two_level_iterator.h"
  16. #include "util/coding.h"
  17. #include "util/logging.h"
  18. namespace leveldb {
  19. static const int kTargetFileSize = 2 * 1048576;
  20. // Maximum bytes of overlaps in grandparent (i.e., level+2) before we
  21. // stop building a single file in a level->level+1 compaction.
  22. static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize;
  23. // Maximum number of bytes in all compacted files. We avoid expanding
  24. // the lower level file set of a compaction if it would make the
  25. // total compaction cover more than this many bytes.
  26. static const int64_t kExpandedCompactionByteSizeLimit = 25 * kTargetFileSize;
  27. static double MaxBytesForLevel(int level) {
  28. // Note: the result for level zero is not really used since we set
  29. // the level-0 compaction threshold based on number of files.
  30. double result = 10 * 1048576.0; // Result for both level-0 and level-1
  31. while (level > 1) {
  32. result *= 10;
  33. level--;
  34. }
  35. return result;
  36. }
  37. static uint64_t MaxFileSizeForLevel(int level) {
  38. return kTargetFileSize; // We could vary per level to reduce number of files?
  39. }
  40. static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
  41. int64_t sum = 0;
  42. for (size_t i = 0; i < files.size(); i++) {
  43. sum += files[i]->file_size;
  44. }
  45. return sum;
  46. }
  47. Version::~Version() {
  48. assert(refs_ == 0);
  49. // Remove from linked list
  50. prev_->next_ = next_;
  51. next_->prev_ = prev_;
  52. // Drop references to files
  53. for (int level = 0; level < config::kNumLevels; level++) {
  54. for (size_t i = 0; i < files_[level].size(); i++) {
  55. FileMetaData* f = files_[level][i];
  56. assert(f->refs > 0);
  57. f->refs--;
  58. if (f->refs <= 0) {
  59. delete f;
  60. }
  61. }
  62. }
  63. }
  64. int FindFile(const InternalKeyComparator& icmp,
  65. const std::vector<FileMetaData*>& files,
  66. const Slice& key) {
  67. uint32_t left = 0;
  68. uint32_t right = files.size();
  69. while (left < right) {
  70. uint32_t mid = (left + right) / 2;
  71. const FileMetaData* f = files[mid];
  72. if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) {
  73. // Key at "mid.largest" is < "target". Therefore all
  74. // files at or before "mid" are uninteresting.
  75. left = mid + 1;
  76. } else {
  77. // Key at "mid.largest" is >= "target". Therefore all files
  78. // after "mid" are uninteresting.
  79. right = mid;
  80. }
  81. }
  82. return right;
  83. }
  84. static bool AfterFile(const Comparator* ucmp,
  85. const Slice* user_key, const FileMetaData* f) {
  86. // NULL user_key occurs before all keys and is therefore never after *f
  87. return (user_key != NULL &&
  88. ucmp->Compare(*user_key, f->largest.user_key()) > 0);
  89. }
  90. static bool BeforeFile(const Comparator* ucmp,
  91. const Slice* user_key, const FileMetaData* f) {
  92. // NULL user_key occurs after all keys and is therefore never before *f
  93. return (user_key != NULL &&
  94. ucmp->Compare(*user_key, f->smallest.user_key()) < 0);
  95. }
  96. bool SomeFileOverlapsRange(
  97. const InternalKeyComparator& icmp,
  98. bool disjoint_sorted_files,
  99. const std::vector<FileMetaData*>& files,
  100. const Slice* smallest_user_key,
  101. const Slice* largest_user_key) {
  102. const Comparator* ucmp = icmp.user_comparator();
  103. if (!disjoint_sorted_files) {
  104. // Need to check against all files
  105. for (size_t i = 0; i < files.size(); i++) {
  106. const FileMetaData* f = files[i];
  107. if (AfterFile(ucmp, smallest_user_key, f) ||
  108. BeforeFile(ucmp, largest_user_key, f)) {
  109. // No overlap
  110. } else {
  111. return true; // Overlap
  112. }
  113. }
  114. return false;
  115. }
  116. // Binary search over file list
  117. uint32_t index = 0;
  118. if (smallest_user_key != NULL) {
  119. // Find the earliest possible internal key for smallest_user_key
  120. InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek);
  121. index = FindFile(icmp, files, small.Encode());
  122. }
  123. if (index >= files.size()) {
  124. // beginning of range is after all files, so no overlap.
  125. return false;
  126. }
  127. return !BeforeFile(ucmp, largest_user_key, files[index]);
  128. }
  129. // An internal iterator. For a given version/level pair, yields
  130. // information about the files in the level. For a given entry, key()
  131. // is the largest key that occurs in the file, and value() is an
  132. // 16-byte value containing the file number and file size, both
  133. // encoded using EncodeFixed64.
  134. class Version::LevelFileNumIterator : public Iterator {
  135. public:
  136. LevelFileNumIterator(const InternalKeyComparator& icmp,
  137. const std::vector<FileMetaData*>* flist)
  138. : icmp_(icmp),
  139. flist_(flist),
  140. index_(flist->size()) { // Marks as invalid
  141. }
  142. virtual bool Valid() const {
  143. return index_ < flist_->size();
  144. }
  145. virtual void Seek(const Slice& target) {
  146. index_ = FindFile(icmp_, *flist_, target);
  147. }
  148. virtual void SeekToFirst() { index_ = 0; }
  149. virtual void SeekToLast() {
  150. index_ = flist_->empty() ? 0 : flist_->size() - 1;
  151. }
  152. virtual void Next() {
  153. assert(Valid());
  154. index_++;
  155. }
  156. virtual void Prev() {
  157. assert(Valid());
  158. if (index_ == 0) {
  159. index_ = flist_->size(); // Marks as invalid
  160. } else {
  161. index_--;
  162. }
  163. }
  164. Slice key() const {
  165. assert(Valid());
  166. return (*flist_)[index_]->largest.Encode();
  167. }
  168. Slice value() const {
  169. assert(Valid());
  170. EncodeFixed64(value_buf_, (*flist_)[index_]->number);
  171. EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size);
  172. return Slice(value_buf_, sizeof(value_buf_));
  173. }
  174. virtual Status status() const { return Status::OK(); }
  175. private:
  176. const InternalKeyComparator icmp_;
  177. const std::vector<FileMetaData*>* const flist_;
  178. uint32_t index_;
  179. // Backing store for value(). Holds the file number and size.
  180. mutable char value_buf_[16];
  181. };
  182. static Iterator* GetFileIterator(void* arg,
  183. const ReadOptions& options,
  184. const Slice& file_value) {
  185. TableCache* cache = reinterpret_cast<TableCache*>(arg);
  186. if (file_value.size() != 16) {
  187. return NewErrorIterator(
  188. Status::Corruption("FileReader invoked with unexpected value"));
  189. } else {
  190. return cache->NewIterator(options,
  191. DecodeFixed64(file_value.data()),
  192. DecodeFixed64(file_value.data() + 8));
  193. }
  194. }
  195. Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
  196. int level) const {
  197. return NewTwoLevelIterator(
  198. new LevelFileNumIterator(vset_->icmp_, &files_[level]),
  199. &GetFileIterator, vset_->table_cache_, options);
  200. }
  201. void Version::AddIterators(const ReadOptions& options,
  202. std::vector<Iterator*>* iters) {
  203. // Merge all level zero files together since they may overlap
  204. for (size_t i = 0; i < files_[0].size(); i++) {
  205. iters->push_back(
  206. vset_->table_cache_->NewIterator(
  207. options, files_[0][i]->number, files_[0][i]->file_size));
  208. }
  209. // For levels > 0, we can use a concatenating iterator that sequentially
  210. // walks through the non-overlapping files in the level, opening them
  211. // lazily.
  212. for (int level = 1; level < config::kNumLevels; level++) {
  213. if (!files_[level].empty()) {
  214. iters->push_back(NewConcatenatingIterator(options, level));
  215. }
  216. }
  217. }
  218. // Callback from TableCache::Get()
  219. namespace {
  220. enum SaverState {
  221. kNotFound,
  222. kFound,
  223. kDeleted,
  224. kCorrupt,
  225. };
  226. struct Saver {
  227. SaverState state;
  228. const Comparator* ucmp;
  229. Slice user_key;
  230. std::string* value;
  231. };
  232. }
  233. static void SaveValue(void* arg, const Slice& ikey, const Slice& v) {
  234. Saver* s = reinterpret_cast<Saver*>(arg);
  235. ParsedInternalKey parsed_key;
  236. if (!ParseInternalKey(ikey, &parsed_key)) {
  237. s->state = kCorrupt;
  238. } else {
  239. if (s->ucmp->Compare(parsed_key.user_key, s->user_key) == 0) {
  240. s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted;
  241. if (s->state == kFound) {
  242. s->value->assign(v.data(), v.size());
  243. }
  244. }
  245. }
  246. }
  247. static bool NewestFirst(FileMetaData* a, FileMetaData* b) {
  248. return a->number > b->number;
  249. }
  250. void Version::ForEachOverlapping(Slice user_key, Slice internal_key,
  251. void* arg,
  252. bool (*func)(void*, int, FileMetaData*)) {
  253. // TODO(sanjay): Change Version::Get() to use this function.
  254. const Comparator* ucmp = vset_->icmp_.user_comparator();
  255. // Search level-0 in order from newest to oldest.
  256. std::vector<FileMetaData*> tmp;
  257. tmp.reserve(files_[0].size());
  258. for (uint32_t i = 0; i < files_[0].size(); i++) {
  259. FileMetaData* f = files_[0][i];
  260. if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
  261. ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
  262. tmp.push_back(f);
  263. }
  264. }
  265. if (!tmp.empty()) {
  266. std::sort(tmp.begin(), tmp.end(), NewestFirst);
  267. for (uint32_t i = 0; i < tmp.size(); i++) {
  268. if (!(*func)(arg, 0, tmp[i])) {
  269. return;
  270. }
  271. }
  272. }
  273. // Search other levels.
  274. for (int level = 1; level < config::kNumLevels; level++) {
  275. size_t num_files = files_[level].size();
  276. if (num_files == 0) continue;
  277. // Binary search to find earliest index whose largest key >= internal_key.
  278. uint32_t index = FindFile(vset_->icmp_, files_[level], internal_key);
  279. if (index < num_files) {
  280. FileMetaData* f = files_[level][index];
  281. if (ucmp->Compare(user_key, f->smallest.user_key()) < 0) {
  282. // All of "f" is past any data for user_key
  283. } else {
  284. if (!(*func)(arg, level, f)) {
  285. return;
  286. }
  287. }
  288. }
  289. }
  290. }
  291. Status Version::Get(const ReadOptions& options,
  292. const LookupKey& k,
  293. std::string* value,
  294. GetStats* stats) {
  295. Slice ikey = k.internal_key();
  296. Slice user_key = k.user_key();
  297. const Comparator* ucmp = vset_->icmp_.user_comparator();
  298. Status s;
  299. stats->seek_file = NULL;
  300. stats->seek_file_level = -1;
  301. FileMetaData* last_file_read = NULL;
  302. int last_file_read_level = -1;
  303. // We can search level-by-level since entries never hop across
  304. // levels. Therefore we are guaranteed that if we find data
  305. // in an smaller level, later levels are irrelevant.
  306. std::vector<FileMetaData*> tmp;
  307. FileMetaData* tmp2;
  308. for (int level = 0; level < config::kNumLevels; level++) {
  309. size_t num_files = files_[level].size();
  310. if (num_files == 0) continue;
  311. // Get the list of files to search in this level
  312. FileMetaData* const* files = &files_[level][0];
  313. if (level == 0) {
  314. // Level-0 files may overlap each other. Find all files that
  315. // overlap user_key and process them in order from newest to oldest.
  316. tmp.reserve(num_files);
  317. for (uint32_t i = 0; i < num_files; i++) {
  318. FileMetaData* f = files[i];
  319. if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
  320. ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
  321. tmp.push_back(f);
  322. }
  323. }
  324. if (tmp.empty()) continue;
  325. std::sort(tmp.begin(), tmp.end(), NewestFirst);
  326. files = &tmp[0];
  327. num_files = tmp.size();
  328. } else {
  329. // Binary search to find earliest index whose largest key >= ikey.
  330. uint32_t index = FindFile(vset_->icmp_, files_[level], ikey);
  331. if (index >= num_files) {
  332. files = NULL;
  333. num_files = 0;
  334. } else {
  335. tmp2 = files[index];
  336. if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
  337. // All of "tmp2" is past any data for user_key
  338. files = NULL;
  339. num_files = 0;
  340. } else {
  341. files = &tmp2;
  342. num_files = 1;
  343. }
  344. }
  345. }
  346. for (uint32_t i = 0; i < num_files; ++i) {
  347. if (last_file_read != NULL && stats->seek_file == NULL) {
  348. // We have had more than one seek for this read. Charge the 1st file.
  349. stats->seek_file = last_file_read;
  350. stats->seek_file_level = last_file_read_level;
  351. }
  352. FileMetaData* f = files[i];
  353. last_file_read = f;
  354. last_file_read_level = level;
  355. Saver saver;
  356. saver.state = kNotFound;
  357. saver.ucmp = ucmp;
  358. saver.user_key = user_key;
  359. saver.value = value;
  360. s = vset_->table_cache_->Get(options, f->number, f->file_size,
  361. ikey, &saver, SaveValue);
  362. if (!s.ok()) {
  363. return s;
  364. }
  365. switch (saver.state) {
  366. case kNotFound:
  367. break; // Keep searching in other files
  368. case kFound:
  369. return s;
  370. case kDeleted:
  371. s = Status::NotFound(Slice()); // Use empty error message for speed
  372. return s;
  373. case kCorrupt:
  374. s = Status::Corruption("corrupted key for ", user_key);
  375. return s;
  376. }
  377. }
  378. }
  379. return Status::NotFound(Slice()); // Use an empty error message for speed
  380. }
  381. bool Version::UpdateStats(const GetStats& stats) {
  382. FileMetaData* f = stats.seek_file;
  383. if (f != NULL) {
  384. f->allowed_seeks--;
  385. if (f->allowed_seeks <= 0 && file_to_compact_ == NULL) {
  386. file_to_compact_ = f;
  387. file_to_compact_level_ = stats.seek_file_level;
  388. return true;
  389. }
  390. }
  391. return false;
  392. }
  393. bool Version::RecordReadSample(Slice internal_key) {
  394. ParsedInternalKey ikey;
  395. if (!ParseInternalKey(internal_key, &ikey)) {
  396. return false;
  397. }
  398. struct State {
  399. GetStats stats; // Holds first matching file
  400. int matches;
  401. static bool Match(void* arg, int level, FileMetaData* f) {
  402. State* state = reinterpret_cast<State*>(arg);
  403. state->matches++;
  404. if (state->matches == 1) {
  405. // Remember first match.
  406. state->stats.seek_file = f;
  407. state->stats.seek_file_level = level;
  408. }
  409. // We can stop iterating once we have a second match.
  410. return state->matches < 2;
  411. }
  412. };
  413. State state;
  414. state.matches = 0;
  415. ForEachOverlapping(ikey.user_key, internal_key, &state, &State::Match);
  416. // Must have at least two matches since we want to merge across
  417. // files. But what if we have a single file that contains many
  418. // overwrites and deletions? Should we have another mechanism for
  419. // finding such files?
  420. if (state.matches >= 2) {
  421. // 1MB cost is about 1 seek (see comment in Builder::Apply).
  422. return UpdateStats(state.stats);
  423. }
  424. return false;
  425. }
  426. void Version::Ref() {
  427. ++refs_;
  428. }
  429. void Version::Unref() {
  430. assert(this != &vset_->dummy_versions_);
  431. assert(refs_ >= 1);
  432. --refs_;
  433. if (refs_ == 0) {
  434. delete this;
  435. }
  436. }
  437. bool Version::OverlapInLevel(int level,
  438. const Slice* smallest_user_key,
  439. const Slice* largest_user_key) {
  440. return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level],
  441. smallest_user_key, largest_user_key);
  442. }
  443. int Version::PickLevelForMemTableOutput(
  444. const Slice& smallest_user_key,
  445. const Slice& largest_user_key) {
  446. int level = 0;
  447. if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) {
  448. // Push to next level if there is no overlap in next level,
  449. // and the #bytes overlapping in the level after that are limited.
  450. InternalKey start(smallest_user_key, kMaxSequenceNumber, kValueTypeForSeek);
  451. InternalKey limit(largest_user_key, 0, static_cast<ValueType>(0));
  452. std::vector<FileMetaData*> overlaps;
  453. while (level < config::kMaxMemCompactLevel) {
  454. if (OverlapInLevel(level + 1, &smallest_user_key, &largest_user_key)) {
  455. break;
  456. }
  457. if (level + 2 < config::kNumLevels) {
  458. // Check that file does not overlap too many grandparent bytes.
  459. GetOverlappingInputs(level + 2, &start, &limit, &overlaps);
  460. const int64_t sum = TotalFileSize(overlaps);
  461. if (sum > kMaxGrandParentOverlapBytes) {
  462. break;
  463. }
  464. }
  465. level++;
  466. }
  467. }
  468. return level;
  469. }
  470. // Store in "*inputs" all files in "level" that overlap [begin,end]
  471. void Version::GetOverlappingInputs(
  472. int level,
  473. const InternalKey* begin,
  474. const InternalKey* end,
  475. std::vector<FileMetaData*>* inputs) {
  476. assert(level >= 0);
  477. assert(level < config::kNumLevels);
  478. inputs->clear();
  479. Slice user_begin, user_end;
  480. if (begin != NULL) {
  481. user_begin = begin->user_key();
  482. }
  483. if (end != NULL) {
  484. user_end = end->user_key();
  485. }
  486. const Comparator* user_cmp = vset_->icmp_.user_comparator();
  487. for (size_t i = 0; i < files_[level].size(); ) {
  488. FileMetaData* f = files_[level][i++];
  489. const Slice file_start = f->smallest.user_key();
  490. const Slice file_limit = f->largest.user_key();
  491. if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) {
  492. // "f" is completely before specified range; skip it
  493. } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) {
  494. // "f" is completely after specified range; skip it
  495. } else {
  496. inputs->push_back(f);
  497. if (level == 0) {
  498. // Level-0 files may overlap each other. So check if the newly
  499. // added file has expanded the range. If so, restart search.
  500. if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) {
  501. user_begin = file_start;
  502. inputs->clear();
  503. i = 0;
  504. } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) {
  505. user_end = file_limit;
  506. inputs->clear();
  507. i = 0;
  508. }
  509. }
  510. }
  511. }
  512. }
  513. std::string Version::DebugString() const {
  514. std::string r;
  515. for (int level = 0; level < config::kNumLevels; level++) {
  516. // E.g.,
  517. // --- level 1 ---
  518. // 17:123['a' .. 'd']
  519. // 20:43['e' .. 'g']
  520. r.append("--- level ");
  521. AppendNumberTo(&r, level);
  522. r.append(" ---\n");
  523. const std::vector<FileMetaData*>& files = files_[level];
  524. for (size_t i = 0; i < files.size(); i++) {
  525. r.push_back(' ');
  526. AppendNumberTo(&r, files[i]->number);
  527. r.push_back(':');
  528. AppendNumberTo(&r, files[i]->file_size);
  529. r.append("[");
  530. r.append(files[i]->smallest.DebugString());
  531. r.append(" .. ");
  532. r.append(files[i]->largest.DebugString());
  533. r.append("]\n");
  534. }
  535. }
  536. return r;
  537. }
  538. // A helper class so we can efficiently apply a whole sequence
  539. // of edits to a particular state without creating intermediate
  540. // Versions that contain full copies of the intermediate state.
  541. class VersionSet::Builder {
  542. private:
  543. // Helper to sort by v->files_[file_number].smallest
  544. struct BySmallestKey {
  545. const InternalKeyComparator* internal_comparator;
  546. bool operator()(FileMetaData* f1, FileMetaData* f2) const {
  547. int r = internal_comparator->Compare(f1->smallest, f2->smallest);
  548. if (r != 0) {
  549. return (r < 0);
  550. } else {
  551. // Break ties by file number
  552. return (f1->number < f2->number);
  553. }
  554. }
  555. };
  556. typedef std::set<FileMetaData*, BySmallestKey> FileSet;
  557. struct LevelState {
  558. std::set<uint64_t> deleted_files;
  559. FileSet* added_files;
  560. };
  561. VersionSet* vset_;
  562. Version* base_;
  563. LevelState levels_[config::kNumLevels];
  564. public:
  565. // Initialize a builder with the files from *base and other info from *vset
  566. Builder(VersionSet* vset, Version* base)
  567. : vset_(vset),
  568. base_(base) {
  569. base_->Ref();
  570. BySmallestKey cmp;
  571. cmp.internal_comparator = &vset_->icmp_;
  572. for (int level = 0; level < config::kNumLevels; level++) {
  573. levels_[level].added_files = new FileSet(cmp);
  574. }
  575. }
  576. ~Builder() {
  577. for (int level = 0; level < config::kNumLevels; level++) {
  578. const FileSet* added = levels_[level].added_files;
  579. std::vector<FileMetaData*> to_unref;
  580. to_unref.reserve(added->size());
  581. for (FileSet::const_iterator it = added->begin();
  582. it != added->end(); ++it) {
  583. to_unref.push_back(*it);
  584. }
  585. delete added;
  586. for (uint32_t i = 0; i < to_unref.size(); i++) {
  587. FileMetaData* f = to_unref[i];
  588. f->refs--;
  589. if (f->refs <= 0) {
  590. delete f;
  591. }
  592. }
  593. }
  594. base_->Unref();
  595. }
  596. // Apply all of the edits in *edit to the current state.
  597. void Apply(VersionEdit* edit) {
  598. // Update compaction pointers
  599. for (size_t i = 0; i < edit->compact_pointers_.size(); i++) {
  600. const int level = edit->compact_pointers_[i].first;
  601. vset_->compact_pointer_[level] =
  602. edit->compact_pointers_[i].second.Encode().ToString();
  603. }
  604. // Delete files
  605. const VersionEdit::DeletedFileSet& del = edit->deleted_files_;
  606. for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin();
  607. iter != del.end();
  608. ++iter) {
  609. const int level = iter->first;
  610. const uint64_t number = iter->second;
  611. levels_[level].deleted_files.insert(number);
  612. }
  613. // Add new files
  614. for (size_t i = 0; i < edit->new_files_.size(); i++) {
  615. const int level = edit->new_files_[i].first;
  616. FileMetaData* f = new FileMetaData(edit->new_files_[i].second);
  617. f->refs = 1;
  618. // We arrange to automatically compact this file after
  619. // a certain number of seeks. Let's assume:
  620. // (1) One seek costs 10ms
  621. // (2) Writing or reading 1MB costs 10ms (100MB/s)
  622. // (3) A compaction of 1MB does 25MB of IO:
  623. // 1MB read from this level
  624. // 10-12MB read from next level (boundaries may be misaligned)
  625. // 10-12MB written to next level
  626. // This implies that 25 seeks cost the same as the compaction
  627. // of 1MB of data. I.e., one seek costs approximately the
  628. // same as the compaction of 40KB of data. We are a little
  629. // conservative and allow approximately one seek for every 16KB
  630. // of data before triggering a compaction.
  631. f->allowed_seeks = (f->file_size / 16384);
  632. if (f->allowed_seeks < 100) f->allowed_seeks = 100;
  633. levels_[level].deleted_files.erase(f->number);
  634. levels_[level].added_files->insert(f);
  635. }
  636. }
  637. // Save the current state in *v.
  638. void SaveTo(Version* v) {
  639. BySmallestKey cmp;
  640. cmp.internal_comparator = &vset_->icmp_;
  641. for (int level = 0; level < config::kNumLevels; level++) {
  642. // Merge the set of added files with the set of pre-existing files.
  643. // Drop any deleted files. Store the result in *v.
  644. const std::vector<FileMetaData*>& base_files = base_->files_[level];
  645. std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin();
  646. std::vector<FileMetaData*>::const_iterator base_end = base_files.end();
  647. const FileSet* added = levels_[level].added_files;
  648. v->files_[level].reserve(base_files.size() + added->size());
  649. for (FileSet::const_iterator added_iter = added->begin();
  650. added_iter != added->end();
  651. ++added_iter) {
  652. // Add all smaller files listed in base_
  653. for (std::vector<FileMetaData*>::const_iterator bpos
  654. = std::upper_bound(base_iter, base_end, *added_iter, cmp);
  655. base_iter != bpos;
  656. ++base_iter) {
  657. MaybeAddFile(v, level, *base_iter);
  658. }
  659. MaybeAddFile(v, level, *added_iter);
  660. }
  661. // Add remaining base files
  662. for (; base_iter != base_end; ++base_iter) {
  663. MaybeAddFile(v, level, *base_iter);
  664. }
  665. #ifndef NDEBUG
  666. // Make sure there is no overlap in levels > 0
  667. if (level > 0) {
  668. for (uint32_t i = 1; i < v->files_[level].size(); i++) {
  669. const InternalKey& prev_end = v->files_[level][i-1]->largest;
  670. const InternalKey& this_begin = v->files_[level][i]->smallest;
  671. if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) {
  672. fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
  673. prev_end.DebugString().c_str(),
  674. this_begin.DebugString().c_str());
  675. abort();
  676. }
  677. }
  678. }
  679. #endif
  680. }
  681. }
  682. void MaybeAddFile(Version* v, int level, FileMetaData* f) {
  683. if (levels_[level].deleted_files.count(f->number) > 0) {
  684. // File is deleted: do nothing
  685. } else {
  686. std::vector<FileMetaData*>* files = &v->files_[level];
  687. if (level > 0 && !files->empty()) {
  688. // Must not overlap
  689. assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest,
  690. f->smallest) < 0);
  691. }
  692. f->refs++;
  693. files->push_back(f);
  694. }
  695. }
  696. };
  697. VersionSet::VersionSet(const std::string& dbname,
  698. const Options* options,
  699. TableCache* table_cache,
  700. const InternalKeyComparator* cmp)
  701. : env_(options->env),
  702. dbname_(dbname),
  703. options_(options),
  704. table_cache_(table_cache),
  705. icmp_(*cmp),
  706. next_file_number_(2),
  707. manifest_file_number_(0), // Filled by Recover()
  708. last_sequence_(0),
  709. log_number_(0),
  710. prev_log_number_(0),
  711. descriptor_file_(NULL),
  712. descriptor_log_(NULL),
  713. dummy_versions_(this),
  714. current_(NULL) {
  715. AppendVersion(new Version(this));
  716. }
  717. VersionSet::~VersionSet() {
  718. current_->Unref();
  719. assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty
  720. delete descriptor_log_;
  721. delete descriptor_file_;
  722. }
  723. void VersionSet::AppendVersion(Version* v) {
  724. // Make "v" current
  725. assert(v->refs_ == 0);
  726. assert(v != current_);
  727. if (current_ != NULL) {
  728. current_->Unref();
  729. }
  730. current_ = v;
  731. v->Ref();
  732. // Append to linked list
  733. v->prev_ = dummy_versions_.prev_;
  734. v->next_ = &dummy_versions_;
  735. v->prev_->next_ = v;
  736. v->next_->prev_ = v;
  737. }
  738. Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
  739. if (edit->has_log_number_) {
  740. assert(edit->log_number_ >= log_number_);
  741. assert(edit->log_number_ < next_file_number_);
  742. } else {
  743. edit->SetLogNumber(log_number_);
  744. }
  745. if (!edit->has_prev_log_number_) {
  746. edit->SetPrevLogNumber(prev_log_number_);
  747. }
  748. edit->SetNextFile(next_file_number_);
  749. edit->SetLastSequence(last_sequence_);
  750. Version* v = new Version(this);
  751. {
  752. Builder builder(this, current_);
  753. builder.Apply(edit);
  754. builder.SaveTo(v);
  755. }
  756. Finalize(v);
  757. // Initialize new descriptor log file if necessary by creating
  758. // a temporary file that contains a snapshot of the current version.
  759. std::string new_manifest_file;
  760. Status s;
  761. if (descriptor_log_ == NULL) {
  762. // No reason to unlock *mu here since we only hit this path in the
  763. // first call to LogAndApply (when opening the database).
  764. assert(descriptor_file_ == NULL);
  765. new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_);
  766. edit->SetNextFile(next_file_number_);
  767. s = env_->NewWritableFile(new_manifest_file, &descriptor_file_);
  768. if (s.ok()) {
  769. descriptor_log_ = new log::Writer(descriptor_file_);
  770. s = WriteSnapshot(descriptor_log_);
  771. }
  772. }
  773. // Unlock during expensive MANIFEST log write
  774. {
  775. mu->Unlock();
  776. // Write new record to MANIFEST log
  777. if (s.ok()) {
  778. std::string record;
  779. edit->EncodeTo(&record);
  780. s = descriptor_log_->AddRecord(record);
  781. if (s.ok()) {
  782. s = descriptor_file_->Sync();
  783. }
  784. if (!s.ok()) {
  785. Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str());
  786. }
  787. }
  788. // If we just created a new descriptor file, install it by writing a
  789. // new CURRENT file that points to it.
  790. if (s.ok() && !new_manifest_file.empty()) {
  791. s = SetCurrentFile(env_, dbname_, manifest_file_number_);
  792. }
  793. mu->Lock();
  794. }
  795. // Install the new version
  796. if (s.ok()) {
  797. AppendVersion(v);
  798. log_number_ = edit->log_number_;
  799. prev_log_number_ = edit->prev_log_number_;
  800. } else {
  801. delete v;
  802. if (!new_manifest_file.empty()) {
  803. delete descriptor_log_;
  804. delete descriptor_file_;
  805. descriptor_log_ = NULL;
  806. descriptor_file_ = NULL;
  807. env_->DeleteFile(new_manifest_file);
  808. }
  809. }
  810. return s;
  811. }
  812. Status VersionSet::Recover() {
  813. struct LogReporter : public log::Reader::Reporter {
  814. Status* status;
  815. virtual void Corruption(size_t bytes, const Status& s) {
  816. if (this->status->ok()) *this->status = s;
  817. }
  818. };
  819. // Read "CURRENT" file, which contains a pointer to the current manifest file
  820. std::string current;
  821. Status s = ReadFileToString(env_, CurrentFileName(dbname_), &current);
  822. if (!s.ok()) {
  823. return s;
  824. }
  825. if (current.empty() || current[current.size()-1] != '\n') {
  826. return Status::Corruption("CURRENT file does not end with newline");
  827. }
  828. current.resize(current.size() - 1);
  829. std::string dscname = dbname_ + "/" + current;
  830. SequentialFile* file;
  831. s = env_->NewSequentialFile(dscname, &file);
  832. if (!s.ok()) {
  833. return s;
  834. }
  835. bool have_log_number = false;
  836. bool have_prev_log_number = false;
  837. bool have_next_file = false;
  838. bool have_last_sequence = false;
  839. uint64_t next_file = 0;
  840. uint64_t last_sequence = 0;
  841. uint64_t log_number = 0;
  842. uint64_t prev_log_number = 0;
  843. Builder builder(this, current_);
  844. {
  845. LogReporter reporter;
  846. reporter.status = &s;
  847. log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/);
  848. Slice record;
  849. std::string scratch;
  850. while (reader.ReadRecord(&record, &scratch) && s.ok()) {
  851. VersionEdit edit;
  852. s = edit.DecodeFrom(record);
  853. if (s.ok()) {
  854. if (edit.has_comparator_ &&
  855. edit.comparator_ != icmp_.user_comparator()->Name()) {
  856. s = Status::InvalidArgument(
  857. edit.comparator_ + " does not match existing comparator ",
  858. icmp_.user_comparator()->Name());
  859. }
  860. }
  861. if (s.ok()) {
  862. builder.Apply(&edit);
  863. }
  864. if (edit.has_log_number_) {
  865. log_number = edit.log_number_;
  866. have_log_number = true;
  867. }
  868. if (edit.has_prev_log_number_) {
  869. prev_log_number = edit.prev_log_number_;
  870. have_prev_log_number = true;
  871. }
  872. if (edit.has_next_file_number_) {
  873. next_file = edit.next_file_number_;
  874. have_next_file = true;
  875. }
  876. if (edit.has_last_sequence_) {
  877. last_sequence = edit.last_sequence_;
  878. have_last_sequence = true;
  879. }
  880. }
  881. }
  882. delete file;
  883. file = NULL;
  884. if (s.ok()) {
  885. if (!have_next_file) {
  886. s = Status::Corruption("no meta-nextfile entry in descriptor");
  887. } else if (!have_log_number) {
  888. s = Status::Corruption("no meta-lognumber entry in descriptor");
  889. } else if (!have_last_sequence) {
  890. s = Status::Corruption("no last-sequence-number entry in descriptor");
  891. }
  892. if (!have_prev_log_number) {
  893. prev_log_number = 0;
  894. }
  895. MarkFileNumberUsed(prev_log_number);
  896. MarkFileNumberUsed(log_number);
  897. }
  898. if (s.ok()) {
  899. Version* v = new Version(this);
  900. builder.SaveTo(v);
  901. // Install recovered version
  902. Finalize(v);
  903. AppendVersion(v);
  904. manifest_file_number_ = next_file;
  905. next_file_number_ = next_file + 1;
  906. last_sequence_ = last_sequence;
  907. log_number_ = log_number;
  908. prev_log_number_ = prev_log_number;
  909. }
  910. return s;
  911. }
  912. void VersionSet::MarkFileNumberUsed(uint64_t number) {
  913. if (next_file_number_ <= number) {
  914. next_file_number_ = number + 1;
  915. }
  916. }
  917. void VersionSet::Finalize(Version* v) {
  918. // Precomputed best level for next compaction
  919. int best_level = -1;
  920. double best_score = -1;
  921. for (int level = 0; level < config::kNumLevels-1; level++) {
  922. double score;
  923. if (level == 0) {
  924. // We treat level-0 specially by bounding the number of files
  925. // instead of number of bytes for two reasons:
  926. //
  927. // (1) With larger write-buffer sizes, it is nice not to do too
  928. // many level-0 compactions.
  929. //
  930. // (2) The files in level-0 are merged on every read and
  931. // therefore we wish to avoid too many files when the individual
  932. // file size is small (perhaps because of a small write-buffer
  933. // setting, or very high compression ratios, or lots of
  934. // overwrites/deletions).
  935. score = v->files_[level].size() /
  936. static_cast<double>(config::kL0_CompactionTrigger);
  937. } else {
  938. // Compute the ratio of current size to size limit.
  939. const uint64_t level_bytes = TotalFileSize(v->files_[level]);
  940. score = static_cast<double>(level_bytes) / MaxBytesForLevel(level);
  941. }
  942. if (score > best_score) {
  943. best_level = level;
  944. best_score = score;
  945. }
  946. }
  947. v->compaction_level_ = best_level;
  948. v->compaction_score_ = best_score;
  949. }
  950. Status VersionSet::WriteSnapshot(log::Writer* log) {
  951. // TODO: Break up into multiple records to reduce memory usage on recovery?
  952. // Save metadata
  953. VersionEdit edit;
  954. edit.SetComparatorName(icmp_.user_comparator()->Name());
  955. // Save compaction pointers
  956. for (int level = 0; level < config::kNumLevels; level++) {
  957. if (!compact_pointer_[level].empty()) {
  958. InternalKey key;
  959. key.DecodeFrom(compact_pointer_[level]);
  960. edit.SetCompactPointer(level, key);
  961. }
  962. }
  963. // Save files
  964. for (int level = 0; level < config::kNumLevels; level++) {
  965. const std::vector<FileMetaData*>& files = current_->files_[level];
  966. for (size_t i = 0; i < files.size(); i++) {
  967. const FileMetaData* f = files[i];
  968. edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest);
  969. }
  970. }
  971. std::string record;
  972. edit.EncodeTo(&record);
  973. return log->AddRecord(record);
  974. }
  975. int VersionSet::NumLevelFiles(int level) const {
  976. assert(level >= 0);
  977. assert(level < config::kNumLevels);
  978. return current_->files_[level].size();
  979. }
  980. const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
  981. // Update code if kNumLevels changes
  982. assert(config::kNumLevels == 7);
  983. snprintf(scratch->buffer, sizeof(scratch->buffer),
  984. "files[ %d %d %d %d %d %d %d ]",
  985. int(current_->files_[0].size()),
  986. int(current_->files_[1].size()),
  987. int(current_->files_[2].size()),
  988. int(current_->files_[3].size()),
  989. int(current_->files_[4].size()),
  990. int(current_->files_[5].size()),
  991. int(current_->files_[6].size()));
  992. return scratch->buffer;
  993. }
  994. uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
  995. uint64_t result = 0;
  996. for (int level = 0; level < config::kNumLevels; level++) {
  997. const std::vector<FileMetaData*>& files = v->files_[level];
  998. for (size_t i = 0; i < files.size(); i++) {
  999. if (icmp_.Compare(files[i]->largest, ikey) <= 0) {
  1000. // Entire file is before "ikey", so just add the file size
  1001. result += files[i]->file_size;
  1002. } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) {
  1003. // Entire file is after "ikey", so ignore
  1004. if (level > 0) {
  1005. // Files other than level 0 are sorted by meta->smallest, so
  1006. // no further files in this level will contain data for
  1007. // "ikey".
  1008. break;
  1009. }
  1010. } else {
  1011. // "ikey" falls in the range for this table. Add the
  1012. // approximate offset of "ikey" within the table.
  1013. Table* tableptr;
  1014. Iterator* iter = table_cache_->NewIterator(
  1015. ReadOptions(), files[i]->number, files[i]->file_size, &tableptr);
  1016. if (tableptr != NULL) {
  1017. result += tableptr->ApproximateOffsetOf(ikey.Encode());
  1018. }
  1019. delete iter;
  1020. }
  1021. }
  1022. }
  1023. return result;
  1024. }
  1025. void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
  1026. for (Version* v = dummy_versions_.next_;
  1027. v != &dummy_versions_;
  1028. v = v->next_) {
  1029. for (int level = 0; level < config::kNumLevels; level++) {
  1030. const std::vector<FileMetaData*>& files = v->files_[level];
  1031. for (size_t i = 0; i < files.size(); i++) {
  1032. live->insert(files[i]->number);
  1033. }
  1034. }
  1035. }
  1036. }
  1037. int64_t VersionSet::NumLevelBytes(int level) const {
  1038. assert(level >= 0);
  1039. assert(level < config::kNumLevels);
  1040. return TotalFileSize(current_->files_[level]);
  1041. }
  1042. int64_t VersionSet::MaxNextLevelOverlappingBytes() {
  1043. int64_t result = 0;
  1044. std::vector<FileMetaData*> overlaps;
  1045. for (int level = 1; level < config::kNumLevels - 1; level++) {
  1046. for (size_t i = 0; i < current_->files_[level].size(); i++) {
  1047. const FileMetaData* f = current_->files_[level][i];
  1048. current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest,
  1049. &overlaps);
  1050. const int64_t sum = TotalFileSize(overlaps);
  1051. if (sum > result) {
  1052. result = sum;
  1053. }
  1054. }
  1055. }
  1056. return result;
  1057. }
  1058. // Stores the minimal range that covers all entries in inputs in
  1059. // *smallest, *largest.
  1060. // REQUIRES: inputs is not empty
  1061. void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
  1062. InternalKey* smallest,
  1063. InternalKey* largest) {
  1064. assert(!inputs.empty());
  1065. smallest->Clear();
  1066. largest->Clear();
  1067. for (size_t i = 0; i < inputs.size(); i++) {
  1068. FileMetaData* f = inputs[i];
  1069. if (i == 0) {
  1070. *smallest = f->smallest;
  1071. *largest = f->largest;
  1072. } else {
  1073. if (icmp_.Compare(f->smallest, *smallest) < 0) {
  1074. *smallest = f->smallest;
  1075. }
  1076. if (icmp_.Compare(f->largest, *largest) > 0) {
  1077. *largest = f->largest;
  1078. }
  1079. }
  1080. }
  1081. }
  1082. // Stores the minimal range that covers all entries in inputs1 and inputs2
  1083. // in *smallest, *largest.
  1084. // REQUIRES: inputs is not empty
  1085. void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
  1086. const std::vector<FileMetaData*>& inputs2,
  1087. InternalKey* smallest,
  1088. InternalKey* largest) {
  1089. std::vector<FileMetaData*> all = inputs1;
  1090. all.insert(all.end(), inputs2.begin(), inputs2.end());
  1091. GetRange(all, smallest, largest);
  1092. }
  1093. Iterator* VersionSet::MakeInputIterator(Compaction* c) {
  1094. ReadOptions options;
  1095. options.verify_checksums = options_->paranoid_checks;
  1096. options.fill_cache = false;
  1097. // Level-0 files have to be merged together. For other levels,
  1098. // we will make a concatenating iterator per level.
  1099. // TODO(opt): use concatenating iterator for level-0 if there is no overlap
  1100. const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2);
  1101. Iterator** list = new Iterator*[space];
  1102. int num = 0;
  1103. for (int which = 0; which < 2; which++) {
  1104. if (!c->inputs_[which].empty()) {
  1105. if (c->level() + which == 0) {
  1106. const std::vector<FileMetaData*>& files = c->inputs_[which];
  1107. for (size_t i = 0; i < files.size(); i++) {
  1108. list[num++] = table_cache_->NewIterator(
  1109. options, files[i]->number, files[i]->file_size);
  1110. }
  1111. } else {
  1112. // Create concatenating iterator for the files from this level
  1113. list[num++] = NewTwoLevelIterator(
  1114. new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]),
  1115. &GetFileIterator, table_cache_, options);
  1116. }
  1117. }
  1118. }
  1119. assert(num <= space);
  1120. Iterator* result = NewMergingIterator(&icmp_, list, num);
  1121. delete[] list;
  1122. return result;
  1123. }
  1124. Compaction* VersionSet::PickCompaction() {
  1125. Compaction* c;
  1126. int level;
  1127. // We prefer compactions triggered by too much data in a level over
  1128. // the compactions triggered by seeks.
  1129. const bool size_compaction = (current_->compaction_score_ >= 1);
  1130. const bool seek_compaction = (current_->file_to_compact_ != NULL);
  1131. if (size_compaction) {
  1132. level = current_->compaction_level_;
  1133. assert(level >= 0);
  1134. assert(level+1 < config::kNumLevels);
  1135. c = new Compaction(level);
  1136. // Pick the first file that comes after compact_pointer_[level]
  1137. for (size_t i = 0; i < current_->files_[level].size(); i++) {
  1138. FileMetaData* f = current_->files_[level][i];
  1139. if (compact_pointer_[level].empty() ||
  1140. icmp_.Compare(f->largest.Encode(), compact_pointer_[level]) > 0) {
  1141. c->inputs_[0].push_back(f);
  1142. break;
  1143. }
  1144. }
  1145. if (c->inputs_[0].empty()) {
  1146. // Wrap-around to the beginning of the key space
  1147. c->inputs_[0].push_back(current_->files_[level][0]);
  1148. }
  1149. } else if (seek_compaction) {
  1150. level = current_->file_to_compact_level_;
  1151. c = new Compaction(level);
  1152. c->inputs_[0].push_back(current_->file_to_compact_);
  1153. } else {
  1154. return NULL;
  1155. }
  1156. c->input_version_ = current_;
  1157. c->input_version_->Ref();
  1158. // Files in level 0 may overlap each other, so pick up all overlapping ones
  1159. if (level == 0) {
  1160. InternalKey smallest, largest;
  1161. GetRange(c->inputs_[0], &smallest, &largest);
  1162. // Note that the next call will discard the file we placed in
  1163. // c->inputs_[0] earlier and replace it with an overlapping set
  1164. // which will include the picked file.
  1165. current_->GetOverlappingInputs(0, &smallest, &largest, &c->inputs_[0]);
  1166. assert(!c->inputs_[0].empty());
  1167. }
  1168. SetupOtherInputs(c);
  1169. return c;
  1170. }
  1171. void VersionSet::SetupOtherInputs(Compaction* c) {
  1172. const int level = c->level();
  1173. InternalKey smallest, largest;
  1174. GetRange(c->inputs_[0], &smallest, &largest);
  1175. current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]);
  1176. // Get entire range covered by compaction
  1177. InternalKey all_start, all_limit;
  1178. GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
  1179. // See if we can grow the number of inputs in "level" without
  1180. // changing the number of "level+1" files we pick up.
  1181. if (!c->inputs_[1].empty()) {
  1182. std::vector<FileMetaData*> expanded0;
  1183. current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0);
  1184. const int64_t inputs0_size = TotalFileSize(c->inputs_[0]);
  1185. const int64_t inputs1_size = TotalFileSize(c->inputs_[1]);
  1186. const int64_t expanded0_size = TotalFileSize(expanded0);
  1187. if (expanded0.size() > c->inputs_[0].size() &&
  1188. inputs1_size + expanded0_size < kExpandedCompactionByteSizeLimit) {
  1189. InternalKey new_start, new_limit;
  1190. GetRange(expanded0, &new_start, &new_limit);
  1191. std::vector<FileMetaData*> expanded1;
  1192. current_->GetOverlappingInputs(level+1, &new_start, &new_limit,
  1193. &expanded1);
  1194. if (expanded1.size() == c->inputs_[1].size()) {
  1195. Log(options_->info_log,
  1196. "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n",
  1197. level,
  1198. int(c->inputs_[0].size()),
  1199. int(c->inputs_[1].size()),
  1200. long(inputs0_size), long(inputs1_size),
  1201. int(expanded0.size()),
  1202. int(expanded1.size()),
  1203. long(expanded0_size), long(inputs1_size));
  1204. smallest = new_start;
  1205. largest = new_limit;
  1206. c->inputs_[0] = expanded0;
  1207. c->inputs_[1] = expanded1;
  1208. GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
  1209. }
  1210. }
  1211. }
  1212. // Compute the set of grandparent files that overlap this compaction
  1213. // (parent == level+1; grandparent == level+2)
  1214. if (level + 2 < config::kNumLevels) {
  1215. current_->GetOverlappingInputs(level + 2, &all_start, &all_limit,
  1216. &c->grandparents_);
  1217. }
  1218. if (false) {
  1219. Log(options_->info_log, "Compacting %d '%s' .. '%s'",
  1220. level,
  1221. smallest.DebugString().c_str(),
  1222. largest.DebugString().c_str());
  1223. }
  1224. // Update the place where we will do the next compaction for this level.
  1225. // We update this immediately instead of waiting for the VersionEdit
  1226. // to be applied so that if the compaction fails, we will try a different
  1227. // key range next time.
  1228. compact_pointer_[level] = largest.Encode().ToString();
  1229. c->edit_.SetCompactPointer(level, largest);
  1230. }
  1231. Compaction* VersionSet::CompactRange(
  1232. int level,
  1233. const InternalKey* begin,
  1234. const InternalKey* end) {
  1235. std::vector<FileMetaData*> inputs;
  1236. current_->GetOverlappingInputs(level, begin, end, &inputs);
  1237. if (inputs.empty()) {
  1238. return NULL;
  1239. }
  1240. // Avoid compacting too much in one shot in case the range is large.
  1241. // But we cannot do this for level-0 since level-0 files can overlap
  1242. // and we must not pick one file and drop another older file if the
  1243. // two files overlap.
  1244. if (level > 0) {
  1245. const uint64_t limit = MaxFileSizeForLevel(level);
  1246. uint64_t total = 0;
  1247. for (size_t i = 0; i < inputs.size(); i++) {
  1248. uint64_t s = inputs[i]->file_size;
  1249. total += s;
  1250. if (total >= limit) {
  1251. inputs.resize(i + 1);
  1252. break;
  1253. }
  1254. }
  1255. }
  1256. Compaction* c = new Compaction(level);
  1257. c->input_version_ = current_;
  1258. c->input_version_->Ref();
  1259. c->inputs_[0] = inputs;
  1260. SetupOtherInputs(c);
  1261. return c;
  1262. }
  1263. Compaction::Compaction(int level)
  1264. : level_(level),
  1265. max_output_file_size_(MaxFileSizeForLevel(level)),
  1266. input_version_(NULL),
  1267. grandparent_index_(0),
  1268. seen_key_(false),
  1269. overlapped_bytes_(0) {
  1270. for (int i = 0; i < config::kNumLevels; i++) {
  1271. level_ptrs_[i] = 0;
  1272. }
  1273. }
  1274. Compaction::~Compaction() {
  1275. if (input_version_ != NULL) {
  1276. input_version_->Unref();
  1277. }
  1278. }
  1279. bool Compaction::IsTrivialMove() const {
  1280. // Avoid a move if there is lots of overlapping grandparent data.
  1281. // Otherwise, the move could create a parent file that will require
  1282. // a very expensive merge later on.
  1283. return (num_input_files(0) == 1 &&
  1284. num_input_files(1) == 0 &&
  1285. TotalFileSize(grandparents_) <= kMaxGrandParentOverlapBytes);
  1286. }
  1287. void Compaction::AddInputDeletions(VersionEdit* edit) {
  1288. for (int which = 0; which < 2; which++) {
  1289. for (size_t i = 0; i < inputs_[which].size(); i++) {
  1290. edit->DeleteFile(level_ + which, inputs_[which][i]->number);
  1291. }
  1292. }
  1293. }
  1294. bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
  1295. // Maybe use binary search to find right entry instead of linear search?
  1296. const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
  1297. for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) {
  1298. const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
  1299. for (; level_ptrs_[lvl] < files.size(); ) {
  1300. FileMetaData* f = files[level_ptrs_[lvl]];
  1301. if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
  1302. // We've advanced far enough
  1303. if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
  1304. // Key falls in this file's range, so definitely not base level
  1305. return false;
  1306. }
  1307. break;
  1308. }
  1309. level_ptrs_[lvl]++;
  1310. }
  1311. }
  1312. return true;
  1313. }
  1314. bool Compaction::ShouldStopBefore(const Slice& internal_key) {
  1315. // Scan to find earliest grandparent file that contains key.
  1316. const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
  1317. while (grandparent_index_ < grandparents_.size() &&
  1318. icmp->Compare(internal_key,
  1319. grandparents_[grandparent_index_]->largest.Encode()) > 0) {
  1320. if (seen_key_) {
  1321. overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
  1322. }
  1323. grandparent_index_++;
  1324. }
  1325. seen_key_ = true;
  1326. if (overlapped_bytes_ > kMaxGrandParentOverlapBytes) {
  1327. // Too much overlap for current output; start new output
  1328. overlapped_bytes_ = 0;
  1329. return true;
  1330. } else {
  1331. return false;
  1332. }
  1333. }
  1334. void Compaction::ReleaseInputs() {
  1335. if (input_version_ != NULL) {
  1336. input_version_->Unref();
  1337. input_version_ = NULL;
  1338. }
  1339. }
  1340. } // namespace leveldb