小组成员:陈予曈,朱陈媛
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1622 lines
50 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "db/db_impl.h"
  5. #include <algorithm>
  6. #include <atomic>
  7. #include <cstdint>
  8. #include <cstdio>
  9. #include <set>
  10. #include <string>
  11. #include <vector>
  12. // 添加所需头文件-朱陈媛
  13. #include <iostream>
  14. #include <sstream>
  15. #include <iomanip>
  16. #include <ctime>
  17. #include <chrono>
  18. #include "db/builder.h"
  19. #include "db/db_iter.h"
  20. #include "db/dbformat.h"
  21. #include "db/filename.h"
  22. #include "db/log_reader.h"
  23. #include "db/log_writer.h"
  24. #include "db/memtable.h"
  25. #include "db/table_cache.h"
  26. #include "db/version_set.h"
  27. #include "db/write_batch_internal.h"
  28. #include "leveldb/db.h"
  29. #include "leveldb/env.h"
  30. #include "leveldb/status.h"
  31. #include "leveldb/table.h"
  32. #include "leveldb/table_builder.h"
  33. #include "port/port.h"
  34. #include "table/block.h"
  35. #include "table/merger.h"
  36. #include "table/two_level_iterator.h"
  37. #include "util/coding.h"
  38. #include "util/logging.h"
  39. #include "util/mutexlock.h"
  40. namespace leveldb {
  41. const int kNumNonTableCacheFiles = 10;
  42. // Information kept for every waiting writer
  43. struct DBImpl::Writer {
  44. explicit Writer(port::Mutex* mu)
  45. : batch(nullptr), sync(false), done(false), cv(mu) {}
  46. Status status;
  47. WriteBatch* batch;
  48. bool sync;
  49. bool done;
  50. port::CondVar cv;
  51. };
  52. struct DBImpl::CompactionState {
  53. // Files produced by compaction
  54. struct Output {
  55. uint64_t number;
  56. uint64_t file_size;
  57. InternalKey smallest, largest;
  58. };
  59. Output* current_output() { return &outputs[outputs.size() - 1]; }
  60. explicit CompactionState(Compaction* c)
  61. : compaction(c),
  62. smallest_snapshot(0),
  63. outfile(nullptr),
  64. builder(nullptr),
  65. total_bytes(0) {}
  66. Compaction* const compaction;
  67. // Sequence numbers < smallest_snapshot are not significant since we
  68. // will never have to service a snapshot below smallest_snapshot.
  69. // Therefore if we have seen a sequence number S <= smallest_snapshot,
  70. // we can drop all entries for the same key with sequence numbers < S.
  71. SequenceNumber smallest_snapshot;
  72. std::vector<Output> outputs;
  73. // State kept for output being generated
  74. WritableFile* outfile;
  75. TableBuilder* builder;
  76. uint64_t total_bytes;
  77. };
  78. // Fix user-supplied options to be reasonable
  79. template <class T, class V>
  80. static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
  81. if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
  82. if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
  83. }
  84. Options SanitizeOptions(const std::string& dbname,
  85. const InternalKeyComparator* icmp,
  86. const InternalFilterPolicy* ipolicy,
  87. const Options& src) {
  88. Options result = src;
  89. result.comparator = icmp;
  90. result.filter_policy = (src.filter_policy != nullptr) ? ipolicy : nullptr;
  91. ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000);
  92. ClipToRange(&result.write_buffer_size, 64 << 10, 1 << 30);
  93. ClipToRange(&result.max_file_size, 1 << 20, 1 << 30);
  94. ClipToRange(&result.block_size, 1 << 10, 4 << 20);
  95. if (result.info_log == nullptr) {
  96. // Open a log file in the same directory as the db
  97. src.env->CreateDir(dbname); // In case it does not exist
  98. src.env->RenameFile(InfoLogFileName(dbname), OldInfoLogFileName(dbname));
  99. Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log);
  100. if (!s.ok()) {
  101. // No place suitable for logging
  102. result.info_log = nullptr;
  103. }
  104. }
  105. if (result.block_cache == nullptr) {
  106. result.block_cache = NewLRUCache(8 << 20);
  107. }
  108. return result;
  109. }
  110. static int TableCacheSize(const Options& sanitized_options) {
  111. // Reserve ten files or so for other uses and give the rest to TableCache.
  112. return sanitized_options.max_open_files - kNumNonTableCacheFiles;
  113. }
  114. DBImpl::DBImpl(const Options& raw_options, const std::string& dbname)
  115. : env_(raw_options.env),
  116. internal_comparator_(raw_options.comparator),
  117. internal_filter_policy_(raw_options.filter_policy),
  118. options_(SanitizeOptions(dbname, &internal_comparator_,
  119. &internal_filter_policy_, raw_options)),
  120. owns_info_log_(options_.info_log != raw_options.info_log),
  121. owns_cache_(options_.block_cache != raw_options.block_cache),
  122. dbname_(dbname),
  123. table_cache_(new TableCache(dbname_, options_, TableCacheSize(options_))),
  124. db_lock_(nullptr),
  125. shutting_down_(false),
  126. background_work_finished_signal_(&mutex_),
  127. mem_(nullptr),
  128. imm_(nullptr),
  129. has_imm_(false),
  130. logfile_(nullptr),
  131. logfile_number_(0),
  132. log_(nullptr),
  133. seed_(0),
  134. tmp_batch_(new WriteBatch),
  135. background_compaction_scheduled_(false),
  136. manual_compaction_(nullptr),
  137. versions_(new VersionSet(dbname_, &options_, table_cache_,
  138. &internal_comparator_)) {}
  139. DBImpl::~DBImpl() {
  140. // Wait for background work to finish.
  141. mutex_.Lock();
  142. shutting_down_.store(true, std::memory_order_release);
  143. while (background_compaction_scheduled_) {
  144. background_work_finished_signal_.Wait();
  145. }
  146. mutex_.Unlock();
  147. if (db_lock_ != nullptr) {
  148. env_->UnlockFile(db_lock_);
  149. }
  150. delete versions_;
  151. if (mem_ != nullptr) mem_->Unref();
  152. if (imm_ != nullptr) imm_->Unref();
  153. delete tmp_batch_;
  154. delete log_;
  155. delete logfile_;
  156. delete table_cache_;
  157. if (owns_info_log_) {
  158. delete options_.info_log;
  159. }
  160. if (owns_cache_) {
  161. delete options_.block_cache;
  162. }
  163. }
  164. Status DBImpl::NewDB() {
  165. VersionEdit new_db;
  166. new_db.SetComparatorName(user_comparator()->Name());
  167. new_db.SetLogNumber(0);
  168. new_db.SetNextFile(2);
  169. new_db.SetLastSequence(0);
  170. const std::string manifest = DescriptorFileName(dbname_, 1);
  171. WritableFile* file;
  172. Status s = env_->NewWritableFile(manifest, &file);
  173. if (!s.ok()) {
  174. return s;
  175. }
  176. {
  177. log::Writer log(file);
  178. std::string record;
  179. new_db.EncodeTo(&record);
  180. s = log.AddRecord(record);
  181. if (s.ok()) {
  182. s = file->Sync();
  183. }
  184. if (s.ok()) {
  185. s = file->Close();
  186. }
  187. }
  188. delete file;
  189. if (s.ok()) {
  190. // Make "CURRENT" file that points to the new manifest file.
  191. s = SetCurrentFile(env_, dbname_, 1);
  192. } else {
  193. env_->RemoveFile(manifest);
  194. }
  195. return s;
  196. }
  197. void DBImpl::MaybeIgnoreError(Status* s) const {
  198. if (s->ok() || options_.paranoid_checks) {
  199. // No change needed
  200. } else {
  201. Log(options_.info_log, "Ignoring error %s", s->ToString().c_str());
  202. *s = Status::OK();
  203. }
  204. }
  205. void DBImpl::RemoveObsoleteFiles() {
  206. mutex_.AssertHeld();
  207. if (!bg_error_.ok()) {
  208. // After a background error, we don't know whether a new version may
  209. // or may not have been committed, so we cannot safely garbage collect.
  210. return;
  211. }
  212. // Make a set of all of the live files
  213. std::set<uint64_t> live = pending_outputs_;
  214. versions_->AddLiveFiles(&live);
  215. std::vector<std::string> filenames;
  216. env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose
  217. uint64_t number;
  218. FileType type;
  219. std::vector<std::string> files_to_delete;
  220. for (std::string& filename : filenames) {
  221. if (ParseFileName(filename, &number, &type)) {
  222. bool keep = true;
  223. switch (type) {
  224. case kLogFile:
  225. keep = ((number >= versions_->LogNumber()) ||
  226. (number == versions_->PrevLogNumber()));
  227. break;
  228. case kDescriptorFile:
  229. // Keep my manifest file, and any newer incarnations'
  230. // (in case there is a race that allows other incarnations)
  231. keep = (number >= versions_->ManifestFileNumber());
  232. break;
  233. case kTableFile:
  234. keep = (live.find(number) != live.end());
  235. break;
  236. case kTempFile:
  237. // Any temp files that are currently being written to must
  238. // be recorded in pending_outputs_, which is inserted into "live"
  239. keep = (live.find(number) != live.end());
  240. break;
  241. case kCurrentFile:
  242. case kDBLockFile:
  243. case kInfoLogFile:
  244. keep = true;
  245. break;
  246. }
  247. if (!keep) {
  248. files_to_delete.push_back(std::move(filename));
  249. if (type == kTableFile) {
  250. table_cache_->Evict(number);
  251. }
  252. Log(options_.info_log, "Delete type=%d #%lld\n", static_cast<int>(type),
  253. static_cast<unsigned long long>(number));
  254. }
  255. }
  256. }
  257. // While deleting all files unblock other threads. All files being deleted
  258. // have unique names which will not collide with newly created files and
  259. // are therefore safe to delete while allowing other threads to proceed.
  260. mutex_.Unlock();
  261. for (const std::string& filename : files_to_delete) {
  262. env_->RemoveFile(dbname_ + "/" + filename);
  263. }
  264. mutex_.Lock();
  265. }
  266. Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) {
  267. mutex_.AssertHeld();
  268. // Ignore error from CreateDir since the creation of the DB is
  269. // committed only when the descriptor is created, and this directory
  270. // may already exist from a previous failed creation attempt.
  271. env_->CreateDir(dbname_);
  272. assert(db_lock_ == nullptr);
  273. Status s = env_->LockFile(LockFileName(dbname_), &db_lock_);
  274. if (!s.ok()) {
  275. return s;
  276. }
  277. if (!env_->FileExists(CurrentFileName(dbname_))) {
  278. if (options_.create_if_missing) {
  279. Log(options_.info_log, "Creating DB %s since it was missing.",
  280. dbname_.c_str());
  281. s = NewDB();
  282. if (!s.ok()) {
  283. return s;
  284. }
  285. } else {
  286. return Status::InvalidArgument(
  287. dbname_, "does not exist (create_if_missing is false)");
  288. }
  289. } else {
  290. if (options_.error_if_exists) {
  291. return Status::InvalidArgument(dbname_,
  292. "exists (error_if_exists is true)");
  293. }
  294. }
  295. s = versions_->Recover(save_manifest);
  296. if (!s.ok()) {
  297. return s;
  298. }
  299. SequenceNumber max_sequence(0);
  300. // Recover from all newer log files than the ones named in the
  301. // descriptor (new log files may have been added by the previous
  302. // incarnation without registering them in the descriptor).
  303. //
  304. // Note that PrevLogNumber() is no longer used, but we pay
  305. // attention to it in case we are recovering a database
  306. // produced by an older version of leveldb.
  307. const uint64_t min_log = versions_->LogNumber();
  308. const uint64_t prev_log = versions_->PrevLogNumber();
  309. std::vector<std::string> filenames;
  310. s = env_->GetChildren(dbname_, &filenames);
  311. if (!s.ok()) {
  312. return s;
  313. }
  314. std::set<uint64_t> expected;
  315. versions_->AddLiveFiles(&expected);
  316. uint64_t number;
  317. FileType type;
  318. std::vector<uint64_t> logs;
  319. for (size_t i = 0; i < filenames.size(); i++) {
  320. if (ParseFileName(filenames[i], &number, &type)) {
  321. expected.erase(number);
  322. if (type == kLogFile && ((number >= min_log) || (number == prev_log)))
  323. logs.push_back(number);
  324. }
  325. }
  326. if (!expected.empty()) {
  327. char buf[50];
  328. std::snprintf(buf, sizeof(buf), "%d missing files; e.g.",
  329. static_cast<int>(expected.size()));
  330. return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin())));
  331. }
  332. // Recover in the order in which the logs were generated
  333. std::sort(logs.begin(), logs.end());
  334. for (size_t i = 0; i < logs.size(); i++) {
  335. s = RecoverLogFile(logs[i], (i == logs.size() - 1), save_manifest, edit,
  336. &max_sequence);
  337. if (!s.ok()) {
  338. return s;
  339. }
  340. // The previous incarnation may not have written any MANIFEST
  341. // records after allocating this log number. So we manually
  342. // update the file number allocation counter in VersionSet.
  343. versions_->MarkFileNumberUsed(logs[i]);
  344. }
  345. if (versions_->LastSequence() < max_sequence) {
  346. versions_->SetLastSequence(max_sequence);
  347. }
  348. return Status::OK();
  349. }
  350. Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
  351. bool* save_manifest, VersionEdit* edit,
  352. SequenceNumber* max_sequence) {
  353. struct LogReporter : public log::Reader::Reporter {
  354. Env* env;
  355. Logger* info_log;
  356. const char* fname;
  357. Status* status; // null if options_.paranoid_checks==false
  358. void Corruption(size_t bytes, const Status& s) override {
  359. Log(info_log, "%s%s: dropping %d bytes; %s",
  360. (this->status == nullptr ? "(ignoring error) " : ""), fname,
  361. static_cast<int>(bytes), s.ToString().c_str());
  362. if (this->status != nullptr && this->status->ok()) *this->status = s;
  363. }
  364. };
  365. mutex_.AssertHeld();
  366. // Open the log file
  367. std::string fname = LogFileName(dbname_, log_number);
  368. SequentialFile* file;
  369. Status status = env_->NewSequentialFile(fname, &file);
  370. if (!status.ok()) {
  371. MaybeIgnoreError(&status);
  372. return status;
  373. }
  374. // Create the log reader.
  375. LogReporter reporter;
  376. reporter.env = env_;
  377. reporter.info_log = options_.info_log;
  378. reporter.fname = fname.c_str();
  379. reporter.status = (options_.paranoid_checks ? &status : nullptr);
  380. // We intentionally make log::Reader do checksumming even if
  381. // paranoid_checks==false so that corruptions cause entire commits
  382. // to be skipped instead of propagating bad information (like overly
  383. // large sequence numbers).
  384. log::Reader reader(file, &reporter, true /*checksum*/, 0 /*initial_offset*/);
  385. Log(options_.info_log, "Recovering log #%llu",
  386. (unsigned long long)log_number);
  387. // Read all the records and add to a memtable
  388. std::string scratch;
  389. Slice record;
  390. WriteBatch batch;
  391. int compactions = 0;
  392. MemTable* mem = nullptr;
  393. while (reader.ReadRecord(&record, &scratch) && status.ok()) {
  394. if (record.size() < 12) {
  395. reporter.Corruption(record.size(),
  396. Status::Corruption("log record too small"));
  397. continue;
  398. }
  399. WriteBatchInternal::SetContents(&batch, record);
  400. if (mem == nullptr) {
  401. mem = new MemTable(internal_comparator_);
  402. mem->Ref();
  403. }
  404. status = WriteBatchInternal::InsertInto(&batch, mem);
  405. MaybeIgnoreError(&status);
  406. if (!status.ok()) {
  407. break;
  408. }
  409. const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) +
  410. WriteBatchInternal::Count(&batch) - 1;
  411. if (last_seq > *max_sequence) {
  412. *max_sequence = last_seq;
  413. }
  414. if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) {
  415. compactions++;
  416. *save_manifest = true;
  417. status = WriteLevel0Table(mem, edit, nullptr);
  418. mem->Unref();
  419. mem = nullptr;
  420. if (!status.ok()) {
  421. // Reflect errors immediately so that conditions like full
  422. // file-systems cause the DB::Open() to fail.
  423. break;
  424. }
  425. }
  426. }
  427. delete file;
  428. // See if we should keep reusing the last log file.
  429. if (status.ok() && options_.reuse_logs && last_log && compactions == 0) {
  430. assert(logfile_ == nullptr);
  431. assert(log_ == nullptr);
  432. assert(mem_ == nullptr);
  433. uint64_t lfile_size;
  434. if (env_->GetFileSize(fname, &lfile_size).ok() &&
  435. env_->NewAppendableFile(fname, &logfile_).ok()) {
  436. Log(options_.info_log, "Reusing old log %s \n", fname.c_str());
  437. log_ = new log::Writer(logfile_, lfile_size);
  438. logfile_number_ = log_number;
  439. if (mem != nullptr) {
  440. mem_ = mem;
  441. mem = nullptr;
  442. } else {
  443. // mem can be nullptr if lognum exists but was empty.
  444. mem_ = new MemTable(internal_comparator_);
  445. mem_->Ref();
  446. }
  447. }
  448. }
  449. if (mem != nullptr) {
  450. // mem did not get reused; compact it.
  451. if (status.ok()) {
  452. *save_manifest = true;
  453. status = WriteLevel0Table(mem, edit, nullptr);
  454. }
  455. mem->Unref();
  456. }
  457. return status;
  458. }
  459. Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
  460. Version* base) {
  461. mutex_.AssertHeld();
  462. const uint64_t start_micros = env_->NowMicros();
  463. FileMetaData meta;
  464. meta.number = versions_->NewFileNumber();
  465. pending_outputs_.insert(meta.number);
  466. Iterator* iter = mem->NewIterator();
  467. Log(options_.info_log, "Level-0 table #%llu: started",
  468. (unsigned long long)meta.number);
  469. Status s;
  470. {
  471. mutex_.Unlock();
  472. s = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
  473. mutex_.Lock();
  474. }
  475. Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s",
  476. (unsigned long long)meta.number, (unsigned long long)meta.file_size,
  477. s.ToString().c_str());
  478. delete iter;
  479. pending_outputs_.erase(meta.number);
  480. // Note that if file_size is zero, the file has been deleted and
  481. // should not be added to the manifest.
  482. int level = 0;
  483. if (s.ok() && meta.file_size > 0) {
  484. const Slice min_user_key = meta.smallest.user_key();
  485. const Slice max_user_key = meta.largest.user_key();
  486. if (base != nullptr) {
  487. level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
  488. }
  489. edit->AddFile(level, meta.number, meta.file_size, meta.smallest,
  490. meta.largest);
  491. }
  492. CompactionStats stats;
  493. stats.micros = env_->NowMicros() - start_micros;
  494. stats.bytes_written = meta.file_size;
  495. stats_[level].Add(stats);
  496. return s;
  497. }
  498. void DBImpl::CompactMemTable() {
  499. mutex_.AssertHeld();
  500. assert(imm_ != nullptr);
  501. // Save the contents of the memtable as a new Table
  502. VersionEdit edit;
  503. Version* base = versions_->current();
  504. base->Ref();
  505. Status s = WriteLevel0Table(imm_, &edit, base);
  506. base->Unref();
  507. if (s.ok() && shutting_down_.load(std::memory_order_acquire)) {
  508. s = Status::IOError("Deleting DB during memtable compaction");
  509. }
  510. // Replace immutable memtable with the generated Table
  511. if (s.ok()) {
  512. edit.SetPrevLogNumber(0);
  513. edit.SetLogNumber(logfile_number_); // Earlier logs no longer needed
  514. s = versions_->LogAndApply(&edit, &mutex_);
  515. }
  516. if (s.ok()) {
  517. // Commit to the new state
  518. imm_->Unref();
  519. imm_ = nullptr;
  520. has_imm_.store(false, std::memory_order_release);
  521. RemoveObsoleteFiles();
  522. } else {
  523. RecordBackgroundError(s);
  524. }
  525. }
  526. void DBImpl::CompactRange(const Slice* begin, const Slice* end) {
  527. int max_level_with_files = 1;
  528. {
  529. MutexLock l(&mutex_);
  530. Version* base = versions_->current();
  531. for (int level = 1; level < config::kNumLevels; level++) {
  532. if (base->OverlapInLevel(level, begin, end)) {
  533. max_level_with_files = level;
  534. }
  535. }
  536. }
  537. TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap
  538. // 这个改成小于等于,test能过,但这不是改变原本的手动合并逻辑了吗?-朱陈媛
  539. // 改了之后应该也不影响合并
  540. for (int level = 0; level <= max_level_with_files; level++) {
  541. TEST_CompactRange(level, begin, end);
  542. }
  543. }
  544. void DBImpl::TEST_CompactRange(int level, const Slice* begin,
  545. const Slice* end) {
  546. assert(level >= 0);
  547. assert(level + 1 < config::kNumLevels);
  548. InternalKey begin_storage, end_storage;
  549. ManualCompaction manual;
  550. manual.level = level;
  551. manual.done = false;
  552. if (begin == nullptr) {
  553. manual.begin = nullptr;
  554. } else {
  555. begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek);
  556. manual.begin = &begin_storage;
  557. }
  558. if (end == nullptr) {
  559. manual.end = nullptr;
  560. } else {
  561. end_storage = InternalKey(*end, 0, static_cast<ValueType>(0));
  562. manual.end = &end_storage;
  563. }
  564. MutexLock l(&mutex_);
  565. while (!manual.done && !shutting_down_.load(std::memory_order_acquire) &&
  566. bg_error_.ok()) {
  567. if (manual_compaction_ == nullptr) { // Idle
  568. manual_compaction_ = &manual;
  569. MaybeScheduleCompaction();
  570. } else { // Running either my compaction or another compaction.
  571. background_work_finished_signal_.Wait();
  572. }
  573. }
  574. // Finish current background compaction in the case where
  575. // `background_work_finished_signal_` was signalled due to an error.
  576. while (background_compaction_scheduled_) {
  577. background_work_finished_signal_.Wait();
  578. }
  579. if (manual_compaction_ == &manual) {
  580. // Cancel my manual compaction since we aborted early for some reason.
  581. manual_compaction_ = nullptr;
  582. }
  583. }
  584. Status DBImpl::TEST_CompactMemTable() {
  585. // nullptr batch means just wait for earlier writes to be done
  586. Status s = Write(WriteOptions(), nullptr);
  587. if (s.ok()) {
  588. // Wait until the compaction completes
  589. MutexLock l(&mutex_);
  590. while (imm_ != nullptr && bg_error_.ok()) {
  591. background_work_finished_signal_.Wait();
  592. }
  593. if (imm_ != nullptr) {
  594. s = bg_error_;
  595. }
  596. }
  597. return s;
  598. }
  599. void DBImpl::RecordBackgroundError(const Status& s) {
  600. mutex_.AssertHeld();
  601. if (bg_error_.ok()) {
  602. bg_error_ = s;
  603. background_work_finished_signal_.SignalAll();
  604. }
  605. }
  606. void DBImpl::MaybeScheduleCompaction() {
  607. mutex_.AssertHeld();
  608. if (background_compaction_scheduled_) {
  609. // Already scheduled
  610. } else if (shutting_down_.load(std::memory_order_acquire)) {
  611. // DB is being deleted; no more background compactions
  612. } else if (!bg_error_.ok()) {
  613. // Already got an error; no more changes
  614. } else if (imm_ == nullptr && manual_compaction_ == nullptr &&
  615. !versions_->NeedsCompaction()) {
  616. // No work to be done
  617. } else {
  618. background_compaction_scheduled_ = true;
  619. env_->Schedule(&DBImpl::BGWork, this);
  620. }
  621. }
  622. void DBImpl::BGWork(void* db) {
  623. reinterpret_cast<DBImpl*>(db)->BackgroundCall();
  624. }
  625. void DBImpl::BackgroundCall() {
  626. MutexLock l(&mutex_);
  627. assert(background_compaction_scheduled_);
  628. if (shutting_down_.load(std::memory_order_acquire)) {
  629. // No more background work when shutting down.
  630. } else if (!bg_error_.ok()) {
  631. // No more background work after a background error.
  632. } else {
  633. BackgroundCompaction();
  634. }
  635. background_compaction_scheduled_ = false;
  636. // Previous compaction may have produced too many files in a level,
  637. // so reschedule another compaction if needed.
  638. MaybeScheduleCompaction();
  639. background_work_finished_signal_.SignalAll();
  640. }
  641. void DBImpl::BackgroundCompaction() {
  642. mutex_.AssertHeld();
  643. if (imm_ != nullptr) {
  644. CompactMemTable();
  645. return;
  646. }
  647. Compaction* c;
  648. bool is_manual = (manual_compaction_ != nullptr);
  649. InternalKey manual_end;
  650. if (is_manual) {
  651. ManualCompaction* m = manual_compaction_;
  652. c = versions_->CompactRange(m->level, m->begin, m->end);
  653. m->done = (c == nullptr);
  654. if (c != nullptr) {
  655. manual_end = c->input(0, c->num_input_files(0) - 1)->largest;
  656. }
  657. Log(options_.info_log,
  658. "Manual compaction at level-%d from %s .. %s; will stop at %s\n",
  659. m->level, (m->begin ? m->begin->DebugString().c_str() : "(begin)"),
  660. (m->end ? m->end->DebugString().c_str() : "(end)"),
  661. (m->done ? "(end)" : manual_end.DebugString().c_str()));
  662. } else {
  663. c = versions_->PickCompaction();
  664. }
  665. Status status;
  666. if (c == nullptr) {
  667. // Nothing to do
  668. } else if (!is_manual && c->IsTrivialMove()) {
  669. // Move file to next level
  670. assert(c->num_input_files(0) == 1);
  671. FileMetaData* f = c->input(0, 0);
  672. c->edit()->RemoveFile(c->level(), f->number);
  673. c->edit()->AddFile(c->level() + 1, f->number, f->file_size, f->smallest,
  674. f->largest);
  675. status = versions_->LogAndApply(c->edit(), &mutex_);
  676. if (!status.ok()) {
  677. RecordBackgroundError(status);
  678. }
  679. VersionSet::LevelSummaryStorage tmp;
  680. Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n",
  681. static_cast<unsigned long long>(f->number), c->level() + 1,
  682. static_cast<unsigned long long>(f->file_size),
  683. status.ToString().c_str(), versions_->LevelSummary(&tmp));
  684. } else {
  685. CompactionState* compact = new CompactionState(c);
  686. status = DoCompactionWork(compact);
  687. if (!status.ok()) {
  688. RecordBackgroundError(status);
  689. }
  690. CleanupCompaction(compact);
  691. c->ReleaseInputs();
  692. RemoveObsoleteFiles();
  693. }
  694. delete c;
  695. if (status.ok()) {
  696. // Done
  697. } else if (shutting_down_.load(std::memory_order_acquire)) {
  698. // Ignore compaction errors found during shutting down
  699. } else {
  700. Log(options_.info_log, "Compaction error: %s", status.ToString().c_str());
  701. }
  702. if (is_manual) {
  703. ManualCompaction* m = manual_compaction_;
  704. if (!status.ok()) {
  705. m->done = true;
  706. }
  707. if (!m->done) {
  708. // We only compacted part of the requested range. Update *m
  709. // to the range that is left to be compacted.
  710. m->tmp_storage = manual_end;
  711. m->begin = &m->tmp_storage;
  712. }
  713. manual_compaction_ = nullptr;
  714. }
  715. }
  716. void DBImpl::CleanupCompaction(CompactionState* compact) {
  717. mutex_.AssertHeld();
  718. if (compact->builder != nullptr) {
  719. // May happen if we get a shutdown call in the middle of compaction
  720. compact->builder->Abandon();
  721. delete compact->builder;
  722. } else {
  723. assert(compact->outfile == nullptr);
  724. }
  725. delete compact->outfile;
  726. for (size_t i = 0; i < compact->outputs.size(); i++) {
  727. const CompactionState::Output& out = compact->outputs[i];
  728. pending_outputs_.erase(out.number);
  729. }
  730. delete compact;
  731. }
  732. Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
  733. assert(compact != nullptr);
  734. assert(compact->builder == nullptr);
  735. uint64_t file_number;
  736. {
  737. mutex_.Lock();
  738. file_number = versions_->NewFileNumber();
  739. pending_outputs_.insert(file_number);
  740. CompactionState::Output out;
  741. out.number = file_number;
  742. out.smallest.Clear();
  743. out.largest.Clear();
  744. compact->outputs.push_back(out);
  745. mutex_.Unlock();
  746. }
  747. // Make the output file
  748. std::string fname = TableFileName(dbname_, file_number);
  749. Status s = env_->NewWritableFile(fname, &compact->outfile);
  750. if (s.ok()) {
  751. compact->builder = new TableBuilder(options_, compact->outfile);
  752. }
  753. return s;
  754. }
  755. Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
  756. Iterator* input) {
  757. assert(compact != nullptr);
  758. assert(compact->outfile != nullptr);
  759. assert(compact->builder != nullptr);
  760. const uint64_t output_number = compact->current_output()->number;
  761. assert(output_number != 0);
  762. // Check for iterator errors
  763. Status s = input->status();
  764. const uint64_t current_entries = compact->builder->NumEntries();
  765. if (s.ok()) {
  766. s = compact->builder->Finish();
  767. } else {
  768. compact->builder->Abandon();
  769. }
  770. const uint64_t current_bytes = compact->builder->FileSize();
  771. compact->current_output()->file_size = current_bytes;
  772. compact->total_bytes += current_bytes;
  773. delete compact->builder;
  774. compact->builder = nullptr;
  775. // Finish and check for file errors
  776. if (s.ok()) {
  777. s = compact->outfile->Sync();
  778. }
  779. if (s.ok()) {
  780. s = compact->outfile->Close();
  781. }
  782. delete compact->outfile;
  783. compact->outfile = nullptr;
  784. if (s.ok() && current_entries > 0) {
  785. // Verify that the table is usable
  786. Iterator* iter =
  787. table_cache_->NewIterator(ReadOptions(), output_number, current_bytes);
  788. s = iter->status();
  789. delete iter;
  790. if (s.ok()) {
  791. Log(options_.info_log, "Generated table #%llu@%d: %lld keys, %lld bytes",
  792. (unsigned long long)output_number, compact->compaction->level(),
  793. (unsigned long long)current_entries,
  794. (unsigned long long)current_bytes);
  795. }
  796. }
  797. return s;
  798. }
  799. Status DBImpl::InstallCompactionResults(CompactionState* compact) {
  800. mutex_.AssertHeld();
  801. Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes",
  802. compact->compaction->num_input_files(0), compact->compaction->level(),
  803. compact->compaction->num_input_files(1), compact->compaction->level() + 1,
  804. static_cast<long long>(compact->total_bytes));
  805. // Add compaction outputs
  806. compact->compaction->AddInputDeletions(compact->compaction->edit());
  807. const int level = compact->compaction->level();
  808. for (size_t i = 0; i < compact->outputs.size(); i++) {
  809. const CompactionState::Output& out = compact->outputs[i];
  810. compact->compaction->edit()->AddFile(level + 1, out.number, out.file_size,
  811. out.smallest, out.largest);
  812. }
  813. return versions_->LogAndApply(compact->compaction->edit(), &mutex_);
  814. }
  815. Status DBImpl::DoCompactionWork(CompactionState* compact) {
  816. const uint64_t start_micros = env_->NowMicros();
  817. int64_t imm_micros = 0; // Micros spent doing imm_ compactions
  818. Log(options_.info_log, "Compacting %d@%d + %d@%d files",
  819. compact->compaction->num_input_files(0), compact->compaction->level(),
  820. compact->compaction->num_input_files(1),
  821. compact->compaction->level() + 1);
  822. assert(versions_->NumLevelFiles(compact->compaction->level()) > 0);
  823. assert(compact->builder == nullptr);
  824. assert(compact->outfile == nullptr);
  825. if (snapshots_.empty()) {
  826. compact->smallest_snapshot = versions_->LastSequence();
  827. } else {
  828. compact->smallest_snapshot = snapshots_.oldest()->sequence_number();
  829. }
  830. Iterator* input = versions_->MakeInputIterator(compact->compaction);
  831. // Release mutex while we're actually doing the compaction work
  832. mutex_.Unlock();
  833. input->SeekToFirst();
  834. Status status;
  835. ParsedInternalKey ikey;
  836. std::string current_user_key;
  837. bool has_current_user_key = false;
  838. SequenceNumber last_sequence_for_key = kMaxSequenceNumber;
  839. while (input->Valid() && !shutting_down_.load(std::memory_order_acquire)) {
  840. // Prioritize immutable compaction work
  841. if (has_imm_.load(std::memory_order_relaxed)) {
  842. const uint64_t imm_start = env_->NowMicros();
  843. mutex_.Lock();
  844. if (imm_ != nullptr) {
  845. CompactMemTable();
  846. // Wake up MakeRoomForWrite() if necessary.
  847. background_work_finished_signal_.SignalAll();
  848. }
  849. mutex_.Unlock();
  850. imm_micros += (env_->NowMicros() - imm_start);
  851. }
  852. Slice key = input->key();
  853. if (compact->compaction->ShouldStopBefore(key) &&
  854. compact->builder != nullptr) {
  855. status = FinishCompactionOutputFile(compact, input);
  856. if (!status.ok()) {
  857. break;
  858. }
  859. }
  860. // Handle key/value, add to state, etc.
  861. bool drop = false;
  862. // 检查数据是否过期-朱陈媛
  863. Slice value = input->value();
  864. std::string value_with_ttl(value.data(), value.size());
  865. if (value_with_ttl.size() >= 19) {
  866. std::string expiration_time_str = value_with_ttl.substr(value_with_ttl.size() - 19); // 提取过期时间戳
  867. std::tm tm = {};
  868. char* res = strptime(expiration_time_str.c_str(), "%Y-%m-%d %H:%M:%S", &tm);
  869. if (res == nullptr) {
  870. std::cerr << "Failed to parse expiration time: " << expiration_time_str << std::endl;
  871. drop = false; // 解析失败则视为有效
  872. } else {
  873. std::time_t expiration_time = std::mktime(&tm);
  874. std::time_t current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
  875. if (expiration_time <= current_time) {
  876. drop = true; // 数据过期,标记为丢弃
  877. }
  878. else
  879. {
  880. drop = false;
  881. }
  882. }
  883. } else {
  884. std::cerr << "Invalid value length for expiration time extraction." << std::endl;
  885. drop = false; // 无法提取到过期时间,则视为有效
  886. }
  887. if (!ParseInternalKey(key, &ikey)) {
  888. // Do not hide error keys
  889. current_user_key.clear();
  890. has_current_user_key = false;
  891. last_sequence_for_key = kMaxSequenceNumber;
  892. } else {
  893. if (!has_current_user_key ||
  894. user_comparator()->Compare(ikey.user_key, Slice(current_user_key)) !=
  895. 0) {
  896. // First occurrence of this user key
  897. current_user_key.assign(ikey.user_key.data(), ikey.user_key.size());
  898. has_current_user_key = true;
  899. last_sequence_for_key = kMaxSequenceNumber;
  900. }
  901. if (last_sequence_for_key <= compact->smallest_snapshot) {
  902. // Hidden by an newer entry for same user key
  903. drop = true; // (A)
  904. } else if (ikey.type == kTypeDeletion &&
  905. ikey.sequence <= compact->smallest_snapshot &&
  906. compact->compaction->IsBaseLevelForKey(ikey.user_key)) {
  907. // For this user key:
  908. // (1) there is no data in higher levels
  909. // (2) data in lower levels will have larger sequence numbers
  910. // (3) data in layers that are being compacted here and have
  911. // smaller sequence numbers will be dropped in the next
  912. // few iterations of this loop (by rule (A) above).
  913. // Therefore this deletion marker is obsolete and can be dropped.
  914. drop = true;
  915. }
  916. last_sequence_for_key = ikey.sequence;
  917. }
  918. #if 0
  919. Log(options_.info_log,
  920. " Compact: %s, seq %d, type: %d %d, drop: %d, is_base: %d, "
  921. "%d smallest_snapshot: %d",
  922. ikey.user_key.ToString().c_str(),
  923. (int)ikey.sequence, ikey.type, kTypeValue, drop,
  924. compact->compaction->IsBaseLevelForKey(ikey.user_key),
  925. (int)last_sequence_for_key, (int)compact->smallest_snapshot);
  926. #endif
  927. if (!drop) {
  928. // Open output file if necessary
  929. if (compact->builder == nullptr) {
  930. status = OpenCompactionOutputFile(compact);
  931. if (!status.ok()) {
  932. break;
  933. }
  934. }
  935. if (compact->builder->NumEntries() == 0) {
  936. compact->current_output()->smallest.DecodeFrom(key);
  937. }
  938. compact->current_output()->largest.DecodeFrom(key);
  939. compact->builder->Add(key, input->value());
  940. // Close output file if it is big enough
  941. if (compact->builder->FileSize() >=
  942. compact->compaction->MaxOutputFileSize()) {
  943. status = FinishCompactionOutputFile(compact, input);
  944. if (!status.ok()) {
  945. break;
  946. }
  947. }
  948. }
  949. input->Next();
  950. }
  951. if (status.ok() && shutting_down_.load(std::memory_order_acquire)) {
  952. status = Status::IOError("Deleting DB during compaction");
  953. }
  954. if (status.ok() && compact->builder != nullptr) {
  955. status = FinishCompactionOutputFile(compact, input);
  956. }
  957. if (status.ok()) {
  958. status = input->status();
  959. }
  960. delete input;
  961. input = nullptr;
  962. CompactionStats stats;
  963. stats.micros = env_->NowMicros() - start_micros - imm_micros;
  964. for (int which = 0; which < 2; which++) {
  965. for (int i = 0; i < compact->compaction->num_input_files(which); i++) {
  966. stats.bytes_read += compact->compaction->input(which, i)->file_size;
  967. }
  968. }
  969. for (size_t i = 0; i < compact->outputs.size(); i++) {
  970. stats.bytes_written += compact->outputs[i].file_size;
  971. }
  972. mutex_.Lock();
  973. stats_[compact->compaction->level() + 1].Add(stats);
  974. if (status.ok()) {
  975. status = InstallCompactionResults(compact);
  976. }
  977. if (!status.ok()) {
  978. RecordBackgroundError(status);
  979. }
  980. VersionSet::LevelSummaryStorage tmp;
  981. Log(options_.info_log, "compacted to: %s", versions_->LevelSummary(&tmp));
  982. return status;
  983. }
  984. namespace {
  985. struct IterState {
  986. port::Mutex* const mu;
  987. Version* const version GUARDED_BY(mu);
  988. MemTable* const mem GUARDED_BY(mu);
  989. MemTable* const imm GUARDED_BY(mu);
  990. IterState(port::Mutex* mutex, MemTable* mem, MemTable* imm, Version* version)
  991. : mu(mutex), version(version), mem(mem), imm(imm) {}
  992. };
  993. static void CleanupIteratorState(void* arg1, void* arg2) {
  994. IterState* state = reinterpret_cast<IterState*>(arg1);
  995. state->mu->Lock();
  996. state->mem->Unref();
  997. if (state->imm != nullptr) state->imm->Unref();
  998. state->version->Unref();
  999. state->mu->Unlock();
  1000. delete state;
  1001. }
  1002. } // anonymous namespace
  1003. Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
  1004. SequenceNumber* latest_snapshot,
  1005. uint32_t* seed) {
  1006. mutex_.Lock();
  1007. *latest_snapshot = versions_->LastSequence();
  1008. // Collect together all needed child iterators
  1009. std::vector<Iterator*> list;
  1010. list.push_back(mem_->NewIterator());
  1011. mem_->Ref();
  1012. if (imm_ != nullptr) {
  1013. list.push_back(imm_->NewIterator());
  1014. imm_->Ref();
  1015. }
  1016. versions_->current()->AddIterators(options, &list);
  1017. Iterator* internal_iter =
  1018. NewMergingIterator(&internal_comparator_, &list[0], list.size());
  1019. versions_->current()->Ref();
  1020. IterState* cleanup = new IterState(&mutex_, mem_, imm_, versions_->current());
  1021. internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, nullptr);
  1022. *seed = ++seed_;
  1023. mutex_.Unlock();
  1024. return internal_iter;
  1025. }
  1026. Iterator* DBImpl::TEST_NewInternalIterator() {
  1027. SequenceNumber ignored;
  1028. uint32_t ignored_seed;
  1029. return NewInternalIterator(ReadOptions(), &ignored, &ignored_seed);
  1030. }
  1031. int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() {
  1032. MutexLock l(&mutex_);
  1033. return versions_->MaxNextLevelOverlappingBytes();
  1034. }
  1035. Status DBImpl::Get(const ReadOptions& options, const Slice& key,
  1036. std::string* value) {
  1037. Status s;
  1038. MutexLock l(&mutex_);
  1039. SequenceNumber snapshot;
  1040. if (options.snapshot != nullptr) {
  1041. snapshot =
  1042. static_cast<const SnapshotImpl*>(options.snapshot)->sequence_number();
  1043. } else {
  1044. snapshot = versions_->LastSequence();
  1045. }
  1046. MemTable* mem = mem_;
  1047. MemTable* imm = imm_;
  1048. Version* current = versions_->current();
  1049. mem->Ref();
  1050. if (imm != nullptr) imm->Ref();
  1051. current->Ref();
  1052. bool have_stat_update = false;
  1053. Version::GetStats stats;
  1054. // Unlock while reading from files and memtables
  1055. {
  1056. mutex_.Unlock();
  1057. // First look in the memtable, then in the immutable memtable (if any).
  1058. LookupKey lkey(key, snapshot);
  1059. if (mem->Get(lkey, value, &s)) {
  1060. // Done
  1061. } else if (imm != nullptr && imm->Get(lkey, value, &s)) {
  1062. // Done
  1063. } else {
  1064. s = current->Get(options, lkey, value, &stats);
  1065. have_stat_update = true;
  1066. }
  1067. mutex_.Lock();
  1068. }
  1069. if (have_stat_update && current->UpdateStats(stats)) {
  1070. MaybeScheduleCompaction();
  1071. }
  1072. mem->Unref();
  1073. if (imm != nullptr) imm->Unref();
  1074. current->Unref();
  1075. return s;
  1076. }
  1077. Iterator* DBImpl::NewIterator(const ReadOptions& options) {
  1078. SequenceNumber latest_snapshot;
  1079. uint32_t seed;
  1080. Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed);
  1081. return NewDBIterator(this, user_comparator(), iter,
  1082. (options.snapshot != nullptr
  1083. ? static_cast<const SnapshotImpl*>(options.snapshot)
  1084. ->sequence_number()
  1085. : latest_snapshot),
  1086. seed);
  1087. }
  1088. void DBImpl::RecordReadSample(Slice key) {
  1089. MutexLock l(&mutex_);
  1090. if (versions_->current()->RecordReadSample(key)) {
  1091. MaybeScheduleCompaction();
  1092. }
  1093. }
  1094. const Snapshot* DBImpl::GetSnapshot() {
  1095. MutexLock l(&mutex_);
  1096. return snapshots_.New(versions_->LastSequence());
  1097. }
  1098. void DBImpl::ReleaseSnapshot(const Snapshot* snapshot) {
  1099. MutexLock l(&mutex_);
  1100. snapshots_.Delete(static_cast<const SnapshotImpl*>(snapshot));
  1101. }
  1102. Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val) {
  1103. return DB::Put(o, key, val);
  1104. }
  1105. // 添加ttl-朱陈媛
  1106. // Convenience methods
  1107. Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val, uint64_t ttl) {
  1108. return DB::Put(o, key, val, ttl);
  1109. }
  1110. Status DBImpl::Delete(const WriteOptions& options, const Slice& key) {
  1111. return DB::Delete(options, key);
  1112. }
  1113. Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
  1114. Writer w(&mutex_);
  1115. w.batch = updates;
  1116. w.sync = options.sync;
  1117. w.done = false;
  1118. MutexLock l(&mutex_);
  1119. writers_.push_back(&w);
  1120. while (!w.done && &w != writers_.front()) {
  1121. w.cv.Wait();
  1122. }
  1123. if (w.done) {
  1124. return w.status;
  1125. }
  1126. // May temporarily unlock and wait.
  1127. Status status = MakeRoomForWrite(updates == nullptr);
  1128. uint64_t last_sequence = versions_->LastSequence();
  1129. Writer* last_writer = &w;
  1130. if (status.ok() && updates != nullptr) { // nullptr batch is for compactions
  1131. WriteBatch* write_batch = BuildBatchGroup(&last_writer);
  1132. WriteBatchInternal::SetSequence(write_batch, last_sequence + 1);
  1133. last_sequence += WriteBatchInternal::Count(write_batch);
  1134. // Add to log and apply to memtable. We can release the lock
  1135. // during this phase since &w is currently responsible for logging
  1136. // and protects against concurrent loggers and concurrent writes
  1137. // into mem_.
  1138. {
  1139. mutex_.Unlock();
  1140. status = log_->AddRecord(WriteBatchInternal::Contents(write_batch));
  1141. bool sync_error = false;
  1142. if (status.ok() && options.sync) {
  1143. status = logfile_->Sync();
  1144. if (!status.ok()) {
  1145. sync_error = true;
  1146. }
  1147. }
  1148. if (status.ok()) {
  1149. status = WriteBatchInternal::InsertInto(write_batch, mem_);
  1150. }
  1151. mutex_.Lock();
  1152. if (sync_error) {
  1153. // The state of the log file is indeterminate: the log record we
  1154. // just added may or may not show up when the DB is re-opened.
  1155. // So we force the DB into a mode where all future writes fail.
  1156. RecordBackgroundError(status);
  1157. }
  1158. }
  1159. if (write_batch == tmp_batch_) tmp_batch_->Clear();
  1160. versions_->SetLastSequence(last_sequence);
  1161. }
  1162. while (true) {
  1163. Writer* ready = writers_.front();
  1164. writers_.pop_front();
  1165. if (ready != &w) {
  1166. ready->status = status;
  1167. ready->done = true;
  1168. ready->cv.Signal();
  1169. }
  1170. if (ready == last_writer) break;
  1171. }
  1172. // Notify new head of write queue
  1173. if (!writers_.empty()) {
  1174. writers_.front()->cv.Signal();
  1175. }
  1176. return status;
  1177. }
  1178. // REQUIRES: Writer list must be non-empty
  1179. // REQUIRES: First writer must have a non-null batch
  1180. WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
  1181. mutex_.AssertHeld();
  1182. assert(!writers_.empty());
  1183. Writer* first = writers_.front();
  1184. WriteBatch* result = first->batch;
  1185. assert(result != nullptr);
  1186. size_t size = WriteBatchInternal::ByteSize(first->batch);
  1187. // Allow the group to grow up to a maximum size, but if the
  1188. // original write is small, limit the growth so we do not slow
  1189. // down the small write too much.
  1190. size_t max_size = 1 << 20;
  1191. if (size <= (128 << 10)) {
  1192. max_size = size + (128 << 10);
  1193. }
  1194. *last_writer = first;
  1195. std::deque<Writer*>::iterator iter = writers_.begin();
  1196. ++iter; // Advance past "first"
  1197. for (; iter != writers_.end(); ++iter) {
  1198. Writer* w = *iter;
  1199. if (w->sync && !first->sync) {
  1200. // Do not include a sync write into a batch handled by a non-sync write.
  1201. break;
  1202. }
  1203. if (w->batch != nullptr) {
  1204. size += WriteBatchInternal::ByteSize(w->batch);
  1205. if (size > max_size) {
  1206. // Do not make batch too big
  1207. break;
  1208. }
  1209. // Append to *result
  1210. if (result == first->batch) {
  1211. // Switch to temporary batch instead of disturbing caller's batch
  1212. result = tmp_batch_;
  1213. assert(WriteBatchInternal::Count(result) == 0);
  1214. WriteBatchInternal::Append(result, first->batch);
  1215. }
  1216. WriteBatchInternal::Append(result, w->batch);
  1217. }
  1218. *last_writer = w;
  1219. }
  1220. return result;
  1221. }
  1222. // REQUIRES: mutex_ is held
  1223. // REQUIRES: this thread is currently at the front of the writer queue
  1224. Status DBImpl::MakeRoomForWrite(bool force) {
  1225. mutex_.AssertHeld();
  1226. assert(!writers_.empty());
  1227. bool allow_delay = !force;
  1228. Status s;
  1229. while (true) {
  1230. if (!bg_error_.ok()) {
  1231. // Yield previous error
  1232. s = bg_error_;
  1233. break;
  1234. } else if (allow_delay && versions_->NumLevelFiles(0) >=
  1235. config::kL0_SlowdownWritesTrigger) {
  1236. // We are getting close to hitting a hard limit on the number of
  1237. // L0 files. Rather than delaying a single write by several
  1238. // seconds when we hit the hard limit, start delaying each
  1239. // individual write by 1ms to reduce latency variance. Also,
  1240. // this delay hands over some CPU to the compaction thread in
  1241. // case it is sharing the same core as the writer.
  1242. mutex_.Unlock();
  1243. env_->SleepForMicroseconds(1000);
  1244. allow_delay = false; // Do not delay a single write more than once
  1245. mutex_.Lock();
  1246. } else if (!force &&
  1247. (mem_->ApproximateMemoryUsage() <= options_.write_buffer_size)) {
  1248. // There is room in current memtable
  1249. break;
  1250. } else if (imm_ != nullptr) {
  1251. // We have filled up the current memtable, but the previous
  1252. // one is still being compacted, so we wait.
  1253. Log(options_.info_log, "Current memtable full; waiting...\n");
  1254. background_work_finished_signal_.Wait();
  1255. } else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) {
  1256. // There are too many level-0 files.
  1257. Log(options_.info_log, "Too many L0 files; waiting...\n");
  1258. background_work_finished_signal_.Wait();
  1259. } else {
  1260. // Attempt to switch to a new memtable and trigger compaction of old
  1261. assert(versions_->PrevLogNumber() == 0);
  1262. uint64_t new_log_number = versions_->NewFileNumber();
  1263. WritableFile* lfile = nullptr;
  1264. s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile);
  1265. if (!s.ok()) {
  1266. // Avoid chewing through file number space in a tight loop.
  1267. versions_->ReuseFileNumber(new_log_number);
  1268. break;
  1269. }
  1270. delete log_;
  1271. s = logfile_->Close();
  1272. if (!s.ok()) {
  1273. // We may have lost some data written to the previous log file.
  1274. // Switch to the new log file anyway, but record as a background
  1275. // error so we do not attempt any more writes.
  1276. //
  1277. // We could perhaps attempt to save the memtable corresponding
  1278. // to log file and suppress the error if that works, but that
  1279. // would add more complexity in a critical code path.
  1280. RecordBackgroundError(s);
  1281. }
  1282. delete logfile_;
  1283. logfile_ = lfile;
  1284. logfile_number_ = new_log_number;
  1285. log_ = new log::Writer(lfile);
  1286. imm_ = mem_;
  1287. has_imm_.store(true, std::memory_order_release);
  1288. mem_ = new MemTable(internal_comparator_);
  1289. mem_->Ref();
  1290. force = false; // Do not force another compaction if have room
  1291. MaybeScheduleCompaction();
  1292. }
  1293. }
  1294. return s;
  1295. }
  1296. bool DBImpl::GetProperty(const Slice& property, std::string* value) {
  1297. value->clear();
  1298. MutexLock l(&mutex_);
  1299. Slice in = property;
  1300. Slice prefix("leveldb.");
  1301. if (!in.starts_with(prefix)) return false;
  1302. in.remove_prefix(prefix.size());
  1303. if (in.starts_with("num-files-at-level")) {
  1304. in.remove_prefix(strlen("num-files-at-level"));
  1305. uint64_t level;
  1306. bool ok = ConsumeDecimalNumber(&in, &level) && in.empty();
  1307. if (!ok || level >= config::kNumLevels) {
  1308. return false;
  1309. } else {
  1310. char buf[100];
  1311. std::snprintf(buf, sizeof(buf), "%d",
  1312. versions_->NumLevelFiles(static_cast<int>(level)));
  1313. *value = buf;
  1314. return true;
  1315. }
  1316. } else if (in == "stats") {
  1317. char buf[200];
  1318. std::snprintf(buf, sizeof(buf),
  1319. " Compactions\n"
  1320. "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n"
  1321. "--------------------------------------------------\n");
  1322. value->append(buf);
  1323. for (int level = 0; level < config::kNumLevels; level++) {
  1324. int files = versions_->NumLevelFiles(level);
  1325. if (stats_[level].micros > 0 || files > 0) {
  1326. std::snprintf(buf, sizeof(buf), "%3d %8d %8.0f %9.0f %8.0f %9.0f\n",
  1327. level, files, versions_->NumLevelBytes(level) / 1048576.0,
  1328. stats_[level].micros / 1e6,
  1329. stats_[level].bytes_read / 1048576.0,
  1330. stats_[level].bytes_written / 1048576.0);
  1331. value->append(buf);
  1332. }
  1333. }
  1334. return true;
  1335. } else if (in == "sstables") {
  1336. *value = versions_->current()->DebugString();
  1337. return true;
  1338. } else if (in == "approximate-memory-usage") {
  1339. size_t total_usage = options_.block_cache->TotalCharge();
  1340. if (mem_) {
  1341. total_usage += mem_->ApproximateMemoryUsage();
  1342. }
  1343. if (imm_) {
  1344. total_usage += imm_->ApproximateMemoryUsage();
  1345. }
  1346. char buf[50];
  1347. std::snprintf(buf, sizeof(buf), "%llu",
  1348. static_cast<unsigned long long>(total_usage));
  1349. value->append(buf);
  1350. return true;
  1351. }
  1352. return false;
  1353. }
  1354. void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) {
  1355. // TODO(opt): better implementation
  1356. MutexLock l(&mutex_);
  1357. Version* v = versions_->current();
  1358. v->Ref();
  1359. for (int i = 0; i < n; i++) {
  1360. // Convert user_key into a corresponding internal key.
  1361. InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek);
  1362. InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek);
  1363. uint64_t start = versions_->ApproximateOffsetOf(v, k1);
  1364. uint64_t limit = versions_->ApproximateOffsetOf(v, k2);
  1365. sizes[i] = (limit >= start ? limit - start : 0);
  1366. }
  1367. v->Unref();
  1368. }
  1369. // Default implementations of convenience methods that subclasses of DB
  1370. // can call if they wish
  1371. Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) {
  1372. WriteBatch batch;
  1373. batch.Put(key, value);
  1374. return Write(opt, &batch);
  1375. }
  1376. // 添加ttl-朱陈媛
  1377. Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value, uint64_t ttl) {
  1378. WriteBatch batch;
  1379. batch.Put(key, value, ttl);
  1380. return Write(opt, &batch);
  1381. }
  1382. Status DB::Delete(const WriteOptions& opt, const Slice& key) {
  1383. WriteBatch batch;
  1384. batch.Delete(key);
  1385. return Write(opt, &batch);
  1386. }
  1387. DB::~DB() = default;
  1388. Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
  1389. *dbptr = nullptr;
  1390. DBImpl* impl = new DBImpl(options, dbname);
  1391. impl->mutex_.Lock();
  1392. VersionEdit edit;
  1393. // Recover handles create_if_missing, error_if_exists
  1394. bool save_manifest = false;
  1395. Status s = impl->Recover(&edit, &save_manifest);
  1396. if (s.ok() && impl->mem_ == nullptr) {
  1397. // Create new log and a corresponding memtable.
  1398. uint64_t new_log_number = impl->versions_->NewFileNumber();
  1399. WritableFile* lfile;
  1400. s = options.env->NewWritableFile(LogFileName(dbname, new_log_number),
  1401. &lfile);
  1402. if (s.ok()) {
  1403. edit.SetLogNumber(new_log_number);
  1404. impl->logfile_ = lfile;
  1405. impl->logfile_number_ = new_log_number;
  1406. impl->log_ = new log::Writer(lfile);
  1407. impl->mem_ = new MemTable(impl->internal_comparator_);
  1408. impl->mem_->Ref();
  1409. }
  1410. }
  1411. if (s.ok() && save_manifest) {
  1412. edit.SetPrevLogNumber(0); // No older logs needed after recovery.
  1413. edit.SetLogNumber(impl->logfile_number_);
  1414. s = impl->versions_->LogAndApply(&edit, &impl->mutex_);
  1415. }
  1416. if (s.ok()) {
  1417. impl->RemoveObsoleteFiles();
  1418. impl->MaybeScheduleCompaction();
  1419. }
  1420. impl->mutex_.Unlock();
  1421. if (s.ok()) {
  1422. assert(impl->mem_ != nullptr);
  1423. *dbptr = impl;
  1424. } else {
  1425. delete impl;
  1426. }
  1427. return s;
  1428. }
  1429. Snapshot::~Snapshot() = default;
  1430. Status DestroyDB(const std::string& dbname, const Options& options) {
  1431. Env* env = options.env;
  1432. std::vector<std::string> filenames;
  1433. Status result = env->GetChildren(dbname, &filenames);
  1434. if (!result.ok()) {
  1435. // Ignore error in case directory does not exist
  1436. return Status::OK();
  1437. }
  1438. FileLock* lock;
  1439. const std::string lockname = LockFileName(dbname);
  1440. result = env->LockFile(lockname, &lock);
  1441. if (result.ok()) {
  1442. uint64_t number;
  1443. FileType type;
  1444. for (size_t i = 0; i < filenames.size(); i++) {
  1445. if (ParseFileName(filenames[i], &number, &type) &&
  1446. type != kDBLockFile) { // Lock file will be deleted at end
  1447. Status del = env->RemoveFile(dbname + "/" + filenames[i]);
  1448. if (result.ok() && !del.ok()) {
  1449. result = del;
  1450. }
  1451. }
  1452. }
  1453. env->UnlockFile(lock); // Ignore error since state is already gone
  1454. env->RemoveFile(lockname);
  1455. env->RemoveDir(dbname); // Ignore error in case dir contains other files
  1456. }
  1457. return result;
  1458. }
  1459. } // namespace leveldb