Browse Source
LevelDB now attempts to reuse the preceding MANIFEST and log file when re-opened.
(Based on a suggestion by cmumford.) "open" benchmark on my workstation speeds up significantly since we can now avoid three fdatasync calls and a compaction per open: Before: ~80000 microseconds After: ~130 microseconds Details: (1) Added Options::reuse_logs (currently defaults to false) to control new behavior. The intention is to change the default to true after some baking. (2) Added Env::NewAppendableFile() whose default implementation returns a not-supported error. (3) VersionSet::Recovery attempts to reuse the MANIFEST from which it is recovering. (4) DBImpl recovery attempts to reuse the last log file and memtable. (5) db_test.cc now tests a new configuration that sets reuse_logs to true. (6) fault_injection_test also tests a reuse_logs==true config. (7) Added a new recovery_test.baseline
Sanjay Ghemawat
10 years ago
committed by
Chris Mumford
Chris Mumford
22 changed files with 707 additions and 155 deletions
Split View
Diff Options
-
+4 -0Makefile
-
+1 -1db/corruption_test.cc
-
+7 -0db/db_bench.cc
-
+109 -69db/db_impl.cc
-
+4 -4db/db_impl.h
-
+20 -1db/db_test.cc
-
+80 -68db/fault_injection_test.cc
-
+21 -3db/log_test.cc
-
+13 -4db/log_writer.cc
-
+6 -0db/log_writer.h
-
+324 -0db/recovery_test.cc
-
+36 -1db/version_set.cc
-
+3 -1db/version_set.h
-
+13 -0helpers/memenv/memenv.cc
-
+11 -2helpers/memenv/memenv_test.cc
-
+18 -0include/leveldb/env.h
-
+6 -0include/leveldb/options.h
-
+3 -0include/leveldb/status.h
-
+4 -0util/env.cc
-
+13 -0util/env_posix.cc
-
+1 -1util/options.cc
-
+10 -0util/testutil.h
+ 4
- 0
Makefile
View File
+ 1
- 1
db/corruption_test.cc
View File
+ 7
- 0
db/db_bench.cc
View File
+ 109
- 69
db/db_impl.cc
View File
+ 4
- 4
db/db_impl.h
View File
+ 20
- 1
db/db_test.cc
View File
+ 80
- 68
db/fault_injection_test.cc
View File
+ 21
- 3
db/log_test.cc
View File
+ 13
- 4
db/log_writer.cc
View File
+ 6
- 0
db/log_writer.h
View File
+ 324
- 0
db/recovery_test.cc
View File
| @ -0,0 +1,324 @@ | |||
| // Copyright (c) 2014 The LevelDB Authors. All rights reserved. | |||
| // Use of this source code is governed by a BSD-style license that can be | |||
| // found in the LICENSE file. See the AUTHORS file for names of contributors. | |||
| #include "db/db_impl.h" | |||
| #include "db/filename.h" | |||
| #include "db/version_set.h" | |||
| #include "db/write_batch_internal.h" | |||
| #include "leveldb/db.h" | |||
| #include "leveldb/env.h" | |||
| #include "leveldb/write_batch.h" | |||
| #include "util/logging.h" | |||
| #include "util/testharness.h" | |||
| #include "util/testutil.h" | |||
| namespace leveldb { | |||
| class RecoveryTest { | |||
| public: | |||
| RecoveryTest() : env_(Env::Default()), db_(NULL) { | |||
| dbname_ = test::TmpDir() + "/recovery_test"; | |||
| DestroyDB(dbname_, Options()); | |||
| Open(); | |||
| } | |||
| ~RecoveryTest() { | |||
| Close(); | |||
| DestroyDB(dbname_, Options()); | |||
| } | |||
| DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); } | |||
| Env* env() const { return env_; } | |||
| bool CanAppend() { | |||
| WritableFile* tmp; | |||
| Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp); | |||
| delete tmp; | |||
| if (s.IsNotSupportedError()) { | |||
| return false; | |||
| } else { | |||
| return true; | |||
| } | |||
| } | |||
| void Close() { | |||
| delete db_; | |||
| db_ = NULL; | |||
| } | |||
| void Open(Options* options = NULL) { | |||
| Close(); | |||
| Options opts; | |||
| if (options != NULL) { | |||
| opts = *options; | |||
| } else { | |||
| opts.reuse_logs = true; // TODO(sanjay): test both ways | |||
| opts.create_if_missing = true; | |||
| } | |||
| if (opts.env == NULL) { | |||
| opts.env = env_; | |||
| } | |||
| ASSERT_OK(DB::Open(opts, dbname_, &db_)); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| } | |||
| Status Put(const std::string& k, const std::string& v) { | |||
| return db_->Put(WriteOptions(), k, v); | |||
| } | |||
| std::string Get(const std::string& k, const Snapshot* snapshot = NULL) { | |||
| std::string result; | |||
| Status s = db_->Get(ReadOptions(), k, &result); | |||
| if (s.IsNotFound()) { | |||
| result = "NOT_FOUND"; | |||
| } else if (!s.ok()) { | |||
| result = s.ToString(); | |||
| } | |||
| return result; | |||
| } | |||
| std::string ManifestFileName() { | |||
| std::string current; | |||
| ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), ¤t)); | |||
| size_t len = current.size(); | |||
| if (len > 0 && current[len-1] == '\n') { | |||
| current.resize(len - 1); | |||
| } | |||
| return dbname_ + "/" + current; | |||
| } | |||
| std::string LogName(uint64_t number) { | |||
| return LogFileName(dbname_, number); | |||
| } | |||
| size_t DeleteLogFiles() { | |||
| std::vector<uint64_t> logs = GetFiles(kLogFile); | |||
| for (size_t i = 0; i < logs.size(); i++) { | |||
| ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]); | |||
| } | |||
| return logs.size(); | |||
| } | |||
| uint64_t FirstLogFile() { | |||
| return GetFiles(kLogFile)[0]; | |||
| } | |||
| std::vector<std::uint64_t> GetFiles(FileType t) { | |||
| std::vector<std::string> filenames; | |||
| ASSERT_OK(env_->GetChildren(dbname_, &filenames)); | |||
| std::vector<std::uint64_t> result; | |||
| for (size_t i = 0; i < filenames.size(); i++) { | |||
| uint64_t number; | |||
| FileType type; | |||
| if (ParseFileName(filenames[i], &number, &type) && type == t) { | |||
| result.push_back(number); | |||
| } | |||
| } | |||
| return result; | |||
| } | |||
| int NumLogs() { | |||
| return GetFiles(kLogFile).size(); | |||
| } | |||
| int NumTables() { | |||
| return GetFiles(kTableFile).size(); | |||
| } | |||
| uint64_t FileSize(const std::string& fname) { | |||
| uint64_t result; | |||
| ASSERT_OK(env_->GetFileSize(fname, &result)) << fname; | |||
| return result; | |||
| } | |||
| void CompactMemTable() { | |||
| dbfull()->TEST_CompactMemTable(); | |||
| } | |||
| // Directly construct a log file that sets key to val. | |||
| void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) { | |||
| std::string fname = LogFileName(dbname_, lognum); | |||
| WritableFile* file; | |||
| ASSERT_OK(env_->NewWritableFile(fname, &file)); | |||
| log::Writer writer(file); | |||
| WriteBatch batch; | |||
| batch.Put(key, val); | |||
| WriteBatchInternal::SetSequence(&batch, seq); | |||
| ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); | |||
| ASSERT_OK(file->Flush()); | |||
| delete file; | |||
| } | |||
| private: | |||
| std::string dbname_; | |||
| Env* env_; | |||
| DB* db_; | |||
| }; | |||
| TEST(RecoveryTest, ManifestReused) { | |||
| if (!CanAppend()) { | |||
| fprintf(stderr, "skipping test because env does not support appending\n"); | |||
| return; | |||
| } | |||
| ASSERT_OK(Put("foo", "bar")); | |||
| Close(); | |||
| std::string old_manifest = ManifestFileName(); | |||
| Open(); | |||
| ASSERT_EQ(old_manifest, ManifestFileName()); | |||
| ASSERT_EQ("bar", Get("foo")); | |||
| Open(); | |||
| ASSERT_EQ(old_manifest, ManifestFileName()); | |||
| ASSERT_EQ("bar", Get("foo")); | |||
| } | |||
| TEST(RecoveryTest, LargeManifestCompacted) { | |||
| if (!CanAppend()) { | |||
| fprintf(stderr, "skipping test because env does not support appending\n"); | |||
| return; | |||
| } | |||
| ASSERT_OK(Put("foo", "bar")); | |||
| Close(); | |||
| std::string old_manifest = ManifestFileName(); | |||
| // Pad with zeroes to make manifest file very big. | |||
| { | |||
| uint64_t len = FileSize(old_manifest); | |||
| WritableFile* file; | |||
| ASSERT_OK(env()->NewAppendableFile(old_manifest, &file)); | |||
| std::string zeroes(3*1048576 - static_cast<size_t>(len), 0); | |||
| ASSERT_OK(file->Append(zeroes)); | |||
| ASSERT_OK(file->Flush()); | |||
| delete file; | |||
| } | |||
| Open(); | |||
| std::string new_manifest = ManifestFileName(); | |||
| ASSERT_NE(old_manifest, new_manifest); | |||
| ASSERT_GT(10000, FileSize(new_manifest)); | |||
| ASSERT_EQ("bar", Get("foo")); | |||
| Open(); | |||
| ASSERT_EQ(new_manifest, ManifestFileName()); | |||
| ASSERT_EQ("bar", Get("foo")); | |||
| } | |||
| TEST(RecoveryTest, NoLogFiles) { | |||
| ASSERT_OK(Put("foo", "bar")); | |||
| ASSERT_EQ(1, DeleteLogFiles()); | |||
| Open(); | |||
| ASSERT_EQ("NOT_FOUND", Get("foo")); | |||
| Open(); | |||
| ASSERT_EQ("NOT_FOUND", Get("foo")); | |||
| } | |||
| TEST(RecoveryTest, LogFileReuse) { | |||
| if (!CanAppend()) { | |||
| fprintf(stderr, "skipping test because env does not support appending\n"); | |||
| return; | |||
| } | |||
| for (int i = 0; i < 2; i++) { | |||
| ASSERT_OK(Put("foo", "bar")); | |||
| if (i == 0) { | |||
| // Compact to ensure current log is empty | |||
| CompactMemTable(); | |||
| } | |||
| Close(); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| uint64_t number = FirstLogFile(); | |||
| if (i == 0) { | |||
| ASSERT_EQ(0, FileSize(LogName(number))); | |||
| } else { | |||
| ASSERT_LT(0, FileSize(LogName(number))); | |||
| } | |||
| Open(); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; | |||
| ASSERT_EQ("bar", Get("foo")); | |||
| Open(); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; | |||
| ASSERT_EQ("bar", Get("foo")); | |||
| } | |||
| } | |||
| TEST(RecoveryTest, MultipleMemTables) { | |||
| // Make a large log. | |||
| const int kNum = 1000; | |||
| for (int i = 0; i < kNum; i++) { | |||
| char buf[100]; | |||
| snprintf(buf, sizeof(buf), "%050d", i); | |||
| ASSERT_OK(Put(buf, buf)); | |||
| } | |||
| ASSERT_EQ(0, NumTables()); | |||
| Close(); | |||
| ASSERT_EQ(0, NumTables()); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| uint64_t old_log_file = FirstLogFile(); | |||
| // Force creation of multiple memtables by reducing the write buffer size. | |||
| Options opt; | |||
| opt.reuse_logs = true; | |||
| opt.write_buffer_size = (kNum*100) / 2; | |||
| Open(&opt); | |||
| ASSERT_LE(2, NumTables()); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log"; | |||
| for (int i = 0; i < kNum; i++) { | |||
| char buf[100]; | |||
| snprintf(buf, sizeof(buf), "%050d", i); | |||
| ASSERT_EQ(buf, Get(buf)); | |||
| } | |||
| } | |||
| TEST(RecoveryTest, MultipleLogFiles) { | |||
| ASSERT_OK(Put("foo", "bar")); | |||
| Close(); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| // Make a bunch of uncompacted log files. | |||
| uint64_t old_log = FirstLogFile(); | |||
| MakeLogFile(old_log+1, 1000, "hello", "world"); | |||
| MakeLogFile(old_log+2, 1001, "hi", "there"); | |||
| MakeLogFile(old_log+3, 1002, "foo", "bar2"); | |||
| // Recover and check that all log files were processed. | |||
| Open(); | |||
| ASSERT_LE(1, NumTables()); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| uint64_t new_log = FirstLogFile(); | |||
| ASSERT_LE(old_log+3, new_log); | |||
| ASSERT_EQ("bar2", Get("foo")); | |||
| ASSERT_EQ("world", Get("hello")); | |||
| ASSERT_EQ("there", Get("hi")); | |||
| // Test that previous recovery produced recoverable state. | |||
| Open(); | |||
| ASSERT_LE(1, NumTables()); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| if (CanAppend()) { | |||
| ASSERT_EQ(new_log, FirstLogFile()); | |||
| } | |||
| ASSERT_EQ("bar2", Get("foo")); | |||
| ASSERT_EQ("world", Get("hello")); | |||
| ASSERT_EQ("there", Get("hi")); | |||
| // Check that introducing an older log file does not cause it to be re-read. | |||
| Close(); | |||
| MakeLogFile(old_log+1, 2000, "hello", "stale write"); | |||
| Open(); | |||
| ASSERT_LE(1, NumTables()); | |||
| ASSERT_EQ(1, NumLogs()); | |||
| if (CanAppend()) { | |||
| ASSERT_EQ(new_log, FirstLogFile()); | |||
| } | |||
| ASSERT_EQ("bar2", Get("foo")); | |||
| ASSERT_EQ("world", Get("hello")); | |||
| ASSERT_EQ("there", Get("hi")); | |||
| } | |||
| } // namespace leveldb | |||
| int main(int argc, char** argv) { | |||
| return leveldb::test::RunAllTests(); | |||
| } | |||