Browse Source
LevelDB now attempts to reuse the preceding MANIFEST and log file when re-opened.
(Based on a suggestion by cmumford.) "open" benchmark on my workstation speeds up significantly since we can now avoid three fdatasync calls and a compaction per open: Before: ~80000 microseconds After: ~130 microseconds Details: (1) Added Options::reuse_logs (currently defaults to false) to control new behavior. The intention is to change the default to true after some baking. (2) Added Env::NewAppendableFile() whose default implementation returns a not-supported error. (3) VersionSet::Recovery attempts to reuse the MANIFEST from which it is recovering. (4) DBImpl recovery attempts to reuse the last log file and memtable. (5) db_test.cc now tests a new configuration that sets reuse_logs to true. (6) fault_injection_test also tests a reuse_logs==true config. (7) Added a new recovery_test.master
Sanjay Ghemawat
10 years ago
committed by
Chris Mumford
Chris Mumford
22 changed files with 707 additions and 155 deletions
Unified View
Diff Options
-
+4 -0Makefile
-
+1 -1db/corruption_test.cc
-
+7 -0db/db_bench.cc
-
+109 -69db/db_impl.cc
-
+4 -4db/db_impl.h
-
+20 -1db/db_test.cc
-
+80 -68db/fault_injection_test.cc
-
+21 -3db/log_test.cc
-
+13 -4db/log_writer.cc
-
+6 -0db/log_writer.h
-
+324 -0db/recovery_test.cc
-
+36 -1db/version_set.cc
-
+3 -1db/version_set.h
-
+13 -0helpers/memenv/memenv.cc
-
+11 -2helpers/memenv/memenv_test.cc
-
+18 -0include/leveldb/env.h
-
+6 -0include/leveldb/options.h
-
+3 -0include/leveldb/status.h
-
+4 -0util/env.cc
-
+13 -0util/env_posix.cc
-
+1 -1util/options.cc
-
+10 -0util/testutil.h
+ 4
- 0
Makefile
View File
+ 1
- 1
db/corruption_test.cc
View File
+ 7
- 0
db/db_bench.cc
View File
+ 109
- 69
db/db_impl.cc
View File
+ 4
- 4
db/db_impl.h
View File
+ 20
- 1
db/db_test.cc
View File
+ 80
- 68
db/fault_injection_test.cc
View File
+ 21
- 3
db/log_test.cc
View File
+ 13
- 4
db/log_writer.cc
View File
+ 6
- 0
db/log_writer.h
View File
+ 324
- 0
db/recovery_test.cc
View File
| @ -0,0 +1,324 @@ | |||||
| // Copyright (c) 2014 The LevelDB Authors. All rights reserved. | |||||
| // Use of this source code is governed by a BSD-style license that can be | |||||
| // found in the LICENSE file. See the AUTHORS file for names of contributors. | |||||
| #include "db/db_impl.h" | |||||
| #include "db/filename.h" | |||||
| #include "db/version_set.h" | |||||
| #include "db/write_batch_internal.h" | |||||
| #include "leveldb/db.h" | |||||
| #include "leveldb/env.h" | |||||
| #include "leveldb/write_batch.h" | |||||
| #include "util/logging.h" | |||||
| #include "util/testharness.h" | |||||
| #include "util/testutil.h" | |||||
| namespace leveldb { | |||||
| class RecoveryTest { | |||||
| public: | |||||
| RecoveryTest() : env_(Env::Default()), db_(NULL) { | |||||
| dbname_ = test::TmpDir() + "/recovery_test"; | |||||
| DestroyDB(dbname_, Options()); | |||||
| Open(); | |||||
| } | |||||
| ~RecoveryTest() { | |||||
| Close(); | |||||
| DestroyDB(dbname_, Options()); | |||||
| } | |||||
| DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); } | |||||
| Env* env() const { return env_; } | |||||
| bool CanAppend() { | |||||
| WritableFile* tmp; | |||||
| Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp); | |||||
| delete tmp; | |||||
| if (s.IsNotSupportedError()) { | |||||
| return false; | |||||
| } else { | |||||
| return true; | |||||
| } | |||||
| } | |||||
| void Close() { | |||||
| delete db_; | |||||
| db_ = NULL; | |||||
| } | |||||
| void Open(Options* options = NULL) { | |||||
| Close(); | |||||
| Options opts; | |||||
| if (options != NULL) { | |||||
| opts = *options; | |||||
| } else { | |||||
| opts.reuse_logs = true; // TODO(sanjay): test both ways | |||||
| opts.create_if_missing = true; | |||||
| } | |||||
| if (opts.env == NULL) { | |||||
| opts.env = env_; | |||||
| } | |||||
| ASSERT_OK(DB::Open(opts, dbname_, &db_)); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| } | |||||
| Status Put(const std::string& k, const std::string& v) { | |||||
| return db_->Put(WriteOptions(), k, v); | |||||
| } | |||||
| std::string Get(const std::string& k, const Snapshot* snapshot = NULL) { | |||||
| std::string result; | |||||
| Status s = db_->Get(ReadOptions(), k, &result); | |||||
| if (s.IsNotFound()) { | |||||
| result = "NOT_FOUND"; | |||||
| } else if (!s.ok()) { | |||||
| result = s.ToString(); | |||||
| } | |||||
| return result; | |||||
| } | |||||
| std::string ManifestFileName() { | |||||
| std::string current; | |||||
| ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), ¤t)); | |||||
| size_t len = current.size(); | |||||
| if (len > 0 && current[len-1] == '\n') { | |||||
| current.resize(len - 1); | |||||
| } | |||||
| return dbname_ + "/" + current; | |||||
| } | |||||
| std::string LogName(uint64_t number) { | |||||
| return LogFileName(dbname_, number); | |||||
| } | |||||
| size_t DeleteLogFiles() { | |||||
| std::vector<uint64_t> logs = GetFiles(kLogFile); | |||||
| for (size_t i = 0; i < logs.size(); i++) { | |||||
| ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]); | |||||
| } | |||||
| return logs.size(); | |||||
| } | |||||
| uint64_t FirstLogFile() { | |||||
| return GetFiles(kLogFile)[0]; | |||||
| } | |||||
| std::vector<std::uint64_t> GetFiles(FileType t) { | |||||
| std::vector<std::string> filenames; | |||||
| ASSERT_OK(env_->GetChildren(dbname_, &filenames)); | |||||
| std::vector<std::uint64_t> result; | |||||
| for (size_t i = 0; i < filenames.size(); i++) { | |||||
| uint64_t number; | |||||
| FileType type; | |||||
| if (ParseFileName(filenames[i], &number, &type) && type == t) { | |||||
| result.push_back(number); | |||||
| } | |||||
| } | |||||
| return result; | |||||
| } | |||||
| int NumLogs() { | |||||
| return GetFiles(kLogFile).size(); | |||||
| } | |||||
| int NumTables() { | |||||
| return GetFiles(kTableFile).size(); | |||||
| } | |||||
| uint64_t FileSize(const std::string& fname) { | |||||
| uint64_t result; | |||||
| ASSERT_OK(env_->GetFileSize(fname, &result)) << fname; | |||||
| return result; | |||||
| } | |||||
| void CompactMemTable() { | |||||
| dbfull()->TEST_CompactMemTable(); | |||||
| } | |||||
| // Directly construct a log file that sets key to val. | |||||
| void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) { | |||||
| std::string fname = LogFileName(dbname_, lognum); | |||||
| WritableFile* file; | |||||
| ASSERT_OK(env_->NewWritableFile(fname, &file)); | |||||
| log::Writer writer(file); | |||||
| WriteBatch batch; | |||||
| batch.Put(key, val); | |||||
| WriteBatchInternal::SetSequence(&batch, seq); | |||||
| ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); | |||||
| ASSERT_OK(file->Flush()); | |||||
| delete file; | |||||
| } | |||||
| private: | |||||
| std::string dbname_; | |||||
| Env* env_; | |||||
| DB* db_; | |||||
| }; | |||||
| TEST(RecoveryTest, ManifestReused) { | |||||
| if (!CanAppend()) { | |||||
| fprintf(stderr, "skipping test because env does not support appending\n"); | |||||
| return; | |||||
| } | |||||
| ASSERT_OK(Put("foo", "bar")); | |||||
| Close(); | |||||
| std::string old_manifest = ManifestFileName(); | |||||
| Open(); | |||||
| ASSERT_EQ(old_manifest, ManifestFileName()); | |||||
| ASSERT_EQ("bar", Get("foo")); | |||||
| Open(); | |||||
| ASSERT_EQ(old_manifest, ManifestFileName()); | |||||
| ASSERT_EQ("bar", Get("foo")); | |||||
| } | |||||
| TEST(RecoveryTest, LargeManifestCompacted) { | |||||
| if (!CanAppend()) { | |||||
| fprintf(stderr, "skipping test because env does not support appending\n"); | |||||
| return; | |||||
| } | |||||
| ASSERT_OK(Put("foo", "bar")); | |||||
| Close(); | |||||
| std::string old_manifest = ManifestFileName(); | |||||
| // Pad with zeroes to make manifest file very big. | |||||
| { | |||||
| uint64_t len = FileSize(old_manifest); | |||||
| WritableFile* file; | |||||
| ASSERT_OK(env()->NewAppendableFile(old_manifest, &file)); | |||||
| std::string zeroes(3*1048576 - static_cast<size_t>(len), 0); | |||||
| ASSERT_OK(file->Append(zeroes)); | |||||
| ASSERT_OK(file->Flush()); | |||||
| delete file; | |||||
| } | |||||
| Open(); | |||||
| std::string new_manifest = ManifestFileName(); | |||||
| ASSERT_NE(old_manifest, new_manifest); | |||||
| ASSERT_GT(10000, FileSize(new_manifest)); | |||||
| ASSERT_EQ("bar", Get("foo")); | |||||
| Open(); | |||||
| ASSERT_EQ(new_manifest, ManifestFileName()); | |||||
| ASSERT_EQ("bar", Get("foo")); | |||||
| } | |||||
| TEST(RecoveryTest, NoLogFiles) { | |||||
| ASSERT_OK(Put("foo", "bar")); | |||||
| ASSERT_EQ(1, DeleteLogFiles()); | |||||
| Open(); | |||||
| ASSERT_EQ("NOT_FOUND", Get("foo")); | |||||
| Open(); | |||||
| ASSERT_EQ("NOT_FOUND", Get("foo")); | |||||
| } | |||||
| TEST(RecoveryTest, LogFileReuse) { | |||||
| if (!CanAppend()) { | |||||
| fprintf(stderr, "skipping test because env does not support appending\n"); | |||||
| return; | |||||
| } | |||||
| for (int i = 0; i < 2; i++) { | |||||
| ASSERT_OK(Put("foo", "bar")); | |||||
| if (i == 0) { | |||||
| // Compact to ensure current log is empty | |||||
| CompactMemTable(); | |||||
| } | |||||
| Close(); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| uint64_t number = FirstLogFile(); | |||||
| if (i == 0) { | |||||
| ASSERT_EQ(0, FileSize(LogName(number))); | |||||
| } else { | |||||
| ASSERT_LT(0, FileSize(LogName(number))); | |||||
| } | |||||
| Open(); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; | |||||
| ASSERT_EQ("bar", Get("foo")); | |||||
| Open(); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file"; | |||||
| ASSERT_EQ("bar", Get("foo")); | |||||
| } | |||||
| } | |||||
| TEST(RecoveryTest, MultipleMemTables) { | |||||
| // Make a large log. | |||||
| const int kNum = 1000; | |||||
| for (int i = 0; i < kNum; i++) { | |||||
| char buf[100]; | |||||
| snprintf(buf, sizeof(buf), "%050d", i); | |||||
| ASSERT_OK(Put(buf, buf)); | |||||
| } | |||||
| ASSERT_EQ(0, NumTables()); | |||||
| Close(); | |||||
| ASSERT_EQ(0, NumTables()); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| uint64_t old_log_file = FirstLogFile(); | |||||
| // Force creation of multiple memtables by reducing the write buffer size. | |||||
| Options opt; | |||||
| opt.reuse_logs = true; | |||||
| opt.write_buffer_size = (kNum*100) / 2; | |||||
| Open(&opt); | |||||
| ASSERT_LE(2, NumTables()); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log"; | |||||
| for (int i = 0; i < kNum; i++) { | |||||
| char buf[100]; | |||||
| snprintf(buf, sizeof(buf), "%050d", i); | |||||
| ASSERT_EQ(buf, Get(buf)); | |||||
| } | |||||
| } | |||||
| TEST(RecoveryTest, MultipleLogFiles) { | |||||
| ASSERT_OK(Put("foo", "bar")); | |||||
| Close(); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| // Make a bunch of uncompacted log files. | |||||
| uint64_t old_log = FirstLogFile(); | |||||
| MakeLogFile(old_log+1, 1000, "hello", "world"); | |||||
| MakeLogFile(old_log+2, 1001, "hi", "there"); | |||||
| MakeLogFile(old_log+3, 1002, "foo", "bar2"); | |||||
| // Recover and check that all log files were processed. | |||||
| Open(); | |||||
| ASSERT_LE(1, NumTables()); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| uint64_t new_log = FirstLogFile(); | |||||
| ASSERT_LE(old_log+3, new_log); | |||||
| ASSERT_EQ("bar2", Get("foo")); | |||||
| ASSERT_EQ("world", Get("hello")); | |||||
| ASSERT_EQ("there", Get("hi")); | |||||
| // Test that previous recovery produced recoverable state. | |||||
| Open(); | |||||
| ASSERT_LE(1, NumTables()); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| if (CanAppend()) { | |||||
| ASSERT_EQ(new_log, FirstLogFile()); | |||||
| } | |||||
| ASSERT_EQ("bar2", Get("foo")); | |||||
| ASSERT_EQ("world", Get("hello")); | |||||
| ASSERT_EQ("there", Get("hi")); | |||||
| // Check that introducing an older log file does not cause it to be re-read. | |||||
| Close(); | |||||
| MakeLogFile(old_log+1, 2000, "hello", "stale write"); | |||||
| Open(); | |||||
| ASSERT_LE(1, NumTables()); | |||||
| ASSERT_EQ(1, NumLogs()); | |||||
| if (CanAppend()) { | |||||
| ASSERT_EQ(new_log, FirstLogFile()); | |||||
| } | |||||
| ASSERT_EQ("bar2", Get("foo")); | |||||
| ASSERT_EQ("world", Get("hello")); | |||||
| ASSERT_EQ("there", Get("hi")); | |||||
| } | |||||
| } // namespace leveldb | |||||
| int main(int argc, char** argv) { | |||||
| return leveldb::test::RunAllTests(); | |||||
| } | |||||