姚凯文 姜嘉琪
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

339 lines
9.1 KiB

  1. // Copyright (c) 2014 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "gtest/gtest.h"
  5. #include "db/db_impl.h"
  6. #include "db/filename.h"
  7. #include "db/version_set.h"
  8. #include "db/write_batch_internal.h"
  9. #include "leveldb/db.h"
  10. #include "leveldb/env.h"
  11. #include "leveldb/write_batch.h"
  12. #include "util/logging.h"
  13. #include "util/testutil.h"
  14. namespace leveldb {
  15. class RecoveryTest : public testing::Test {
  16. public:
  17. RecoveryTest() : env_(Env::Default()), db_(nullptr) {
  18. dbname_ = testing::TempDir() + "recovery_test";
  19. DestroyDB(dbname_, Options());
  20. Open();
  21. }
  22. ~RecoveryTest() {
  23. Close();
  24. DestroyDB(dbname_, Options());
  25. }
  26. DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); }
  27. Env* env() const { return env_; }
  28. bool CanAppend() {
  29. WritableFile* tmp;
  30. Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp);
  31. delete tmp;
  32. if (s.IsNotSupportedError()) {
  33. return false;
  34. } else {
  35. return true;
  36. }
  37. }
  38. void Close() {
  39. delete db_;
  40. db_ = nullptr;
  41. }
  42. Status OpenWithStatus(Options* options = nullptr) {
  43. Close();
  44. Options opts;
  45. if (options != nullptr) {
  46. opts = *options;
  47. } else {
  48. opts.reuse_logs = true; // TODO(sanjay): test both ways
  49. opts.create_if_missing = true;
  50. }
  51. if (opts.env == nullptr) {
  52. opts.env = env_;
  53. }
  54. return DB::Open(opts, dbname_, &db_);
  55. }
  56. void Open(Options* options = nullptr) {
  57. ASSERT_LEVELDB_OK(OpenWithStatus(options));
  58. ASSERT_EQ(1, NumLogs());
  59. }
  60. Status Put(const std::string& k, const std::string& v) {
  61. return db_->Put(WriteOptions(), k, v);
  62. }
  63. std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
  64. std::string result;
  65. Status s = db_->Get(ReadOptions(), k, &result);
  66. if (s.IsNotFound()) {
  67. result = "NOT_FOUND";
  68. } else if (!s.ok()) {
  69. result = s.ToString();
  70. }
  71. return result;
  72. }
  73. std::string ManifestFileName() {
  74. std::string current;
  75. EXPECT_LEVELDB_OK(
  76. ReadFileToString(env_, CurrentFileName(dbname_), &current));
  77. size_t len = current.size();
  78. if (len > 0 && current[len - 1] == '\n') {
  79. current.resize(len - 1);
  80. }
  81. return dbname_ + "/" + current;
  82. }
  83. std::string LogName(uint64_t number) { return LogFileName(dbname_, number); }
  84. size_t RemoveLogFiles() {
  85. // Linux allows unlinking open files, but Windows does not.
  86. // Closing the db allows for file deletion.
  87. Close();
  88. std::vector<uint64_t> logs = GetFiles(kLogFile);
  89. for (size_t i = 0; i < logs.size(); i++) {
  90. EXPECT_LEVELDB_OK(env_->RemoveFile(LogName(logs[i]))) << LogName(logs[i]);
  91. }
  92. return logs.size();
  93. }
  94. void RemoveManifestFile() {
  95. ASSERT_LEVELDB_OK(env_->RemoveFile(ManifestFileName()));
  96. }
  97. uint64_t FirstLogFile() { return GetFiles(kLogFile)[0]; }
  98. std::vector<uint64_t> GetFiles(FileType t) {
  99. std::vector<std::string> filenames;
  100. EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames));
  101. std::vector<uint64_t> result;
  102. for (size_t i = 0; i < filenames.size(); i++) {
  103. uint64_t number;
  104. FileType type;
  105. if (ParseFileName(filenames[i], &number, &type) && type == t) {
  106. result.push_back(number);
  107. }
  108. }
  109. return result;
  110. }
  111. int NumLogs() { return GetFiles(kLogFile).size(); }
  112. int NumTables() { return GetFiles(kTableFile).size(); }
  113. uint64_t FileSize(const std::string& fname) {
  114. uint64_t result;
  115. EXPECT_LEVELDB_OK(env_->GetFileSize(fname, &result)) << fname;
  116. return result;
  117. }
  118. void CompactMemTable() { dbfull()->TEST_CompactMemTable(); }
  119. // Directly construct a log file that sets key to val.
  120. void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
  121. std::string fname = LogFileName(dbname_, lognum);
  122. WritableFile* file;
  123. ASSERT_LEVELDB_OK(env_->NewWritableFile(fname, &file));
  124. log::Writer writer(file);
  125. WriteBatch batch;
  126. batch.Put(key, val);
  127. WriteBatchInternal::SetSequence(&batch, seq);
  128. ASSERT_LEVELDB_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
  129. ASSERT_LEVELDB_OK(file->Flush());
  130. delete file;
  131. }
  132. private:
  133. std::string dbname_;
  134. Env* env_;
  135. DB* db_;
  136. };
  137. TEST_F(RecoveryTest, ManifestReused) {
  138. if (!CanAppend()) {
  139. std::fprintf(stderr,
  140. "skipping test because env does not support appending\n");
  141. return;
  142. }
  143. ASSERT_LEVELDB_OK(Put("foo", "bar"));
  144. Close();
  145. std::string old_manifest = ManifestFileName();
  146. Open();
  147. ASSERT_EQ(old_manifest, ManifestFileName());
  148. ASSERT_EQ("bar", Get("foo"));
  149. Open();
  150. ASSERT_EQ(old_manifest, ManifestFileName());
  151. ASSERT_EQ("bar", Get("foo"));
  152. }
  153. TEST_F(RecoveryTest, LargeManifestCompacted) {
  154. if (!CanAppend()) {
  155. std::fprintf(stderr,
  156. "skipping test because env does not support appending\n");
  157. return;
  158. }
  159. ASSERT_LEVELDB_OK(Put("foo", "bar"));
  160. Close();
  161. std::string old_manifest = ManifestFileName();
  162. // Pad with zeroes to make manifest file very big.
  163. {
  164. uint64_t len = FileSize(old_manifest);
  165. WritableFile* file;
  166. ASSERT_LEVELDB_OK(env()->NewAppendableFile(old_manifest, &file));
  167. std::string zeroes(3 * 1048576 - static_cast<size_t>(len), 0);
  168. ASSERT_LEVELDB_OK(file->Append(zeroes));
  169. ASSERT_LEVELDB_OK(file->Flush());
  170. delete file;
  171. }
  172. Open();
  173. std::string new_manifest = ManifestFileName();
  174. ASSERT_NE(old_manifest, new_manifest);
  175. ASSERT_GT(10000, FileSize(new_manifest));
  176. ASSERT_EQ("bar", Get("foo"));
  177. Open();
  178. ASSERT_EQ(new_manifest, ManifestFileName());
  179. ASSERT_EQ("bar", Get("foo"));
  180. }
  181. TEST_F(RecoveryTest, NoLogFiles) {
  182. ASSERT_LEVELDB_OK(Put("foo", "bar"));
  183. ASSERT_EQ(1, RemoveLogFiles());
  184. Open();
  185. ASSERT_EQ("NOT_FOUND", Get("foo"));
  186. Open();
  187. ASSERT_EQ("NOT_FOUND", Get("foo"));
  188. }
  189. TEST_F(RecoveryTest, LogFileReuse) {
  190. if (!CanAppend()) {
  191. std::fprintf(stderr,
  192. "skipping test because env does not support appending\n");
  193. return;
  194. }
  195. for (int i = 0; i < 2; i++) {
  196. ASSERT_LEVELDB_OK(Put("foo", "bar"));
  197. if (i == 0) {
  198. // Compact to ensure current log is empty
  199. CompactMemTable();
  200. }
  201. Close();
  202. ASSERT_EQ(1, NumLogs());
  203. uint64_t number = FirstLogFile();
  204. if (i == 0) {
  205. ASSERT_EQ(0, FileSize(LogName(number)));
  206. } else {
  207. ASSERT_LT(0, FileSize(LogName(number)));
  208. }
  209. Open();
  210. ASSERT_EQ(1, NumLogs());
  211. ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
  212. ASSERT_EQ("bar", Get("foo"));
  213. Open();
  214. ASSERT_EQ(1, NumLogs());
  215. ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
  216. ASSERT_EQ("bar", Get("foo"));
  217. }
  218. }
  219. TEST_F(RecoveryTest, MultipleMemTables) {
  220. // Make a large log.
  221. const int kNum = 1000;
  222. for (int i = 0; i < kNum; i++) {
  223. char buf[100];
  224. std::snprintf(buf, sizeof(buf), "%050d", i);
  225. ASSERT_LEVELDB_OK(Put(buf, buf));
  226. }
  227. ASSERT_EQ(0, NumTables());
  228. Close();
  229. ASSERT_EQ(0, NumTables());
  230. ASSERT_EQ(1, NumLogs());
  231. uint64_t old_log_file = FirstLogFile();
  232. // Force creation of multiple memtables by reducing the write buffer size.
  233. Options opt;
  234. opt.reuse_logs = true;
  235. opt.write_buffer_size = (kNum * 100) / 2;
  236. Open(&opt);
  237. ASSERT_LE(2, NumTables());
  238. ASSERT_EQ(1, NumLogs());
  239. ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log";
  240. for (int i = 0; i < kNum; i++) {
  241. char buf[100];
  242. std::snprintf(buf, sizeof(buf), "%050d", i);
  243. ASSERT_EQ(buf, Get(buf));
  244. }
  245. }
  246. TEST_F(RecoveryTest, MultipleLogFiles) {
  247. ASSERT_LEVELDB_OK(Put("foo", "bar"));
  248. Close();
  249. ASSERT_EQ(1, NumLogs());
  250. // Make a bunch of uncompacted log files.
  251. uint64_t old_log = FirstLogFile();
  252. MakeLogFile(old_log + 1, 1000, "hello", "world");
  253. MakeLogFile(old_log + 2, 1001, "hi", "there");
  254. MakeLogFile(old_log + 3, 1002, "foo", "bar2");
  255. // Recover and check that all log files were processed.
  256. Open();
  257. ASSERT_LE(1, NumTables());
  258. ASSERT_EQ(1, NumLogs());
  259. uint64_t new_log = FirstLogFile();
  260. ASSERT_LE(old_log + 3, new_log);
  261. ASSERT_EQ("bar2", Get("foo"));
  262. ASSERT_EQ("world", Get("hello"));
  263. ASSERT_EQ("there", Get("hi"));
  264. // Test that previous recovery produced recoverable state.
  265. Open();
  266. ASSERT_LE(1, NumTables());
  267. ASSERT_EQ(1, NumLogs());
  268. if (CanAppend()) {
  269. ASSERT_EQ(new_log, FirstLogFile());
  270. }
  271. ASSERT_EQ("bar2", Get("foo"));
  272. ASSERT_EQ("world", Get("hello"));
  273. ASSERT_EQ("there", Get("hi"));
  274. // Check that introducing an older log file does not cause it to be re-read.
  275. Close();
  276. MakeLogFile(old_log + 1, 2000, "hello", "stale write");
  277. Open();
  278. ASSERT_LE(1, NumTables());
  279. ASSERT_EQ(1, NumLogs());
  280. if (CanAppend()) {
  281. ASSERT_EQ(new_log, FirstLogFile());
  282. }
  283. ASSERT_EQ("bar2", Get("foo"));
  284. ASSERT_EQ("world", Get("hello"));
  285. ASSERT_EQ("there", Get("hi"));
  286. }
  287. TEST_F(RecoveryTest, ManifestMissing) {
  288. ASSERT_LEVELDB_OK(Put("foo", "bar"));
  289. Close();
  290. RemoveManifestFile();
  291. Status status = OpenWithStatus();
  292. #if defined(LEVELDB_PLATFORM_CHROMIUM)
  293. // TODO(crbug.com/760362): See comment in MakeIOError() from env_chromium.cc.
  294. ASSERT_TRUE(status.IsIOError());
  295. #else
  296. ASSERT_TRUE(status.IsCorruption());
  297. #endif // defined(LEVELDB_PLATFORM_CHROMIUM)
  298. }
  299. } // namespace leveldb