作者: 谢瑞阳 10225101483 徐翔宇 10225101535

344 рядки
8.8 KiB

  1. // Copyright (c) 2014 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "db/db_impl.h"
  5. #include "db/filename.h"
  6. #include "db/version_set.h"
  7. #include "db/write_batch_internal.h"
  8. #include "leveldb/db.h"
  9. #include "leveldb/env.h"
  10. #include "leveldb/write_batch.h"
  11. #include "util/logging.h"
  12. #include "util/testharness.h"
  13. #include "util/testutil.h"
  14. namespace leveldb {
  15. class RecoveryTest {
  16. public:
  17. RecoveryTest() : env_(Env::Default()), db_(nullptr) {
  18. dbname_ = test::TmpDir() + "/recovery_test";
  19. DestroyDB(dbname_, Options());
  20. Open();
  21. }
  22. ~RecoveryTest() {
  23. Close();
  24. DestroyDB(dbname_, Options());
  25. }
  26. DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); }
  27. Env* env() const { return env_; }
  28. bool CanAppend() {
  29. WritableFile* tmp;
  30. Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp);
  31. delete tmp;
  32. if (s.IsNotSupportedError()) {
  33. return false;
  34. } else {
  35. return true;
  36. }
  37. }
  38. void Close() {
  39. delete db_;
  40. db_ = nullptr;
  41. }
  42. Status OpenWithStatus(Options* options = nullptr) {
  43. Close();
  44. Options opts;
  45. if (options != nullptr) {
  46. opts = *options;
  47. } else {
  48. opts.reuse_logs = true; // TODO(sanjay): test both ways
  49. opts.create_if_missing = true;
  50. }
  51. if (opts.env == nullptr) {
  52. opts.env = env_;
  53. }
  54. return DB::Open(opts, dbname_, &db_);
  55. }
  56. void Open(Options* options = nullptr) {
  57. ASSERT_OK(OpenWithStatus(options));
  58. ASSERT_EQ(1, NumLogs());
  59. }
  60. Status Put(const std::string& k, const std::string& v) {
  61. return db_->Put(WriteOptions(), k, v);
  62. }
  63. std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
  64. std::string result;
  65. Status s = db_->Get(ReadOptions(), k, &result);
  66. if (s.IsNotFound()) {
  67. result = "NOT_FOUND";
  68. } else if (!s.ok()) {
  69. result = s.ToString();
  70. }
  71. return result;
  72. }
  73. std::string ManifestFileName() {
  74. std::string current;
  75. ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), &current));
  76. size_t len = current.size();
  77. if (len > 0 && current[len-1] == '\n') {
  78. current.resize(len - 1);
  79. }
  80. return dbname_ + "/" + current;
  81. }
  82. std::string LogName(uint64_t number) {
  83. return LogFileName(dbname_, number);
  84. }
  85. size_t DeleteLogFiles() {
  86. // Linux allows unlinking open files, but Windows does not.
  87. // Closing the db allows for file deletion.
  88. Close();
  89. std::vector<uint64_t> logs = GetFiles(kLogFile);
  90. for (size_t i = 0; i < logs.size(); i++) {
  91. ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]);
  92. }
  93. return logs.size();
  94. }
  95. void DeleteManifestFile() {
  96. ASSERT_OK(env_->DeleteFile(ManifestFileName()));
  97. }
  98. uint64_t FirstLogFile() {
  99. return GetFiles(kLogFile)[0];
  100. }
  101. std::vector<uint64_t> GetFiles(FileType t) {
  102. std::vector<std::string> filenames;
  103. ASSERT_OK(env_->GetChildren(dbname_, &filenames));
  104. std::vector<uint64_t> result;
  105. for (size_t i = 0; i < filenames.size(); i++) {
  106. uint64_t number;
  107. FileType type;
  108. if (ParseFileName(filenames[i], &number, &type) && type == t) {
  109. result.push_back(number);
  110. }
  111. }
  112. return result;
  113. }
  114. int NumLogs() {
  115. return GetFiles(kLogFile).size();
  116. }
  117. int NumTables() {
  118. return GetFiles(kTableFile).size();
  119. }
  120. uint64_t FileSize(const std::string& fname) {
  121. uint64_t result;
  122. ASSERT_OK(env_->GetFileSize(fname, &result)) << fname;
  123. return result;
  124. }
  125. void CompactMemTable() {
  126. dbfull()->TEST_CompactMemTable();
  127. }
  128. // Directly construct a log file that sets key to val.
  129. void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
  130. std::string fname = LogFileName(dbname_, lognum);
  131. WritableFile* file;
  132. ASSERT_OK(env_->NewWritableFile(fname, &file));
  133. log::Writer writer(file);
  134. WriteBatch batch;
  135. batch.Put(key, val);
  136. WriteBatchInternal::SetSequence(&batch, seq);
  137. ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
  138. ASSERT_OK(file->Flush());
  139. delete file;
  140. }
  141. private:
  142. std::string dbname_;
  143. Env* env_;
  144. DB* db_;
  145. };
  146. TEST(RecoveryTest, ManifestReused) {
  147. if (!CanAppend()) {
  148. fprintf(stderr, "skipping test because env does not support appending\n");
  149. return;
  150. }
  151. ASSERT_OK(Put("foo", "bar"));
  152. Close();
  153. std::string old_manifest = ManifestFileName();
  154. Open();
  155. ASSERT_EQ(old_manifest, ManifestFileName());
  156. ASSERT_EQ("bar", Get("foo"));
  157. Open();
  158. ASSERT_EQ(old_manifest, ManifestFileName());
  159. ASSERT_EQ("bar", Get("foo"));
  160. }
  161. TEST(RecoveryTest, LargeManifestCompacted) {
  162. if (!CanAppend()) {
  163. fprintf(stderr, "skipping test because env does not support appending\n");
  164. return;
  165. }
  166. ASSERT_OK(Put("foo", "bar"));
  167. Close();
  168. std::string old_manifest = ManifestFileName();
  169. // Pad with zeroes to make manifest file very big.
  170. {
  171. uint64_t len = FileSize(old_manifest);
  172. WritableFile* file;
  173. ASSERT_OK(env()->NewAppendableFile(old_manifest, &file));
  174. std::string zeroes(3*1048576 - static_cast<size_t>(len), 0);
  175. ASSERT_OK(file->Append(zeroes));
  176. ASSERT_OK(file->Flush());
  177. delete file;
  178. }
  179. Open();
  180. std::string new_manifest = ManifestFileName();
  181. ASSERT_NE(old_manifest, new_manifest);
  182. ASSERT_GT(10000, FileSize(new_manifest));
  183. ASSERT_EQ("bar", Get("foo"));
  184. Open();
  185. ASSERT_EQ(new_manifest, ManifestFileName());
  186. ASSERT_EQ("bar", Get("foo"));
  187. }
  188. TEST(RecoveryTest, NoLogFiles) {
  189. ASSERT_OK(Put("foo", "bar"));
  190. ASSERT_EQ(1, DeleteLogFiles());
  191. Open();
  192. ASSERT_EQ("NOT_FOUND", Get("foo"));
  193. Open();
  194. ASSERT_EQ("NOT_FOUND", Get("foo"));
  195. }
  196. TEST(RecoveryTest, LogFileReuse) {
  197. if (!CanAppend()) {
  198. fprintf(stderr, "skipping test because env does not support appending\n");
  199. return;
  200. }
  201. for (int i = 0; i < 2; i++) {
  202. ASSERT_OK(Put("foo", "bar"));
  203. if (i == 0) {
  204. // Compact to ensure current log is empty
  205. CompactMemTable();
  206. }
  207. Close();
  208. ASSERT_EQ(1, NumLogs());
  209. uint64_t number = FirstLogFile();
  210. if (i == 0) {
  211. ASSERT_EQ(0, FileSize(LogName(number)));
  212. } else {
  213. ASSERT_LT(0, FileSize(LogName(number)));
  214. }
  215. Open();
  216. ASSERT_EQ(1, NumLogs());
  217. ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
  218. ASSERT_EQ("bar", Get("foo"));
  219. Open();
  220. ASSERT_EQ(1, NumLogs());
  221. ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
  222. ASSERT_EQ("bar", Get("foo"));
  223. }
  224. }
  225. TEST(RecoveryTest, MultipleMemTables) {
  226. // Make a large log.
  227. const int kNum = 1000;
  228. for (int i = 0; i < kNum; i++) {
  229. char buf[100];
  230. snprintf(buf, sizeof(buf), "%050d", i);
  231. ASSERT_OK(Put(buf, buf));
  232. }
  233. ASSERT_EQ(0, NumTables());
  234. Close();
  235. ASSERT_EQ(0, NumTables());
  236. ASSERT_EQ(1, NumLogs());
  237. uint64_t old_log_file = FirstLogFile();
  238. // Force creation of multiple memtables by reducing the write buffer size.
  239. Options opt;
  240. opt.reuse_logs = true;
  241. opt.write_buffer_size = (kNum*100) / 2;
  242. Open(&opt);
  243. ASSERT_LE(2, NumTables());
  244. ASSERT_EQ(1, NumLogs());
  245. ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log";
  246. for (int i = 0; i < kNum; i++) {
  247. char buf[100];
  248. snprintf(buf, sizeof(buf), "%050d", i);
  249. ASSERT_EQ(buf, Get(buf));
  250. }
  251. }
  252. TEST(RecoveryTest, MultipleLogFiles) {
  253. ASSERT_OK(Put("foo", "bar"));
  254. Close();
  255. ASSERT_EQ(1, NumLogs());
  256. // Make a bunch of uncompacted log files.
  257. uint64_t old_log = FirstLogFile();
  258. MakeLogFile(old_log+1, 1000, "hello", "world");
  259. MakeLogFile(old_log+2, 1001, "hi", "there");
  260. MakeLogFile(old_log+3, 1002, "foo", "bar2");
  261. // Recover and check that all log files were processed.
  262. Open();
  263. ASSERT_LE(1, NumTables());
  264. ASSERT_EQ(1, NumLogs());
  265. uint64_t new_log = FirstLogFile();
  266. ASSERT_LE(old_log+3, new_log);
  267. ASSERT_EQ("bar2", Get("foo"));
  268. ASSERT_EQ("world", Get("hello"));
  269. ASSERT_EQ("there", Get("hi"));
  270. // Test that previous recovery produced recoverable state.
  271. Open();
  272. ASSERT_LE(1, NumTables());
  273. ASSERT_EQ(1, NumLogs());
  274. if (CanAppend()) {
  275. ASSERT_EQ(new_log, FirstLogFile());
  276. }
  277. ASSERT_EQ("bar2", Get("foo"));
  278. ASSERT_EQ("world", Get("hello"));
  279. ASSERT_EQ("there", Get("hi"));
  280. // Check that introducing an older log file does not cause it to be re-read.
  281. Close();
  282. MakeLogFile(old_log+1, 2000, "hello", "stale write");
  283. Open();
  284. ASSERT_LE(1, NumTables());
  285. ASSERT_EQ(1, NumLogs());
  286. if (CanAppend()) {
  287. ASSERT_EQ(new_log, FirstLogFile());
  288. }
  289. ASSERT_EQ("bar2", Get("foo"));
  290. ASSERT_EQ("world", Get("hello"));
  291. ASSERT_EQ("there", Get("hi"));
  292. }
  293. TEST(RecoveryTest, ManifestMissing) {
  294. ASSERT_OK(Put("foo", "bar"));
  295. Close();
  296. DeleteManifestFile();
  297. Status status = OpenWithStatus();
  298. ASSERT_TRUE(status.IsCorruption());
  299. }
  300. } // namespace leveldb
  301. int main(int argc, char** argv) {
  302. return leveldb::test::RunAllTests();
  303. }