作者: 谢瑞阳 10225101483 徐翔宇 10225101535

324 righe
8.3 KiB

  1. // Copyright (c) 2014 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "db/db_impl.h"
  5. #include "db/filename.h"
  6. #include "db/version_set.h"
  7. #include "db/write_batch_internal.h"
  8. #include "leveldb/db.h"
  9. #include "leveldb/env.h"
  10. #include "leveldb/write_batch.h"
  11. #include "util/logging.h"
  12. #include "util/testharness.h"
  13. #include "util/testutil.h"
  14. namespace leveldb {
  15. class RecoveryTest {
  16. public:
  17. RecoveryTest() : env_(Env::Default()), db_(NULL) {
  18. dbname_ = test::TmpDir() + "/recovery_test";
  19. DestroyDB(dbname_, Options());
  20. Open();
  21. }
  22. ~RecoveryTest() {
  23. Close();
  24. DestroyDB(dbname_, Options());
  25. }
  26. DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); }
  27. Env* env() const { return env_; }
  28. bool CanAppend() {
  29. WritableFile* tmp;
  30. Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp);
  31. delete tmp;
  32. if (s.IsNotSupportedError()) {
  33. return false;
  34. } else {
  35. return true;
  36. }
  37. }
  38. void Close() {
  39. delete db_;
  40. db_ = NULL;
  41. }
  42. void Open(Options* options = NULL) {
  43. Close();
  44. Options opts;
  45. if (options != NULL) {
  46. opts = *options;
  47. } else {
  48. opts.reuse_logs = true; // TODO(sanjay): test both ways
  49. opts.create_if_missing = true;
  50. }
  51. if (opts.env == NULL) {
  52. opts.env = env_;
  53. }
  54. ASSERT_OK(DB::Open(opts, dbname_, &db_));
  55. ASSERT_EQ(1, NumLogs());
  56. }
  57. Status Put(const std::string& k, const std::string& v) {
  58. return db_->Put(WriteOptions(), k, v);
  59. }
  60. std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
  61. std::string result;
  62. Status s = db_->Get(ReadOptions(), k, &result);
  63. if (s.IsNotFound()) {
  64. result = "NOT_FOUND";
  65. } else if (!s.ok()) {
  66. result = s.ToString();
  67. }
  68. return result;
  69. }
  70. std::string ManifestFileName() {
  71. std::string current;
  72. ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), &current));
  73. size_t len = current.size();
  74. if (len > 0 && current[len-1] == '\n') {
  75. current.resize(len - 1);
  76. }
  77. return dbname_ + "/" + current;
  78. }
  79. std::string LogName(uint64_t number) {
  80. return LogFileName(dbname_, number);
  81. }
  82. size_t DeleteLogFiles() {
  83. std::vector<uint64_t> logs = GetFiles(kLogFile);
  84. for (size_t i = 0; i < logs.size(); i++) {
  85. ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]);
  86. }
  87. return logs.size();
  88. }
  89. uint64_t FirstLogFile() {
  90. return GetFiles(kLogFile)[0];
  91. }
  92. std::vector<std::uint64_t> GetFiles(FileType t) {
  93. std::vector<std::string> filenames;
  94. ASSERT_OK(env_->GetChildren(dbname_, &filenames));
  95. std::vector<std::uint64_t> result;
  96. for (size_t i = 0; i < filenames.size(); i++) {
  97. uint64_t number;
  98. FileType type;
  99. if (ParseFileName(filenames[i], &number, &type) && type == t) {
  100. result.push_back(number);
  101. }
  102. }
  103. return result;
  104. }
  105. int NumLogs() {
  106. return GetFiles(kLogFile).size();
  107. }
  108. int NumTables() {
  109. return GetFiles(kTableFile).size();
  110. }
  111. uint64_t FileSize(const std::string& fname) {
  112. uint64_t result;
  113. ASSERT_OK(env_->GetFileSize(fname, &result)) << fname;
  114. return result;
  115. }
  116. void CompactMemTable() {
  117. dbfull()->TEST_CompactMemTable();
  118. }
  119. // Directly construct a log file that sets key to val.
  120. void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
  121. std::string fname = LogFileName(dbname_, lognum);
  122. WritableFile* file;
  123. ASSERT_OK(env_->NewWritableFile(fname, &file));
  124. log::Writer writer(file);
  125. WriteBatch batch;
  126. batch.Put(key, val);
  127. WriteBatchInternal::SetSequence(&batch, seq);
  128. ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
  129. ASSERT_OK(file->Flush());
  130. delete file;
  131. }
  132. private:
  133. std::string dbname_;
  134. Env* env_;
  135. DB* db_;
  136. };
  137. TEST(RecoveryTest, ManifestReused) {
  138. if (!CanAppend()) {
  139. fprintf(stderr, "skipping test because env does not support appending\n");
  140. return;
  141. }
  142. ASSERT_OK(Put("foo", "bar"));
  143. Close();
  144. std::string old_manifest = ManifestFileName();
  145. Open();
  146. ASSERT_EQ(old_manifest, ManifestFileName());
  147. ASSERT_EQ("bar", Get("foo"));
  148. Open();
  149. ASSERT_EQ(old_manifest, ManifestFileName());
  150. ASSERT_EQ("bar", Get("foo"));
  151. }
  152. TEST(RecoveryTest, LargeManifestCompacted) {
  153. if (!CanAppend()) {
  154. fprintf(stderr, "skipping test because env does not support appending\n");
  155. return;
  156. }
  157. ASSERT_OK(Put("foo", "bar"));
  158. Close();
  159. std::string old_manifest = ManifestFileName();
  160. // Pad with zeroes to make manifest file very big.
  161. {
  162. uint64_t len = FileSize(old_manifest);
  163. WritableFile* file;
  164. ASSERT_OK(env()->NewAppendableFile(old_manifest, &file));
  165. std::string zeroes(3*1048576 - static_cast<size_t>(len), 0);
  166. ASSERT_OK(file->Append(zeroes));
  167. ASSERT_OK(file->Flush());
  168. delete file;
  169. }
  170. Open();
  171. std::string new_manifest = ManifestFileName();
  172. ASSERT_NE(old_manifest, new_manifest);
  173. ASSERT_GT(10000, FileSize(new_manifest));
  174. ASSERT_EQ("bar", Get("foo"));
  175. Open();
  176. ASSERT_EQ(new_manifest, ManifestFileName());
  177. ASSERT_EQ("bar", Get("foo"));
  178. }
  179. TEST(RecoveryTest, NoLogFiles) {
  180. ASSERT_OK(Put("foo", "bar"));
  181. ASSERT_EQ(1, DeleteLogFiles());
  182. Open();
  183. ASSERT_EQ("NOT_FOUND", Get("foo"));
  184. Open();
  185. ASSERT_EQ("NOT_FOUND", Get("foo"));
  186. }
  187. TEST(RecoveryTest, LogFileReuse) {
  188. if (!CanAppend()) {
  189. fprintf(stderr, "skipping test because env does not support appending\n");
  190. return;
  191. }
  192. for (int i = 0; i < 2; i++) {
  193. ASSERT_OK(Put("foo", "bar"));
  194. if (i == 0) {
  195. // Compact to ensure current log is empty
  196. CompactMemTable();
  197. }
  198. Close();
  199. ASSERT_EQ(1, NumLogs());
  200. uint64_t number = FirstLogFile();
  201. if (i == 0) {
  202. ASSERT_EQ(0, FileSize(LogName(number)));
  203. } else {
  204. ASSERT_LT(0, FileSize(LogName(number)));
  205. }
  206. Open();
  207. ASSERT_EQ(1, NumLogs());
  208. ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
  209. ASSERT_EQ("bar", Get("foo"));
  210. Open();
  211. ASSERT_EQ(1, NumLogs());
  212. ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
  213. ASSERT_EQ("bar", Get("foo"));
  214. }
  215. }
  216. TEST(RecoveryTest, MultipleMemTables) {
  217. // Make a large log.
  218. const int kNum = 1000;
  219. for (int i = 0; i < kNum; i++) {
  220. char buf[100];
  221. snprintf(buf, sizeof(buf), "%050d", i);
  222. ASSERT_OK(Put(buf, buf));
  223. }
  224. ASSERT_EQ(0, NumTables());
  225. Close();
  226. ASSERT_EQ(0, NumTables());
  227. ASSERT_EQ(1, NumLogs());
  228. uint64_t old_log_file = FirstLogFile();
  229. // Force creation of multiple memtables by reducing the write buffer size.
  230. Options opt;
  231. opt.reuse_logs = true;
  232. opt.write_buffer_size = (kNum*100) / 2;
  233. Open(&opt);
  234. ASSERT_LE(2, NumTables());
  235. ASSERT_EQ(1, NumLogs());
  236. ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log";
  237. for (int i = 0; i < kNum; i++) {
  238. char buf[100];
  239. snprintf(buf, sizeof(buf), "%050d", i);
  240. ASSERT_EQ(buf, Get(buf));
  241. }
  242. }
  243. TEST(RecoveryTest, MultipleLogFiles) {
  244. ASSERT_OK(Put("foo", "bar"));
  245. Close();
  246. ASSERT_EQ(1, NumLogs());
  247. // Make a bunch of uncompacted log files.
  248. uint64_t old_log = FirstLogFile();
  249. MakeLogFile(old_log+1, 1000, "hello", "world");
  250. MakeLogFile(old_log+2, 1001, "hi", "there");
  251. MakeLogFile(old_log+3, 1002, "foo", "bar2");
  252. // Recover and check that all log files were processed.
  253. Open();
  254. ASSERT_LE(1, NumTables());
  255. ASSERT_EQ(1, NumLogs());
  256. uint64_t new_log = FirstLogFile();
  257. ASSERT_LE(old_log+3, new_log);
  258. ASSERT_EQ("bar2", Get("foo"));
  259. ASSERT_EQ("world", Get("hello"));
  260. ASSERT_EQ("there", Get("hi"));
  261. // Test that previous recovery produced recoverable state.
  262. Open();
  263. ASSERT_LE(1, NumTables());
  264. ASSERT_EQ(1, NumLogs());
  265. if (CanAppend()) {
  266. ASSERT_EQ(new_log, FirstLogFile());
  267. }
  268. ASSERT_EQ("bar2", Get("foo"));
  269. ASSERT_EQ("world", Get("hello"));
  270. ASSERT_EQ("there", Get("hi"));
  271. // Check that introducing an older log file does not cause it to be re-read.
  272. Close();
  273. MakeLogFile(old_log+1, 2000, "hello", "stale write");
  274. Open();
  275. ASSERT_LE(1, NumTables());
  276. ASSERT_EQ(1, NumLogs());
  277. if (CanAppend()) {
  278. ASSERT_EQ(new_log, FirstLogFile());
  279. }
  280. ASSERT_EQ("bar2", Get("foo"));
  281. ASSERT_EQ("world", Get("hello"));
  282. ASSERT_EQ("there", Get("hi"));
  283. }
  284. } // namespace leveldb
  285. int main(int argc, char** argv) {
  286. return leveldb::test::RunAllTests();
  287. }