作者: 谢瑞阳 10225101483 徐翔宇 10225101535
選択できるのは25トピックまでです。 トピックは、先頭が英数字で、英数字とダッシュ('-')を使用した35文字以内のものにしてください。

360 行
9.7 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include <sys/types.h>
  5. #include "db/db_impl.h"
  6. #include "db/filename.h"
  7. #include "db/log_format.h"
  8. #include "db/version_set.h"
  9. #include "leveldb/cache.h"
  10. #include "leveldb/db.h"
  11. #include "leveldb/table.h"
  12. #include "leveldb/write_batch.h"
  13. #include "util/logging.h"
  14. #include "util/testharness.h"
  15. #include "util/testutil.h"
  16. namespace leveldb {
  17. static const int kValueSize = 1000;
  18. class CorruptionTest {
  19. public:
  20. test::ErrorEnv env_;
  21. std::string dbname_;
  22. Cache* tiny_cache_;
  23. Options options_;
  24. DB* db_;
  25. CorruptionTest() {
  26. tiny_cache_ = NewLRUCache(100);
  27. options_.env = &env_;
  28. options_.block_cache = tiny_cache_;
  29. dbname_ = "/memenv/corruption_test";
  30. DestroyDB(dbname_, options_);
  31. db_ = nullptr;
  32. options_.create_if_missing = true;
  33. Reopen();
  34. options_.create_if_missing = false;
  35. }
  36. ~CorruptionTest() {
  37. delete db_;
  38. delete tiny_cache_;
  39. }
  40. Status TryReopen() {
  41. delete db_;
  42. db_ = nullptr;
  43. return DB::Open(options_, dbname_, &db_);
  44. }
  45. void Reopen() { ASSERT_OK(TryReopen()); }
  46. void RepairDB() {
  47. delete db_;
  48. db_ = nullptr;
  49. ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
  50. }
  51. void Build(int n) {
  52. std::string key_space, value_space;
  53. WriteBatch batch;
  54. for (int i = 0; i < n; i++) {
  55. // if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
  56. Slice key = Key(i, &key_space);
  57. batch.Clear();
  58. batch.Put(key, Value(i, &value_space));
  59. WriteOptions options;
  60. // Corrupt() doesn't work without this sync on windows; stat reports 0 for
  61. // the file size.
  62. if (i == n - 1) {
  63. options.sync = true;
  64. }
  65. ASSERT_OK(db_->Write(options, &batch));
  66. }
  67. }
  68. void Check(int min_expected, int max_expected) {
  69. int next_expected = 0;
  70. int missed = 0;
  71. int bad_keys = 0;
  72. int bad_values = 0;
  73. int correct = 0;
  74. std::string value_space;
  75. Iterator* iter = db_->NewIterator(ReadOptions());
  76. for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
  77. uint64_t key;
  78. Slice in(iter->key());
  79. if (in == "" || in == "~") {
  80. // Ignore boundary keys.
  81. continue;
  82. }
  83. if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
  84. key < next_expected) {
  85. bad_keys++;
  86. continue;
  87. }
  88. missed += (key - next_expected);
  89. next_expected = key + 1;
  90. if (iter->value() != Value(key, &value_space)) {
  91. bad_values++;
  92. } else {
  93. correct++;
  94. }
  95. }
  96. delete iter;
  97. fprintf(stderr,
  98. "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
  99. min_expected, max_expected, correct, bad_keys, bad_values, missed);
  100. ASSERT_LE(min_expected, correct);
  101. ASSERT_GE(max_expected, correct);
  102. }
  103. void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
  104. // Pick file to corrupt
  105. std::vector<std::string> filenames;
  106. ASSERT_OK(env_.target()->GetChildren(dbname_, &filenames));
  107. uint64_t number;
  108. FileType type;
  109. std::string fname;
  110. int picked_number = -1;
  111. for (size_t i = 0; i < filenames.size(); i++) {
  112. if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
  113. int(number) > picked_number) { // Pick latest file
  114. fname = dbname_ + "/" + filenames[i];
  115. picked_number = number;
  116. }
  117. }
  118. ASSERT_TRUE(!fname.empty()) << filetype;
  119. uint64_t file_size;
  120. ASSERT_OK(env_.target()->GetFileSize(fname, &file_size));
  121. if (offset < 0) {
  122. // Relative to end of file; make it absolute
  123. if (-offset > file_size) {
  124. offset = 0;
  125. } else {
  126. offset = file_size + offset;
  127. }
  128. }
  129. if (offset > file_size) {
  130. offset = file_size;
  131. }
  132. if (offset + bytes_to_corrupt > file_size) {
  133. bytes_to_corrupt = file_size - offset;
  134. }
  135. // Do it
  136. std::string contents;
  137. Status s = ReadFileToString(env_.target(), fname, &contents);
  138. ASSERT_TRUE(s.ok()) << s.ToString();
  139. for (int i = 0; i < bytes_to_corrupt; i++) {
  140. contents[i + offset] ^= 0x80;
  141. }
  142. s = WriteStringToFile(env_.target(), contents, fname);
  143. ASSERT_TRUE(s.ok()) << s.ToString();
  144. }
  145. int Property(const std::string& name) {
  146. std::string property;
  147. int result;
  148. if (db_->GetProperty(name, &property) &&
  149. sscanf(property.c_str(), "%d", &result) == 1) {
  150. return result;
  151. } else {
  152. return -1;
  153. }
  154. }
  155. // Return the ith key
  156. Slice Key(int i, std::string* storage) {
  157. char buf[100];
  158. snprintf(buf, sizeof(buf), "%016d", i);
  159. storage->assign(buf, strlen(buf));
  160. return Slice(*storage);
  161. }
  162. // Return the value to associate with the specified key
  163. Slice Value(int k, std::string* storage) {
  164. Random r(k);
  165. return test::RandomString(&r, kValueSize, storage);
  166. }
  167. };
  168. TEST(CorruptionTest, Recovery) {
  169. Build(100);
  170. Check(100, 100);
  171. Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
  172. Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block
  173. Reopen();
  174. // The 64 records in the first two log blocks are completely lost.
  175. Check(36, 36);
  176. }
  177. TEST(CorruptionTest, RecoverWriteError) {
  178. env_.writable_file_error_ = true;
  179. Status s = TryReopen();
  180. ASSERT_TRUE(!s.ok());
  181. }
  182. TEST(CorruptionTest, NewFileErrorDuringWrite) {
  183. // Do enough writing to force minor compaction
  184. env_.writable_file_error_ = true;
  185. const int num = 3 + (Options().write_buffer_size / kValueSize);
  186. std::string value_storage;
  187. Status s;
  188. for (int i = 0; s.ok() && i < num; i++) {
  189. WriteBatch batch;
  190. batch.Put("a", Value(100, &value_storage));
  191. s = db_->Write(WriteOptions(), &batch);
  192. }
  193. ASSERT_TRUE(!s.ok());
  194. ASSERT_GE(env_.num_writable_file_errors_, 1);
  195. env_.writable_file_error_ = false;
  196. Reopen();
  197. }
  198. TEST(CorruptionTest, TableFile) {
  199. Build(100);
  200. DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  201. dbi->TEST_CompactMemTable();
  202. dbi->TEST_CompactRange(0, nullptr, nullptr);
  203. dbi->TEST_CompactRange(1, nullptr, nullptr);
  204. Corrupt(kTableFile, 100, 1);
  205. Check(90, 99);
  206. }
  207. TEST(CorruptionTest, TableFileRepair) {
  208. options_.block_size = 2 * kValueSize; // Limit scope of corruption
  209. options_.paranoid_checks = true;
  210. Reopen();
  211. Build(100);
  212. DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  213. dbi->TEST_CompactMemTable();
  214. dbi->TEST_CompactRange(0, nullptr, nullptr);
  215. dbi->TEST_CompactRange(1, nullptr, nullptr);
  216. Corrupt(kTableFile, 100, 1);
  217. RepairDB();
  218. Reopen();
  219. Check(95, 99);
  220. }
  221. TEST(CorruptionTest, TableFileIndexData) {
  222. Build(10000); // Enough to build multiple Tables
  223. DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  224. dbi->TEST_CompactMemTable();
  225. Corrupt(kTableFile, -2000, 500);
  226. Reopen();
  227. Check(5000, 9999);
  228. }
  229. TEST(CorruptionTest, MissingDescriptor) {
  230. Build(1000);
  231. RepairDB();
  232. Reopen();
  233. Check(1000, 1000);
  234. }
  235. TEST(CorruptionTest, SequenceNumberRecovery) {
  236. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
  237. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
  238. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3"));
  239. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4"));
  240. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5"));
  241. RepairDB();
  242. Reopen();
  243. std::string v;
  244. ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
  245. ASSERT_EQ("v5", v);
  246. // Write something. If sequence number was not recovered properly,
  247. // it will be hidden by an earlier write.
  248. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6"));
  249. ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
  250. ASSERT_EQ("v6", v);
  251. Reopen();
  252. ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
  253. ASSERT_EQ("v6", v);
  254. }
  255. TEST(CorruptionTest, CorruptedDescriptor) {
  256. ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
  257. DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  258. dbi->TEST_CompactMemTable();
  259. dbi->TEST_CompactRange(0, nullptr, nullptr);
  260. Corrupt(kDescriptorFile, 0, 1000);
  261. Status s = TryReopen();
  262. ASSERT_TRUE(!s.ok());
  263. RepairDB();
  264. Reopen();
  265. std::string v;
  266. ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
  267. ASSERT_EQ("hello", v);
  268. }
  269. TEST(CorruptionTest, CompactionInputError) {
  270. Build(10);
  271. DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  272. dbi->TEST_CompactMemTable();
  273. const int last = config::kMaxMemCompactLevel;
  274. ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
  275. Corrupt(kTableFile, 100, 1);
  276. Check(5, 9);
  277. // Force compactions by writing lots of values
  278. Build(10000);
  279. Check(10000, 10000);
  280. }
  281. TEST(CorruptionTest, CompactionInputErrorParanoid) {
  282. options_.paranoid_checks = true;
  283. options_.write_buffer_size = 512 << 10;
  284. Reopen();
  285. DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  286. // Make multiple inputs so we need to compact.
  287. for (int i = 0; i < 2; i++) {
  288. Build(10);
  289. dbi->TEST_CompactMemTable();
  290. Corrupt(kTableFile, 100, 1);
  291. env_.SleepForMicroseconds(100000);
  292. }
  293. dbi->CompactRange(nullptr, nullptr);
  294. // Write must fail because of corrupted table
  295. std::string tmp1, tmp2;
  296. Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
  297. ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
  298. }
  299. TEST(CorruptionTest, UnrelatedKeys) {
  300. Build(10);
  301. DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  302. dbi->TEST_CompactMemTable();
  303. Corrupt(kTableFile, 100, 1);
  304. std::string tmp1, tmp2;
  305. ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
  306. std::string v;
  307. ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
  308. ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
  309. dbi->TEST_CompactMemTable();
  310. ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
  311. ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
  312. }
  313. } // namespace leveldb
  314. int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }