Du kannst nicht mehr als 25 Themen auswählen Themen müssen entweder mit einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.

139 Zeilen
3.5 KiB

Fix snapshot compaction bug Closes google/leveldb#320 During compaction it was possible that records from a block b1=(l1,u1) would be pushed down from level i to level i+1. If there is a block b2=(l2,u2) at level i with k1 = user_key(u1) = user_key(l2) then a subsequent search for k1 will yield the record l2 which has a smaller sequence number than u1 because the sort order for records sorts increasing by user key but decreaing by sequence number. This change add a call to a new function AddBoundaryInputs to SetupOtherInputs. AddBoundaryInputs searches for a block b2 matching the criteria above and adds it to the set of files to be compacted. Whenever AddBoundaryInputs is called it is important that the compaction fileset in level i+1 (known as c->inputs_[1] in the code) be recomputed. Each call to AddBoundaryInputs is followed by a call to GetOverlappingInputs. SetupOtherInputs is called on both manual and automated compaction passes. It is called for both level zero and for levels greater than 0. The original change posted in https://github.com/google/leveldb/pull/339 has been modified to also include changed made by Chris Mumford<cmumford@google.com> in https://github.com/cmumford/leveldb/commit/4b72cb14f8da2aab12451c24b8e205aff686e9dc 1. Releasing snapshots during test cleanup to avoid memory leak warnings. 2. Refactored test to use testutil.h to be in line with other issue tests and to create the test database in the correct temporary location. 3. Added copyright banner. Otherwise, just minor formatting and limiting character width to 80 characters. Additionally the change was rebased on top of current master and changes previously made to the Makefile were ported to the CMakeLists.txt. Testing Done: A test program (issue320_test) was constructed that performs mutations while snapshots are active. issue320_test fails without this bug fix after 64k writes. It passes with this bug fix. It was run with 200M writes and passed. Unit tests were written for the new function that was added to the code. Make test was run and seen to pass. Signed-off-by: Richard Cole <richcole@amazon.com>
vor 9 Jahren
  1. // Copyright (c) 2019 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include <iostream>
  5. #include <sstream>
  6. #include <string>
  7. #include <map>
  8. #include <vector>
  9. #include <memory>
  10. #include <math.h>
  11. #include <leveldb/db.h>
  12. #include <leveldb/write_batch.h>
  13. #include <util/testharness.h>
  14. using namespace std;
  15. namespace leveldb {
  16. namespace {
  17. unsigned int random(unsigned int max) {
  18. return std::rand() % max;
  19. }
  20. string newString(int32_t index) {
  21. const unsigned int len = 1024;
  22. char bytes[len];
  23. unsigned int i = 0;
  24. while (i < 8) {
  25. bytes[i] = 'a' + ((index >> (4 * i)) & 0xf);
  26. ++i;
  27. }
  28. while (i < sizeof(bytes)) {
  29. bytes[i] = 'a' + random(26);
  30. ++i;
  31. }
  32. return string(bytes, sizeof(bytes));
  33. }
  34. } // namespace
  35. class Issue320 { };
  36. TEST(Issue320, Test) {
  37. std::srand(0);
  38. bool delete_before_put = false;
  39. bool keep_snapshots = true;
  40. vector<pair<string, string>*> test_map(10000, nullptr);
  41. vector<Snapshot const*> snapshots(100, nullptr);
  42. DB* db;
  43. Options options;
  44. options.create_if_missing = true;
  45. std::string dbpath = test::TmpDir() + "/leveldb_issue320_test";
  46. ASSERT_OK(DB::Open(options, dbpath, &db));
  47. unsigned int target_size = 10000;
  48. unsigned int num_items = 0;
  49. unsigned long count = 0;
  50. string key;
  51. string value, old_value;
  52. WriteOptions writeOptions;
  53. ReadOptions readOptions;
  54. while (count < 200000) {
  55. if ((++count % 1000) == 0) {
  56. cout << "count: " << count << endl;
  57. }
  58. unsigned int index = random(test_map.size());
  59. WriteBatch batch;
  60. if (test_map[index] == nullptr) {
  61. num_items++;
  62. test_map[index] =
  63. new pair<string, string>(newString(index), newString(index));
  64. batch.Put(test_map[index]->first, test_map[index]->second);
  65. } else {
  66. ASSERT_OK(db->Get(readOptions, test_map[index]->first, &old_value));
  67. if (old_value != test_map[index]->second) {
  68. cout << "ERROR incorrect value returned by Get" << endl;
  69. cout << " count=" << count << endl;
  70. cout << " old value=" << old_value << endl;
  71. cout << " test_map[index]->second=" << test_map[index]->second << endl;
  72. cout << " test_map[index]->first=" << test_map[index]->first << endl;
  73. cout << " index=" << index << endl;
  74. ASSERT_EQ(old_value, test_map[index]->second);
  75. }
  76. if (num_items >= target_size && random(100) > 30) {
  77. batch.Delete(test_map[index]->first);
  78. delete test_map[index];
  79. test_map[index] = nullptr;
  80. --num_items;
  81. } else {
  82. test_map[index]->second = newString(index);
  83. if (delete_before_put) batch.Delete(test_map[index]->first);
  84. batch.Put(test_map[index]->first, test_map[index]->second);
  85. }
  86. }
  87. ASSERT_OK(db->Write(writeOptions, &batch));
  88. if (keep_snapshots && random(10) == 0) {
  89. unsigned int i = random(snapshots.size());
  90. if (snapshots[i] != nullptr) {
  91. db->ReleaseSnapshot(snapshots[i]);
  92. }
  93. snapshots[i] = db->GetSnapshot();
  94. }
  95. }
  96. for (Snapshot const* snapshot : snapshots) {
  97. if (snapshot) {
  98. db->ReleaseSnapshot(snapshot);
  99. }
  100. }
  101. for (size_t i = 0; i < test_map.size(); ++i) {
  102. if (test_map[i] != nullptr) {
  103. delete test_map[i];
  104. test_map[i] = nullptr;
  105. }
  106. }
  107. delete db;
  108. DestroyDB(dbpath, options);
  109. }
  110. } // namespace leveldb
  111. int main(int argc, char** argv) {
  112. return leveldb::test::RunAllTests();
  113. }