Nie możesz wybrać więcej, niż 25 tematów Tematy muszą się zaczynać od litery lub cyfry, mogą zawierać myślniki ('-') i mogą mieć do 35 znaków.

1387 wiersze
40 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "leveldb/db.h"
  5. #include "db/db_impl.h"
  6. #include "db/filename.h"
  7. #include "db/version_set.h"
  8. #include "db/write_batch_internal.h"
  9. #include "leveldb/env.h"
  10. #include "leveldb/table.h"
  11. #include "util/logging.h"
  12. #include "util/testharness.h"
  13. #include "util/testutil.h"
  14. namespace leveldb {
  15. static std::string RandomString(Random* rnd, int len) {
  16. std::string r;
  17. test::RandomString(rnd, len, &r);
  18. return r;
  19. }
  20. // Special Env used to delay background operations
  21. class SpecialEnv : public EnvWrapper {
  22. public:
  23. // sstable Sync() calls are blocked while this pointer is non-NULL.
  24. port::AtomicPointer delay_sstable_sync_;
  25. explicit SpecialEnv(Env* base) : EnvWrapper(base) {
  26. delay_sstable_sync_.Release_Store(NULL);
  27. }
  28. Status NewWritableFile(const std::string& f, WritableFile** r) {
  29. class SSTableFile : public WritableFile {
  30. private:
  31. SpecialEnv* env_;
  32. WritableFile* base_;
  33. public:
  34. SSTableFile(SpecialEnv* env, WritableFile* base)
  35. : env_(env),
  36. base_(base) {
  37. }
  38. ~SSTableFile() { delete base_; }
  39. Status Append(const Slice& data) { return base_->Append(data); }
  40. Status Close() { return base_->Close(); }
  41. Status Flush() { return base_->Flush(); }
  42. Status Sync() {
  43. while (env_->delay_sstable_sync_.Acquire_Load() != NULL) {
  44. env_->SleepForMicroseconds(100000);
  45. }
  46. return base_->Sync();
  47. }
  48. };
  49. Status s = target()->NewWritableFile(f, r);
  50. if (s.ok()) {
  51. if (strstr(f.c_str(), ".sst") != NULL) {
  52. *r = new SSTableFile(this, *r);
  53. }
  54. }
  55. return s;
  56. }
  57. };
  58. class DBTest {
  59. public:
  60. std::string dbname_;
  61. SpecialEnv* env_;
  62. DB* db_;
  63. Options last_options_;
  64. DBTest() : env_(new SpecialEnv(Env::Default())) {
  65. dbname_ = test::TmpDir() + "/db_test";
  66. DestroyDB(dbname_, Options());
  67. db_ = NULL;
  68. Reopen();
  69. }
  70. ~DBTest() {
  71. delete db_;
  72. DestroyDB(dbname_, Options());
  73. delete env_;
  74. }
  75. DBImpl* dbfull() {
  76. return reinterpret_cast<DBImpl*>(db_);
  77. }
  78. void Reopen(Options* options = NULL) {
  79. ASSERT_OK(TryReopen(options));
  80. }
  81. void DestroyAndReopen(Options* options = NULL) {
  82. delete db_;
  83. db_ = NULL;
  84. DestroyDB(dbname_, Options());
  85. ASSERT_OK(TryReopen(options));
  86. }
  87. Status TryReopen(Options* options) {
  88. delete db_;
  89. db_ = NULL;
  90. Options opts;
  91. if (options != NULL) {
  92. opts = *options;
  93. } else {
  94. opts.create_if_missing = true;
  95. }
  96. last_options_ = opts;
  97. return DB::Open(opts, dbname_, &db_);
  98. }
  99. Status Put(const std::string& k, const std::string& v) {
  100. return db_->Put(WriteOptions(), k, v);
  101. }
  102. Status Delete(const std::string& k) {
  103. return db_->Delete(WriteOptions(), k);
  104. }
  105. std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
  106. ReadOptions options;
  107. options.snapshot = snapshot;
  108. std::string result;
  109. Status s = db_->Get(options, k, &result);
  110. if (s.IsNotFound()) {
  111. result = "NOT_FOUND";
  112. } else if (!s.ok()) {
  113. result = s.ToString();
  114. }
  115. return result;
  116. }
  117. std::string AllEntriesFor(const Slice& user_key) {
  118. Iterator* iter = dbfull()->TEST_NewInternalIterator();
  119. InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
  120. iter->Seek(target.Encode());
  121. std::string result;
  122. if (!iter->status().ok()) {
  123. result = iter->status().ToString();
  124. } else {
  125. result = "[ ";
  126. bool first = true;
  127. while (iter->Valid()) {
  128. ParsedInternalKey ikey;
  129. if (!ParseInternalKey(iter->key(), &ikey)) {
  130. result += "CORRUPTED";
  131. } else {
  132. if (last_options_.comparator->Compare(
  133. ikey.user_key, user_key) != 0) {
  134. break;
  135. }
  136. if (!first) {
  137. result += ", ";
  138. }
  139. first = false;
  140. switch (ikey.type) {
  141. case kTypeValue:
  142. result += iter->value().ToString();
  143. break;
  144. case kTypeDeletion:
  145. result += "DEL";
  146. break;
  147. }
  148. }
  149. iter->Next();
  150. }
  151. if (!first) {
  152. result += " ";
  153. }
  154. result += "]";
  155. }
  156. delete iter;
  157. return result;
  158. }
  159. int NumTableFilesAtLevel(int level) {
  160. std::string property;
  161. ASSERT_TRUE(
  162. db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level),
  163. &property));
  164. return atoi(property.c_str());
  165. }
  166. int TotalTableFiles() {
  167. int result = 0;
  168. for (int level = 0; level < config::kNumLevels; level++) {
  169. result += NumTableFilesAtLevel(level);
  170. }
  171. return result;
  172. }
  173. uint64_t Size(const Slice& start, const Slice& limit) {
  174. Range r(start, limit);
  175. uint64_t size;
  176. db_->GetApproximateSizes(&r, 1, &size);
  177. return size;
  178. }
  179. void Compact(const Slice& start, const Slice& limit) {
  180. dbfull()->TEST_CompactMemTable();
  181. int max_level_with_files = 1;
  182. for (int level = 1; level < config::kNumLevels; level++) {
  183. if (NumTableFilesAtLevel(level) > 0) {
  184. max_level_with_files = level;
  185. }
  186. }
  187. for (int level = 0; level < max_level_with_files; level++) {
  188. dbfull()->TEST_CompactRange(level, "", "~");
  189. }
  190. }
  191. // Prevent pushing of new sstables into deeper levels by adding
  192. // tables that cover a specified range to all levels.
  193. void FillLevels(const std::string& smallest, const std::string& largest) {
  194. for (int level = 0; level < config::kNumLevels; level++) {
  195. Put(smallest, "begin");
  196. Put(largest, "end");
  197. dbfull()->TEST_CompactMemTable();
  198. }
  199. }
  200. void DumpFileCounts(const char* label) {
  201. fprintf(stderr, "---\n%s:\n", label);
  202. fprintf(stderr, "maxoverlap: %lld\n",
  203. static_cast<long long>(
  204. dbfull()->TEST_MaxNextLevelOverlappingBytes()));
  205. for (int level = 0; level < config::kNumLevels; level++) {
  206. int num = NumTableFilesAtLevel(level);
  207. if (num > 0) {
  208. fprintf(stderr, " level %3d : %d files\n", level, num);
  209. }
  210. }
  211. }
  212. std::string IterStatus(Iterator* iter) {
  213. std::string result;
  214. if (iter->Valid()) {
  215. result = iter->key().ToString() + "->" + iter->value().ToString();
  216. } else {
  217. result = "(invalid)";
  218. }
  219. return result;
  220. }
  221. };
  222. TEST(DBTest, Empty) {
  223. ASSERT_TRUE(db_ != NULL);
  224. ASSERT_EQ("NOT_FOUND", Get("foo"));
  225. }
  226. TEST(DBTest, ReadWrite) {
  227. ASSERT_OK(Put("foo", "v1"));
  228. ASSERT_EQ("v1", Get("foo"));
  229. ASSERT_OK(Put("bar", "v2"));
  230. ASSERT_OK(Put("foo", "v3"));
  231. ASSERT_EQ("v3", Get("foo"));
  232. ASSERT_EQ("v2", Get("bar"));
  233. }
  234. TEST(DBTest, PutDeleteGet) {
  235. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
  236. ASSERT_EQ("v1", Get("foo"));
  237. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
  238. ASSERT_EQ("v2", Get("foo"));
  239. ASSERT_OK(db_->Delete(WriteOptions(), "foo"));
  240. ASSERT_EQ("NOT_FOUND", Get("foo"));
  241. }
  242. TEST(DBTest, GetFromImmutableLayer) {
  243. Options options;
  244. options.env = env_;
  245. options.write_buffer_size = 100000; // Small write buffer
  246. Reopen(&options);
  247. ASSERT_OK(Put("foo", "v1"));
  248. ASSERT_EQ("v1", Get("foo"));
  249. env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
  250. Put("k1", std::string(100000, 'x')); // Fill memtable
  251. Put("k2", std::string(100000, 'y')); // Trigger compaction
  252. ASSERT_EQ("v1", Get("foo"));
  253. env_->delay_sstable_sync_.Release_Store(NULL); // Release sync calls
  254. }
  255. TEST(DBTest, GetFromVersions) {
  256. ASSERT_OK(Put("foo", "v1"));
  257. dbfull()->TEST_CompactMemTable();
  258. ASSERT_EQ("v1", Get("foo"));
  259. }
  260. TEST(DBTest, GetSnapshot) {
  261. // Try with both a short key and a long key
  262. for (int i = 0; i < 2; i++) {
  263. std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
  264. ASSERT_OK(Put(key, "v1"));
  265. const Snapshot* s1 = db_->GetSnapshot();
  266. ASSERT_OK(Put(key, "v2"));
  267. ASSERT_EQ("v2", Get(key));
  268. ASSERT_EQ("v1", Get(key, s1));
  269. dbfull()->TEST_CompactMemTable();
  270. ASSERT_EQ("v2", Get(key));
  271. ASSERT_EQ("v1", Get(key, s1));
  272. db_->ReleaseSnapshot(s1);
  273. }
  274. }
  275. TEST(DBTest, GetLevel0Ordering) {
  276. // Check that we process level-0 files in correct order. The code
  277. // below generates two level-0 files where the earlier one comes
  278. // before the later one in the level-0 file list since the earlier
  279. // one has a smaller "smallest" key.
  280. ASSERT_OK(Put("bar", "b"));
  281. ASSERT_OK(Put("foo", "v1"));
  282. dbfull()->TEST_CompactMemTable();
  283. ASSERT_OK(Put("foo", "v2"));
  284. dbfull()->TEST_CompactMemTable();
  285. ASSERT_EQ("v2", Get("foo"));
  286. }
  287. TEST(DBTest, GetOrderedByLevels) {
  288. ASSERT_OK(Put("foo", "v1"));
  289. Compact("a", "z");
  290. ASSERT_EQ("v1", Get("foo"));
  291. ASSERT_OK(Put("foo", "v2"));
  292. ASSERT_EQ("v2", Get("foo"));
  293. dbfull()->TEST_CompactMemTable();
  294. ASSERT_EQ("v2", Get("foo"));
  295. }
  296. TEST(DBTest, GetPicksCorrectFile) {
  297. // Arrange to have multiple files in a non-level-0 level.
  298. ASSERT_OK(Put("a", "va"));
  299. Compact("a", "b");
  300. ASSERT_OK(Put("x", "vx"));
  301. Compact("x", "y");
  302. ASSERT_OK(Put("f", "vf"));
  303. Compact("f", "g");
  304. ASSERT_EQ("va", Get("a"));
  305. ASSERT_EQ("vf", Get("f"));
  306. ASSERT_EQ("vx", Get("x"));
  307. }
  308. TEST(DBTest, IterEmpty) {
  309. Iterator* iter = db_->NewIterator(ReadOptions());
  310. iter->SeekToFirst();
  311. ASSERT_EQ(IterStatus(iter), "(invalid)");
  312. iter->SeekToLast();
  313. ASSERT_EQ(IterStatus(iter), "(invalid)");
  314. iter->Seek("foo");
  315. ASSERT_EQ(IterStatus(iter), "(invalid)");
  316. delete iter;
  317. }
  318. TEST(DBTest, IterSingle) {
  319. ASSERT_OK(Put("a", "va"));
  320. Iterator* iter = db_->NewIterator(ReadOptions());
  321. iter->SeekToFirst();
  322. ASSERT_EQ(IterStatus(iter), "a->va");
  323. iter->Next();
  324. ASSERT_EQ(IterStatus(iter), "(invalid)");
  325. iter->SeekToFirst();
  326. ASSERT_EQ(IterStatus(iter), "a->va");
  327. iter->Prev();
  328. ASSERT_EQ(IterStatus(iter), "(invalid)");
  329. iter->SeekToLast();
  330. ASSERT_EQ(IterStatus(iter), "a->va");
  331. iter->Next();
  332. ASSERT_EQ(IterStatus(iter), "(invalid)");
  333. iter->SeekToLast();
  334. ASSERT_EQ(IterStatus(iter), "a->va");
  335. iter->Prev();
  336. ASSERT_EQ(IterStatus(iter), "(invalid)");
  337. iter->Seek("");
  338. ASSERT_EQ(IterStatus(iter), "a->va");
  339. iter->Next();
  340. ASSERT_EQ(IterStatus(iter), "(invalid)");
  341. iter->Seek("a");
  342. ASSERT_EQ(IterStatus(iter), "a->va");
  343. iter->Next();
  344. ASSERT_EQ(IterStatus(iter), "(invalid)");
  345. iter->Seek("b");
  346. ASSERT_EQ(IterStatus(iter), "(invalid)");
  347. delete iter;
  348. }
  349. TEST(DBTest, IterMulti) {
  350. ASSERT_OK(Put("a", "va"));
  351. ASSERT_OK(Put("b", "vb"));
  352. ASSERT_OK(Put("c", "vc"));
  353. Iterator* iter = db_->NewIterator(ReadOptions());
  354. iter->SeekToFirst();
  355. ASSERT_EQ(IterStatus(iter), "a->va");
  356. iter->Next();
  357. ASSERT_EQ(IterStatus(iter), "b->vb");
  358. iter->Next();
  359. ASSERT_EQ(IterStatus(iter), "c->vc");
  360. iter->Next();
  361. ASSERT_EQ(IterStatus(iter), "(invalid)");
  362. iter->SeekToFirst();
  363. ASSERT_EQ(IterStatus(iter), "a->va");
  364. iter->Prev();
  365. ASSERT_EQ(IterStatus(iter), "(invalid)");
  366. iter->SeekToLast();
  367. ASSERT_EQ(IterStatus(iter), "c->vc");
  368. iter->Prev();
  369. ASSERT_EQ(IterStatus(iter), "b->vb");
  370. iter->Prev();
  371. ASSERT_EQ(IterStatus(iter), "a->va");
  372. iter->Prev();
  373. ASSERT_EQ(IterStatus(iter), "(invalid)");
  374. iter->SeekToLast();
  375. ASSERT_EQ(IterStatus(iter), "c->vc");
  376. iter->Next();
  377. ASSERT_EQ(IterStatus(iter), "(invalid)");
  378. iter->Seek("");
  379. ASSERT_EQ(IterStatus(iter), "a->va");
  380. iter->Seek("a");
  381. ASSERT_EQ(IterStatus(iter), "a->va");
  382. iter->Seek("ax");
  383. ASSERT_EQ(IterStatus(iter), "b->vb");
  384. iter->Seek("b");
  385. ASSERT_EQ(IterStatus(iter), "b->vb");
  386. iter->Seek("z");
  387. ASSERT_EQ(IterStatus(iter), "(invalid)");
  388. // Switch from reverse to forward
  389. iter->SeekToLast();
  390. iter->Prev();
  391. iter->Prev();
  392. iter->Next();
  393. ASSERT_EQ(IterStatus(iter), "b->vb");
  394. // Switch from forward to reverse
  395. iter->SeekToFirst();
  396. iter->Next();
  397. iter->Next();
  398. iter->Prev();
  399. ASSERT_EQ(IterStatus(iter), "b->vb");
  400. // Make sure iter stays at snapshot
  401. ASSERT_OK(Put("a", "va2"));
  402. ASSERT_OK(Put("a2", "va3"));
  403. ASSERT_OK(Put("b", "vb2"));
  404. ASSERT_OK(Put("c", "vc2"));
  405. ASSERT_OK(Delete("b"));
  406. iter->SeekToFirst();
  407. ASSERT_EQ(IterStatus(iter), "a->va");
  408. iter->Next();
  409. ASSERT_EQ(IterStatus(iter), "b->vb");
  410. iter->Next();
  411. ASSERT_EQ(IterStatus(iter), "c->vc");
  412. iter->Next();
  413. ASSERT_EQ(IterStatus(iter), "(invalid)");
  414. iter->SeekToLast();
  415. ASSERT_EQ(IterStatus(iter), "c->vc");
  416. iter->Prev();
  417. ASSERT_EQ(IterStatus(iter), "b->vb");
  418. iter->Prev();
  419. ASSERT_EQ(IterStatus(iter), "a->va");
  420. iter->Prev();
  421. ASSERT_EQ(IterStatus(iter), "(invalid)");
  422. delete iter;
  423. }
  424. TEST(DBTest, IterSmallAndLargeMix) {
  425. ASSERT_OK(Put("a", "va"));
  426. ASSERT_OK(Put("b", std::string(100000, 'b')));
  427. ASSERT_OK(Put("c", "vc"));
  428. ASSERT_OK(Put("d", std::string(100000, 'd')));
  429. ASSERT_OK(Put("e", std::string(100000, 'e')));
  430. Iterator* iter = db_->NewIterator(ReadOptions());
  431. iter->SeekToFirst();
  432. ASSERT_EQ(IterStatus(iter), "a->va");
  433. iter->Next();
  434. ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
  435. iter->Next();
  436. ASSERT_EQ(IterStatus(iter), "c->vc");
  437. iter->Next();
  438. ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
  439. iter->Next();
  440. ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
  441. iter->Next();
  442. ASSERT_EQ(IterStatus(iter), "(invalid)");
  443. iter->SeekToLast();
  444. ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
  445. iter->Prev();
  446. ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
  447. iter->Prev();
  448. ASSERT_EQ(IterStatus(iter), "c->vc");
  449. iter->Prev();
  450. ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
  451. iter->Prev();
  452. ASSERT_EQ(IterStatus(iter), "a->va");
  453. iter->Prev();
  454. ASSERT_EQ(IterStatus(iter), "(invalid)");
  455. delete iter;
  456. }
  457. TEST(DBTest, Recover) {
  458. ASSERT_OK(Put("foo", "v1"));
  459. ASSERT_OK(Put("baz", "v5"));
  460. Reopen();
  461. ASSERT_EQ("v1", Get("foo"));
  462. ASSERT_EQ("v1", Get("foo"));
  463. ASSERT_EQ("v5", Get("baz"));
  464. ASSERT_OK(Put("bar", "v2"));
  465. ASSERT_OK(Put("foo", "v3"));
  466. Reopen();
  467. ASSERT_EQ("v3", Get("foo"));
  468. ASSERT_OK(Put("foo", "v4"));
  469. ASSERT_EQ("v4", Get("foo"));
  470. ASSERT_EQ("v2", Get("bar"));
  471. ASSERT_EQ("v5", Get("baz"));
  472. }
  473. TEST(DBTest, RecoveryWithEmptyLog) {
  474. ASSERT_OK(Put("foo", "v1"));
  475. ASSERT_OK(Put("foo", "v2"));
  476. Reopen();
  477. Reopen();
  478. ASSERT_OK(Put("foo", "v3"));
  479. Reopen();
  480. ASSERT_EQ("v3", Get("foo"));
  481. }
  482. // Check that writes done during a memtable compaction are recovered
  483. // if the database is shutdown during the memtable compaction.
  484. TEST(DBTest, RecoverDuringMemtableCompaction) {
  485. Options options;
  486. options.env = env_;
  487. options.write_buffer_size = 1000000;
  488. Reopen(&options);
  489. // Trigger a long memtable compaction and reopen the database during it
  490. ASSERT_OK(Put("foo", "v1")); // Goes to 1st log file
  491. ASSERT_OK(Put("big1", std::string(10000000, 'x'))); // Fills memtable
  492. ASSERT_OK(Put("big2", std::string(1000, 'y'))); // Triggers compaction
  493. ASSERT_OK(Put("bar", "v2")); // Goes to new log file
  494. Reopen(&options);
  495. ASSERT_EQ("v1", Get("foo"));
  496. ASSERT_EQ("v2", Get("bar"));
  497. ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
  498. ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
  499. }
  500. static std::string Key(int i) {
  501. char buf[100];
  502. snprintf(buf, sizeof(buf), "key%06d", i);
  503. return std::string(buf);
  504. }
  505. TEST(DBTest, MinorCompactionsHappen) {
  506. Options options;
  507. options.write_buffer_size = 10000;
  508. Reopen(&options);
  509. const int N = 500;
  510. int starting_num_tables = TotalTableFiles();
  511. for (int i = 0; i < N; i++) {
  512. ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v')));
  513. }
  514. int ending_num_tables = TotalTableFiles();
  515. ASSERT_GT(ending_num_tables, starting_num_tables);
  516. for (int i = 0; i < N; i++) {
  517. ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
  518. }
  519. Reopen();
  520. for (int i = 0; i < N; i++) {
  521. ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
  522. }
  523. }
  524. TEST(DBTest, RecoverWithLargeLog) {
  525. {
  526. Options options;
  527. Reopen(&options);
  528. ASSERT_OK(Put("big1", std::string(200000, '1')));
  529. ASSERT_OK(Put("big2", std::string(200000, '2')));
  530. ASSERT_OK(Put("small3", std::string(10, '3')));
  531. ASSERT_OK(Put("small4", std::string(10, '4')));
  532. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  533. }
  534. // Make sure that if we re-open with a small write buffer size that
  535. // we flush table files in the middle of a large log file.
  536. Options options;
  537. options.write_buffer_size = 100000;
  538. Reopen(&options);
  539. ASSERT_EQ(NumTableFilesAtLevel(0), 3);
  540. ASSERT_EQ(std::string(200000, '1'), Get("big1"));
  541. ASSERT_EQ(std::string(200000, '2'), Get("big2"));
  542. ASSERT_EQ(std::string(10, '3'), Get("small3"));
  543. ASSERT_EQ(std::string(10, '4'), Get("small4"));
  544. ASSERT_GT(NumTableFilesAtLevel(0), 1);
  545. }
  546. TEST(DBTest, CompactionsGenerateMultipleFiles) {
  547. Options options;
  548. options.write_buffer_size = 100000000; // Large write buffer
  549. Reopen(&options);
  550. Random rnd(301);
  551. // Write 8MB (80 values, each 100K)
  552. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  553. std::vector<std::string> values;
  554. for (int i = 0; i < 80; i++) {
  555. values.push_back(RandomString(&rnd, 100000));
  556. ASSERT_OK(Put(Key(i), values[i]));
  557. }
  558. // Reopening moves updates to level-0
  559. Reopen(&options);
  560. dbfull()->TEST_CompactRange(0, "", Key(100000));
  561. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  562. ASSERT_GT(NumTableFilesAtLevel(1), 1);
  563. for (int i = 0; i < 80; i++) {
  564. ASSERT_EQ(Get(Key(i)), values[i]);
  565. }
  566. }
  567. TEST(DBTest, RepeatedWritesToSameKey) {
  568. Options options;
  569. options.env = env_;
  570. options.write_buffer_size = 100000; // Small write buffer
  571. Reopen(&options);
  572. // We must have at most one file per level except for level-0,
  573. // which may have up to kL0_StopWritesTrigger files.
  574. const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger;
  575. Random rnd(301);
  576. std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
  577. for (int i = 0; i < 5 * kMaxFiles; i++) {
  578. Put("key", value);
  579. ASSERT_LE(TotalTableFiles(), kMaxFiles);
  580. fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles());
  581. }
  582. }
  583. TEST(DBTest, SparseMerge) {
  584. Options options;
  585. options.compression = kNoCompression;
  586. Reopen(&options);
  587. FillLevels("A", "Z");
  588. // Suppose there is:
  589. // small amount of data with prefix A
  590. // large amount of data with prefix B
  591. // small amount of data with prefix C
  592. // and that recent updates have made small changes to all three prefixes.
  593. // Check that we do not do a compaction that merges all of B in one shot.
  594. const std::string value(1000, 'x');
  595. Put("A", "va");
  596. // Write approximately 100MB of "B" values
  597. for (int i = 0; i < 100000; i++) {
  598. char key[100];
  599. snprintf(key, sizeof(key), "B%010d", i);
  600. Put(key, value);
  601. }
  602. Put("C", "vc");
  603. dbfull()->TEST_CompactMemTable();
  604. dbfull()->TEST_CompactRange(0, "A", "Z");
  605. // Make sparse update
  606. Put("A", "va2");
  607. Put("B100", "bvalue2");
  608. Put("C", "vc2");
  609. dbfull()->TEST_CompactMemTable();
  610. // Compactions should not cause us to create a situation where
  611. // a file overlaps too much data at the next level.
  612. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  613. dbfull()->TEST_CompactRange(0, "", "z");
  614. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  615. dbfull()->TEST_CompactRange(1, "", "z");
  616. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  617. }
  618. static bool Between(uint64_t val, uint64_t low, uint64_t high) {
  619. bool result = (val >= low) && (val <= high);
  620. if (!result) {
  621. fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
  622. (unsigned long long)(val),
  623. (unsigned long long)(low),
  624. (unsigned long long)(high));
  625. }
  626. return result;
  627. }
  628. TEST(DBTest, ApproximateSizes) {
  629. Options options;
  630. options.write_buffer_size = 100000000; // Large write buffer
  631. options.compression = kNoCompression;
  632. DestroyAndReopen();
  633. ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
  634. Reopen(&options);
  635. ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
  636. // Write 8MB (80 values, each 100K)
  637. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  638. const int N = 80;
  639. Random rnd(301);
  640. for (int i = 0; i < N; i++) {
  641. ASSERT_OK(Put(Key(i), RandomString(&rnd, 100000)));
  642. }
  643. // 0 because GetApproximateSizes() does not account for memtable space
  644. ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
  645. // Check sizes across recovery by reopening a few times
  646. for (int run = 0; run < 3; run++) {
  647. Reopen(&options);
  648. for (int compact_start = 0; compact_start < N; compact_start += 10) {
  649. for (int i = 0; i < N; i += 10) {
  650. ASSERT_TRUE(Between(Size("", Key(i)), 100000*i, 100000*i + 10000));
  651. ASSERT_TRUE(Between(Size("", Key(i)+".suffix"),
  652. 100000 * (i+1), 100000 * (i+1) + 10000));
  653. ASSERT_TRUE(Between(Size(Key(i), Key(i+10)),
  654. 100000 * 10, 100000 * 10 + 10000));
  655. }
  656. ASSERT_TRUE(Between(Size("", Key(50)), 5000000, 5010000));
  657. ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), 5100000, 5110000));
  658. dbfull()->TEST_CompactRange(0,
  659. Key(compact_start),
  660. Key(compact_start + 9));
  661. }
  662. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  663. ASSERT_GT(NumTableFilesAtLevel(1), 0);
  664. }
  665. }
  666. TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
  667. Options options;
  668. options.compression = kNoCompression;
  669. Reopen();
  670. Random rnd(301);
  671. std::string big1 = RandomString(&rnd, 100000);
  672. ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
  673. ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
  674. ASSERT_OK(Put(Key(2), big1));
  675. ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
  676. ASSERT_OK(Put(Key(4), big1));
  677. ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
  678. ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
  679. ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
  680. // Check sizes across recovery by reopening a few times
  681. for (int run = 0; run < 3; run++) {
  682. Reopen(&options);
  683. ASSERT_TRUE(Between(Size("", Key(0)), 0, 0));
  684. ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000));
  685. ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000));
  686. ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000));
  687. ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000));
  688. ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000));
  689. ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000));
  690. ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000));
  691. ASSERT_TRUE(Between(Size("", Key(8)), 550000, 551000));
  692. ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
  693. dbfull()->TEST_CompactRange(0, Key(0), Key(100));
  694. }
  695. }
  696. TEST(DBTest, IteratorPinsRef) {
  697. Put("foo", "hello");
  698. // Get iterator that will yield the current contents of the DB.
  699. Iterator* iter = db_->NewIterator(ReadOptions());
  700. // Write to force compactions
  701. Put("foo", "newvalue1");
  702. for (int i = 0; i < 100; i++) {
  703. ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values
  704. }
  705. Put("foo", "newvalue2");
  706. iter->SeekToFirst();
  707. ASSERT_TRUE(iter->Valid());
  708. ASSERT_EQ("foo", iter->key().ToString());
  709. ASSERT_EQ("hello", iter->value().ToString());
  710. iter->Next();
  711. ASSERT_TRUE(!iter->Valid());
  712. delete iter;
  713. }
  714. TEST(DBTest, Snapshot) {
  715. Put("foo", "v1");
  716. const Snapshot* s1 = db_->GetSnapshot();
  717. Put("foo", "v2");
  718. const Snapshot* s2 = db_->GetSnapshot();
  719. Put("foo", "v3");
  720. const Snapshot* s3 = db_->GetSnapshot();
  721. Put("foo", "v4");
  722. ASSERT_EQ("v1", Get("foo", s1));
  723. ASSERT_EQ("v2", Get("foo", s2));
  724. ASSERT_EQ("v3", Get("foo", s3));
  725. ASSERT_EQ("v4", Get("foo"));
  726. db_->ReleaseSnapshot(s3);
  727. ASSERT_EQ("v1", Get("foo", s1));
  728. ASSERT_EQ("v2", Get("foo", s2));
  729. ASSERT_EQ("v4", Get("foo"));
  730. db_->ReleaseSnapshot(s1);
  731. ASSERT_EQ("v2", Get("foo", s2));
  732. ASSERT_EQ("v4", Get("foo"));
  733. db_->ReleaseSnapshot(s2);
  734. ASSERT_EQ("v4", Get("foo"));
  735. }
  736. TEST(DBTest, HiddenValuesAreRemoved) {
  737. Random rnd(301);
  738. FillLevels("a", "z");
  739. std::string big = RandomString(&rnd, 50000);
  740. Put("foo", big);
  741. Put("pastfoo", "v");
  742. const Snapshot* snapshot = db_->GetSnapshot();
  743. Put("foo", "tiny");
  744. Put("pastfoo2", "v2"); // Advance sequence number one more
  745. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  746. ASSERT_GT(NumTableFilesAtLevel(0), 0);
  747. ASSERT_EQ(big, Get("foo", snapshot));
  748. ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
  749. db_->ReleaseSnapshot(snapshot);
  750. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
  751. dbfull()->TEST_CompactRange(0, "", "x");
  752. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
  753. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  754. ASSERT_GE(NumTableFilesAtLevel(1), 1);
  755. dbfull()->TEST_CompactRange(1, "", "x");
  756. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
  757. ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
  758. }
  759. TEST(DBTest, DeletionMarkers1) {
  760. Put("foo", "v1");
  761. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  762. const int last = config::kMaxMemCompactLevel;
  763. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
  764. // Place a table at level last-1 to prevent merging with preceding mutation
  765. Put("a", "begin");
  766. Put("z", "end");
  767. dbfull()->TEST_CompactMemTable();
  768. ASSERT_EQ(NumTableFilesAtLevel(last), 1);
  769. ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
  770. Delete("foo");
  771. Put("foo", "v2");
  772. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
  773. ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
  774. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
  775. dbfull()->TEST_CompactRange(last-2, "", "z");
  776. // DEL eliminated, but v1 remains because we aren't compacting that level
  777. // (DEL can be eliminated because v2 hides v1).
  778. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
  779. dbfull()->TEST_CompactRange(last-1, "", "z");
  780. // Merging last-1 w/ last, so we are the base level for "foo", so
  781. // DEL is removed. (as is v1).
  782. ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]");
  783. }
  784. TEST(DBTest, DeletionMarkers2) {
  785. Put("foo", "v1");
  786. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  787. const int last = config::kMaxMemCompactLevel;
  788. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
  789. // Place a table at level last-1 to prevent merging with preceding mutation
  790. Put("a", "begin");
  791. Put("z", "end");
  792. dbfull()->TEST_CompactMemTable();
  793. ASSERT_EQ(NumTableFilesAtLevel(last), 1);
  794. ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
  795. Delete("foo");
  796. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  797. ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
  798. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  799. dbfull()->TEST_CompactRange(last-2, "", "z");
  800. // DEL kept: "last" file overlaps
  801. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  802. dbfull()->TEST_CompactRange(last-1, "", "z");
  803. // Merging last-1 w/ last, so we are the base level for "foo", so
  804. // DEL is removed. (as is v1).
  805. ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
  806. }
  807. TEST(DBTest, ComparatorCheck) {
  808. class NewComparator : public Comparator {
  809. public:
  810. virtual const char* Name() const { return "leveldb.NewComparator"; }
  811. virtual int Compare(const Slice& a, const Slice& b) const {
  812. return BytewiseComparator()->Compare(a, b);
  813. }
  814. virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
  815. BytewiseComparator()->FindShortestSeparator(s, l);
  816. }
  817. virtual void FindShortSuccessor(std::string* key) const {
  818. BytewiseComparator()->FindShortSuccessor(key);
  819. }
  820. };
  821. NewComparator cmp;
  822. Options new_options;
  823. new_options.comparator = &cmp;
  824. Status s = TryReopen(&new_options);
  825. ASSERT_TRUE(!s.ok());
  826. ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos)
  827. << s.ToString();
  828. }
  829. TEST(DBTest, DBOpen_Options) {
  830. std::string dbname = test::TmpDir() + "/db_options_test";
  831. DestroyDB(dbname, Options());
  832. // Does not exist, and create_if_missing == false: error
  833. DB* db = NULL;
  834. Options opts;
  835. opts.create_if_missing = false;
  836. Status s = DB::Open(opts, dbname, &db);
  837. ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL);
  838. ASSERT_TRUE(db == NULL);
  839. // Does not exist, and create_if_missing == true: OK
  840. opts.create_if_missing = true;
  841. s = DB::Open(opts, dbname, &db);
  842. ASSERT_OK(s);
  843. ASSERT_TRUE(db != NULL);
  844. delete db;
  845. db = NULL;
  846. // Does exist, and error_if_exists == true: error
  847. opts.create_if_missing = false;
  848. opts.error_if_exists = true;
  849. s = DB::Open(opts, dbname, &db);
  850. ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL);
  851. ASSERT_TRUE(db == NULL);
  852. // Does exist, and error_if_exists == false: OK
  853. opts.create_if_missing = true;
  854. opts.error_if_exists = false;
  855. s = DB::Open(opts, dbname, &db);
  856. ASSERT_OK(s);
  857. ASSERT_TRUE(db != NULL);
  858. delete db;
  859. db = NULL;
  860. }
  861. // Multi-threaded test:
  862. namespace {
  863. static const int kNumThreads = 4;
  864. static const int kTestSeconds = 10;
  865. static const int kNumKeys = 1000;
  866. struct MTState {
  867. DBTest* test;
  868. port::AtomicPointer stop;
  869. port::AtomicPointer counter[kNumThreads];
  870. port::AtomicPointer thread_done[kNumThreads];
  871. };
  872. struct MTThread {
  873. MTState* state;
  874. int id;
  875. };
  876. static void MTThreadBody(void* arg) {
  877. MTThread* t = reinterpret_cast<MTThread*>(arg);
  878. DB* db = t->state->test->db_;
  879. uintptr_t counter = 0;
  880. fprintf(stderr, "... starting thread %d\n", t->id);
  881. Random rnd(1000 + t->id);
  882. std::string value;
  883. char valbuf[1500];
  884. while (t->state->stop.Acquire_Load() == NULL) {
  885. t->state->counter[t->id].Release_Store(reinterpret_cast<void*>(counter));
  886. int key = rnd.Uniform(kNumKeys);
  887. char keybuf[20];
  888. snprintf(keybuf, sizeof(keybuf), "%016d", key);
  889. if (rnd.OneIn(2)) {
  890. // Write values of the form <key, my id, counter>.
  891. // We add some padding for force compactions.
  892. snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d",
  893. key, t->id, static_cast<int>(counter));
  894. ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf)));
  895. } else {
  896. // Read a value and verify that it matches the pattern written above.
  897. Status s = db->Get(ReadOptions(), Slice(keybuf), &value);
  898. if (s.IsNotFound()) {
  899. // Key has not yet been written
  900. } else {
  901. // Check that the writer thread counter is >= the counter in the value
  902. ASSERT_OK(s);
  903. int k, w, c;
  904. ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value;
  905. ASSERT_EQ(k, key);
  906. ASSERT_GE(w, 0);
  907. ASSERT_LT(w, kNumThreads);
  908. ASSERT_LE(c, reinterpret_cast<uintptr_t>(
  909. t->state->counter[w].Acquire_Load()));
  910. }
  911. }
  912. counter++;
  913. }
  914. t->state->thread_done[t->id].Release_Store(t);
  915. fprintf(stderr, "... stopping thread %d after %d ops\n", t->id, int(counter));
  916. }
  917. }
  918. TEST(DBTest, MultiThreaded) {
  919. // Initialize state
  920. MTState mt;
  921. mt.test = this;
  922. mt.stop.Release_Store(0);
  923. for (int id = 0; id < kNumThreads; id++) {
  924. mt.counter[id].Release_Store(0);
  925. mt.thread_done[id].Release_Store(0);
  926. }
  927. // Start threads
  928. MTThread thread[kNumThreads];
  929. for (int id = 0; id < kNumThreads; id++) {
  930. thread[id].state = &mt;
  931. thread[id].id = id;
  932. env_->StartThread(MTThreadBody, &thread[id]);
  933. }
  934. // Let them run for a while
  935. env_->SleepForMicroseconds(kTestSeconds * 1000000);
  936. // Stop the threads and wait for them to finish
  937. mt.stop.Release_Store(&mt);
  938. for (int id = 0; id < kNumThreads; id++) {
  939. while (mt.thread_done[id].Acquire_Load() == NULL) {
  940. env_->SleepForMicroseconds(100000);
  941. }
  942. }
  943. }
  944. namespace {
  945. typedef std::map<std::string, std::string> KVMap;
  946. }
  947. class ModelDB: public DB {
  948. public:
  949. class ModelSnapshot : public Snapshot {
  950. public:
  951. KVMap map_;
  952. };
  953. explicit ModelDB(const Options& options): options_(options) { }
  954. ~ModelDB() { }
  955. virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
  956. return DB::Put(o, k, v);
  957. }
  958. virtual Status Delete(const WriteOptions& o, const Slice& key) {
  959. return DB::Delete(o, key);
  960. }
  961. virtual Status Get(const ReadOptions& options,
  962. const Slice& key, std::string* value) {
  963. assert(false); // Not implemented
  964. return Status::NotFound(key);
  965. }
  966. virtual Iterator* NewIterator(const ReadOptions& options) {
  967. if (options.snapshot == NULL) {
  968. KVMap* saved = new KVMap;
  969. *saved = map_;
  970. return new ModelIter(saved, true);
  971. } else {
  972. const KVMap* snapshot_state =
  973. &(reinterpret_cast<const ModelSnapshot*>(options.snapshot)->map_);
  974. return new ModelIter(snapshot_state, false);
  975. }
  976. }
  977. virtual const Snapshot* GetSnapshot() {
  978. ModelSnapshot* snapshot = new ModelSnapshot;
  979. snapshot->map_ = map_;
  980. return snapshot;
  981. }
  982. virtual void ReleaseSnapshot(const Snapshot* snapshot) {
  983. delete reinterpret_cast<const ModelSnapshot*>(snapshot);
  984. }
  985. virtual Status Write(const WriteOptions& options, WriteBatch* batch) {
  986. assert(options.post_write_snapshot == NULL); // Not supported
  987. class Handler : public WriteBatch::Handler {
  988. public:
  989. KVMap* map_;
  990. virtual void Put(const Slice& key, const Slice& value) {
  991. (*map_)[key.ToString()] = value.ToString();
  992. }
  993. virtual void Delete(const Slice& key) {
  994. map_->erase(key.ToString());
  995. }
  996. };
  997. Handler handler;
  998. handler.map_ = &map_;
  999. return batch->Iterate(&handler);
  1000. }
  1001. virtual bool GetProperty(const Slice& property, std::string* value) {
  1002. return false;
  1003. }
  1004. virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) {
  1005. for (int i = 0; i < n; i++) {
  1006. sizes[i] = 0;
  1007. }
  1008. }
  1009. private:
  1010. class ModelIter: public Iterator {
  1011. public:
  1012. ModelIter(const KVMap* map, bool owned)
  1013. : map_(map), owned_(owned), iter_(map_->end()) {
  1014. }
  1015. ~ModelIter() {
  1016. if (owned_) delete map_;
  1017. }
  1018. virtual bool Valid() const { return iter_ != map_->end(); }
  1019. virtual void SeekToFirst() { iter_ = map_->begin(); }
  1020. virtual void SeekToLast() {
  1021. if (map_->empty()) {
  1022. iter_ = map_->end();
  1023. } else {
  1024. iter_ = map_->find(map_->rbegin()->first);
  1025. }
  1026. }
  1027. virtual void Seek(const Slice& k) {
  1028. iter_ = map_->lower_bound(k.ToString());
  1029. }
  1030. virtual void Next() { ++iter_; }
  1031. virtual void Prev() { --iter_; }
  1032. virtual Slice key() const { return iter_->first; }
  1033. virtual Slice value() const { return iter_->second; }
  1034. virtual Status status() const { return Status::OK(); }
  1035. private:
  1036. const KVMap* const map_;
  1037. const bool owned_; // Do we own map_
  1038. KVMap::const_iterator iter_;
  1039. };
  1040. const Options options_;
  1041. KVMap map_;
  1042. };
  1043. static std::string RandomKey(Random* rnd) {
  1044. int len = (rnd->OneIn(3)
  1045. ? 1 // Short sometimes to encourage collisions
  1046. : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
  1047. return test::RandomKey(rnd, len);
  1048. }
  1049. static bool CompareIterators(int step,
  1050. DB* model,
  1051. DB* db,
  1052. const Snapshot* model_snap,
  1053. const Snapshot* db_snap) {
  1054. ReadOptions options;
  1055. options.snapshot = model_snap;
  1056. Iterator* miter = model->NewIterator(options);
  1057. options.snapshot = db_snap;
  1058. Iterator* dbiter = db->NewIterator(options);
  1059. bool ok = true;
  1060. int count = 0;
  1061. for (miter->SeekToFirst(), dbiter->SeekToFirst();
  1062. ok && miter->Valid() && dbiter->Valid();
  1063. miter->Next(), dbiter->Next()) {
  1064. count++;
  1065. if (miter->key().compare(dbiter->key()) != 0) {
  1066. fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
  1067. step,
  1068. EscapeString(miter->key()).c_str(),
  1069. EscapeString(dbiter->key()).c_str());
  1070. ok = false;
  1071. break;
  1072. }
  1073. if (miter->value().compare(dbiter->value()) != 0) {
  1074. fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
  1075. step,
  1076. EscapeString(miter->key()).c_str(),
  1077. EscapeString(miter->value()).c_str(),
  1078. EscapeString(miter->value()).c_str());
  1079. ok = false;
  1080. }
  1081. }
  1082. if (ok) {
  1083. if (miter->Valid() != dbiter->Valid()) {
  1084. fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n",
  1085. step, miter->Valid(), dbiter->Valid());
  1086. ok = false;
  1087. }
  1088. }
  1089. fprintf(stderr, "%d entries compared: ok=%d\n", count, ok);
  1090. delete miter;
  1091. delete dbiter;
  1092. return ok;
  1093. }
  1094. TEST(DBTest, Randomized) {
  1095. Random rnd(test::RandomSeed());
  1096. ModelDB model(last_options_);
  1097. const int N = 10000;
  1098. const Snapshot* model_snap = NULL;
  1099. const Snapshot* db_snap = NULL;
  1100. std::string k, v;
  1101. for (int step = 0; step < N; step++) {
  1102. if (step % 100 == 0) {
  1103. fprintf(stderr, "Step %d of %d\n", step, N);
  1104. }
  1105. int p = rnd.Uniform(100);
  1106. if (p < 45) { // Put
  1107. k = RandomKey(&rnd);
  1108. v = RandomString(&rnd,
  1109. rnd.OneIn(20)
  1110. ? 100 + rnd.Uniform(100)
  1111. : rnd.Uniform(8));
  1112. ASSERT_OK(model.Put(WriteOptions(), k, v));
  1113. ASSERT_OK(db_->Put(WriteOptions(), k, v));
  1114. } else if (p < 90) { // Delete
  1115. k = RandomKey(&rnd);
  1116. ASSERT_OK(model.Delete(WriteOptions(), k));
  1117. ASSERT_OK(db_->Delete(WriteOptions(), k));
  1118. } else { // Multi-element batch
  1119. WriteBatch b;
  1120. const int num = rnd.Uniform(8);
  1121. for (int i = 0; i < num; i++) {
  1122. if (i == 0 || !rnd.OneIn(10)) {
  1123. k = RandomKey(&rnd);
  1124. } else {
  1125. // Periodically re-use the same key from the previous iter, so
  1126. // we have multiple entries in the write batch for the same key
  1127. }
  1128. if (rnd.OneIn(2)) {
  1129. v = RandomString(&rnd, rnd.Uniform(10));
  1130. b.Put(k, v);
  1131. } else {
  1132. b.Delete(k);
  1133. }
  1134. }
  1135. ASSERT_OK(model.Write(WriteOptions(), &b));
  1136. ASSERT_OK(db_->Write(WriteOptions(), &b));
  1137. }
  1138. if ((step % 100) == 0) {
  1139. ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
  1140. ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
  1141. // Save a snapshot from each DB this time that we'll use next
  1142. // time we compare things, to make sure the current state is
  1143. // preserved with the snapshot
  1144. if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
  1145. if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
  1146. Reopen();
  1147. ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
  1148. model_snap = model.GetSnapshot();
  1149. db_snap = db_->GetSnapshot();
  1150. }
  1151. }
  1152. if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
  1153. if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
  1154. }
  1155. std::string MakeKey(unsigned int num) {
  1156. char buf[30];
  1157. snprintf(buf, sizeof(buf), "%016u", num);
  1158. return std::string(buf);
  1159. }
  1160. void BM_LogAndApply(int iters, int num_base_files) {
  1161. std::string dbname = test::TmpDir() + "/leveldb_test_benchmark";
  1162. DestroyDB(dbname, Options());
  1163. DB* db = NULL;
  1164. Options opts;
  1165. opts.create_if_missing = true;
  1166. Status s = DB::Open(opts, dbname, &db);
  1167. ASSERT_OK(s);
  1168. ASSERT_TRUE(db != NULL);
  1169. delete db;
  1170. db = NULL;
  1171. Env* env = Env::Default();
  1172. InternalKeyComparator cmp(BytewiseComparator());
  1173. Options options;
  1174. VersionSet vset(dbname, &options, NULL, &cmp);
  1175. ASSERT_OK(vset.Recover());
  1176. VersionEdit vbase;
  1177. uint64_t fnum = 1;
  1178. for (int i = 0; i < num_base_files; i++) {
  1179. InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
  1180. InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
  1181. vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
  1182. }
  1183. ASSERT_OK(vset.LogAndApply(&vbase));
  1184. uint64_t start_micros = env->NowMicros();
  1185. for (int i = 0; i < iters; i++) {
  1186. VersionEdit vedit;
  1187. vedit.DeleteFile(2, fnum);
  1188. InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
  1189. InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
  1190. vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
  1191. vset.LogAndApply(&vedit);
  1192. }
  1193. uint64_t stop_micros = env->NowMicros();
  1194. unsigned int us = stop_micros - start_micros;
  1195. char buf[16];
  1196. snprintf(buf, sizeof(buf), "%d", num_base_files);
  1197. fprintf(stderr,
  1198. "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n",
  1199. buf, iters, us, ((float)us) / iters);
  1200. }
  1201. }
  1202. int main(int argc, char** argv) {
  1203. if (argc > 1 && std::string(argv[1]) == "--benchmark") {
  1204. leveldb::BM_LogAndApply(1000, 1);
  1205. leveldb::BM_LogAndApply(1000, 100);
  1206. leveldb::BM_LogAndApply(1000, 10000);
  1207. leveldb::BM_LogAndApply(100, 100000);
  1208. return 0;
  1209. }
  1210. return leveldb::test::RunAllTests();
  1211. }