作者: 韩晨旭 10225101440 李畅 10225102463
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1386 lines
40 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "leveldb/db.h"
  5. #include "db/db_impl.h"
  6. #include "db/filename.h"
  7. #include "db/version_set.h"
  8. #include "db/write_batch_internal.h"
  9. #include "leveldb/env.h"
  10. #include "leveldb/table.h"
  11. #include "util/logging.h"
  12. #include "util/testharness.h"
  13. #include "util/testutil.h"
  14. namespace leveldb {
  15. static std::string RandomString(Random* rnd, int len) {
  16. std::string r;
  17. test::RandomString(rnd, len, &r);
  18. return r;
  19. }
  20. // Special Env used to delay background operations
  21. class SpecialEnv : public EnvWrapper {
  22. public:
  23. // sstable Sync() calls are blocked while this pointer is non-NULL.
  24. port::AtomicPointer delay_sstable_sync_;
  25. explicit SpecialEnv(Env* base) : EnvWrapper(base) {
  26. delay_sstable_sync_.Release_Store(NULL);
  27. }
  28. Status NewWritableFile(const std::string& f, WritableFile** r) {
  29. class SSTableFile : public WritableFile {
  30. private:
  31. SpecialEnv* env_;
  32. WritableFile* base_;
  33. public:
  34. SSTableFile(SpecialEnv* env, WritableFile* base)
  35. : env_(env),
  36. base_(base) {
  37. }
  38. Status Append(const Slice& data) { return base_->Append(data); }
  39. Status Close() { return base_->Close(); }
  40. Status Flush() { return base_->Flush(); }
  41. Status Sync() {
  42. while (env_->delay_sstable_sync_.Acquire_Load() != NULL) {
  43. env_->SleepForMicroseconds(100000);
  44. }
  45. return base_->Sync();
  46. }
  47. };
  48. Status s = target()->NewWritableFile(f, r);
  49. if (s.ok()) {
  50. if (strstr(f.c_str(), ".sst") != NULL) {
  51. *r = new SSTableFile(this, *r);
  52. }
  53. }
  54. return s;
  55. }
  56. };
  57. class DBTest {
  58. public:
  59. std::string dbname_;
  60. SpecialEnv* env_;
  61. DB* db_;
  62. Options last_options_;
  63. DBTest() : env_(new SpecialEnv(Env::Default())) {
  64. dbname_ = test::TmpDir() + "/db_test";
  65. DestroyDB(dbname_, Options());
  66. db_ = NULL;
  67. Reopen();
  68. }
  69. ~DBTest() {
  70. delete db_;
  71. DestroyDB(dbname_, Options());
  72. delete env_;
  73. }
  74. DBImpl* dbfull() {
  75. return reinterpret_cast<DBImpl*>(db_);
  76. }
  77. void Reopen(Options* options = NULL) {
  78. ASSERT_OK(TryReopen(options));
  79. }
  80. void DestroyAndReopen(Options* options = NULL) {
  81. delete db_;
  82. db_ = NULL;
  83. DestroyDB(dbname_, Options());
  84. ASSERT_OK(TryReopen(options));
  85. }
  86. Status TryReopen(Options* options) {
  87. delete db_;
  88. db_ = NULL;
  89. Options opts;
  90. if (options != NULL) {
  91. opts = *options;
  92. } else {
  93. opts.create_if_missing = true;
  94. }
  95. last_options_ = opts;
  96. return DB::Open(opts, dbname_, &db_);
  97. }
  98. Status Put(const std::string& k, const std::string& v) {
  99. return db_->Put(WriteOptions(), k, v);
  100. }
  101. Status Delete(const std::string& k) {
  102. return db_->Delete(WriteOptions(), k);
  103. }
  104. std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
  105. ReadOptions options;
  106. options.snapshot = snapshot;
  107. std::string result;
  108. Status s = db_->Get(options, k, &result);
  109. if (s.IsNotFound()) {
  110. result = "NOT_FOUND";
  111. } else if (!s.ok()) {
  112. result = s.ToString();
  113. }
  114. return result;
  115. }
  116. std::string AllEntriesFor(const Slice& user_key) {
  117. Iterator* iter = dbfull()->TEST_NewInternalIterator();
  118. InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
  119. iter->Seek(target.Encode());
  120. std::string result;
  121. if (!iter->status().ok()) {
  122. result = iter->status().ToString();
  123. } else {
  124. result = "[ ";
  125. bool first = true;
  126. while (iter->Valid()) {
  127. ParsedInternalKey ikey;
  128. if (!ParseInternalKey(iter->key(), &ikey)) {
  129. result += "CORRUPTED";
  130. } else {
  131. if (last_options_.comparator->Compare(
  132. ikey.user_key, user_key) != 0) {
  133. break;
  134. }
  135. if (!first) {
  136. result += ", ";
  137. }
  138. first = false;
  139. switch (ikey.type) {
  140. case kTypeValue:
  141. result += iter->value().ToString();
  142. break;
  143. case kTypeDeletion:
  144. result += "DEL";
  145. break;
  146. }
  147. }
  148. iter->Next();
  149. }
  150. if (!first) {
  151. result += " ";
  152. }
  153. result += "]";
  154. }
  155. delete iter;
  156. return result;
  157. }
  158. int NumTableFilesAtLevel(int level) {
  159. std::string property;
  160. ASSERT_TRUE(
  161. db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level),
  162. &property));
  163. return atoi(property.c_str());
  164. }
  165. int TotalTableFiles() {
  166. int result = 0;
  167. for (int level = 0; level < config::kNumLevels; level++) {
  168. result += NumTableFilesAtLevel(level);
  169. }
  170. return result;
  171. }
  172. uint64_t Size(const Slice& start, const Slice& limit) {
  173. Range r(start, limit);
  174. uint64_t size;
  175. db_->GetApproximateSizes(&r, 1, &size);
  176. return size;
  177. }
  178. void Compact(const Slice& start, const Slice& limit) {
  179. dbfull()->TEST_CompactMemTable();
  180. int max_level_with_files = 1;
  181. for (int level = 1; level < config::kNumLevels; level++) {
  182. if (NumTableFilesAtLevel(level) > 0) {
  183. max_level_with_files = level;
  184. }
  185. }
  186. for (int level = 0; level < max_level_with_files; level++) {
  187. dbfull()->TEST_CompactRange(level, "", "~");
  188. }
  189. }
  190. // Prevent pushing of new sstables into deeper levels by adding
  191. // tables that cover a specified range to all levels.
  192. void FillLevels(const std::string& smallest, const std::string& largest) {
  193. for (int level = 0; level < config::kNumLevels; level++) {
  194. Put(smallest, "begin");
  195. Put(largest, "end");
  196. dbfull()->TEST_CompactMemTable();
  197. }
  198. }
  199. void DumpFileCounts(const char* label) {
  200. fprintf(stderr, "---\n%s:\n", label);
  201. fprintf(stderr, "maxoverlap: %lld\n",
  202. static_cast<long long>(
  203. dbfull()->TEST_MaxNextLevelOverlappingBytes()));
  204. for (int level = 0; level < config::kNumLevels; level++) {
  205. int num = NumTableFilesAtLevel(level);
  206. if (num > 0) {
  207. fprintf(stderr, " level %3d : %d files\n", level, num);
  208. }
  209. }
  210. }
  211. std::string IterStatus(Iterator* iter) {
  212. std::string result;
  213. if (iter->Valid()) {
  214. result = iter->key().ToString() + "->" + iter->value().ToString();
  215. } else {
  216. result = "(invalid)";
  217. }
  218. return result;
  219. }
  220. };
  221. TEST(DBTest, Empty) {
  222. ASSERT_TRUE(db_ != NULL);
  223. ASSERT_EQ("NOT_FOUND", Get("foo"));
  224. }
  225. TEST(DBTest, ReadWrite) {
  226. ASSERT_OK(Put("foo", "v1"));
  227. ASSERT_EQ("v1", Get("foo"));
  228. ASSERT_OK(Put("bar", "v2"));
  229. ASSERT_OK(Put("foo", "v3"));
  230. ASSERT_EQ("v3", Get("foo"));
  231. ASSERT_EQ("v2", Get("bar"));
  232. }
  233. TEST(DBTest, PutDeleteGet) {
  234. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
  235. ASSERT_EQ("v1", Get("foo"));
  236. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
  237. ASSERT_EQ("v2", Get("foo"));
  238. ASSERT_OK(db_->Delete(WriteOptions(), "foo"));
  239. ASSERT_EQ("NOT_FOUND", Get("foo"));
  240. }
  241. TEST(DBTest, GetFromImmutableLayer) {
  242. Options options;
  243. options.env = env_;
  244. options.write_buffer_size = 100000; // Small write buffer
  245. Reopen(&options);
  246. ASSERT_OK(Put("foo", "v1"));
  247. ASSERT_EQ("v1", Get("foo"));
  248. env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
  249. Put("k1", std::string(100000, 'x')); // Fill memtable
  250. Put("k2", std::string(100000, 'y')); // Trigger compaction
  251. ASSERT_EQ("v1", Get("foo"));
  252. env_->delay_sstable_sync_.Release_Store(NULL); // Release sync calls
  253. }
  254. TEST(DBTest, GetFromVersions) {
  255. ASSERT_OK(Put("foo", "v1"));
  256. dbfull()->TEST_CompactMemTable();
  257. ASSERT_EQ("v1", Get("foo"));
  258. }
  259. TEST(DBTest, GetSnapshot) {
  260. // Try with both a short key and a long key
  261. for (int i = 0; i < 2; i++) {
  262. std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
  263. ASSERT_OK(Put(key, "v1"));
  264. const Snapshot* s1 = db_->GetSnapshot();
  265. ASSERT_OK(Put(key, "v2"));
  266. ASSERT_EQ("v2", Get(key));
  267. ASSERT_EQ("v1", Get(key, s1));
  268. dbfull()->TEST_CompactMemTable();
  269. ASSERT_EQ("v2", Get(key));
  270. ASSERT_EQ("v1", Get(key, s1));
  271. db_->ReleaseSnapshot(s1);
  272. }
  273. }
  274. TEST(DBTest, GetLevel0Ordering) {
  275. // Check that we process level-0 files in correct order. The code
  276. // below generates two level-0 files where the earlier one comes
  277. // before the later one in the level-0 file list since the earlier
  278. // one has a smaller "smallest" key.
  279. ASSERT_OK(Put("bar", "b"));
  280. ASSERT_OK(Put("foo", "v1"));
  281. dbfull()->TEST_CompactMemTable();
  282. ASSERT_OK(Put("foo", "v2"));
  283. dbfull()->TEST_CompactMemTable();
  284. ASSERT_EQ("v2", Get("foo"));
  285. }
  286. TEST(DBTest, GetOrderedByLevels) {
  287. ASSERT_OK(Put("foo", "v1"));
  288. Compact("a", "z");
  289. ASSERT_EQ("v1", Get("foo"));
  290. ASSERT_OK(Put("foo", "v2"));
  291. ASSERT_EQ("v2", Get("foo"));
  292. dbfull()->TEST_CompactMemTable();
  293. ASSERT_EQ("v2", Get("foo"));
  294. }
  295. TEST(DBTest, GetPicksCorrectFile) {
  296. // Arrange to have multiple files in a non-level-0 level.
  297. ASSERT_OK(Put("a", "va"));
  298. Compact("a", "b");
  299. ASSERT_OK(Put("x", "vx"));
  300. Compact("x", "y");
  301. ASSERT_OK(Put("f", "vf"));
  302. Compact("f", "g");
  303. ASSERT_EQ("va", Get("a"));
  304. ASSERT_EQ("vf", Get("f"));
  305. ASSERT_EQ("vx", Get("x"));
  306. }
  307. TEST(DBTest, IterEmpty) {
  308. Iterator* iter = db_->NewIterator(ReadOptions());
  309. iter->SeekToFirst();
  310. ASSERT_EQ(IterStatus(iter), "(invalid)");
  311. iter->SeekToLast();
  312. ASSERT_EQ(IterStatus(iter), "(invalid)");
  313. iter->Seek("foo");
  314. ASSERT_EQ(IterStatus(iter), "(invalid)");
  315. delete iter;
  316. }
  317. TEST(DBTest, IterSingle) {
  318. ASSERT_OK(Put("a", "va"));
  319. Iterator* iter = db_->NewIterator(ReadOptions());
  320. iter->SeekToFirst();
  321. ASSERT_EQ(IterStatus(iter), "a->va");
  322. iter->Next();
  323. ASSERT_EQ(IterStatus(iter), "(invalid)");
  324. iter->SeekToFirst();
  325. ASSERT_EQ(IterStatus(iter), "a->va");
  326. iter->Prev();
  327. ASSERT_EQ(IterStatus(iter), "(invalid)");
  328. iter->SeekToLast();
  329. ASSERT_EQ(IterStatus(iter), "a->va");
  330. iter->Next();
  331. ASSERT_EQ(IterStatus(iter), "(invalid)");
  332. iter->SeekToLast();
  333. ASSERT_EQ(IterStatus(iter), "a->va");
  334. iter->Prev();
  335. ASSERT_EQ(IterStatus(iter), "(invalid)");
  336. iter->Seek("");
  337. ASSERT_EQ(IterStatus(iter), "a->va");
  338. iter->Next();
  339. ASSERT_EQ(IterStatus(iter), "(invalid)");
  340. iter->Seek("a");
  341. ASSERT_EQ(IterStatus(iter), "a->va");
  342. iter->Next();
  343. ASSERT_EQ(IterStatus(iter), "(invalid)");
  344. iter->Seek("b");
  345. ASSERT_EQ(IterStatus(iter), "(invalid)");
  346. delete iter;
  347. }
  348. TEST(DBTest, IterMulti) {
  349. ASSERT_OK(Put("a", "va"));
  350. ASSERT_OK(Put("b", "vb"));
  351. ASSERT_OK(Put("c", "vc"));
  352. Iterator* iter = db_->NewIterator(ReadOptions());
  353. iter->SeekToFirst();
  354. ASSERT_EQ(IterStatus(iter), "a->va");
  355. iter->Next();
  356. ASSERT_EQ(IterStatus(iter), "b->vb");
  357. iter->Next();
  358. ASSERT_EQ(IterStatus(iter), "c->vc");
  359. iter->Next();
  360. ASSERT_EQ(IterStatus(iter), "(invalid)");
  361. iter->SeekToFirst();
  362. ASSERT_EQ(IterStatus(iter), "a->va");
  363. iter->Prev();
  364. ASSERT_EQ(IterStatus(iter), "(invalid)");
  365. iter->SeekToLast();
  366. ASSERT_EQ(IterStatus(iter), "c->vc");
  367. iter->Prev();
  368. ASSERT_EQ(IterStatus(iter), "b->vb");
  369. iter->Prev();
  370. ASSERT_EQ(IterStatus(iter), "a->va");
  371. iter->Prev();
  372. ASSERT_EQ(IterStatus(iter), "(invalid)");
  373. iter->SeekToLast();
  374. ASSERT_EQ(IterStatus(iter), "c->vc");
  375. iter->Next();
  376. ASSERT_EQ(IterStatus(iter), "(invalid)");
  377. iter->Seek("");
  378. ASSERT_EQ(IterStatus(iter), "a->va");
  379. iter->Seek("a");
  380. ASSERT_EQ(IterStatus(iter), "a->va");
  381. iter->Seek("ax");
  382. ASSERT_EQ(IterStatus(iter), "b->vb");
  383. iter->Seek("b");
  384. ASSERT_EQ(IterStatus(iter), "b->vb");
  385. iter->Seek("z");
  386. ASSERT_EQ(IterStatus(iter), "(invalid)");
  387. // Switch from reverse to forward
  388. iter->SeekToLast();
  389. iter->Prev();
  390. iter->Prev();
  391. iter->Next();
  392. ASSERT_EQ(IterStatus(iter), "b->vb");
  393. // Switch from forward to reverse
  394. iter->SeekToFirst();
  395. iter->Next();
  396. iter->Next();
  397. iter->Prev();
  398. ASSERT_EQ(IterStatus(iter), "b->vb");
  399. // Make sure iter stays at snapshot
  400. ASSERT_OK(Put("a", "va2"));
  401. ASSERT_OK(Put("a2", "va3"));
  402. ASSERT_OK(Put("b", "vb2"));
  403. ASSERT_OK(Put("c", "vc2"));
  404. ASSERT_OK(Delete("b"));
  405. iter->SeekToFirst();
  406. ASSERT_EQ(IterStatus(iter), "a->va");
  407. iter->Next();
  408. ASSERT_EQ(IterStatus(iter), "b->vb");
  409. iter->Next();
  410. ASSERT_EQ(IterStatus(iter), "c->vc");
  411. iter->Next();
  412. ASSERT_EQ(IterStatus(iter), "(invalid)");
  413. iter->SeekToLast();
  414. ASSERT_EQ(IterStatus(iter), "c->vc");
  415. iter->Prev();
  416. ASSERT_EQ(IterStatus(iter), "b->vb");
  417. iter->Prev();
  418. ASSERT_EQ(IterStatus(iter), "a->va");
  419. iter->Prev();
  420. ASSERT_EQ(IterStatus(iter), "(invalid)");
  421. delete iter;
  422. }
  423. TEST(DBTest, IterSmallAndLargeMix) {
  424. ASSERT_OK(Put("a", "va"));
  425. ASSERT_OK(Put("b", std::string(100000, 'b')));
  426. ASSERT_OK(Put("c", "vc"));
  427. ASSERT_OK(Put("d", std::string(100000, 'd')));
  428. ASSERT_OK(Put("e", std::string(100000, 'e')));
  429. Iterator* iter = db_->NewIterator(ReadOptions());
  430. iter->SeekToFirst();
  431. ASSERT_EQ(IterStatus(iter), "a->va");
  432. iter->Next();
  433. ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
  434. iter->Next();
  435. ASSERT_EQ(IterStatus(iter), "c->vc");
  436. iter->Next();
  437. ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
  438. iter->Next();
  439. ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
  440. iter->Next();
  441. ASSERT_EQ(IterStatus(iter), "(invalid)");
  442. iter->SeekToLast();
  443. ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
  444. iter->Prev();
  445. ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
  446. iter->Prev();
  447. ASSERT_EQ(IterStatus(iter), "c->vc");
  448. iter->Prev();
  449. ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
  450. iter->Prev();
  451. ASSERT_EQ(IterStatus(iter), "a->va");
  452. iter->Prev();
  453. ASSERT_EQ(IterStatus(iter), "(invalid)");
  454. delete iter;
  455. }
  456. TEST(DBTest, Recover) {
  457. ASSERT_OK(Put("foo", "v1"));
  458. ASSERT_OK(Put("baz", "v5"));
  459. Reopen();
  460. ASSERT_EQ("v1", Get("foo"));
  461. ASSERT_EQ("v1", Get("foo"));
  462. ASSERT_EQ("v5", Get("baz"));
  463. ASSERT_OK(Put("bar", "v2"));
  464. ASSERT_OK(Put("foo", "v3"));
  465. Reopen();
  466. ASSERT_EQ("v3", Get("foo"));
  467. ASSERT_OK(Put("foo", "v4"));
  468. ASSERT_EQ("v4", Get("foo"));
  469. ASSERT_EQ("v2", Get("bar"));
  470. ASSERT_EQ("v5", Get("baz"));
  471. }
  472. TEST(DBTest, RecoveryWithEmptyLog) {
  473. ASSERT_OK(Put("foo", "v1"));
  474. ASSERT_OK(Put("foo", "v2"));
  475. Reopen();
  476. Reopen();
  477. ASSERT_OK(Put("foo", "v3"));
  478. Reopen();
  479. ASSERT_EQ("v3", Get("foo"));
  480. }
  481. // Check that writes done during a memtable compaction are recovered
  482. // if the database is shutdown during the memtable compaction.
  483. TEST(DBTest, RecoverDuringMemtableCompaction) {
  484. Options options;
  485. options.env = env_;
  486. options.write_buffer_size = 1000000;
  487. Reopen(&options);
  488. // Trigger a long memtable compaction and reopen the database during it
  489. ASSERT_OK(Put("foo", "v1")); // Goes to 1st log file
  490. ASSERT_OK(Put("big1", std::string(10000000, 'x'))); // Fills memtable
  491. ASSERT_OK(Put("big2", std::string(1000, 'y'))); // Triggers compaction
  492. ASSERT_OK(Put("bar", "v2")); // Goes to new log file
  493. Reopen(&options);
  494. ASSERT_EQ("v1", Get("foo"));
  495. ASSERT_EQ("v2", Get("bar"));
  496. ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
  497. ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
  498. }
  499. static std::string Key(int i) {
  500. char buf[100];
  501. snprintf(buf, sizeof(buf), "key%06d", i);
  502. return std::string(buf);
  503. }
  504. TEST(DBTest, MinorCompactionsHappen) {
  505. Options options;
  506. options.write_buffer_size = 10000;
  507. Reopen(&options);
  508. const int N = 500;
  509. int starting_num_tables = TotalTableFiles();
  510. for (int i = 0; i < N; i++) {
  511. ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v')));
  512. }
  513. int ending_num_tables = TotalTableFiles();
  514. ASSERT_GT(ending_num_tables, starting_num_tables);
  515. for (int i = 0; i < N; i++) {
  516. ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
  517. }
  518. Reopen();
  519. for (int i = 0; i < N; i++) {
  520. ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
  521. }
  522. }
  523. TEST(DBTest, RecoverWithLargeLog) {
  524. {
  525. Options options;
  526. Reopen(&options);
  527. ASSERT_OK(Put("big1", std::string(200000, '1')));
  528. ASSERT_OK(Put("big2", std::string(200000, '2')));
  529. ASSERT_OK(Put("small3", std::string(10, '3')));
  530. ASSERT_OK(Put("small4", std::string(10, '4')));
  531. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  532. }
  533. // Make sure that if we re-open with a small write buffer size that
  534. // we flush table files in the middle of a large log file.
  535. Options options;
  536. options.write_buffer_size = 100000;
  537. Reopen(&options);
  538. ASSERT_EQ(NumTableFilesAtLevel(0), 3);
  539. ASSERT_EQ(std::string(200000, '1'), Get("big1"));
  540. ASSERT_EQ(std::string(200000, '2'), Get("big2"));
  541. ASSERT_EQ(std::string(10, '3'), Get("small3"));
  542. ASSERT_EQ(std::string(10, '4'), Get("small4"));
  543. ASSERT_GT(NumTableFilesAtLevel(0), 1);
  544. }
  545. TEST(DBTest, CompactionsGenerateMultipleFiles) {
  546. Options options;
  547. options.write_buffer_size = 100000000; // Large write buffer
  548. Reopen(&options);
  549. Random rnd(301);
  550. // Write 8MB (80 values, each 100K)
  551. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  552. std::vector<std::string> values;
  553. for (int i = 0; i < 80; i++) {
  554. values.push_back(RandomString(&rnd, 100000));
  555. ASSERT_OK(Put(Key(i), values[i]));
  556. }
  557. // Reopening moves updates to level-0
  558. Reopen(&options);
  559. dbfull()->TEST_CompactRange(0, "", Key(100000));
  560. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  561. ASSERT_GT(NumTableFilesAtLevel(1), 1);
  562. for (int i = 0; i < 80; i++) {
  563. ASSERT_EQ(Get(Key(i)), values[i]);
  564. }
  565. }
  566. TEST(DBTest, RepeatedWritesToSameKey) {
  567. Options options;
  568. options.env = env_;
  569. options.write_buffer_size = 100000; // Small write buffer
  570. Reopen(&options);
  571. // We must have at most one file per level except for level-0,
  572. // which may have up to kL0_StopWritesTrigger files.
  573. const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger;
  574. Random rnd(301);
  575. std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
  576. for (int i = 0; i < 5 * kMaxFiles; i++) {
  577. Put("key", value);
  578. ASSERT_LE(TotalTableFiles(), kMaxFiles);
  579. fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles());
  580. }
  581. }
  582. TEST(DBTest, SparseMerge) {
  583. Options options;
  584. options.compression = kNoCompression;
  585. Reopen(&options);
  586. FillLevels("A", "Z");
  587. // Suppose there is:
  588. // small amount of data with prefix A
  589. // large amount of data with prefix B
  590. // small amount of data with prefix C
  591. // and that recent updates have made small changes to all three prefixes.
  592. // Check that we do not do a compaction that merges all of B in one shot.
  593. const std::string value(1000, 'x');
  594. Put("A", "va");
  595. // Write approximately 100MB of "B" values
  596. for (int i = 0; i < 100000; i++) {
  597. char key[100];
  598. snprintf(key, sizeof(key), "B%010d", i);
  599. Put(key, value);
  600. }
  601. Put("C", "vc");
  602. dbfull()->TEST_CompactMemTable();
  603. dbfull()->TEST_CompactRange(0, "A", "Z");
  604. // Make sparse update
  605. Put("A", "va2");
  606. Put("B100", "bvalue2");
  607. Put("C", "vc2");
  608. dbfull()->TEST_CompactMemTable();
  609. // Compactions should not cause us to create a situation where
  610. // a file overlaps too much data at the next level.
  611. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  612. dbfull()->TEST_CompactRange(0, "", "z");
  613. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  614. dbfull()->TEST_CompactRange(1, "", "z");
  615. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  616. }
  617. static bool Between(uint64_t val, uint64_t low, uint64_t high) {
  618. bool result = (val >= low) && (val <= high);
  619. if (!result) {
  620. fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
  621. (unsigned long long)(val),
  622. (unsigned long long)(low),
  623. (unsigned long long)(high));
  624. }
  625. return result;
  626. }
  627. TEST(DBTest, ApproximateSizes) {
  628. Options options;
  629. options.write_buffer_size = 100000000; // Large write buffer
  630. options.compression = kNoCompression;
  631. DestroyAndReopen();
  632. ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
  633. Reopen(&options);
  634. ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
  635. // Write 8MB (80 values, each 100K)
  636. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  637. const int N = 80;
  638. Random rnd(301);
  639. for (int i = 0; i < N; i++) {
  640. ASSERT_OK(Put(Key(i), RandomString(&rnd, 100000)));
  641. }
  642. // 0 because GetApproximateSizes() does not account for memtable space
  643. ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
  644. // Check sizes across recovery by reopening a few times
  645. for (int run = 0; run < 3; run++) {
  646. Reopen(&options);
  647. for (int compact_start = 0; compact_start < N; compact_start += 10) {
  648. for (int i = 0; i < N; i += 10) {
  649. ASSERT_TRUE(Between(Size("", Key(i)), 100000*i, 100000*i + 10000));
  650. ASSERT_TRUE(Between(Size("", Key(i)+".suffix"),
  651. 100000 * (i+1), 100000 * (i+1) + 10000));
  652. ASSERT_TRUE(Between(Size(Key(i), Key(i+10)),
  653. 100000 * 10, 100000 * 10 + 10000));
  654. }
  655. ASSERT_TRUE(Between(Size("", Key(50)), 5000000, 5010000));
  656. ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), 5100000, 5110000));
  657. dbfull()->TEST_CompactRange(0,
  658. Key(compact_start),
  659. Key(compact_start + 9));
  660. }
  661. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  662. ASSERT_GT(NumTableFilesAtLevel(1), 0);
  663. }
  664. }
  665. TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
  666. Options options;
  667. options.compression = kNoCompression;
  668. Reopen();
  669. Random rnd(301);
  670. std::string big1 = RandomString(&rnd, 100000);
  671. ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
  672. ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
  673. ASSERT_OK(Put(Key(2), big1));
  674. ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
  675. ASSERT_OK(Put(Key(4), big1));
  676. ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
  677. ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
  678. ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
  679. // Check sizes across recovery by reopening a few times
  680. for (int run = 0; run < 3; run++) {
  681. Reopen(&options);
  682. ASSERT_TRUE(Between(Size("", Key(0)), 0, 0));
  683. ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000));
  684. ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000));
  685. ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000));
  686. ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000));
  687. ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000));
  688. ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000));
  689. ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000));
  690. ASSERT_TRUE(Between(Size("", Key(8)), 550000, 551000));
  691. ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
  692. dbfull()->TEST_CompactRange(0, Key(0), Key(100));
  693. }
  694. }
  695. TEST(DBTest, IteratorPinsRef) {
  696. Put("foo", "hello");
  697. // Get iterator that will yield the current contents of the DB.
  698. Iterator* iter = db_->NewIterator(ReadOptions());
  699. // Write to force compactions
  700. Put("foo", "newvalue1");
  701. for (int i = 0; i < 100; i++) {
  702. ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values
  703. }
  704. Put("foo", "newvalue2");
  705. iter->SeekToFirst();
  706. ASSERT_TRUE(iter->Valid());
  707. ASSERT_EQ("foo", iter->key().ToString());
  708. ASSERT_EQ("hello", iter->value().ToString());
  709. iter->Next();
  710. ASSERT_TRUE(!iter->Valid());
  711. delete iter;
  712. }
  713. TEST(DBTest, Snapshot) {
  714. Put("foo", "v1");
  715. const Snapshot* s1 = db_->GetSnapshot();
  716. Put("foo", "v2");
  717. const Snapshot* s2 = db_->GetSnapshot();
  718. Put("foo", "v3");
  719. const Snapshot* s3 = db_->GetSnapshot();
  720. Put("foo", "v4");
  721. ASSERT_EQ("v1", Get("foo", s1));
  722. ASSERT_EQ("v2", Get("foo", s2));
  723. ASSERT_EQ("v3", Get("foo", s3));
  724. ASSERT_EQ("v4", Get("foo"));
  725. db_->ReleaseSnapshot(s3);
  726. ASSERT_EQ("v1", Get("foo", s1));
  727. ASSERT_EQ("v2", Get("foo", s2));
  728. ASSERT_EQ("v4", Get("foo"));
  729. db_->ReleaseSnapshot(s1);
  730. ASSERT_EQ("v2", Get("foo", s2));
  731. ASSERT_EQ("v4", Get("foo"));
  732. db_->ReleaseSnapshot(s2);
  733. ASSERT_EQ("v4", Get("foo"));
  734. }
  735. TEST(DBTest, HiddenValuesAreRemoved) {
  736. Random rnd(301);
  737. FillLevels("a", "z");
  738. std::string big = RandomString(&rnd, 50000);
  739. Put("foo", big);
  740. Put("pastfoo", "v");
  741. const Snapshot* snapshot = db_->GetSnapshot();
  742. Put("foo", "tiny");
  743. Put("pastfoo2", "v2"); // Advance sequence number one more
  744. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  745. ASSERT_GT(NumTableFilesAtLevel(0), 0);
  746. ASSERT_EQ(big, Get("foo", snapshot));
  747. ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
  748. db_->ReleaseSnapshot(snapshot);
  749. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
  750. dbfull()->TEST_CompactRange(0, "", "x");
  751. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
  752. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  753. ASSERT_GE(NumTableFilesAtLevel(1), 1);
  754. dbfull()->TEST_CompactRange(1, "", "x");
  755. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
  756. ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
  757. }
  758. TEST(DBTest, DeletionMarkers1) {
  759. Put("foo", "v1");
  760. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  761. const int last = config::kMaxMemCompactLevel;
  762. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
  763. // Place a table at level last-1 to prevent merging with preceding mutation
  764. Put("a", "begin");
  765. Put("z", "end");
  766. dbfull()->TEST_CompactMemTable();
  767. ASSERT_EQ(NumTableFilesAtLevel(last), 1);
  768. ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
  769. Delete("foo");
  770. Put("foo", "v2");
  771. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
  772. ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
  773. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
  774. dbfull()->TEST_CompactRange(last-2, "", "z");
  775. // DEL eliminated, but v1 remains because we aren't compacting that level
  776. // (DEL can be eliminated because v2 hides v1).
  777. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
  778. dbfull()->TEST_CompactRange(last-1, "", "z");
  779. // Merging last-1 w/ last, so we are the base level for "foo", so
  780. // DEL is removed. (as is v1).
  781. ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]");
  782. }
  783. TEST(DBTest, DeletionMarkers2) {
  784. Put("foo", "v1");
  785. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  786. const int last = config::kMaxMemCompactLevel;
  787. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
  788. // Place a table at level last-1 to prevent merging with preceding mutation
  789. Put("a", "begin");
  790. Put("z", "end");
  791. dbfull()->TEST_CompactMemTable();
  792. ASSERT_EQ(NumTableFilesAtLevel(last), 1);
  793. ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
  794. Delete("foo");
  795. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  796. ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
  797. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  798. dbfull()->TEST_CompactRange(last-2, "", "z");
  799. // DEL kept: "last" file overlaps
  800. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  801. dbfull()->TEST_CompactRange(last-1, "", "z");
  802. // Merging last-1 w/ last, so we are the base level for "foo", so
  803. // DEL is removed. (as is v1).
  804. ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
  805. }
  806. TEST(DBTest, ComparatorCheck) {
  807. class NewComparator : public Comparator {
  808. public:
  809. virtual const char* Name() const { return "leveldb.NewComparator"; }
  810. virtual int Compare(const Slice& a, const Slice& b) const {
  811. return BytewiseComparator()->Compare(a, b);
  812. }
  813. virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
  814. BytewiseComparator()->FindShortestSeparator(s, l);
  815. }
  816. virtual void FindShortSuccessor(std::string* key) const {
  817. BytewiseComparator()->FindShortSuccessor(key);
  818. }
  819. };
  820. NewComparator cmp;
  821. Options new_options;
  822. new_options.comparator = &cmp;
  823. Status s = TryReopen(&new_options);
  824. ASSERT_TRUE(!s.ok());
  825. ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos)
  826. << s.ToString();
  827. }
  828. TEST(DBTest, DBOpen_Options) {
  829. std::string dbname = test::TmpDir() + "/db_options_test";
  830. DestroyDB(dbname, Options());
  831. // Does not exist, and create_if_missing == false: error
  832. DB* db = NULL;
  833. Options opts;
  834. opts.create_if_missing = false;
  835. Status s = DB::Open(opts, dbname, &db);
  836. ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL);
  837. ASSERT_TRUE(db == NULL);
  838. // Does not exist, and create_if_missing == true: OK
  839. opts.create_if_missing = true;
  840. s = DB::Open(opts, dbname, &db);
  841. ASSERT_OK(s);
  842. ASSERT_TRUE(db != NULL);
  843. delete db;
  844. db = NULL;
  845. // Does exist, and error_if_exists == true: error
  846. opts.create_if_missing = false;
  847. opts.error_if_exists = true;
  848. s = DB::Open(opts, dbname, &db);
  849. ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL);
  850. ASSERT_TRUE(db == NULL);
  851. // Does exist, and error_if_exists == false: OK
  852. opts.create_if_missing = true;
  853. opts.error_if_exists = false;
  854. s = DB::Open(opts, dbname, &db);
  855. ASSERT_OK(s);
  856. ASSERT_TRUE(db != NULL);
  857. delete db;
  858. db = NULL;
  859. }
  860. // Multi-threaded test:
  861. namespace {
  862. static const int kNumThreads = 4;
  863. static const int kTestSeconds = 10;
  864. static const int kNumKeys = 1000;
  865. struct MTState {
  866. DBTest* test;
  867. port::AtomicPointer stop;
  868. port::AtomicPointer counter[kNumThreads];
  869. port::AtomicPointer thread_done[kNumThreads];
  870. };
  871. struct MTThread {
  872. MTState* state;
  873. int id;
  874. };
  875. static void MTThreadBody(void* arg) {
  876. MTThread* t = reinterpret_cast<MTThread*>(arg);
  877. DB* db = t->state->test->db_;
  878. uintptr_t counter = 0;
  879. fprintf(stderr, "... starting thread %d\n", t->id);
  880. Random rnd(1000 + t->id);
  881. std::string value;
  882. char valbuf[1500];
  883. while (t->state->stop.Acquire_Load() == NULL) {
  884. t->state->counter[t->id].Release_Store(reinterpret_cast<void*>(counter));
  885. int key = rnd.Uniform(kNumKeys);
  886. char keybuf[20];
  887. snprintf(keybuf, sizeof(keybuf), "%016d", key);
  888. if (rnd.OneIn(2)) {
  889. // Write values of the form <key, my id, counter>.
  890. // We add some padding for force compactions.
  891. snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d",
  892. key, t->id, static_cast<int>(counter));
  893. ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf)));
  894. } else {
  895. // Read a value and verify that it matches the pattern written above.
  896. Status s = db->Get(ReadOptions(), Slice(keybuf), &value);
  897. if (s.IsNotFound()) {
  898. // Key has not yet been written
  899. } else {
  900. // Check that the writer thread counter is >= the counter in the value
  901. ASSERT_OK(s);
  902. int k, w, c;
  903. ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value;
  904. ASSERT_EQ(k, key);
  905. ASSERT_GE(w, 0);
  906. ASSERT_LT(w, kNumThreads);
  907. ASSERT_LE(c, reinterpret_cast<uintptr_t>(
  908. t->state->counter[w].Acquire_Load()));
  909. }
  910. }
  911. counter++;
  912. }
  913. t->state->thread_done[t->id].Release_Store(t);
  914. fprintf(stderr, "... stopping thread %d after %d ops\n", t->id, int(counter));
  915. }
  916. }
  917. TEST(DBTest, MultiThreaded) {
  918. // Initialize state
  919. MTState mt;
  920. mt.test = this;
  921. mt.stop.Release_Store(0);
  922. for (int id = 0; id < kNumThreads; id++) {
  923. mt.counter[id].Release_Store(0);
  924. mt.thread_done[id].Release_Store(0);
  925. }
  926. // Start threads
  927. MTThread thread[kNumThreads];
  928. for (int id = 0; id < kNumThreads; id++) {
  929. thread[id].state = &mt;
  930. thread[id].id = id;
  931. env_->StartThread(MTThreadBody, &thread[id]);
  932. }
  933. // Let them run for a while
  934. env_->SleepForMicroseconds(kTestSeconds * 1000000);
  935. // Stop the threads and wait for them to finish
  936. mt.stop.Release_Store(&mt);
  937. for (int id = 0; id < kNumThreads; id++) {
  938. while (mt.thread_done[id].Acquire_Load() == NULL) {
  939. env_->SleepForMicroseconds(100000);
  940. }
  941. }
  942. }
  943. namespace {
  944. typedef std::map<std::string, std::string> KVMap;
  945. }
  946. class ModelDB: public DB {
  947. public:
  948. class ModelSnapshot : public Snapshot {
  949. public:
  950. KVMap map_;
  951. };
  952. explicit ModelDB(const Options& options): options_(options) { }
  953. ~ModelDB() { }
  954. virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
  955. return DB::Put(o, k, v);
  956. }
  957. virtual Status Delete(const WriteOptions& o, const Slice& key) {
  958. return DB::Delete(o, key);
  959. }
  960. virtual Status Get(const ReadOptions& options,
  961. const Slice& key, std::string* value) {
  962. assert(false); // Not implemented
  963. return Status::NotFound(key);
  964. }
  965. virtual Iterator* NewIterator(const ReadOptions& options) {
  966. if (options.snapshot == NULL) {
  967. KVMap* saved = new KVMap;
  968. *saved = map_;
  969. return new ModelIter(saved, true);
  970. } else {
  971. const KVMap* snapshot_state =
  972. &(reinterpret_cast<const ModelSnapshot*>(options.snapshot)->map_);
  973. return new ModelIter(snapshot_state, false);
  974. }
  975. }
  976. virtual const Snapshot* GetSnapshot() {
  977. ModelSnapshot* snapshot = new ModelSnapshot;
  978. snapshot->map_ = map_;
  979. return snapshot;
  980. }
  981. virtual void ReleaseSnapshot(const Snapshot* snapshot) {
  982. delete reinterpret_cast<const ModelSnapshot*>(snapshot);
  983. }
  984. virtual Status Write(const WriteOptions& options, WriteBatch* batch) {
  985. assert(options.post_write_snapshot == NULL); // Not supported
  986. class Handler : public WriteBatch::Handler {
  987. public:
  988. KVMap* map_;
  989. virtual void Put(const Slice& key, const Slice& value) {
  990. (*map_)[key.ToString()] = value.ToString();
  991. }
  992. virtual void Delete(const Slice& key) {
  993. map_->erase(key.ToString());
  994. }
  995. };
  996. Handler handler;
  997. handler.map_ = &map_;
  998. return batch->Iterate(&handler);
  999. }
  1000. virtual bool GetProperty(const Slice& property, std::string* value) {
  1001. return false;
  1002. }
  1003. virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) {
  1004. for (int i = 0; i < n; i++) {
  1005. sizes[i] = 0;
  1006. }
  1007. }
  1008. private:
  1009. class ModelIter: public Iterator {
  1010. public:
  1011. ModelIter(const KVMap* map, bool owned)
  1012. : map_(map), owned_(owned), iter_(map_->end()) {
  1013. }
  1014. ~ModelIter() {
  1015. if (owned_) delete map_;
  1016. }
  1017. virtual bool Valid() const { return iter_ != map_->end(); }
  1018. virtual void SeekToFirst() { iter_ = map_->begin(); }
  1019. virtual void SeekToLast() {
  1020. if (map_->empty()) {
  1021. iter_ = map_->end();
  1022. } else {
  1023. iter_ = map_->find(map_->rbegin()->first);
  1024. }
  1025. }
  1026. virtual void Seek(const Slice& k) {
  1027. iter_ = map_->lower_bound(k.ToString());
  1028. }
  1029. virtual void Next() { ++iter_; }
  1030. virtual void Prev() { --iter_; }
  1031. virtual Slice key() const { return iter_->first; }
  1032. virtual Slice value() const { return iter_->second; }
  1033. virtual Status status() const { return Status::OK(); }
  1034. private:
  1035. const KVMap* const map_;
  1036. const bool owned_; // Do we own map_
  1037. KVMap::const_iterator iter_;
  1038. };
  1039. const Options options_;
  1040. KVMap map_;
  1041. };
  1042. static std::string RandomKey(Random* rnd) {
  1043. int len = (rnd->OneIn(3)
  1044. ? 1 // Short sometimes to encourage collisions
  1045. : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
  1046. return test::RandomKey(rnd, len);
  1047. }
  1048. static bool CompareIterators(int step,
  1049. DB* model,
  1050. DB* db,
  1051. const Snapshot* model_snap,
  1052. const Snapshot* db_snap) {
  1053. ReadOptions options;
  1054. options.snapshot = model_snap;
  1055. Iterator* miter = model->NewIterator(options);
  1056. options.snapshot = db_snap;
  1057. Iterator* dbiter = db->NewIterator(options);
  1058. bool ok = true;
  1059. int count = 0;
  1060. for (miter->SeekToFirst(), dbiter->SeekToFirst();
  1061. ok && miter->Valid() && dbiter->Valid();
  1062. miter->Next(), dbiter->Next()) {
  1063. count++;
  1064. if (miter->key().compare(dbiter->key()) != 0) {
  1065. fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
  1066. step,
  1067. EscapeString(miter->key()).c_str(),
  1068. EscapeString(dbiter->key()).c_str());
  1069. ok = false;
  1070. break;
  1071. }
  1072. if (miter->value().compare(dbiter->value()) != 0) {
  1073. fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
  1074. step,
  1075. EscapeString(miter->key()).c_str(),
  1076. EscapeString(miter->value()).c_str(),
  1077. EscapeString(miter->value()).c_str());
  1078. ok = false;
  1079. }
  1080. }
  1081. if (ok) {
  1082. if (miter->Valid() != dbiter->Valid()) {
  1083. fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n",
  1084. step, miter->Valid(), dbiter->Valid());
  1085. ok = false;
  1086. }
  1087. }
  1088. fprintf(stderr, "%d entries compared: ok=%d\n", count, ok);
  1089. delete miter;
  1090. delete dbiter;
  1091. return ok;
  1092. }
  1093. TEST(DBTest, Randomized) {
  1094. Random rnd(test::RandomSeed());
  1095. ModelDB model(last_options_);
  1096. const int N = 10000;
  1097. const Snapshot* model_snap = NULL;
  1098. const Snapshot* db_snap = NULL;
  1099. std::string k, v;
  1100. for (int step = 0; step < N; step++) {
  1101. if (step % 100 == 0) {
  1102. fprintf(stderr, "Step %d of %d\n", step, N);
  1103. }
  1104. int p = rnd.Uniform(100);
  1105. if (p < 45) { // Put
  1106. k = RandomKey(&rnd);
  1107. v = RandomString(&rnd,
  1108. rnd.OneIn(20)
  1109. ? 100 + rnd.Uniform(100)
  1110. : rnd.Uniform(8));
  1111. ASSERT_OK(model.Put(WriteOptions(), k, v));
  1112. ASSERT_OK(db_->Put(WriteOptions(), k, v));
  1113. } else if (p < 90) { // Delete
  1114. k = RandomKey(&rnd);
  1115. ASSERT_OK(model.Delete(WriteOptions(), k));
  1116. ASSERT_OK(db_->Delete(WriteOptions(), k));
  1117. } else { // Multi-element batch
  1118. WriteBatch b;
  1119. const int num = rnd.Uniform(8);
  1120. for (int i = 0; i < num; i++) {
  1121. if (i == 0 || !rnd.OneIn(10)) {
  1122. k = RandomKey(&rnd);
  1123. } else {
  1124. // Periodically re-use the same key from the previous iter, so
  1125. // we have multiple entries in the write batch for the same key
  1126. }
  1127. if (rnd.OneIn(2)) {
  1128. v = RandomString(&rnd, rnd.Uniform(10));
  1129. b.Put(k, v);
  1130. } else {
  1131. b.Delete(k);
  1132. }
  1133. }
  1134. ASSERT_OK(model.Write(WriteOptions(), &b));
  1135. ASSERT_OK(db_->Write(WriteOptions(), &b));
  1136. }
  1137. if ((step % 100) == 0) {
  1138. ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
  1139. ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
  1140. // Save a snapshot from each DB this time that we'll use next
  1141. // time we compare things, to make sure the current state is
  1142. // preserved with the snapshot
  1143. if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
  1144. if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
  1145. Reopen();
  1146. ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
  1147. model_snap = model.GetSnapshot();
  1148. db_snap = db_->GetSnapshot();
  1149. }
  1150. }
  1151. if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
  1152. if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
  1153. }
  1154. std::string MakeKey(unsigned int num) {
  1155. char buf[30];
  1156. snprintf(buf, sizeof(buf), "%016u", num);
  1157. return std::string(buf);
  1158. }
  1159. void BM_LogAndApply(int iters, int num_base_files) {
  1160. std::string dbname = test::TmpDir() + "/leveldb_test_benchmark";
  1161. DestroyDB(dbname, Options());
  1162. DB* db = NULL;
  1163. Options opts;
  1164. opts.create_if_missing = true;
  1165. Status s = DB::Open(opts, dbname, &db);
  1166. ASSERT_OK(s);
  1167. ASSERT_TRUE(db != NULL);
  1168. delete db;
  1169. db = NULL;
  1170. Env* env = Env::Default();
  1171. InternalKeyComparator cmp(BytewiseComparator());
  1172. Options options;
  1173. VersionSet vset(dbname, &options, NULL, &cmp);
  1174. ASSERT_OK(vset.Recover());
  1175. VersionEdit vbase;
  1176. uint64_t fnum = 1;
  1177. for (int i = 0; i < num_base_files; i++) {
  1178. InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
  1179. InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
  1180. vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
  1181. }
  1182. ASSERT_OK(vset.LogAndApply(&vbase));
  1183. uint64_t start_micros = env->NowMicros();
  1184. for (int i = 0; i < iters; i++) {
  1185. VersionEdit vedit;
  1186. vedit.DeleteFile(2, fnum);
  1187. InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
  1188. InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
  1189. vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
  1190. vset.LogAndApply(&vedit);
  1191. }
  1192. uint64_t stop_micros = env->NowMicros();
  1193. unsigned int us = stop_micros - start_micros;
  1194. char buf[16];
  1195. snprintf(buf, sizeof(buf), "%d", num_base_files);
  1196. fprintf(stderr,
  1197. "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n",
  1198. buf, iters, us, ((float)us) / iters);
  1199. }
  1200. }
  1201. int main(int argc, char** argv) {
  1202. if (argc > 1 && std::string(argv[1]) == "--benchmark") {
  1203. leveldb::BM_LogAndApply(1000, 1);
  1204. leveldb::BM_LogAndApply(1000, 100);
  1205. leveldb::BM_LogAndApply(1000, 10000);
  1206. leveldb::BM_LogAndApply(100, 100000);
  1207. return 0;
  1208. }
  1209. return leveldb::test::RunAllTests();
  1210. }