2092 rivejä
60 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "leveldb/db.h"
  5. #include "leveldb/filter_policy.h"
  6. #include "db/db_impl.h"
  7. #include "db/filename.h"
  8. #include "db/version_set.h"
  9. #include "db/write_batch_internal.h"
  10. #include "leveldb/cache.h"
  11. #include "leveldb/env.h"
  12. #include "leveldb/table.h"
  13. #include "util/hash.h"
  14. #include "util/logging.h"
  15. #include "util/mutexlock.h"
  16. #include "util/testharness.h"
  17. #include "util/testutil.h"
  18. namespace leveldb {
  19. static std::string RandomString(Random* rnd, int len) {
  20. std::string r;
  21. test::RandomString(rnd, len, &r);
  22. return r;
  23. }
  24. namespace {
  25. class AtomicCounter {
  26. private:
  27. port::Mutex mu_;
  28. int count_;
  29. public:
  30. AtomicCounter() : count_(0) { }
  31. void Increment() {
  32. IncrementBy(1);
  33. }
  34. void IncrementBy(int count) {
  35. MutexLock l(&mu_);
  36. count_ += count;
  37. }
  38. int Read() {
  39. MutexLock l(&mu_);
  40. return count_;
  41. }
  42. void Reset() {
  43. MutexLock l(&mu_);
  44. count_ = 0;
  45. }
  46. };
  47. void DelayMilliseconds(int millis) {
  48. Env::Default()->SleepForMicroseconds(millis * 1000);
  49. }
  50. }
  51. // Special Env used to delay background operations
  52. class SpecialEnv : public EnvWrapper {
  53. public:
  54. // sstable Sync() calls are blocked while this pointer is non-NULL.
  55. port::AtomicPointer delay_sstable_sync_;
  56. // Simulate no-space errors while this pointer is non-NULL.
  57. port::AtomicPointer no_space_;
  58. // Simulate non-writable file system while this pointer is non-NULL
  59. port::AtomicPointer non_writable_;
  60. // Force sync of manifest files to fail while this pointer is non-NULL
  61. port::AtomicPointer manifest_sync_error_;
  62. // Force write to manifest files to fail while this pointer is non-NULL
  63. port::AtomicPointer manifest_write_error_;
  64. bool count_random_reads_;
  65. AtomicCounter random_read_counter_;
  66. AtomicCounter sleep_counter_;
  67. AtomicCounter sleep_time_counter_;
  68. explicit SpecialEnv(Env* base) : EnvWrapper(base) {
  69. delay_sstable_sync_.Release_Store(NULL);
  70. no_space_.Release_Store(NULL);
  71. non_writable_.Release_Store(NULL);
  72. count_random_reads_ = false;
  73. manifest_sync_error_.Release_Store(NULL);
  74. manifest_write_error_.Release_Store(NULL);
  75. }
  76. Status NewWritableFile(const std::string& f, WritableFile** r) {
  77. class SSTableFile : public WritableFile {
  78. private:
  79. SpecialEnv* env_;
  80. WritableFile* base_;
  81. public:
  82. SSTableFile(SpecialEnv* env, WritableFile* base)
  83. : env_(env),
  84. base_(base) {
  85. }
  86. ~SSTableFile() { delete base_; }
  87. Status Append(const Slice& data) {
  88. if (env_->no_space_.Acquire_Load() != NULL) {
  89. // Drop writes on the floor
  90. return Status::OK();
  91. } else {
  92. return base_->Append(data);
  93. }
  94. }
  95. Status Close() { return base_->Close(); }
  96. Status Flush() { return base_->Flush(); }
  97. Status Sync() {
  98. while (env_->delay_sstable_sync_.Acquire_Load() != NULL) {
  99. DelayMilliseconds(100);
  100. }
  101. return base_->Sync();
  102. }
  103. };
  104. class ManifestFile : public WritableFile {
  105. private:
  106. SpecialEnv* env_;
  107. WritableFile* base_;
  108. public:
  109. ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { }
  110. ~ManifestFile() { delete base_; }
  111. Status Append(const Slice& data) {
  112. if (env_->manifest_write_error_.Acquire_Load() != NULL) {
  113. return Status::IOError("simulated writer error");
  114. } else {
  115. return base_->Append(data);
  116. }
  117. }
  118. Status Close() { return base_->Close(); }
  119. Status Flush() { return base_->Flush(); }
  120. Status Sync() {
  121. if (env_->manifest_sync_error_.Acquire_Load() != NULL) {
  122. return Status::IOError("simulated sync error");
  123. } else {
  124. return base_->Sync();
  125. }
  126. }
  127. };
  128. if (non_writable_.Acquire_Load() != NULL) {
  129. return Status::IOError("simulated write error");
  130. }
  131. Status s = target()->NewWritableFile(f, r);
  132. if (s.ok()) {
  133. if (strstr(f.c_str(), ".sst") != NULL) {
  134. *r = new SSTableFile(this, *r);
  135. } else if (strstr(f.c_str(), "MANIFEST") != NULL) {
  136. *r = new ManifestFile(this, *r);
  137. }
  138. }
  139. return s;
  140. }
  141. Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) {
  142. class CountingFile : public RandomAccessFile {
  143. private:
  144. RandomAccessFile* target_;
  145. AtomicCounter* counter_;
  146. public:
  147. CountingFile(RandomAccessFile* target, AtomicCounter* counter)
  148. : target_(target), counter_(counter) {
  149. }
  150. virtual ~CountingFile() { delete target_; }
  151. virtual Status Read(uint64_t offset, size_t n, Slice* result,
  152. char* scratch) const {
  153. counter_->Increment();
  154. return target_->Read(offset, n, result, scratch);
  155. }
  156. };
  157. Status s = target()->NewRandomAccessFile(f, r);
  158. if (s.ok() && count_random_reads_) {
  159. *r = new CountingFile(*r, &random_read_counter_);
  160. }
  161. return s;
  162. }
  163. virtual void SleepForMicroseconds(int micros) {
  164. sleep_counter_.Increment();
  165. sleep_time_counter_.IncrementBy(micros);
  166. }
  167. };
  168. class DBTest {
  169. private:
  170. const FilterPolicy* filter_policy_;
  171. // Sequence of option configurations to try
  172. enum OptionConfig {
  173. kDefault,
  174. kFilter,
  175. kUncompressed,
  176. kEnd
  177. };
  178. int option_config_;
  179. public:
  180. std::string dbname_;
  181. SpecialEnv* env_;
  182. DB* db_;
  183. Options last_options_;
  184. DBTest() : option_config_(kDefault),
  185. env_(new SpecialEnv(Env::Default())) {
  186. filter_policy_ = NewBloomFilterPolicy(10);
  187. dbname_ = test::TmpDir() + "/db_test";
  188. DestroyDB(dbname_, Options());
  189. db_ = NULL;
  190. Reopen();
  191. }
  192. ~DBTest() {
  193. delete db_;
  194. DestroyDB(dbname_, Options());
  195. delete env_;
  196. delete filter_policy_;
  197. }
  198. // Switch to a fresh database with the next option configuration to
  199. // test. Return false if there are no more configurations to test.
  200. bool ChangeOptions() {
  201. option_config_++;
  202. if (option_config_ >= kEnd) {
  203. return false;
  204. } else {
  205. DestroyAndReopen();
  206. return true;
  207. }
  208. }
  209. // Return the current option configuration.
  210. Options CurrentOptions() {
  211. Options options;
  212. switch (option_config_) {
  213. case kFilter:
  214. options.filter_policy = filter_policy_;
  215. break;
  216. case kUncompressed:
  217. options.compression = kNoCompression;
  218. break;
  219. default:
  220. break;
  221. }
  222. return options;
  223. }
  224. DBImpl* dbfull() {
  225. return reinterpret_cast<DBImpl*>(db_);
  226. }
  227. void Reopen(Options* options = NULL) {
  228. ASSERT_OK(TryReopen(options));
  229. }
  230. void Close() {
  231. delete db_;
  232. db_ = NULL;
  233. }
  234. void DestroyAndReopen(Options* options = NULL) {
  235. delete db_;
  236. db_ = NULL;
  237. DestroyDB(dbname_, Options());
  238. ASSERT_OK(TryReopen(options));
  239. }
  240. Status TryReopen(Options* options) {
  241. delete db_;
  242. db_ = NULL;
  243. Options opts;
  244. if (options != NULL) {
  245. opts = *options;
  246. } else {
  247. opts = CurrentOptions();
  248. opts.create_if_missing = true;
  249. }
  250. last_options_ = opts;
  251. return DB::Open(opts, dbname_, &db_);
  252. }
  253. Status Put(const std::string& k, const std::string& v) {
  254. return db_->Put(WriteOptions(), k, v);
  255. }
  256. Status Delete(const std::string& k) {
  257. return db_->Delete(WriteOptions(), k);
  258. }
  259. std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
  260. ReadOptions options;
  261. options.snapshot = snapshot;
  262. std::string result;
  263. Status s = db_->Get(options, k, &result);
  264. if (s.IsNotFound()) {
  265. result = "NOT_FOUND";
  266. } else if (!s.ok()) {
  267. result = s.ToString();
  268. }
  269. return result;
  270. }
  271. // Return a string that contains all key,value pairs in order,
  272. // formatted like "(k1->v1)(k2->v2)".
  273. std::string Contents() {
  274. std::vector<std::string> forward;
  275. std::string result;
  276. Iterator* iter = db_->NewIterator(ReadOptions());
  277. for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
  278. std::string s = IterStatus(iter);
  279. result.push_back('(');
  280. result.append(s);
  281. result.push_back(')');
  282. forward.push_back(s);
  283. }
  284. // Check reverse iteration results are the reverse of forward results
  285. int matched = 0;
  286. for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
  287. ASSERT_LT(matched, forward.size());
  288. ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
  289. matched++;
  290. }
  291. ASSERT_EQ(matched, forward.size());
  292. delete iter;
  293. return result;
  294. }
  295. std::string AllEntriesFor(const Slice& user_key) {
  296. Iterator* iter = dbfull()->TEST_NewInternalIterator();
  297. InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
  298. iter->Seek(target.Encode());
  299. std::string result;
  300. if (!iter->status().ok()) {
  301. result = iter->status().ToString();
  302. } else {
  303. result = "[ ";
  304. bool first = true;
  305. while (iter->Valid()) {
  306. ParsedInternalKey ikey;
  307. if (!ParseInternalKey(iter->key(), &ikey)) {
  308. result += "CORRUPTED";
  309. } else {
  310. if (last_options_.comparator->Compare(ikey.user_key, user_key) != 0) {
  311. break;
  312. }
  313. if (!first) {
  314. result += ", ";
  315. }
  316. first = false;
  317. switch (ikey.type) {
  318. case kTypeValue:
  319. result += iter->value().ToString();
  320. break;
  321. case kTypeDeletion:
  322. result += "DEL";
  323. break;
  324. }
  325. }
  326. iter->Next();
  327. }
  328. if (!first) {
  329. result += " ";
  330. }
  331. result += "]";
  332. }
  333. delete iter;
  334. return result;
  335. }
  336. int NumTableFilesAtLevel(int level) {
  337. std::string property;
  338. ASSERT_TRUE(
  339. db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level),
  340. &property));
  341. return atoi(property.c_str());
  342. }
  343. int TotalTableFiles() {
  344. int result = 0;
  345. for (int level = 0; level < config::kNumLevels; level++) {
  346. result += NumTableFilesAtLevel(level);
  347. }
  348. return result;
  349. }
  350. // Return spread of files per level
  351. std::string FilesPerLevel() {
  352. std::string result;
  353. int last_non_zero_offset = 0;
  354. for (int level = 0; level < config::kNumLevels; level++) {
  355. int f = NumTableFilesAtLevel(level);
  356. char buf[100];
  357. snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
  358. result += buf;
  359. if (f > 0) {
  360. last_non_zero_offset = result.size();
  361. }
  362. }
  363. result.resize(last_non_zero_offset);
  364. return result;
  365. }
  366. int CountFiles() {
  367. std::vector<std::string> files;
  368. env_->GetChildren(dbname_, &files);
  369. return static_cast<int>(files.size());
  370. }
  371. uint64_t Size(const Slice& start, const Slice& limit) {
  372. Range r(start, limit);
  373. uint64_t size;
  374. db_->GetApproximateSizes(&r, 1, &size);
  375. return size;
  376. }
  377. void Compact(const Slice& start, const Slice& limit) {
  378. db_->CompactRange(&start, &limit);
  379. }
  380. // Do n memtable compactions, each of which produces an sstable
  381. // covering the range [small,large].
  382. void MakeTables(int n, const std::string& small, const std::string& large) {
  383. for (int i = 0; i < n; i++) {
  384. Put(small, "begin");
  385. Put(large, "end");
  386. dbfull()->TEST_CompactMemTable();
  387. }
  388. }
  389. // Prevent pushing of new sstables into deeper levels by adding
  390. // tables that cover a specified range to all levels.
  391. void FillLevels(const std::string& smallest, const std::string& largest) {
  392. MakeTables(config::kNumLevels, smallest, largest);
  393. }
  394. void DumpFileCounts(const char* label) {
  395. fprintf(stderr, "---\n%s:\n", label);
  396. fprintf(stderr, "maxoverlap: %lld\n",
  397. static_cast<long long>(
  398. dbfull()->TEST_MaxNextLevelOverlappingBytes()));
  399. for (int level = 0; level < config::kNumLevels; level++) {
  400. int num = NumTableFilesAtLevel(level);
  401. if (num > 0) {
  402. fprintf(stderr, " level %3d : %d files\n", level, num);
  403. }
  404. }
  405. }
  406. std::string DumpSSTableList() {
  407. std::string property;
  408. db_->GetProperty("leveldb.sstables", &property);
  409. return property;
  410. }
  411. std::string IterStatus(Iterator* iter) {
  412. std::string result;
  413. if (iter->Valid()) {
  414. result = iter->key().ToString() + "->" + iter->value().ToString();
  415. } else {
  416. result = "(invalid)";
  417. }
  418. return result;
  419. }
  420. bool DeleteAnSSTFile() {
  421. std::vector<std::string> filenames;
  422. ASSERT_OK(env_->GetChildren(dbname_, &filenames));
  423. uint64_t number;
  424. FileType type;
  425. for (size_t i = 0; i < filenames.size(); i++) {
  426. if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
  427. ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number)));
  428. return true;
  429. }
  430. }
  431. return false;
  432. }
  433. };
  434. TEST(DBTest, Empty) {
  435. do {
  436. ASSERT_TRUE(db_ != NULL);
  437. ASSERT_EQ("NOT_FOUND", Get("foo"));
  438. } while (ChangeOptions());
  439. }
  440. TEST(DBTest, ReadWrite) {
  441. do {
  442. ASSERT_OK(Put("foo", "v1"));
  443. ASSERT_EQ("v1", Get("foo"));
  444. ASSERT_OK(Put("bar", "v2"));
  445. ASSERT_OK(Put("foo", "v3"));
  446. ASSERT_EQ("v3", Get("foo"));
  447. ASSERT_EQ("v2", Get("bar"));
  448. } while (ChangeOptions());
  449. }
  450. TEST(DBTest, PutDeleteGet) {
  451. do {
  452. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
  453. ASSERT_EQ("v1", Get("foo"));
  454. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
  455. ASSERT_EQ("v2", Get("foo"));
  456. ASSERT_OK(db_->Delete(WriteOptions(), "foo"));
  457. ASSERT_EQ("NOT_FOUND", Get("foo"));
  458. } while (ChangeOptions());
  459. }
  460. TEST(DBTest, GetFromImmutableLayer) {
  461. do {
  462. Options options = CurrentOptions();
  463. options.env = env_;
  464. options.write_buffer_size = 100000; // Small write buffer
  465. Reopen(&options);
  466. ASSERT_OK(Put("foo", "v1"));
  467. ASSERT_EQ("v1", Get("foo"));
  468. env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
  469. Put("k1", std::string(100000, 'x')); // Fill memtable
  470. Put("k2", std::string(100000, 'y')); // Trigger compaction
  471. ASSERT_EQ("v1", Get("foo"));
  472. env_->delay_sstable_sync_.Release_Store(NULL); // Release sync calls
  473. } while (ChangeOptions());
  474. }
  475. TEST(DBTest, GetFromVersions) {
  476. do {
  477. ASSERT_OK(Put("foo", "v1"));
  478. dbfull()->TEST_CompactMemTable();
  479. ASSERT_EQ("v1", Get("foo"));
  480. } while (ChangeOptions());
  481. }
  482. TEST(DBTest, GetSnapshot) {
  483. do {
  484. // Try with both a short key and a long key
  485. for (int i = 0; i < 2; i++) {
  486. std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
  487. ASSERT_OK(Put(key, "v1"));
  488. const Snapshot* s1 = db_->GetSnapshot();
  489. ASSERT_OK(Put(key, "v2"));
  490. ASSERT_EQ("v2", Get(key));
  491. ASSERT_EQ("v1", Get(key, s1));
  492. dbfull()->TEST_CompactMemTable();
  493. ASSERT_EQ("v2", Get(key));
  494. ASSERT_EQ("v1", Get(key, s1));
  495. db_->ReleaseSnapshot(s1);
  496. }
  497. } while (ChangeOptions());
  498. }
  499. TEST(DBTest, GetLevel0Ordering) {
  500. do {
  501. // Check that we process level-0 files in correct order. The code
  502. // below generates two level-0 files where the earlier one comes
  503. // before the later one in the level-0 file list since the earlier
  504. // one has a smaller "smallest" key.
  505. ASSERT_OK(Put("bar", "b"));
  506. ASSERT_OK(Put("foo", "v1"));
  507. dbfull()->TEST_CompactMemTable();
  508. ASSERT_OK(Put("foo", "v2"));
  509. dbfull()->TEST_CompactMemTable();
  510. ASSERT_EQ("v2", Get("foo"));
  511. } while (ChangeOptions());
  512. }
  513. TEST(DBTest, GetOrderedByLevels) {
  514. do {
  515. ASSERT_OK(Put("foo", "v1"));
  516. Compact("a", "z");
  517. ASSERT_EQ("v1", Get("foo"));
  518. ASSERT_OK(Put("foo", "v2"));
  519. ASSERT_EQ("v2", Get("foo"));
  520. dbfull()->TEST_CompactMemTable();
  521. ASSERT_EQ("v2", Get("foo"));
  522. } while (ChangeOptions());
  523. }
  524. TEST(DBTest, GetPicksCorrectFile) {
  525. do {
  526. // Arrange to have multiple files in a non-level-0 level.
  527. ASSERT_OK(Put("a", "va"));
  528. Compact("a", "b");
  529. ASSERT_OK(Put("x", "vx"));
  530. Compact("x", "y");
  531. ASSERT_OK(Put("f", "vf"));
  532. Compact("f", "g");
  533. ASSERT_EQ("va", Get("a"));
  534. ASSERT_EQ("vf", Get("f"));
  535. ASSERT_EQ("vx", Get("x"));
  536. } while (ChangeOptions());
  537. }
  538. TEST(DBTest, GetEncountersEmptyLevel) {
  539. do {
  540. // Arrange for the following to happen:
  541. // * sstable A in level 0
  542. // * nothing in level 1
  543. // * sstable B in level 2
  544. // Then do enough Get() calls to arrange for an automatic compaction
  545. // of sstable A. A bug would cause the compaction to be marked as
  546. // occuring at level 1 (instead of the correct level 0).
  547. // Step 1: First place sstables in levels 0 and 2
  548. int compaction_count = 0;
  549. while (NumTableFilesAtLevel(0) == 0 ||
  550. NumTableFilesAtLevel(2) == 0) {
  551. ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2";
  552. compaction_count++;
  553. Put("a", "begin");
  554. Put("z", "end");
  555. dbfull()->TEST_CompactMemTable();
  556. }
  557. // Step 2: clear level 1 if necessary.
  558. dbfull()->TEST_CompactRange(1, NULL, NULL);
  559. ASSERT_EQ(NumTableFilesAtLevel(0), 1);
  560. ASSERT_EQ(NumTableFilesAtLevel(1), 0);
  561. ASSERT_EQ(NumTableFilesAtLevel(2), 1);
  562. // Step 3: read a bunch of times
  563. for (int i = 0; i < 1000; i++) {
  564. ASSERT_EQ("NOT_FOUND", Get("missing"));
  565. }
  566. // Step 4: Wait for compaction to finish
  567. DelayMilliseconds(1000);
  568. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  569. } while (ChangeOptions());
  570. }
  571. TEST(DBTest, IterEmpty) {
  572. Iterator* iter = db_->NewIterator(ReadOptions());
  573. iter->SeekToFirst();
  574. ASSERT_EQ(IterStatus(iter), "(invalid)");
  575. iter->SeekToLast();
  576. ASSERT_EQ(IterStatus(iter), "(invalid)");
  577. iter->Seek("foo");
  578. ASSERT_EQ(IterStatus(iter), "(invalid)");
  579. delete iter;
  580. }
  581. TEST(DBTest, IterSingle) {
  582. ASSERT_OK(Put("a", "va"));
  583. Iterator* iter = db_->NewIterator(ReadOptions());
  584. iter->SeekToFirst();
  585. ASSERT_EQ(IterStatus(iter), "a->va");
  586. iter->Next();
  587. ASSERT_EQ(IterStatus(iter), "(invalid)");
  588. iter->SeekToFirst();
  589. ASSERT_EQ(IterStatus(iter), "a->va");
  590. iter->Prev();
  591. ASSERT_EQ(IterStatus(iter), "(invalid)");
  592. iter->SeekToLast();
  593. ASSERT_EQ(IterStatus(iter), "a->va");
  594. iter->Next();
  595. ASSERT_EQ(IterStatus(iter), "(invalid)");
  596. iter->SeekToLast();
  597. ASSERT_EQ(IterStatus(iter), "a->va");
  598. iter->Prev();
  599. ASSERT_EQ(IterStatus(iter), "(invalid)");
  600. iter->Seek("");
  601. ASSERT_EQ(IterStatus(iter), "a->va");
  602. iter->Next();
  603. ASSERT_EQ(IterStatus(iter), "(invalid)");
  604. iter->Seek("a");
  605. ASSERT_EQ(IterStatus(iter), "a->va");
  606. iter->Next();
  607. ASSERT_EQ(IterStatus(iter), "(invalid)");
  608. iter->Seek("b");
  609. ASSERT_EQ(IterStatus(iter), "(invalid)");
  610. delete iter;
  611. }
  612. TEST(DBTest, IterMulti) {
  613. ASSERT_OK(Put("a", "va"));
  614. ASSERT_OK(Put("b", "vb"));
  615. ASSERT_OK(Put("c", "vc"));
  616. Iterator* iter = db_->NewIterator(ReadOptions());
  617. iter->SeekToFirst();
  618. ASSERT_EQ(IterStatus(iter), "a->va");
  619. iter->Next();
  620. ASSERT_EQ(IterStatus(iter), "b->vb");
  621. iter->Next();
  622. ASSERT_EQ(IterStatus(iter), "c->vc");
  623. iter->Next();
  624. ASSERT_EQ(IterStatus(iter), "(invalid)");
  625. iter->SeekToFirst();
  626. ASSERT_EQ(IterStatus(iter), "a->va");
  627. iter->Prev();
  628. ASSERT_EQ(IterStatus(iter), "(invalid)");
  629. iter->SeekToLast();
  630. ASSERT_EQ(IterStatus(iter), "c->vc");
  631. iter->Prev();
  632. ASSERT_EQ(IterStatus(iter), "b->vb");
  633. iter->Prev();
  634. ASSERT_EQ(IterStatus(iter), "a->va");
  635. iter->Prev();
  636. ASSERT_EQ(IterStatus(iter), "(invalid)");
  637. iter->SeekToLast();
  638. ASSERT_EQ(IterStatus(iter), "c->vc");
  639. iter->Next();
  640. ASSERT_EQ(IterStatus(iter), "(invalid)");
  641. iter->Seek("");
  642. ASSERT_EQ(IterStatus(iter), "a->va");
  643. iter->Seek("a");
  644. ASSERT_EQ(IterStatus(iter), "a->va");
  645. iter->Seek("ax");
  646. ASSERT_EQ(IterStatus(iter), "b->vb");
  647. iter->Seek("b");
  648. ASSERT_EQ(IterStatus(iter), "b->vb");
  649. iter->Seek("z");
  650. ASSERT_EQ(IterStatus(iter), "(invalid)");
  651. // Switch from reverse to forward
  652. iter->SeekToLast();
  653. iter->Prev();
  654. iter->Prev();
  655. iter->Next();
  656. ASSERT_EQ(IterStatus(iter), "b->vb");
  657. // Switch from forward to reverse
  658. iter->SeekToFirst();
  659. iter->Next();
  660. iter->Next();
  661. iter->Prev();
  662. ASSERT_EQ(IterStatus(iter), "b->vb");
  663. // Make sure iter stays at snapshot
  664. ASSERT_OK(Put("a", "va2"));
  665. ASSERT_OK(Put("a2", "va3"));
  666. ASSERT_OK(Put("b", "vb2"));
  667. ASSERT_OK(Put("c", "vc2"));
  668. ASSERT_OK(Delete("b"));
  669. iter->SeekToFirst();
  670. ASSERT_EQ(IterStatus(iter), "a->va");
  671. iter->Next();
  672. ASSERT_EQ(IterStatus(iter), "b->vb");
  673. iter->Next();
  674. ASSERT_EQ(IterStatus(iter), "c->vc");
  675. iter->Next();
  676. ASSERT_EQ(IterStatus(iter), "(invalid)");
  677. iter->SeekToLast();
  678. ASSERT_EQ(IterStatus(iter), "c->vc");
  679. iter->Prev();
  680. ASSERT_EQ(IterStatus(iter), "b->vb");
  681. iter->Prev();
  682. ASSERT_EQ(IterStatus(iter), "a->va");
  683. iter->Prev();
  684. ASSERT_EQ(IterStatus(iter), "(invalid)");
  685. delete iter;
  686. }
  687. TEST(DBTest, IterSmallAndLargeMix) {
  688. ASSERT_OK(Put("a", "va"));
  689. ASSERT_OK(Put("b", std::string(100000, 'b')));
  690. ASSERT_OK(Put("c", "vc"));
  691. ASSERT_OK(Put("d", std::string(100000, 'd')));
  692. ASSERT_OK(Put("e", std::string(100000, 'e')));
  693. Iterator* iter = db_->NewIterator(ReadOptions());
  694. iter->SeekToFirst();
  695. ASSERT_EQ(IterStatus(iter), "a->va");
  696. iter->Next();
  697. ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
  698. iter->Next();
  699. ASSERT_EQ(IterStatus(iter), "c->vc");
  700. iter->Next();
  701. ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
  702. iter->Next();
  703. ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
  704. iter->Next();
  705. ASSERT_EQ(IterStatus(iter), "(invalid)");
  706. iter->SeekToLast();
  707. ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
  708. iter->Prev();
  709. ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
  710. iter->Prev();
  711. ASSERT_EQ(IterStatus(iter), "c->vc");
  712. iter->Prev();
  713. ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
  714. iter->Prev();
  715. ASSERT_EQ(IterStatus(iter), "a->va");
  716. iter->Prev();
  717. ASSERT_EQ(IterStatus(iter), "(invalid)");
  718. delete iter;
  719. }
  720. TEST(DBTest, IterMultiWithDelete) {
  721. do {
  722. ASSERT_OK(Put("a", "va"));
  723. ASSERT_OK(Put("b", "vb"));
  724. ASSERT_OK(Put("c", "vc"));
  725. ASSERT_OK(Delete("b"));
  726. ASSERT_EQ("NOT_FOUND", Get("b"));
  727. Iterator* iter = db_->NewIterator(ReadOptions());
  728. iter->Seek("c");
  729. ASSERT_EQ(IterStatus(iter), "c->vc");
  730. iter->Prev();
  731. ASSERT_EQ(IterStatus(iter), "a->va");
  732. delete iter;
  733. } while (ChangeOptions());
  734. }
  735. TEST(DBTest, Recover) {
  736. do {
  737. ASSERT_OK(Put("foo", "v1"));
  738. ASSERT_OK(Put("baz", "v5"));
  739. Reopen();
  740. ASSERT_EQ("v1", Get("foo"));
  741. ASSERT_EQ("v1", Get("foo"));
  742. ASSERT_EQ("v5", Get("baz"));
  743. ASSERT_OK(Put("bar", "v2"));
  744. ASSERT_OK(Put("foo", "v3"));
  745. Reopen();
  746. ASSERT_EQ("v3", Get("foo"));
  747. ASSERT_OK(Put("foo", "v4"));
  748. ASSERT_EQ("v4", Get("foo"));
  749. ASSERT_EQ("v2", Get("bar"));
  750. ASSERT_EQ("v5", Get("baz"));
  751. } while (ChangeOptions());
  752. }
  753. TEST(DBTest, RecoveryWithEmptyLog) {
  754. do {
  755. ASSERT_OK(Put("foo", "v1"));
  756. ASSERT_OK(Put("foo", "v2"));
  757. Reopen();
  758. Reopen();
  759. ASSERT_OK(Put("foo", "v3"));
  760. Reopen();
  761. ASSERT_EQ("v3", Get("foo"));
  762. } while (ChangeOptions());
  763. }
  764. // Check that writes done during a memtable compaction are recovered
  765. // if the database is shutdown during the memtable compaction.
  766. TEST(DBTest, RecoverDuringMemtableCompaction) {
  767. do {
  768. Options options = CurrentOptions();
  769. options.env = env_;
  770. options.write_buffer_size = 1000000;
  771. Reopen(&options);
  772. // Trigger a long memtable compaction and reopen the database during it
  773. ASSERT_OK(Put("foo", "v1")); // Goes to 1st log file
  774. ASSERT_OK(Put("big1", std::string(10000000, 'x'))); // Fills memtable
  775. ASSERT_OK(Put("big2", std::string(1000, 'y'))); // Triggers compaction
  776. ASSERT_OK(Put("bar", "v2")); // Goes to new log file
  777. Reopen(&options);
  778. ASSERT_EQ("v1", Get("foo"));
  779. ASSERT_EQ("v2", Get("bar"));
  780. ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
  781. ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
  782. } while (ChangeOptions());
  783. }
  784. static std::string Key(int i) {
  785. char buf[100];
  786. snprintf(buf, sizeof(buf), "key%06d", i);
  787. return std::string(buf);
  788. }
  789. TEST(DBTest, MinorCompactionsHappen) {
  790. Options options = CurrentOptions();
  791. options.write_buffer_size = 10000;
  792. Reopen(&options);
  793. const int N = 500;
  794. int starting_num_tables = TotalTableFiles();
  795. for (int i = 0; i < N; i++) {
  796. ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v')));
  797. }
  798. int ending_num_tables = TotalTableFiles();
  799. ASSERT_GT(ending_num_tables, starting_num_tables);
  800. for (int i = 0; i < N; i++) {
  801. ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
  802. }
  803. Reopen();
  804. for (int i = 0; i < N; i++) {
  805. ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
  806. }
  807. }
  808. TEST(DBTest, RecoverWithLargeLog) {
  809. {
  810. Options options = CurrentOptions();
  811. Reopen(&options);
  812. ASSERT_OK(Put("big1", std::string(200000, '1')));
  813. ASSERT_OK(Put("big2", std::string(200000, '2')));
  814. ASSERT_OK(Put("small3", std::string(10, '3')));
  815. ASSERT_OK(Put("small4", std::string(10, '4')));
  816. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  817. }
  818. // Make sure that if we re-open with a small write buffer size that
  819. // we flush table files in the middle of a large log file.
  820. Options options = CurrentOptions();
  821. options.write_buffer_size = 100000;
  822. Reopen(&options);
  823. ASSERT_EQ(NumTableFilesAtLevel(0), 3);
  824. ASSERT_EQ(std::string(200000, '1'), Get("big1"));
  825. ASSERT_EQ(std::string(200000, '2'), Get("big2"));
  826. ASSERT_EQ(std::string(10, '3'), Get("small3"));
  827. ASSERT_EQ(std::string(10, '4'), Get("small4"));
  828. ASSERT_GT(NumTableFilesAtLevel(0), 1);
  829. }
  830. TEST(DBTest, CompactionsGenerateMultipleFiles) {
  831. Options options = CurrentOptions();
  832. options.write_buffer_size = 100000000; // Large write buffer
  833. Reopen(&options);
  834. Random rnd(301);
  835. // Write 8MB (80 values, each 100K)
  836. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  837. std::vector<std::string> values;
  838. for (int i = 0; i < 80; i++) {
  839. values.push_back(RandomString(&rnd, 100000));
  840. ASSERT_OK(Put(Key(i), values[i]));
  841. }
  842. // Reopening moves updates to level-0
  843. Reopen(&options);
  844. dbfull()->TEST_CompactRange(0, NULL, NULL);
  845. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  846. ASSERT_GT(NumTableFilesAtLevel(1), 1);
  847. for (int i = 0; i < 80; i++) {
  848. ASSERT_EQ(Get(Key(i)), values[i]);
  849. }
  850. }
  851. TEST(DBTest, RepeatedWritesToSameKey) {
  852. Options options = CurrentOptions();
  853. options.env = env_;
  854. options.write_buffer_size = 100000; // Small write buffer
  855. Reopen(&options);
  856. // We must have at most one file per level except for level-0,
  857. // which may have up to kL0_StopWritesTrigger files.
  858. const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger;
  859. Random rnd(301);
  860. std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
  861. for (int i = 0; i < 5 * kMaxFiles; i++) {
  862. Put("key", value);
  863. ASSERT_LE(TotalTableFiles(), kMaxFiles);
  864. fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles());
  865. }
  866. }
  867. TEST(DBTest, SparseMerge) {
  868. Options options = CurrentOptions();
  869. options.compression = kNoCompression;
  870. Reopen(&options);
  871. FillLevels("A", "Z");
  872. // Suppose there is:
  873. // small amount of data with prefix A
  874. // large amount of data with prefix B
  875. // small amount of data with prefix C
  876. // and that recent updates have made small changes to all three prefixes.
  877. // Check that we do not do a compaction that merges all of B in one shot.
  878. const std::string value(1000, 'x');
  879. Put("A", "va");
  880. // Write approximately 100MB of "B" values
  881. for (int i = 0; i < 100000; i++) {
  882. char key[100];
  883. snprintf(key, sizeof(key), "B%010d", i);
  884. Put(key, value);
  885. }
  886. Put("C", "vc");
  887. dbfull()->TEST_CompactMemTable();
  888. dbfull()->TEST_CompactRange(0, NULL, NULL);
  889. // Make sparse update
  890. Put("A", "va2");
  891. Put("B100", "bvalue2");
  892. Put("C", "vc2");
  893. dbfull()->TEST_CompactMemTable();
  894. // Compactions should not cause us to create a situation where
  895. // a file overlaps too much data at the next level.
  896. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  897. dbfull()->TEST_CompactRange(0, NULL, NULL);
  898. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  899. dbfull()->TEST_CompactRange(1, NULL, NULL);
  900. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  901. }
  902. static bool Between(uint64_t val, uint64_t low, uint64_t high) {
  903. bool result = (val >= low) && (val <= high);
  904. if (!result) {
  905. fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
  906. (unsigned long long)(val),
  907. (unsigned long long)(low),
  908. (unsigned long long)(high));
  909. }
  910. return result;
  911. }
  912. TEST(DBTest, ApproximateSizes) {
  913. do {
  914. Options options = CurrentOptions();
  915. options.write_buffer_size = 100000000; // Large write buffer
  916. options.compression = kNoCompression;
  917. DestroyAndReopen();
  918. ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
  919. Reopen(&options);
  920. ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
  921. // Write 8MB (80 values, each 100K)
  922. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  923. const int N = 80;
  924. static const int S1 = 100000;
  925. static const int S2 = 105000; // Allow some expansion from metadata
  926. Random rnd(301);
  927. for (int i = 0; i < N; i++) {
  928. ASSERT_OK(Put(Key(i), RandomString(&rnd, S1)));
  929. }
  930. // 0 because GetApproximateSizes() does not account for memtable space
  931. ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
  932. // Check sizes across recovery by reopening a few times
  933. for (int run = 0; run < 3; run++) {
  934. Reopen(&options);
  935. for (int compact_start = 0; compact_start < N; compact_start += 10) {
  936. for (int i = 0; i < N; i += 10) {
  937. ASSERT_TRUE(Between(Size("", Key(i)), S1*i, S2*i));
  938. ASSERT_TRUE(Between(Size("", Key(i)+".suffix"), S1*(i+1), S2*(i+1)));
  939. ASSERT_TRUE(Between(Size(Key(i), Key(i+10)), S1*10, S2*10));
  940. }
  941. ASSERT_TRUE(Between(Size("", Key(50)), S1*50, S2*50));
  942. ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), S1*50, S2*50));
  943. std::string cstart_str = Key(compact_start);
  944. std::string cend_str = Key(compact_start + 9);
  945. Slice cstart = cstart_str;
  946. Slice cend = cend_str;
  947. dbfull()->TEST_CompactRange(0, &cstart, &cend);
  948. }
  949. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  950. ASSERT_GT(NumTableFilesAtLevel(1), 0);
  951. }
  952. } while (ChangeOptions());
  953. }
  954. TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
  955. do {
  956. Options options = CurrentOptions();
  957. options.compression = kNoCompression;
  958. Reopen();
  959. Random rnd(301);
  960. std::string big1 = RandomString(&rnd, 100000);
  961. ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
  962. ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
  963. ASSERT_OK(Put(Key(2), big1));
  964. ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
  965. ASSERT_OK(Put(Key(4), big1));
  966. ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
  967. ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
  968. ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
  969. // Check sizes across recovery by reopening a few times
  970. for (int run = 0; run < 3; run++) {
  971. Reopen(&options);
  972. ASSERT_TRUE(Between(Size("", Key(0)), 0, 0));
  973. ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000));
  974. ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000));
  975. ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000));
  976. ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000));
  977. ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000));
  978. ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000));
  979. ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000));
  980. ASSERT_TRUE(Between(Size("", Key(8)), 550000, 560000));
  981. ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
  982. dbfull()->TEST_CompactRange(0, NULL, NULL);
  983. }
  984. } while (ChangeOptions());
  985. }
  986. TEST(DBTest, IteratorPinsRef) {
  987. Put("foo", "hello");
  988. // Get iterator that will yield the current contents of the DB.
  989. Iterator* iter = db_->NewIterator(ReadOptions());
  990. // Write to force compactions
  991. Put("foo", "newvalue1");
  992. for (int i = 0; i < 100; i++) {
  993. ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values
  994. }
  995. Put("foo", "newvalue2");
  996. iter->SeekToFirst();
  997. ASSERT_TRUE(iter->Valid());
  998. ASSERT_EQ("foo", iter->key().ToString());
  999. ASSERT_EQ("hello", iter->value().ToString());
  1000. iter->Next();
  1001. ASSERT_TRUE(!iter->Valid());
  1002. delete iter;
  1003. }
  1004. TEST(DBTest, Snapshot) {
  1005. do {
  1006. Put("foo", "v1");
  1007. const Snapshot* s1 = db_->GetSnapshot();
  1008. Put("foo", "v2");
  1009. const Snapshot* s2 = db_->GetSnapshot();
  1010. Put("foo", "v3");
  1011. const Snapshot* s3 = db_->GetSnapshot();
  1012. Put("foo", "v4");
  1013. ASSERT_EQ("v1", Get("foo", s1));
  1014. ASSERT_EQ("v2", Get("foo", s2));
  1015. ASSERT_EQ("v3", Get("foo", s3));
  1016. ASSERT_EQ("v4", Get("foo"));
  1017. db_->ReleaseSnapshot(s3);
  1018. ASSERT_EQ("v1", Get("foo", s1));
  1019. ASSERT_EQ("v2", Get("foo", s2));
  1020. ASSERT_EQ("v4", Get("foo"));
  1021. db_->ReleaseSnapshot(s1);
  1022. ASSERT_EQ("v2", Get("foo", s2));
  1023. ASSERT_EQ("v4", Get("foo"));
  1024. db_->ReleaseSnapshot(s2);
  1025. ASSERT_EQ("v4", Get("foo"));
  1026. } while (ChangeOptions());
  1027. }
  1028. TEST(DBTest, HiddenValuesAreRemoved) {
  1029. do {
  1030. Random rnd(301);
  1031. FillLevels("a", "z");
  1032. std::string big = RandomString(&rnd, 50000);
  1033. Put("foo", big);
  1034. Put("pastfoo", "v");
  1035. const Snapshot* snapshot = db_->GetSnapshot();
  1036. Put("foo", "tiny");
  1037. Put("pastfoo2", "v2"); // Advance sequence number one more
  1038. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  1039. ASSERT_GT(NumTableFilesAtLevel(0), 0);
  1040. ASSERT_EQ(big, Get("foo", snapshot));
  1041. ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
  1042. db_->ReleaseSnapshot(snapshot);
  1043. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
  1044. Slice x("x");
  1045. dbfull()->TEST_CompactRange(0, NULL, &x);
  1046. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
  1047. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  1048. ASSERT_GE(NumTableFilesAtLevel(1), 1);
  1049. dbfull()->TEST_CompactRange(1, NULL, &x);
  1050. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
  1051. ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
  1052. } while (ChangeOptions());
  1053. }
  1054. TEST(DBTest, DeletionMarkers1) {
  1055. Put("foo", "v1");
  1056. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  1057. const int last = config::kMaxMemCompactLevel;
  1058. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
  1059. // Place a table at level last-1 to prevent merging with preceding mutation
  1060. Put("a", "begin");
  1061. Put("z", "end");
  1062. dbfull()->TEST_CompactMemTable();
  1063. ASSERT_EQ(NumTableFilesAtLevel(last), 1);
  1064. ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
  1065. Delete("foo");
  1066. Put("foo", "v2");
  1067. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
  1068. ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
  1069. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
  1070. Slice z("z");
  1071. dbfull()->TEST_CompactRange(last-2, NULL, &z);
  1072. // DEL eliminated, but v1 remains because we aren't compacting that level
  1073. // (DEL can be eliminated because v2 hides v1).
  1074. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
  1075. dbfull()->TEST_CompactRange(last-1, NULL, NULL);
  1076. // Merging last-1 w/ last, so we are the base level for "foo", so
  1077. // DEL is removed. (as is v1).
  1078. ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]");
  1079. }
  1080. TEST(DBTest, DeletionMarkers2) {
  1081. Put("foo", "v1");
  1082. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  1083. const int last = config::kMaxMemCompactLevel;
  1084. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
  1085. // Place a table at level last-1 to prevent merging with preceding mutation
  1086. Put("a", "begin");
  1087. Put("z", "end");
  1088. dbfull()->TEST_CompactMemTable();
  1089. ASSERT_EQ(NumTableFilesAtLevel(last), 1);
  1090. ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
  1091. Delete("foo");
  1092. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  1093. ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
  1094. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  1095. dbfull()->TEST_CompactRange(last-2, NULL, NULL);
  1096. // DEL kept: "last" file overlaps
  1097. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  1098. dbfull()->TEST_CompactRange(last-1, NULL, NULL);
  1099. // Merging last-1 w/ last, so we are the base level for "foo", so
  1100. // DEL is removed. (as is v1).
  1101. ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
  1102. }
  1103. TEST(DBTest, OverlapInLevel0) {
  1104. do {
  1105. ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config";
  1106. // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
  1107. ASSERT_OK(Put("100", "v100"));
  1108. ASSERT_OK(Put("999", "v999"));
  1109. dbfull()->TEST_CompactMemTable();
  1110. ASSERT_OK(Delete("100"));
  1111. ASSERT_OK(Delete("999"));
  1112. dbfull()->TEST_CompactMemTable();
  1113. ASSERT_EQ("0,1,1", FilesPerLevel());
  1114. // Make files spanning the following ranges in level-0:
  1115. // files[0] 200 .. 900
  1116. // files[1] 300 .. 500
  1117. // Note that files are sorted by smallest key.
  1118. ASSERT_OK(Put("300", "v300"));
  1119. ASSERT_OK(Put("500", "v500"));
  1120. dbfull()->TEST_CompactMemTable();
  1121. ASSERT_OK(Put("200", "v200"));
  1122. ASSERT_OK(Put("600", "v600"));
  1123. ASSERT_OK(Put("900", "v900"));
  1124. dbfull()->TEST_CompactMemTable();
  1125. ASSERT_EQ("2,1,1", FilesPerLevel());
  1126. // Compact away the placeholder files we created initially
  1127. dbfull()->TEST_CompactRange(1, NULL, NULL);
  1128. dbfull()->TEST_CompactRange(2, NULL, NULL);
  1129. ASSERT_EQ("2", FilesPerLevel());
  1130. // Do a memtable compaction. Before bug-fix, the compaction would
  1131. // not detect the overlap with level-0 files and would incorrectly place
  1132. // the deletion in a deeper level.
  1133. ASSERT_OK(Delete("600"));
  1134. dbfull()->TEST_CompactMemTable();
  1135. ASSERT_EQ("3", FilesPerLevel());
  1136. ASSERT_EQ("NOT_FOUND", Get("600"));
  1137. } while (ChangeOptions());
  1138. }
  1139. TEST(DBTest, L0_CompactionBug_Issue44_a) {
  1140. Reopen();
  1141. ASSERT_OK(Put("b", "v"));
  1142. Reopen();
  1143. ASSERT_OK(Delete("b"));
  1144. ASSERT_OK(Delete("a"));
  1145. Reopen();
  1146. ASSERT_OK(Delete("a"));
  1147. Reopen();
  1148. ASSERT_OK(Put("a", "v"));
  1149. Reopen();
  1150. Reopen();
  1151. ASSERT_EQ("(a->v)", Contents());
  1152. DelayMilliseconds(1000); // Wait for compaction to finish
  1153. ASSERT_EQ("(a->v)", Contents());
  1154. }
  1155. TEST(DBTest, L0_CompactionBug_Issue44_b) {
  1156. Reopen();
  1157. Put("","");
  1158. Reopen();
  1159. Delete("e");
  1160. Put("","");
  1161. Reopen();
  1162. Put("c", "cv");
  1163. Reopen();
  1164. Put("","");
  1165. Reopen();
  1166. Put("","");
  1167. DelayMilliseconds(1000); // Wait for compaction to finish
  1168. Reopen();
  1169. Put("d","dv");
  1170. Reopen();
  1171. Put("","");
  1172. Reopen();
  1173. Delete("d");
  1174. Delete("b");
  1175. Reopen();
  1176. ASSERT_EQ("(->)(c->cv)", Contents());
  1177. DelayMilliseconds(1000); // Wait for compaction to finish
  1178. ASSERT_EQ("(->)(c->cv)", Contents());
  1179. }
  1180. TEST(DBTest, ComparatorCheck) {
  1181. class NewComparator : public Comparator {
  1182. public:
  1183. virtual const char* Name() const { return "leveldb.NewComparator"; }
  1184. virtual int Compare(const Slice& a, const Slice& b) const {
  1185. return BytewiseComparator()->Compare(a, b);
  1186. }
  1187. virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
  1188. BytewiseComparator()->FindShortestSeparator(s, l);
  1189. }
  1190. virtual void FindShortSuccessor(std::string* key) const {
  1191. BytewiseComparator()->FindShortSuccessor(key);
  1192. }
  1193. };
  1194. NewComparator cmp;
  1195. Options new_options = CurrentOptions();
  1196. new_options.comparator = &cmp;
  1197. Status s = TryReopen(&new_options);
  1198. ASSERT_TRUE(!s.ok());
  1199. ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos)
  1200. << s.ToString();
  1201. }
  1202. TEST(DBTest, CustomComparator) {
  1203. class NumberComparator : public Comparator {
  1204. public:
  1205. virtual const char* Name() const { return "test.NumberComparator"; }
  1206. virtual int Compare(const Slice& a, const Slice& b) const {
  1207. return ToNumber(a) - ToNumber(b);
  1208. }
  1209. virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
  1210. ToNumber(*s); // Check format
  1211. ToNumber(l); // Check format
  1212. }
  1213. virtual void FindShortSuccessor(std::string* key) const {
  1214. ToNumber(*key); // Check format
  1215. }
  1216. private:
  1217. static int ToNumber(const Slice& x) {
  1218. // Check that there are no extra characters.
  1219. ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']')
  1220. << EscapeString(x);
  1221. int val;
  1222. char ignored;
  1223. ASSERT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1)
  1224. << EscapeString(x);
  1225. return val;
  1226. }
  1227. };
  1228. NumberComparator cmp;
  1229. Options new_options = CurrentOptions();
  1230. new_options.create_if_missing = true;
  1231. new_options.comparator = &cmp;
  1232. new_options.filter_policy = NULL; // Cannot use bloom filters
  1233. new_options.write_buffer_size = 1000; // Compact more often
  1234. DestroyAndReopen(&new_options);
  1235. ASSERT_OK(Put("[10]", "ten"));
  1236. ASSERT_OK(Put("[0x14]", "twenty"));
  1237. for (int i = 0; i < 2; i++) {
  1238. ASSERT_EQ("ten", Get("[10]"));
  1239. ASSERT_EQ("ten", Get("[0xa]"));
  1240. ASSERT_EQ("twenty", Get("[20]"));
  1241. ASSERT_EQ("twenty", Get("[0x14]"));
  1242. ASSERT_EQ("NOT_FOUND", Get("[15]"));
  1243. ASSERT_EQ("NOT_FOUND", Get("[0xf]"));
  1244. Compact("[0]", "[9999]");
  1245. }
  1246. for (int run = 0; run < 2; run++) {
  1247. for (int i = 0; i < 1000; i++) {
  1248. char buf[100];
  1249. snprintf(buf, sizeof(buf), "[%d]", i*10);
  1250. ASSERT_OK(Put(buf, buf));
  1251. }
  1252. Compact("[0]", "[1000000]");
  1253. }
  1254. }
  1255. TEST(DBTest, ManualCompaction) {
  1256. ASSERT_EQ(config::kMaxMemCompactLevel, 2)
  1257. << "Need to update this test to match kMaxMemCompactLevel";
  1258. MakeTables(3, "p", "q");
  1259. ASSERT_EQ("1,1,1", FilesPerLevel());
  1260. // Compaction range falls before files
  1261. Compact("", "c");
  1262. ASSERT_EQ("1,1,1", FilesPerLevel());
  1263. // Compaction range falls after files
  1264. Compact("r", "z");
  1265. ASSERT_EQ("1,1,1", FilesPerLevel());
  1266. // Compaction range overlaps files
  1267. Compact("p1", "p9");
  1268. ASSERT_EQ("0,0,1", FilesPerLevel());
  1269. // Populate a different range
  1270. MakeTables(3, "c", "e");
  1271. ASSERT_EQ("1,1,2", FilesPerLevel());
  1272. // Compact just the new range
  1273. Compact("b", "f");
  1274. ASSERT_EQ("0,0,2", FilesPerLevel());
  1275. // Compact all
  1276. MakeTables(1, "a", "z");
  1277. ASSERT_EQ("0,1,2", FilesPerLevel());
  1278. db_->CompactRange(NULL, NULL);
  1279. ASSERT_EQ("0,0,1", FilesPerLevel());
  1280. }
  1281. TEST(DBTest, DBOpen_Options) {
  1282. std::string dbname = test::TmpDir() + "/db_options_test";
  1283. DestroyDB(dbname, Options());
  1284. // Does not exist, and create_if_missing == false: error
  1285. DB* db = NULL;
  1286. Options opts;
  1287. opts.create_if_missing = false;
  1288. Status s = DB::Open(opts, dbname, &db);
  1289. ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL);
  1290. ASSERT_TRUE(db == NULL);
  1291. // Does not exist, and create_if_missing == true: OK
  1292. opts.create_if_missing = true;
  1293. s = DB::Open(opts, dbname, &db);
  1294. ASSERT_OK(s);
  1295. ASSERT_TRUE(db != NULL);
  1296. delete db;
  1297. db = NULL;
  1298. // Does exist, and error_if_exists == true: error
  1299. opts.create_if_missing = false;
  1300. opts.error_if_exists = true;
  1301. s = DB::Open(opts, dbname, &db);
  1302. ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL);
  1303. ASSERT_TRUE(db == NULL);
  1304. // Does exist, and error_if_exists == false: OK
  1305. opts.create_if_missing = true;
  1306. opts.error_if_exists = false;
  1307. s = DB::Open(opts, dbname, &db);
  1308. ASSERT_OK(s);
  1309. ASSERT_TRUE(db != NULL);
  1310. delete db;
  1311. db = NULL;
  1312. }
  1313. TEST(DBTest, Locking) {
  1314. DB* db2 = NULL;
  1315. Status s = DB::Open(CurrentOptions(), dbname_, &db2);
  1316. ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db";
  1317. }
  1318. // Check that number of files does not grow when we are out of space
  1319. TEST(DBTest, NoSpace) {
  1320. Options options = CurrentOptions();
  1321. options.env = env_;
  1322. Reopen(&options);
  1323. ASSERT_OK(Put("foo", "v1"));
  1324. ASSERT_EQ("v1", Get("foo"));
  1325. Compact("a", "z");
  1326. const int num_files = CountFiles();
  1327. env_->no_space_.Release_Store(env_); // Force out-of-space errors
  1328. env_->sleep_counter_.Reset();
  1329. for (int i = 0; i < 5; i++) {
  1330. for (int level = 0; level < config::kNumLevels-1; level++) {
  1331. dbfull()->TEST_CompactRange(level, NULL, NULL);
  1332. }
  1333. }
  1334. env_->no_space_.Release_Store(NULL);
  1335. ASSERT_LT(CountFiles(), num_files + 3);
  1336. // Check that compaction attempts slept after errors
  1337. ASSERT_GE(env_->sleep_counter_.Read(), 5);
  1338. }
  1339. TEST(DBTest, ExponentialBackoff) {
  1340. Options options = CurrentOptions();
  1341. options.env = env_;
  1342. Reopen(&options);
  1343. ASSERT_OK(Put("foo", "v1"));
  1344. ASSERT_EQ("v1", Get("foo"));
  1345. Compact("a", "z");
  1346. env_->non_writable_.Release_Store(env_); // Force errors for new files
  1347. env_->sleep_counter_.Reset();
  1348. env_->sleep_time_counter_.Reset();
  1349. for (int i = 0; i < 5; i++) {
  1350. dbfull()->TEST_CompactRange(2, NULL, NULL);
  1351. }
  1352. env_->non_writable_.Release_Store(NULL);
  1353. // Wait for compaction to finish
  1354. DelayMilliseconds(1000);
  1355. ASSERT_GE(env_->sleep_counter_.Read(), 5);
  1356. ASSERT_LT(env_->sleep_counter_.Read(), 10);
  1357. ASSERT_GE(env_->sleep_time_counter_.Read(), 10e6);
  1358. }
  1359. TEST(DBTest, NonWritableFileSystem) {
  1360. Options options = CurrentOptions();
  1361. options.write_buffer_size = 1000;
  1362. options.env = env_;
  1363. Reopen(&options);
  1364. ASSERT_OK(Put("foo", "v1"));
  1365. env_->non_writable_.Release_Store(env_); // Force errors for new files
  1366. std::string big(100000, 'x');
  1367. int errors = 0;
  1368. for (int i = 0; i < 20; i++) {
  1369. fprintf(stderr, "iter %d; errors %d\n", i, errors);
  1370. if (!Put("foo", big).ok()) {
  1371. errors++;
  1372. DelayMilliseconds(100);
  1373. }
  1374. }
  1375. ASSERT_GT(errors, 0);
  1376. env_->non_writable_.Release_Store(NULL);
  1377. }
  1378. TEST(DBTest, ManifestWriteError) {
  1379. // Test for the following problem:
  1380. // (a) Compaction produces file F
  1381. // (b) Log record containing F is written to MANIFEST file, but Sync() fails
  1382. // (c) GC deletes F
  1383. // (d) After reopening DB, reads fail since deleted F is named in log record
  1384. // We iterate twice. In the second iteration, everything is the
  1385. // same except the log record never makes it to the MANIFEST file.
  1386. for (int iter = 0; iter < 2; iter++) {
  1387. port::AtomicPointer* error_type = (iter == 0)
  1388. ? &env_->manifest_sync_error_
  1389. : &env_->manifest_write_error_;
  1390. // Insert foo=>bar mapping
  1391. Options options = CurrentOptions();
  1392. options.env = env_;
  1393. options.create_if_missing = true;
  1394. options.error_if_exists = false;
  1395. DestroyAndReopen(&options);
  1396. ASSERT_OK(Put("foo", "bar"));
  1397. ASSERT_EQ("bar", Get("foo"));
  1398. // Memtable compaction (will succeed)
  1399. dbfull()->TEST_CompactMemTable();
  1400. ASSERT_EQ("bar", Get("foo"));
  1401. const int last = config::kMaxMemCompactLevel;
  1402. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level
  1403. // Merging compaction (will fail)
  1404. error_type->Release_Store(env_);
  1405. dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail
  1406. ASSERT_EQ("bar", Get("foo"));
  1407. // Recovery: should not lose data
  1408. error_type->Release_Store(NULL);
  1409. Reopen(&options);
  1410. ASSERT_EQ("bar", Get("foo"));
  1411. }
  1412. }
  1413. TEST(DBTest, MissingSSTFile) {
  1414. ASSERT_OK(Put("foo", "bar"));
  1415. ASSERT_EQ("bar", Get("foo"));
  1416. // Dump the memtable to disk.
  1417. dbfull()->TEST_CompactMemTable();
  1418. ASSERT_EQ("bar", Get("foo"));
  1419. Close();
  1420. ASSERT_TRUE(DeleteAnSSTFile());
  1421. Options options = CurrentOptions();
  1422. options.paranoid_checks = true;
  1423. Status s = TryReopen(&options);
  1424. ASSERT_TRUE(!s.ok());
  1425. ASSERT_TRUE(s.ToString().find("issing") != std::string::npos)
  1426. << s.ToString();
  1427. }
  1428. TEST(DBTest, FilesDeletedAfterCompaction) {
  1429. ASSERT_OK(Put("foo", "v2"));
  1430. Compact("a", "z");
  1431. const int num_files = CountFiles();
  1432. for (int i = 0; i < 10; i++) {
  1433. ASSERT_OK(Put("foo", "v2"));
  1434. Compact("a", "z");
  1435. }
  1436. ASSERT_EQ(CountFiles(), num_files);
  1437. }
  1438. TEST(DBTest, BloomFilter) {
  1439. env_->count_random_reads_ = true;
  1440. Options options = CurrentOptions();
  1441. options.env = env_;
  1442. options.block_cache = NewLRUCache(0); // Prevent cache hits
  1443. options.filter_policy = NewBloomFilterPolicy(10);
  1444. Reopen(&options);
  1445. // Populate multiple layers
  1446. const int N = 10000;
  1447. for (int i = 0; i < N; i++) {
  1448. ASSERT_OK(Put(Key(i), Key(i)));
  1449. }
  1450. Compact("a", "z");
  1451. for (int i = 0; i < N; i += 100) {
  1452. ASSERT_OK(Put(Key(i), Key(i)));
  1453. }
  1454. dbfull()->TEST_CompactMemTable();
  1455. // Prevent auto compactions triggered by seeks
  1456. env_->delay_sstable_sync_.Release_Store(env_);
  1457. // Lookup present keys. Should rarely read from small sstable.
  1458. env_->random_read_counter_.Reset();
  1459. for (int i = 0; i < N; i++) {
  1460. ASSERT_EQ(Key(i), Get(Key(i)));
  1461. }
  1462. int reads = env_->random_read_counter_.Read();
  1463. fprintf(stderr, "%d present => %d reads\n", N, reads);
  1464. ASSERT_GE(reads, N);
  1465. ASSERT_LE(reads, N + 2*N/100);
  1466. // Lookup present keys. Should rarely read from either sstable.
  1467. env_->random_read_counter_.Reset();
  1468. for (int i = 0; i < N; i++) {
  1469. ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing"));
  1470. }
  1471. reads = env_->random_read_counter_.Read();
  1472. fprintf(stderr, "%d missing => %d reads\n", N, reads);
  1473. ASSERT_LE(reads, 3*N/100);
  1474. env_->delay_sstable_sync_.Release_Store(NULL);
  1475. Close();
  1476. delete options.block_cache;
  1477. delete options.filter_policy;
  1478. }
  1479. // Multi-threaded test:
  1480. namespace {
  1481. static const int kNumThreads = 4;
  1482. static const int kTestSeconds = 10;
  1483. static const int kNumKeys = 1000;
  1484. struct MTState {
  1485. DBTest* test;
  1486. port::AtomicPointer stop;
  1487. port::AtomicPointer counter[kNumThreads];
  1488. port::AtomicPointer thread_done[kNumThreads];
  1489. };
  1490. struct MTThread {
  1491. MTState* state;
  1492. int id;
  1493. };
  1494. static void MTThreadBody(void* arg) {
  1495. MTThread* t = reinterpret_cast<MTThread*>(arg);
  1496. int id = t->id;
  1497. DB* db = t->state->test->db_;
  1498. uintptr_t counter = 0;
  1499. fprintf(stderr, "... starting thread %d\n", id);
  1500. Random rnd(1000 + id);
  1501. std::string value;
  1502. char valbuf[1500];
  1503. while (t->state->stop.Acquire_Load() == NULL) {
  1504. t->state->counter[id].Release_Store(reinterpret_cast<void*>(counter));
  1505. int key = rnd.Uniform(kNumKeys);
  1506. char keybuf[20];
  1507. snprintf(keybuf, sizeof(keybuf), "%016d", key);
  1508. if (rnd.OneIn(2)) {
  1509. // Write values of the form <key, my id, counter>.
  1510. // We add some padding for force compactions.
  1511. snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d",
  1512. key, id, static_cast<int>(counter));
  1513. ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf)));
  1514. } else {
  1515. // Read a value and verify that it matches the pattern written above.
  1516. Status s = db->Get(ReadOptions(), Slice(keybuf), &value);
  1517. if (s.IsNotFound()) {
  1518. // Key has not yet been written
  1519. } else {
  1520. // Check that the writer thread counter is >= the counter in the value
  1521. ASSERT_OK(s);
  1522. int k, w, c;
  1523. ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value;
  1524. ASSERT_EQ(k, key);
  1525. ASSERT_GE(w, 0);
  1526. ASSERT_LT(w, kNumThreads);
  1527. ASSERT_LE(c, reinterpret_cast<uintptr_t>(
  1528. t->state->counter[w].Acquire_Load()));
  1529. }
  1530. }
  1531. counter++;
  1532. }
  1533. t->state->thread_done[id].Release_Store(t);
  1534. fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
  1535. }
  1536. } // namespace
  1537. TEST(DBTest, MultiThreaded) {
  1538. do {
  1539. // Initialize state
  1540. MTState mt;
  1541. mt.test = this;
  1542. mt.stop.Release_Store(0);
  1543. for (int id = 0; id < kNumThreads; id++) {
  1544. mt.counter[id].Release_Store(0);
  1545. mt.thread_done[id].Release_Store(0);
  1546. }
  1547. // Start threads
  1548. MTThread thread[kNumThreads];
  1549. for (int id = 0; id < kNumThreads; id++) {
  1550. thread[id].state = &mt;
  1551. thread[id].id = id;
  1552. env_->StartThread(MTThreadBody, &thread[id]);
  1553. }
  1554. // Let them run for a while
  1555. DelayMilliseconds(kTestSeconds * 1000);
  1556. // Stop the threads and wait for them to finish
  1557. mt.stop.Release_Store(&mt);
  1558. for (int id = 0; id < kNumThreads; id++) {
  1559. while (mt.thread_done[id].Acquire_Load() == NULL) {
  1560. DelayMilliseconds(100);
  1561. }
  1562. }
  1563. } while (ChangeOptions());
  1564. }
  1565. namespace {
  1566. typedef std::map<std::string, std::string> KVMap;
  1567. }
  1568. class ModelDB: public DB {
  1569. public:
  1570. class ModelSnapshot : public Snapshot {
  1571. public:
  1572. KVMap map_;
  1573. };
  1574. explicit ModelDB(const Options& options): options_(options) { }
  1575. ~ModelDB() { }
  1576. virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
  1577. return DB::Put(o, k, v);
  1578. }
  1579. virtual Status Delete(const WriteOptions& o, const Slice& key) {
  1580. return DB::Delete(o, key);
  1581. }
  1582. virtual Status Get(const ReadOptions& options,
  1583. const Slice& key, std::string* value) {
  1584. assert(false); // Not implemented
  1585. return Status::NotFound(key);
  1586. }
  1587. virtual Iterator* NewIterator(const ReadOptions& options) {
  1588. if (options.snapshot == NULL) {
  1589. KVMap* saved = new KVMap;
  1590. *saved = map_;
  1591. return new ModelIter(saved, true);
  1592. } else {
  1593. const KVMap* snapshot_state =
  1594. &(reinterpret_cast<const ModelSnapshot*>(options.snapshot)->map_);
  1595. return new ModelIter(snapshot_state, false);
  1596. }
  1597. }
  1598. virtual const Snapshot* GetSnapshot() {
  1599. ModelSnapshot* snapshot = new ModelSnapshot;
  1600. snapshot->map_ = map_;
  1601. return snapshot;
  1602. }
  1603. virtual void ReleaseSnapshot(const Snapshot* snapshot) {
  1604. delete reinterpret_cast<const ModelSnapshot*>(snapshot);
  1605. }
  1606. virtual Status Write(const WriteOptions& options, WriteBatch* batch) {
  1607. class Handler : public WriteBatch::Handler {
  1608. public:
  1609. KVMap* map_;
  1610. virtual void Put(const Slice& key, const Slice& value) {
  1611. (*map_)[key.ToString()] = value.ToString();
  1612. }
  1613. virtual void Delete(const Slice& key) {
  1614. map_->erase(key.ToString());
  1615. }
  1616. };
  1617. Handler handler;
  1618. handler.map_ = &map_;
  1619. return batch->Iterate(&handler);
  1620. }
  1621. virtual bool GetProperty(const Slice& property, std::string* value) {
  1622. return false;
  1623. }
  1624. virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) {
  1625. for (int i = 0; i < n; i++) {
  1626. sizes[i] = 0;
  1627. }
  1628. }
  1629. virtual void CompactRange(const Slice* start, const Slice* end) {
  1630. }
  1631. private:
  1632. class ModelIter: public Iterator {
  1633. public:
  1634. ModelIter(const KVMap* map, bool owned)
  1635. : map_(map), owned_(owned), iter_(map_->end()) {
  1636. }
  1637. ~ModelIter() {
  1638. if (owned_) delete map_;
  1639. }
  1640. virtual bool Valid() const { return iter_ != map_->end(); }
  1641. virtual void SeekToFirst() { iter_ = map_->begin(); }
  1642. virtual void SeekToLast() {
  1643. if (map_->empty()) {
  1644. iter_ = map_->end();
  1645. } else {
  1646. iter_ = map_->find(map_->rbegin()->first);
  1647. }
  1648. }
  1649. virtual void Seek(const Slice& k) {
  1650. iter_ = map_->lower_bound(k.ToString());
  1651. }
  1652. virtual void Next() { ++iter_; }
  1653. virtual void Prev() { --iter_; }
  1654. virtual Slice key() const { return iter_->first; }
  1655. virtual Slice value() const { return iter_->second; }
  1656. virtual Status status() const { return Status::OK(); }
  1657. private:
  1658. const KVMap* const map_;
  1659. const bool owned_; // Do we own map_
  1660. KVMap::const_iterator iter_;
  1661. };
  1662. const Options options_;
  1663. KVMap map_;
  1664. };
  1665. static std::string RandomKey(Random* rnd) {
  1666. int len = (rnd->OneIn(3)
  1667. ? 1 // Short sometimes to encourage collisions
  1668. : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
  1669. return test::RandomKey(rnd, len);
  1670. }
  1671. static bool CompareIterators(int step,
  1672. DB* model,
  1673. DB* db,
  1674. const Snapshot* model_snap,
  1675. const Snapshot* db_snap) {
  1676. ReadOptions options;
  1677. options.snapshot = model_snap;
  1678. Iterator* miter = model->NewIterator(options);
  1679. options.snapshot = db_snap;
  1680. Iterator* dbiter = db->NewIterator(options);
  1681. bool ok = true;
  1682. int count = 0;
  1683. for (miter->SeekToFirst(), dbiter->SeekToFirst();
  1684. ok && miter->Valid() && dbiter->Valid();
  1685. miter->Next(), dbiter->Next()) {
  1686. count++;
  1687. if (miter->key().compare(dbiter->key()) != 0) {
  1688. fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
  1689. step,
  1690. EscapeString(miter->key()).c_str(),
  1691. EscapeString(dbiter->key()).c_str());
  1692. ok = false;
  1693. break;
  1694. }
  1695. if (miter->value().compare(dbiter->value()) != 0) {
  1696. fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
  1697. step,
  1698. EscapeString(miter->key()).c_str(),
  1699. EscapeString(miter->value()).c_str(),
  1700. EscapeString(miter->value()).c_str());
  1701. ok = false;
  1702. }
  1703. }
  1704. if (ok) {
  1705. if (miter->Valid() != dbiter->Valid()) {
  1706. fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n",
  1707. step, miter->Valid(), dbiter->Valid());
  1708. ok = false;
  1709. }
  1710. }
  1711. fprintf(stderr, "%d entries compared: ok=%d\n", count, ok);
  1712. delete miter;
  1713. delete dbiter;
  1714. return ok;
  1715. }
  1716. TEST(DBTest, Randomized) {
  1717. Random rnd(test::RandomSeed());
  1718. do {
  1719. ModelDB model(CurrentOptions());
  1720. const int N = 10000;
  1721. const Snapshot* model_snap = NULL;
  1722. const Snapshot* db_snap = NULL;
  1723. std::string k, v;
  1724. for (int step = 0; step < N; step++) {
  1725. if (step % 100 == 0) {
  1726. fprintf(stderr, "Step %d of %d\n", step, N);
  1727. }
  1728. // TODO(sanjay): Test Get() works
  1729. int p = rnd.Uniform(100);
  1730. if (p < 45) { // Put
  1731. k = RandomKey(&rnd);
  1732. v = RandomString(&rnd,
  1733. rnd.OneIn(20)
  1734. ? 100 + rnd.Uniform(100)
  1735. : rnd.Uniform(8));
  1736. ASSERT_OK(model.Put(WriteOptions(), k, v));
  1737. ASSERT_OK(db_->Put(WriteOptions(), k, v));
  1738. } else if (p < 90) { // Delete
  1739. k = RandomKey(&rnd);
  1740. ASSERT_OK(model.Delete(WriteOptions(), k));
  1741. ASSERT_OK(db_->Delete(WriteOptions(), k));
  1742. } else { // Multi-element batch
  1743. WriteBatch b;
  1744. const int num = rnd.Uniform(8);
  1745. for (int i = 0; i < num; i++) {
  1746. if (i == 0 || !rnd.OneIn(10)) {
  1747. k = RandomKey(&rnd);
  1748. } else {
  1749. // Periodically re-use the same key from the previous iter, so
  1750. // we have multiple entries in the write batch for the same key
  1751. }
  1752. if (rnd.OneIn(2)) {
  1753. v = RandomString(&rnd, rnd.Uniform(10));
  1754. b.Put(k, v);
  1755. } else {
  1756. b.Delete(k);
  1757. }
  1758. }
  1759. ASSERT_OK(model.Write(WriteOptions(), &b));
  1760. ASSERT_OK(db_->Write(WriteOptions(), &b));
  1761. }
  1762. if ((step % 100) == 0) {
  1763. ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
  1764. ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
  1765. // Save a snapshot from each DB this time that we'll use next
  1766. // time we compare things, to make sure the current state is
  1767. // preserved with the snapshot
  1768. if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
  1769. if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
  1770. Reopen();
  1771. ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
  1772. model_snap = model.GetSnapshot();
  1773. db_snap = db_->GetSnapshot();
  1774. }
  1775. }
  1776. if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
  1777. if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
  1778. } while (ChangeOptions());
  1779. }
  1780. std::string MakeKey(unsigned int num) {
  1781. char buf[30];
  1782. snprintf(buf, sizeof(buf), "%016u", num);
  1783. return std::string(buf);
  1784. }
  1785. void BM_LogAndApply(int iters, int num_base_files) {
  1786. std::string dbname = test::TmpDir() + "/leveldb_test_benchmark";
  1787. DestroyDB(dbname, Options());
  1788. DB* db = NULL;
  1789. Options opts;
  1790. opts.create_if_missing = true;
  1791. Status s = DB::Open(opts, dbname, &db);
  1792. ASSERT_OK(s);
  1793. ASSERT_TRUE(db != NULL);
  1794. delete db;
  1795. db = NULL;
  1796. Env* env = Env::Default();
  1797. port::Mutex mu;
  1798. MutexLock l(&mu);
  1799. InternalKeyComparator cmp(BytewiseComparator());
  1800. Options options;
  1801. VersionSet vset(dbname, &options, NULL, &cmp);
  1802. ASSERT_OK(vset.Recover());
  1803. VersionEdit vbase;
  1804. uint64_t fnum = 1;
  1805. for (int i = 0; i < num_base_files; i++) {
  1806. InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
  1807. InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
  1808. vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
  1809. }
  1810. ASSERT_OK(vset.LogAndApply(&vbase, &mu));
  1811. uint64_t start_micros = env->NowMicros();
  1812. for (int i = 0; i < iters; i++) {
  1813. VersionEdit vedit;
  1814. vedit.DeleteFile(2, fnum);
  1815. InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
  1816. InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
  1817. vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
  1818. vset.LogAndApply(&vedit, &mu);
  1819. }
  1820. uint64_t stop_micros = env->NowMicros();
  1821. unsigned int us = stop_micros - start_micros;
  1822. char buf[16];
  1823. snprintf(buf, sizeof(buf), "%d", num_base_files);
  1824. fprintf(stderr,
  1825. "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n",
  1826. buf, iters, us, ((float)us) / iters);
  1827. }
  1828. } // namespace leveldb
  1829. int main(int argc, char** argv) {
  1830. if (argc > 1 && std::string(argv[1]) == "--benchmark") {
  1831. leveldb::BM_LogAndApply(1000, 1);
  1832. leveldb::BM_LogAndApply(1000, 100);
  1833. leveldb::BM_LogAndApply(1000, 10000);
  1834. leveldb::BM_LogAndApply(100, 100000);
  1835. return 0;
  1836. }
  1837. return leveldb::test::RunAllTests();
  1838. }