作者: 韩晨旭 10225101440 李畅 10225102463
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1711 lines
49 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "leveldb/db.h"
  5. #include "db/db_impl.h"
  6. #include "db/filename.h"
  7. #include "db/version_set.h"
  8. #include "db/write_batch_internal.h"
  9. #include "leveldb/env.h"
  10. #include "leveldb/table.h"
  11. #include "util/logging.h"
  12. #include "util/mutexlock.h"
  13. #include "util/testharness.h"
  14. #include "util/testutil.h"
  15. namespace leveldb {
  16. static std::string RandomString(Random* rnd, int len) {
  17. std::string r;
  18. test::RandomString(rnd, len, &r);
  19. return r;
  20. }
  21. // Special Env used to delay background operations
  22. class SpecialEnv : public EnvWrapper {
  23. public:
  24. // sstable Sync() calls are blocked while this pointer is non-NULL.
  25. port::AtomicPointer delay_sstable_sync_;
  26. // Simulate no-space errors while this pointer is non-NULL.
  27. port::AtomicPointer no_space_;
  28. explicit SpecialEnv(Env* base) : EnvWrapper(base) {
  29. delay_sstable_sync_.Release_Store(NULL);
  30. no_space_.Release_Store(NULL);
  31. }
  32. Status NewWritableFile(const std::string& f, WritableFile** r) {
  33. class SSTableFile : public WritableFile {
  34. private:
  35. SpecialEnv* env_;
  36. WritableFile* base_;
  37. public:
  38. SSTableFile(SpecialEnv* env, WritableFile* base)
  39. : env_(env),
  40. base_(base) {
  41. }
  42. ~SSTableFile() { delete base_; }
  43. Status Append(const Slice& data) {
  44. if (env_->no_space_.Acquire_Load() != NULL) {
  45. // Drop writes on the floor
  46. return Status::OK();
  47. } else {
  48. return base_->Append(data);
  49. }
  50. }
  51. Status Close() { return base_->Close(); }
  52. Status Flush() { return base_->Flush(); }
  53. Status Sync() {
  54. while (env_->delay_sstable_sync_.Acquire_Load() != NULL) {
  55. env_->SleepForMicroseconds(100000);
  56. }
  57. return base_->Sync();
  58. }
  59. };
  60. Status s = target()->NewWritableFile(f, r);
  61. if (s.ok()) {
  62. if (strstr(f.c_str(), ".sst") != NULL) {
  63. *r = new SSTableFile(this, *r);
  64. }
  65. }
  66. return s;
  67. }
  68. };
  69. class DBTest {
  70. public:
  71. std::string dbname_;
  72. SpecialEnv* env_;
  73. DB* db_;
  74. Options last_options_;
  75. DBTest() : env_(new SpecialEnv(Env::Default())) {
  76. dbname_ = test::TmpDir() + "/db_test";
  77. DestroyDB(dbname_, Options());
  78. db_ = NULL;
  79. Reopen();
  80. }
  81. ~DBTest() {
  82. delete db_;
  83. DestroyDB(dbname_, Options());
  84. delete env_;
  85. }
  86. DBImpl* dbfull() {
  87. return reinterpret_cast<DBImpl*>(db_);
  88. }
  89. void Reopen(Options* options = NULL) {
  90. ASSERT_OK(TryReopen(options));
  91. }
  92. void DestroyAndReopen(Options* options = NULL) {
  93. delete db_;
  94. db_ = NULL;
  95. DestroyDB(dbname_, Options());
  96. ASSERT_OK(TryReopen(options));
  97. }
  98. Status TryReopen(Options* options) {
  99. delete db_;
  100. db_ = NULL;
  101. Options opts;
  102. if (options != NULL) {
  103. opts = *options;
  104. } else {
  105. opts.create_if_missing = true;
  106. }
  107. last_options_ = opts;
  108. return DB::Open(opts, dbname_, &db_);
  109. }
  110. Status Put(const std::string& k, const std::string& v) {
  111. return db_->Put(WriteOptions(), k, v);
  112. }
  113. Status Delete(const std::string& k) {
  114. return db_->Delete(WriteOptions(), k);
  115. }
  116. std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
  117. ReadOptions options;
  118. options.snapshot = snapshot;
  119. std::string result;
  120. Status s = db_->Get(options, k, &result);
  121. if (s.IsNotFound()) {
  122. result = "NOT_FOUND";
  123. } else if (!s.ok()) {
  124. result = s.ToString();
  125. }
  126. return result;
  127. }
  128. // Return a string that contains all key,value pairs in order,
  129. // formatted like "(k1->v1)(k2->v2)".
  130. std::string Contents() {
  131. std::vector<std::string> forward;
  132. std::string result;
  133. Iterator* iter = db_->NewIterator(ReadOptions());
  134. for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
  135. std::string s = IterStatus(iter);
  136. result.push_back('(');
  137. result.append(s);
  138. result.push_back(')');
  139. forward.push_back(s);
  140. }
  141. // Check reverse iteration results are the reverse of forward results
  142. int matched = 0;
  143. for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
  144. ASSERT_LT(matched, forward.size());
  145. ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
  146. matched++;
  147. }
  148. ASSERT_EQ(matched, forward.size());
  149. delete iter;
  150. return result;
  151. }
  152. std::string AllEntriesFor(const Slice& user_key) {
  153. Iterator* iter = dbfull()->TEST_NewInternalIterator();
  154. InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
  155. iter->Seek(target.Encode());
  156. std::string result;
  157. if (!iter->status().ok()) {
  158. result = iter->status().ToString();
  159. } else {
  160. result = "[ ";
  161. bool first = true;
  162. while (iter->Valid()) {
  163. ParsedInternalKey ikey;
  164. if (!ParseInternalKey(iter->key(), &ikey)) {
  165. result += "CORRUPTED";
  166. } else {
  167. if (last_options_.comparator->Compare(
  168. ikey.user_key, user_key) != 0) {
  169. break;
  170. }
  171. if (!first) {
  172. result += ", ";
  173. }
  174. first = false;
  175. switch (ikey.type) {
  176. case kTypeValue:
  177. result += iter->value().ToString();
  178. break;
  179. case kTypeDeletion:
  180. result += "DEL";
  181. break;
  182. }
  183. }
  184. iter->Next();
  185. }
  186. if (!first) {
  187. result += " ";
  188. }
  189. result += "]";
  190. }
  191. delete iter;
  192. return result;
  193. }
  194. int NumTableFilesAtLevel(int level) {
  195. std::string property;
  196. ASSERT_TRUE(
  197. db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level),
  198. &property));
  199. return atoi(property.c_str());
  200. }
  201. int TotalTableFiles() {
  202. int result = 0;
  203. for (int level = 0; level < config::kNumLevels; level++) {
  204. result += NumTableFilesAtLevel(level);
  205. }
  206. return result;
  207. }
  208. // Return spread of files per level
  209. std::string FilesPerLevel() {
  210. std::string result;
  211. int last_non_zero_offset = 0;
  212. for (int level = 0; level < config::kNumLevels; level++) {
  213. int f = NumTableFilesAtLevel(level);
  214. char buf[100];
  215. snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
  216. result += buf;
  217. if (f > 0) {
  218. last_non_zero_offset = result.size();
  219. }
  220. }
  221. result.resize(last_non_zero_offset);
  222. return result;
  223. }
  224. int CountFiles() {
  225. std::vector<std::string> files;
  226. env_->GetChildren(dbname_, &files);
  227. return static_cast<int>(files.size());
  228. }
  229. uint64_t Size(const Slice& start, const Slice& limit) {
  230. Range r(start, limit);
  231. uint64_t size;
  232. db_->GetApproximateSizes(&r, 1, &size);
  233. return size;
  234. }
  235. void Compact(const Slice& start, const Slice& limit) {
  236. db_->CompactRange(&start, &limit);
  237. }
  238. // Do n memtable compactions, each of which produces an sstable
  239. // covering the range [small,large].
  240. void MakeTables(int n, const std::string& small, const std::string& large) {
  241. for (int i = 0; i < n; i++) {
  242. Put(small, "begin");
  243. Put(large, "end");
  244. dbfull()->TEST_CompactMemTable();
  245. }
  246. }
  247. // Prevent pushing of new sstables into deeper levels by adding
  248. // tables that cover a specified range to all levels.
  249. void FillLevels(const std::string& smallest, const std::string& largest) {
  250. MakeTables(config::kNumLevels, smallest, largest);
  251. }
  252. void DumpFileCounts(const char* label) {
  253. fprintf(stderr, "---\n%s:\n", label);
  254. fprintf(stderr, "maxoverlap: %lld\n",
  255. static_cast<long long>(
  256. dbfull()->TEST_MaxNextLevelOverlappingBytes()));
  257. for (int level = 0; level < config::kNumLevels; level++) {
  258. int num = NumTableFilesAtLevel(level);
  259. if (num > 0) {
  260. fprintf(stderr, " level %3d : %d files\n", level, num);
  261. }
  262. }
  263. }
  264. std::string DumpSSTableList() {
  265. std::string property;
  266. db_->GetProperty("leveldb.sstables", &property);
  267. return property;
  268. }
  269. std::string IterStatus(Iterator* iter) {
  270. std::string result;
  271. if (iter->Valid()) {
  272. result = iter->key().ToString() + "->" + iter->value().ToString();
  273. } else {
  274. result = "(invalid)";
  275. }
  276. return result;
  277. }
  278. };
  279. TEST(DBTest, Empty) {
  280. ASSERT_TRUE(db_ != NULL);
  281. ASSERT_EQ("NOT_FOUND", Get("foo"));
  282. }
  283. TEST(DBTest, ReadWrite) {
  284. ASSERT_OK(Put("foo", "v1"));
  285. ASSERT_EQ("v1", Get("foo"));
  286. ASSERT_OK(Put("bar", "v2"));
  287. ASSERT_OK(Put("foo", "v3"));
  288. ASSERT_EQ("v3", Get("foo"));
  289. ASSERT_EQ("v2", Get("bar"));
  290. }
  291. TEST(DBTest, PutDeleteGet) {
  292. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
  293. ASSERT_EQ("v1", Get("foo"));
  294. ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
  295. ASSERT_EQ("v2", Get("foo"));
  296. ASSERT_OK(db_->Delete(WriteOptions(), "foo"));
  297. ASSERT_EQ("NOT_FOUND", Get("foo"));
  298. }
  299. TEST(DBTest, GetFromImmutableLayer) {
  300. Options options;
  301. options.env = env_;
  302. options.write_buffer_size = 100000; // Small write buffer
  303. Reopen(&options);
  304. ASSERT_OK(Put("foo", "v1"));
  305. ASSERT_EQ("v1", Get("foo"));
  306. env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
  307. Put("k1", std::string(100000, 'x')); // Fill memtable
  308. Put("k2", std::string(100000, 'y')); // Trigger compaction
  309. ASSERT_EQ("v1", Get("foo"));
  310. env_->delay_sstable_sync_.Release_Store(NULL); // Release sync calls
  311. }
  312. TEST(DBTest, GetFromVersions) {
  313. ASSERT_OK(Put("foo", "v1"));
  314. dbfull()->TEST_CompactMemTable();
  315. ASSERT_EQ("v1", Get("foo"));
  316. }
  317. TEST(DBTest, GetSnapshot) {
  318. // Try with both a short key and a long key
  319. for (int i = 0; i < 2; i++) {
  320. std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
  321. ASSERT_OK(Put(key, "v1"));
  322. const Snapshot* s1 = db_->GetSnapshot();
  323. ASSERT_OK(Put(key, "v2"));
  324. ASSERT_EQ("v2", Get(key));
  325. ASSERT_EQ("v1", Get(key, s1));
  326. dbfull()->TEST_CompactMemTable();
  327. ASSERT_EQ("v2", Get(key));
  328. ASSERT_EQ("v1", Get(key, s1));
  329. db_->ReleaseSnapshot(s1);
  330. }
  331. }
  332. TEST(DBTest, GetLevel0Ordering) {
  333. // Check that we process level-0 files in correct order. The code
  334. // below generates two level-0 files where the earlier one comes
  335. // before the later one in the level-0 file list since the earlier
  336. // one has a smaller "smallest" key.
  337. ASSERT_OK(Put("bar", "b"));
  338. ASSERT_OK(Put("foo", "v1"));
  339. dbfull()->TEST_CompactMemTable();
  340. ASSERT_OK(Put("foo", "v2"));
  341. dbfull()->TEST_CompactMemTable();
  342. ASSERT_EQ("v2", Get("foo"));
  343. }
  344. TEST(DBTest, GetOrderedByLevels) {
  345. ASSERT_OK(Put("foo", "v1"));
  346. Compact("a", "z");
  347. ASSERT_EQ("v1", Get("foo"));
  348. ASSERT_OK(Put("foo", "v2"));
  349. ASSERT_EQ("v2", Get("foo"));
  350. dbfull()->TEST_CompactMemTable();
  351. ASSERT_EQ("v2", Get("foo"));
  352. }
  353. TEST(DBTest, GetPicksCorrectFile) {
  354. // Arrange to have multiple files in a non-level-0 level.
  355. ASSERT_OK(Put("a", "va"));
  356. Compact("a", "b");
  357. ASSERT_OK(Put("x", "vx"));
  358. Compact("x", "y");
  359. ASSERT_OK(Put("f", "vf"));
  360. Compact("f", "g");
  361. ASSERT_EQ("va", Get("a"));
  362. ASSERT_EQ("vf", Get("f"));
  363. ASSERT_EQ("vx", Get("x"));
  364. }
  365. TEST(DBTest, GetEncountersEmptyLevel) {
  366. // Arrange for the following to happen:
  367. // * sstable A in level 0
  368. // * nothing in level 1
  369. // * sstable B in level 2
  370. // Then do enough Get() calls to arrange for an automatic compaction
  371. // of sstable A. A bug would cause the compaction to be marked as
  372. // occuring at level 1 (instead of the correct level 0).
  373. // Step 1: First place sstables in levels 0 and 2
  374. int compaction_count = 0;
  375. while (NumTableFilesAtLevel(0) == 0 ||
  376. NumTableFilesAtLevel(2) == 0) {
  377. ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2";
  378. compaction_count++;
  379. Put("a", "begin");
  380. Put("z", "end");
  381. dbfull()->TEST_CompactMemTable();
  382. }
  383. // Step 2: clear level 1 if necessary.
  384. dbfull()->TEST_CompactRange(1, NULL, NULL);
  385. ASSERT_EQ(NumTableFilesAtLevel(0), 1);
  386. ASSERT_EQ(NumTableFilesAtLevel(1), 0);
  387. ASSERT_EQ(NumTableFilesAtLevel(2), 1);
  388. // Step 3: read until level 0 compaction disappears.
  389. int read_count = 0;
  390. while (NumTableFilesAtLevel(0) > 0) {
  391. ASSERT_LE(read_count, 10000) << "did not trigger level 0 compaction";
  392. read_count++;
  393. ASSERT_EQ("NOT_FOUND", Get("missing"));
  394. }
  395. }
  396. TEST(DBTest, IterEmpty) {
  397. Iterator* iter = db_->NewIterator(ReadOptions());
  398. iter->SeekToFirst();
  399. ASSERT_EQ(IterStatus(iter), "(invalid)");
  400. iter->SeekToLast();
  401. ASSERT_EQ(IterStatus(iter), "(invalid)");
  402. iter->Seek("foo");
  403. ASSERT_EQ(IterStatus(iter), "(invalid)");
  404. delete iter;
  405. }
  406. TEST(DBTest, IterSingle) {
  407. ASSERT_OK(Put("a", "va"));
  408. Iterator* iter = db_->NewIterator(ReadOptions());
  409. iter->SeekToFirst();
  410. ASSERT_EQ(IterStatus(iter), "a->va");
  411. iter->Next();
  412. ASSERT_EQ(IterStatus(iter), "(invalid)");
  413. iter->SeekToFirst();
  414. ASSERT_EQ(IterStatus(iter), "a->va");
  415. iter->Prev();
  416. ASSERT_EQ(IterStatus(iter), "(invalid)");
  417. iter->SeekToLast();
  418. ASSERT_EQ(IterStatus(iter), "a->va");
  419. iter->Next();
  420. ASSERT_EQ(IterStatus(iter), "(invalid)");
  421. iter->SeekToLast();
  422. ASSERT_EQ(IterStatus(iter), "a->va");
  423. iter->Prev();
  424. ASSERT_EQ(IterStatus(iter), "(invalid)");
  425. iter->Seek("");
  426. ASSERT_EQ(IterStatus(iter), "a->va");
  427. iter->Next();
  428. ASSERT_EQ(IterStatus(iter), "(invalid)");
  429. iter->Seek("a");
  430. ASSERT_EQ(IterStatus(iter), "a->va");
  431. iter->Next();
  432. ASSERT_EQ(IterStatus(iter), "(invalid)");
  433. iter->Seek("b");
  434. ASSERT_EQ(IterStatus(iter), "(invalid)");
  435. delete iter;
  436. }
  437. TEST(DBTest, IterMulti) {
  438. ASSERT_OK(Put("a", "va"));
  439. ASSERT_OK(Put("b", "vb"));
  440. ASSERT_OK(Put("c", "vc"));
  441. Iterator* iter = db_->NewIterator(ReadOptions());
  442. iter->SeekToFirst();
  443. ASSERT_EQ(IterStatus(iter), "a->va");
  444. iter->Next();
  445. ASSERT_EQ(IterStatus(iter), "b->vb");
  446. iter->Next();
  447. ASSERT_EQ(IterStatus(iter), "c->vc");
  448. iter->Next();
  449. ASSERT_EQ(IterStatus(iter), "(invalid)");
  450. iter->SeekToFirst();
  451. ASSERT_EQ(IterStatus(iter), "a->va");
  452. iter->Prev();
  453. ASSERT_EQ(IterStatus(iter), "(invalid)");
  454. iter->SeekToLast();
  455. ASSERT_EQ(IterStatus(iter), "c->vc");
  456. iter->Prev();
  457. ASSERT_EQ(IterStatus(iter), "b->vb");
  458. iter->Prev();
  459. ASSERT_EQ(IterStatus(iter), "a->va");
  460. iter->Prev();
  461. ASSERT_EQ(IterStatus(iter), "(invalid)");
  462. iter->SeekToLast();
  463. ASSERT_EQ(IterStatus(iter), "c->vc");
  464. iter->Next();
  465. ASSERT_EQ(IterStatus(iter), "(invalid)");
  466. iter->Seek("");
  467. ASSERT_EQ(IterStatus(iter), "a->va");
  468. iter->Seek("a");
  469. ASSERT_EQ(IterStatus(iter), "a->va");
  470. iter->Seek("ax");
  471. ASSERT_EQ(IterStatus(iter), "b->vb");
  472. iter->Seek("b");
  473. ASSERT_EQ(IterStatus(iter), "b->vb");
  474. iter->Seek("z");
  475. ASSERT_EQ(IterStatus(iter), "(invalid)");
  476. // Switch from reverse to forward
  477. iter->SeekToLast();
  478. iter->Prev();
  479. iter->Prev();
  480. iter->Next();
  481. ASSERT_EQ(IterStatus(iter), "b->vb");
  482. // Switch from forward to reverse
  483. iter->SeekToFirst();
  484. iter->Next();
  485. iter->Next();
  486. iter->Prev();
  487. ASSERT_EQ(IterStatus(iter), "b->vb");
  488. // Make sure iter stays at snapshot
  489. ASSERT_OK(Put("a", "va2"));
  490. ASSERT_OK(Put("a2", "va3"));
  491. ASSERT_OK(Put("b", "vb2"));
  492. ASSERT_OK(Put("c", "vc2"));
  493. ASSERT_OK(Delete("b"));
  494. iter->SeekToFirst();
  495. ASSERT_EQ(IterStatus(iter), "a->va");
  496. iter->Next();
  497. ASSERT_EQ(IterStatus(iter), "b->vb");
  498. iter->Next();
  499. ASSERT_EQ(IterStatus(iter), "c->vc");
  500. iter->Next();
  501. ASSERT_EQ(IterStatus(iter), "(invalid)");
  502. iter->SeekToLast();
  503. ASSERT_EQ(IterStatus(iter), "c->vc");
  504. iter->Prev();
  505. ASSERT_EQ(IterStatus(iter), "b->vb");
  506. iter->Prev();
  507. ASSERT_EQ(IterStatus(iter), "a->va");
  508. iter->Prev();
  509. ASSERT_EQ(IterStatus(iter), "(invalid)");
  510. delete iter;
  511. }
  512. TEST(DBTest, IterSmallAndLargeMix) {
  513. ASSERT_OK(Put("a", "va"));
  514. ASSERT_OK(Put("b", std::string(100000, 'b')));
  515. ASSERT_OK(Put("c", "vc"));
  516. ASSERT_OK(Put("d", std::string(100000, 'd')));
  517. ASSERT_OK(Put("e", std::string(100000, 'e')));
  518. Iterator* iter = db_->NewIterator(ReadOptions());
  519. iter->SeekToFirst();
  520. ASSERT_EQ(IterStatus(iter), "a->va");
  521. iter->Next();
  522. ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
  523. iter->Next();
  524. ASSERT_EQ(IterStatus(iter), "c->vc");
  525. iter->Next();
  526. ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
  527. iter->Next();
  528. ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
  529. iter->Next();
  530. ASSERT_EQ(IterStatus(iter), "(invalid)");
  531. iter->SeekToLast();
  532. ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
  533. iter->Prev();
  534. ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
  535. iter->Prev();
  536. ASSERT_EQ(IterStatus(iter), "c->vc");
  537. iter->Prev();
  538. ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
  539. iter->Prev();
  540. ASSERT_EQ(IterStatus(iter), "a->va");
  541. iter->Prev();
  542. ASSERT_EQ(IterStatus(iter), "(invalid)");
  543. delete iter;
  544. }
  545. TEST(DBTest, IterMultiWithDelete) {
  546. ASSERT_OK(Put("a", "va"));
  547. ASSERT_OK(Put("b", "vb"));
  548. ASSERT_OK(Put("c", "vc"));
  549. ASSERT_OK(Delete("b"));
  550. ASSERT_EQ("NOT_FOUND", Get("b"));
  551. Iterator* iter = db_->NewIterator(ReadOptions());
  552. iter->Seek("c");
  553. ASSERT_EQ(IterStatus(iter), "c->vc");
  554. iter->Prev();
  555. ASSERT_EQ(IterStatus(iter), "a->va");
  556. delete iter;
  557. }
  558. TEST(DBTest, Recover) {
  559. ASSERT_OK(Put("foo", "v1"));
  560. ASSERT_OK(Put("baz", "v5"));
  561. Reopen();
  562. ASSERT_EQ("v1", Get("foo"));
  563. ASSERT_EQ("v1", Get("foo"));
  564. ASSERT_EQ("v5", Get("baz"));
  565. ASSERT_OK(Put("bar", "v2"));
  566. ASSERT_OK(Put("foo", "v3"));
  567. Reopen();
  568. ASSERT_EQ("v3", Get("foo"));
  569. ASSERT_OK(Put("foo", "v4"));
  570. ASSERT_EQ("v4", Get("foo"));
  571. ASSERT_EQ("v2", Get("bar"));
  572. ASSERT_EQ("v5", Get("baz"));
  573. }
  574. TEST(DBTest, RecoveryWithEmptyLog) {
  575. ASSERT_OK(Put("foo", "v1"));
  576. ASSERT_OK(Put("foo", "v2"));
  577. Reopen();
  578. Reopen();
  579. ASSERT_OK(Put("foo", "v3"));
  580. Reopen();
  581. ASSERT_EQ("v3", Get("foo"));
  582. }
  583. // Check that writes done during a memtable compaction are recovered
  584. // if the database is shutdown during the memtable compaction.
  585. TEST(DBTest, RecoverDuringMemtableCompaction) {
  586. Options options;
  587. options.env = env_;
  588. options.write_buffer_size = 1000000;
  589. Reopen(&options);
  590. // Trigger a long memtable compaction and reopen the database during it
  591. ASSERT_OK(Put("foo", "v1")); // Goes to 1st log file
  592. ASSERT_OK(Put("big1", std::string(10000000, 'x'))); // Fills memtable
  593. ASSERT_OK(Put("big2", std::string(1000, 'y'))); // Triggers compaction
  594. ASSERT_OK(Put("bar", "v2")); // Goes to new log file
  595. Reopen(&options);
  596. ASSERT_EQ("v1", Get("foo"));
  597. ASSERT_EQ("v2", Get("bar"));
  598. ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
  599. ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
  600. }
  601. static std::string Key(int i) {
  602. char buf[100];
  603. snprintf(buf, sizeof(buf), "key%06d", i);
  604. return std::string(buf);
  605. }
  606. TEST(DBTest, MinorCompactionsHappen) {
  607. Options options;
  608. options.write_buffer_size = 10000;
  609. Reopen(&options);
  610. const int N = 500;
  611. int starting_num_tables = TotalTableFiles();
  612. for (int i = 0; i < N; i++) {
  613. ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v')));
  614. }
  615. int ending_num_tables = TotalTableFiles();
  616. ASSERT_GT(ending_num_tables, starting_num_tables);
  617. for (int i = 0; i < N; i++) {
  618. ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
  619. }
  620. Reopen();
  621. for (int i = 0; i < N; i++) {
  622. ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
  623. }
  624. }
  625. TEST(DBTest, RecoverWithLargeLog) {
  626. {
  627. Options options;
  628. Reopen(&options);
  629. ASSERT_OK(Put("big1", std::string(200000, '1')));
  630. ASSERT_OK(Put("big2", std::string(200000, '2')));
  631. ASSERT_OK(Put("small3", std::string(10, '3')));
  632. ASSERT_OK(Put("small4", std::string(10, '4')));
  633. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  634. }
  635. // Make sure that if we re-open with a small write buffer size that
  636. // we flush table files in the middle of a large log file.
  637. Options options;
  638. options.write_buffer_size = 100000;
  639. Reopen(&options);
  640. ASSERT_EQ(NumTableFilesAtLevel(0), 3);
  641. ASSERT_EQ(std::string(200000, '1'), Get("big1"));
  642. ASSERT_EQ(std::string(200000, '2'), Get("big2"));
  643. ASSERT_EQ(std::string(10, '3'), Get("small3"));
  644. ASSERT_EQ(std::string(10, '4'), Get("small4"));
  645. ASSERT_GT(NumTableFilesAtLevel(0), 1);
  646. }
  647. TEST(DBTest, CompactionsGenerateMultipleFiles) {
  648. Options options;
  649. options.write_buffer_size = 100000000; // Large write buffer
  650. Reopen(&options);
  651. Random rnd(301);
  652. // Write 8MB (80 values, each 100K)
  653. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  654. std::vector<std::string> values;
  655. for (int i = 0; i < 80; i++) {
  656. values.push_back(RandomString(&rnd, 100000));
  657. ASSERT_OK(Put(Key(i), values[i]));
  658. }
  659. // Reopening moves updates to level-0
  660. Reopen(&options);
  661. dbfull()->TEST_CompactRange(0, NULL, NULL);
  662. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  663. ASSERT_GT(NumTableFilesAtLevel(1), 1);
  664. for (int i = 0; i < 80; i++) {
  665. ASSERT_EQ(Get(Key(i)), values[i]);
  666. }
  667. }
  668. TEST(DBTest, RepeatedWritesToSameKey) {
  669. Options options;
  670. options.env = env_;
  671. options.write_buffer_size = 100000; // Small write buffer
  672. Reopen(&options);
  673. // We must have at most one file per level except for level-0,
  674. // which may have up to kL0_StopWritesTrigger files.
  675. const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger;
  676. Random rnd(301);
  677. std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
  678. for (int i = 0; i < 5 * kMaxFiles; i++) {
  679. Put("key", value);
  680. ASSERT_LE(TotalTableFiles(), kMaxFiles);
  681. fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles());
  682. }
  683. }
  684. TEST(DBTest, SparseMerge) {
  685. Options options;
  686. options.compression = kNoCompression;
  687. Reopen(&options);
  688. FillLevels("A", "Z");
  689. // Suppose there is:
  690. // small amount of data with prefix A
  691. // large amount of data with prefix B
  692. // small amount of data with prefix C
  693. // and that recent updates have made small changes to all three prefixes.
  694. // Check that we do not do a compaction that merges all of B in one shot.
  695. const std::string value(1000, 'x');
  696. Put("A", "va");
  697. // Write approximately 100MB of "B" values
  698. for (int i = 0; i < 100000; i++) {
  699. char key[100];
  700. snprintf(key, sizeof(key), "B%010d", i);
  701. Put(key, value);
  702. }
  703. Put("C", "vc");
  704. dbfull()->TEST_CompactMemTable();
  705. dbfull()->TEST_CompactRange(0, NULL, NULL);
  706. // Make sparse update
  707. Put("A", "va2");
  708. Put("B100", "bvalue2");
  709. Put("C", "vc2");
  710. dbfull()->TEST_CompactMemTable();
  711. // Compactions should not cause us to create a situation where
  712. // a file overlaps too much data at the next level.
  713. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  714. dbfull()->TEST_CompactRange(0, NULL, NULL);
  715. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  716. dbfull()->TEST_CompactRange(1, NULL, NULL);
  717. ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
  718. }
  719. static bool Between(uint64_t val, uint64_t low, uint64_t high) {
  720. bool result = (val >= low) && (val <= high);
  721. if (!result) {
  722. fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
  723. (unsigned long long)(val),
  724. (unsigned long long)(low),
  725. (unsigned long long)(high));
  726. }
  727. return result;
  728. }
  729. TEST(DBTest, ApproximateSizes) {
  730. Options options;
  731. options.write_buffer_size = 100000000; // Large write buffer
  732. options.compression = kNoCompression;
  733. DestroyAndReopen();
  734. ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
  735. Reopen(&options);
  736. ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
  737. // Write 8MB (80 values, each 100K)
  738. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  739. const int N = 80;
  740. Random rnd(301);
  741. for (int i = 0; i < N; i++) {
  742. ASSERT_OK(Put(Key(i), RandomString(&rnd, 100000)));
  743. }
  744. // 0 because GetApproximateSizes() does not account for memtable space
  745. ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
  746. // Check sizes across recovery by reopening a few times
  747. for (int run = 0; run < 3; run++) {
  748. Reopen(&options);
  749. for (int compact_start = 0; compact_start < N; compact_start += 10) {
  750. for (int i = 0; i < N; i += 10) {
  751. ASSERT_TRUE(Between(Size("", Key(i)), 100000*i, 100000*i + 10000));
  752. ASSERT_TRUE(Between(Size("", Key(i)+".suffix"),
  753. 100000 * (i+1), 100000 * (i+1) + 10000));
  754. ASSERT_TRUE(Between(Size(Key(i), Key(i+10)),
  755. 100000 * 10, 100000 * 10 + 10000));
  756. }
  757. ASSERT_TRUE(Between(Size("", Key(50)), 5000000, 5010000));
  758. ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), 5100000, 5110000));
  759. std::string cstart_str = Key(compact_start);
  760. std::string cend_str = Key(compact_start + 9);
  761. Slice cstart = cstart_str;
  762. Slice cend = cend_str;
  763. dbfull()->TEST_CompactRange(0, &cstart, &cend);
  764. }
  765. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  766. ASSERT_GT(NumTableFilesAtLevel(1), 0);
  767. }
  768. }
  769. TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
  770. Options options;
  771. options.compression = kNoCompression;
  772. Reopen();
  773. Random rnd(301);
  774. std::string big1 = RandomString(&rnd, 100000);
  775. ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
  776. ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
  777. ASSERT_OK(Put(Key(2), big1));
  778. ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
  779. ASSERT_OK(Put(Key(4), big1));
  780. ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
  781. ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
  782. ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
  783. // Check sizes across recovery by reopening a few times
  784. for (int run = 0; run < 3; run++) {
  785. Reopen(&options);
  786. ASSERT_TRUE(Between(Size("", Key(0)), 0, 0));
  787. ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000));
  788. ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000));
  789. ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000));
  790. ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000));
  791. ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000));
  792. ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000));
  793. ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000));
  794. ASSERT_TRUE(Between(Size("", Key(8)), 550000, 551000));
  795. ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
  796. dbfull()->TEST_CompactRange(0, NULL, NULL);
  797. }
  798. }
  799. TEST(DBTest, IteratorPinsRef) {
  800. Put("foo", "hello");
  801. // Get iterator that will yield the current contents of the DB.
  802. Iterator* iter = db_->NewIterator(ReadOptions());
  803. // Write to force compactions
  804. Put("foo", "newvalue1");
  805. for (int i = 0; i < 100; i++) {
  806. ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values
  807. }
  808. Put("foo", "newvalue2");
  809. iter->SeekToFirst();
  810. ASSERT_TRUE(iter->Valid());
  811. ASSERT_EQ("foo", iter->key().ToString());
  812. ASSERT_EQ("hello", iter->value().ToString());
  813. iter->Next();
  814. ASSERT_TRUE(!iter->Valid());
  815. delete iter;
  816. }
  817. TEST(DBTest, Snapshot) {
  818. Put("foo", "v1");
  819. const Snapshot* s1 = db_->GetSnapshot();
  820. Put("foo", "v2");
  821. const Snapshot* s2 = db_->GetSnapshot();
  822. Put("foo", "v3");
  823. const Snapshot* s3 = db_->GetSnapshot();
  824. Put("foo", "v4");
  825. ASSERT_EQ("v1", Get("foo", s1));
  826. ASSERT_EQ("v2", Get("foo", s2));
  827. ASSERT_EQ("v3", Get("foo", s3));
  828. ASSERT_EQ("v4", Get("foo"));
  829. db_->ReleaseSnapshot(s3);
  830. ASSERT_EQ("v1", Get("foo", s1));
  831. ASSERT_EQ("v2", Get("foo", s2));
  832. ASSERT_EQ("v4", Get("foo"));
  833. db_->ReleaseSnapshot(s1);
  834. ASSERT_EQ("v2", Get("foo", s2));
  835. ASSERT_EQ("v4", Get("foo"));
  836. db_->ReleaseSnapshot(s2);
  837. ASSERT_EQ("v4", Get("foo"));
  838. }
  839. TEST(DBTest, HiddenValuesAreRemoved) {
  840. Random rnd(301);
  841. FillLevels("a", "z");
  842. std::string big = RandomString(&rnd, 50000);
  843. Put("foo", big);
  844. Put("pastfoo", "v");
  845. const Snapshot* snapshot = db_->GetSnapshot();
  846. Put("foo", "tiny");
  847. Put("pastfoo2", "v2"); // Advance sequence number one more
  848. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  849. ASSERT_GT(NumTableFilesAtLevel(0), 0);
  850. ASSERT_EQ(big, Get("foo", snapshot));
  851. ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
  852. db_->ReleaseSnapshot(snapshot);
  853. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
  854. Slice x("x");
  855. dbfull()->TEST_CompactRange(0, NULL, &x);
  856. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
  857. ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  858. ASSERT_GE(NumTableFilesAtLevel(1), 1);
  859. dbfull()->TEST_CompactRange(1, NULL, &x);
  860. ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
  861. ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
  862. }
  863. TEST(DBTest, DeletionMarkers1) {
  864. Put("foo", "v1");
  865. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  866. const int last = config::kMaxMemCompactLevel;
  867. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
  868. // Place a table at level last-1 to prevent merging with preceding mutation
  869. Put("a", "begin");
  870. Put("z", "end");
  871. dbfull()->TEST_CompactMemTable();
  872. ASSERT_EQ(NumTableFilesAtLevel(last), 1);
  873. ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
  874. Delete("foo");
  875. Put("foo", "v2");
  876. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
  877. ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
  878. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
  879. Slice z("z");
  880. dbfull()->TEST_CompactRange(last-2, NULL, &z);
  881. // DEL eliminated, but v1 remains because we aren't compacting that level
  882. // (DEL can be eliminated because v2 hides v1).
  883. ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
  884. dbfull()->TEST_CompactRange(last-1, NULL, NULL);
  885. // Merging last-1 w/ last, so we are the base level for "foo", so
  886. // DEL is removed. (as is v1).
  887. ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]");
  888. }
  889. TEST(DBTest, DeletionMarkers2) {
  890. Put("foo", "v1");
  891. ASSERT_OK(dbfull()->TEST_CompactMemTable());
  892. const int last = config::kMaxMemCompactLevel;
  893. ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
  894. // Place a table at level last-1 to prevent merging with preceding mutation
  895. Put("a", "begin");
  896. Put("z", "end");
  897. dbfull()->TEST_CompactMemTable();
  898. ASSERT_EQ(NumTableFilesAtLevel(last), 1);
  899. ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
  900. Delete("foo");
  901. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  902. ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
  903. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  904. dbfull()->TEST_CompactRange(last-2, NULL, NULL);
  905. // DEL kept: "last" file overlaps
  906. ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
  907. dbfull()->TEST_CompactRange(last-1, NULL, NULL);
  908. // Merging last-1 w/ last, so we are the base level for "foo", so
  909. // DEL is removed. (as is v1).
  910. ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
  911. }
  912. TEST(DBTest, OverlapInLevel0) {
  913. ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config";
  914. // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
  915. ASSERT_OK(Put("100", "v100"));
  916. ASSERT_OK(Put("999", "v999"));
  917. dbfull()->TEST_CompactMemTable();
  918. ASSERT_OK(Delete("100"));
  919. ASSERT_OK(Delete("999"));
  920. dbfull()->TEST_CompactMemTable();
  921. ASSERT_EQ("0,1,1", FilesPerLevel());
  922. // Make files spanning the following ranges in level-0:
  923. // files[0] 200 .. 900
  924. // files[1] 300 .. 500
  925. // Note that files are sorted by smallest key.
  926. ASSERT_OK(Put("300", "v300"));
  927. ASSERT_OK(Put("500", "v500"));
  928. dbfull()->TEST_CompactMemTable();
  929. ASSERT_OK(Put("200", "v200"));
  930. ASSERT_OK(Put("600", "v600"));
  931. ASSERT_OK(Put("900", "v900"));
  932. dbfull()->TEST_CompactMemTable();
  933. ASSERT_EQ("2,1,1", FilesPerLevel());
  934. // Compact away the placeholder files we created initially
  935. dbfull()->TEST_CompactRange(1, NULL, NULL);
  936. dbfull()->TEST_CompactRange(2, NULL, NULL);
  937. ASSERT_EQ("2", FilesPerLevel());
  938. // Do a memtable compaction. Before bug-fix, the compaction would
  939. // not detect the overlap with level-0 files and would incorrectly place
  940. // the deletion in a deeper level.
  941. ASSERT_OK(Delete("600"));
  942. dbfull()->TEST_CompactMemTable();
  943. ASSERT_EQ("3", FilesPerLevel());
  944. ASSERT_EQ("NOT_FOUND", Get("600"));
  945. }
  946. TEST(DBTest, L0_CompactionBug_Issue44_a) {
  947. Reopen();
  948. ASSERT_OK(Put("b", "v"));
  949. Reopen();
  950. ASSERT_OK(Delete("b"));
  951. ASSERT_OK(Delete("a"));
  952. Reopen();
  953. ASSERT_OK(Delete("a"));
  954. Reopen();
  955. ASSERT_OK(Put("a", "v"));
  956. Reopen();
  957. Reopen();
  958. ASSERT_EQ("(a->v)", Contents());
  959. env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
  960. ASSERT_EQ("(a->v)", Contents());
  961. }
  962. TEST(DBTest, L0_CompactionBug_Issue44_b) {
  963. Reopen();
  964. Put("","");
  965. Reopen();
  966. Delete("e");
  967. Put("","");
  968. Reopen();
  969. Put("c", "cv");
  970. Reopen();
  971. Put("","");
  972. Reopen();
  973. Put("","");
  974. env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
  975. Reopen();
  976. Put("d","dv");
  977. Reopen();
  978. Put("","");
  979. Reopen();
  980. Delete("d");
  981. Delete("b");
  982. Reopen();
  983. ASSERT_EQ("(->)(c->cv)", Contents());
  984. env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
  985. ASSERT_EQ("(->)(c->cv)", Contents());
  986. }
  987. TEST(DBTest, ComparatorCheck) {
  988. class NewComparator : public Comparator {
  989. public:
  990. virtual const char* Name() const { return "leveldb.NewComparator"; }
  991. virtual int Compare(const Slice& a, const Slice& b) const {
  992. return BytewiseComparator()->Compare(a, b);
  993. }
  994. virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
  995. BytewiseComparator()->FindShortestSeparator(s, l);
  996. }
  997. virtual void FindShortSuccessor(std::string* key) const {
  998. BytewiseComparator()->FindShortSuccessor(key);
  999. }
  1000. };
  1001. NewComparator cmp;
  1002. Options new_options;
  1003. new_options.comparator = &cmp;
  1004. Status s = TryReopen(&new_options);
  1005. ASSERT_TRUE(!s.ok());
  1006. ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos)
  1007. << s.ToString();
  1008. }
  1009. TEST(DBTest, CustomComparator) {
  1010. class NumberComparator : public Comparator {
  1011. public:
  1012. virtual const char* Name() const { return "test.NumberComparator"; }
  1013. virtual int Compare(const Slice& a, const Slice& b) const {
  1014. return ToNumber(a) - ToNumber(b);
  1015. }
  1016. virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
  1017. ToNumber(*s); // Check format
  1018. ToNumber(l); // Check format
  1019. }
  1020. virtual void FindShortSuccessor(std::string* key) const {
  1021. ToNumber(*key); // Check format
  1022. }
  1023. private:
  1024. static int ToNumber(const Slice& x) {
  1025. // Check that there are no extra characters.
  1026. ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']')
  1027. << EscapeString(x);
  1028. int val;
  1029. char ignored;
  1030. ASSERT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1)
  1031. << EscapeString(x);
  1032. return val;
  1033. }
  1034. };
  1035. NumberComparator cmp;
  1036. Options new_options;
  1037. new_options.create_if_missing = true;
  1038. new_options.comparator = &cmp;
  1039. new_options.write_buffer_size = 1000; // Compact more often
  1040. DestroyAndReopen(&new_options);
  1041. ASSERT_OK(Put("[10]", "ten"));
  1042. ASSERT_OK(Put("[0x14]", "twenty"));
  1043. for (int i = 0; i < 2; i++) {
  1044. ASSERT_EQ("ten", Get("[10]"));
  1045. ASSERT_EQ("ten", Get("[0xa]"));
  1046. ASSERT_EQ("twenty", Get("[20]"));
  1047. ASSERT_EQ("twenty", Get("[0x14]"));
  1048. Compact("[0]", "[9999]");
  1049. }
  1050. for (int run = 0; run < 2; run++) {
  1051. for (int i = 0; i < 1000; i++) {
  1052. char buf[100];
  1053. snprintf(buf, sizeof(buf), "[%d]", i*10);
  1054. ASSERT_OK(Put(buf, buf));
  1055. }
  1056. Compact("[0]", "[1000000]");
  1057. }
  1058. }
  1059. TEST(DBTest, ManualCompaction) {
  1060. ASSERT_EQ(config::kMaxMemCompactLevel, 2)
  1061. << "Need to update this test to match kMaxMemCompactLevel";
  1062. MakeTables(3, "p", "q");
  1063. ASSERT_EQ("1,1,1", FilesPerLevel());
  1064. // Compaction range falls before files
  1065. Compact("", "c");
  1066. ASSERT_EQ("1,1,1", FilesPerLevel());
  1067. // Compaction range falls after files
  1068. Compact("r", "z");
  1069. ASSERT_EQ("1,1,1", FilesPerLevel());
  1070. // Compaction range overlaps files
  1071. Compact("p1", "p9");
  1072. ASSERT_EQ("0,0,1", FilesPerLevel());
  1073. // Populate a different range
  1074. MakeTables(3, "c", "e");
  1075. ASSERT_EQ("1,1,2", FilesPerLevel());
  1076. // Compact just the new range
  1077. Compact("b", "f");
  1078. ASSERT_EQ("0,0,2", FilesPerLevel());
  1079. // Compact all
  1080. MakeTables(1, "a", "z");
  1081. ASSERT_EQ("0,1,2", FilesPerLevel());
  1082. db_->CompactRange(NULL, NULL);
  1083. ASSERT_EQ("0,0,1", FilesPerLevel());
  1084. }
  1085. TEST(DBTest, DBOpen_Options) {
  1086. std::string dbname = test::TmpDir() + "/db_options_test";
  1087. DestroyDB(dbname, Options());
  1088. // Does not exist, and create_if_missing == false: error
  1089. DB* db = NULL;
  1090. Options opts;
  1091. opts.create_if_missing = false;
  1092. Status s = DB::Open(opts, dbname, &db);
  1093. ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL);
  1094. ASSERT_TRUE(db == NULL);
  1095. // Does not exist, and create_if_missing == true: OK
  1096. opts.create_if_missing = true;
  1097. s = DB::Open(opts, dbname, &db);
  1098. ASSERT_OK(s);
  1099. ASSERT_TRUE(db != NULL);
  1100. delete db;
  1101. db = NULL;
  1102. // Does exist, and error_if_exists == true: error
  1103. opts.create_if_missing = false;
  1104. opts.error_if_exists = true;
  1105. s = DB::Open(opts, dbname, &db);
  1106. ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL);
  1107. ASSERT_TRUE(db == NULL);
  1108. // Does exist, and error_if_exists == false: OK
  1109. opts.create_if_missing = true;
  1110. opts.error_if_exists = false;
  1111. s = DB::Open(opts, dbname, &db);
  1112. ASSERT_OK(s);
  1113. ASSERT_TRUE(db != NULL);
  1114. delete db;
  1115. db = NULL;
  1116. }
  1117. // Check that number of files does not grow when we are out of space
  1118. TEST(DBTest, NoSpace) {
  1119. Options options;
  1120. options.env = env_;
  1121. Reopen(&options);
  1122. ASSERT_OK(Put("foo", "v1"));
  1123. ASSERT_EQ("v1", Get("foo"));
  1124. Compact("a", "z");
  1125. const int num_files = CountFiles();
  1126. env_->no_space_.Release_Store(env_); // Force out-of-space errors
  1127. for (int i = 0; i < 10; i++) {
  1128. for (int level = 0; level < config::kNumLevels-1; level++) {
  1129. dbfull()->TEST_CompactRange(level, NULL, NULL);
  1130. }
  1131. }
  1132. env_->no_space_.Release_Store(NULL);
  1133. ASSERT_LT(CountFiles(), num_files + 5);
  1134. }
  1135. TEST(DBTest, FilesDeletedAfterCompaction) {
  1136. ASSERT_OK(Put("foo", "v2"));
  1137. Compact("a", "z");
  1138. const int num_files = CountFiles();
  1139. for (int i = 0; i < 10; i++) {
  1140. ASSERT_OK(Put("foo", "v2"));
  1141. Compact("a", "z");
  1142. }
  1143. ASSERT_EQ(CountFiles(), num_files);
  1144. }
  1145. // Multi-threaded test:
  1146. namespace {
  1147. static const int kNumThreads = 4;
  1148. static const int kTestSeconds = 10;
  1149. static const int kNumKeys = 1000;
  1150. struct MTState {
  1151. DBTest* test;
  1152. port::AtomicPointer stop;
  1153. port::AtomicPointer counter[kNumThreads];
  1154. port::AtomicPointer thread_done[kNumThreads];
  1155. };
  1156. struct MTThread {
  1157. MTState* state;
  1158. int id;
  1159. };
  1160. static void MTThreadBody(void* arg) {
  1161. MTThread* t = reinterpret_cast<MTThread*>(arg);
  1162. int id = t->id;
  1163. DB* db = t->state->test->db_;
  1164. uintptr_t counter = 0;
  1165. fprintf(stderr, "... starting thread %d\n", id);
  1166. Random rnd(1000 + id);
  1167. std::string value;
  1168. char valbuf[1500];
  1169. while (t->state->stop.Acquire_Load() == NULL) {
  1170. t->state->counter[id].Release_Store(reinterpret_cast<void*>(counter));
  1171. int key = rnd.Uniform(kNumKeys);
  1172. char keybuf[20];
  1173. snprintf(keybuf, sizeof(keybuf), "%016d", key);
  1174. if (rnd.OneIn(2)) {
  1175. // Write values of the form <key, my id, counter>.
  1176. // We add some padding for force compactions.
  1177. snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d",
  1178. key, id, static_cast<int>(counter));
  1179. ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf)));
  1180. } else {
  1181. // Read a value and verify that it matches the pattern written above.
  1182. Status s = db->Get(ReadOptions(), Slice(keybuf), &value);
  1183. if (s.IsNotFound()) {
  1184. // Key has not yet been written
  1185. } else {
  1186. // Check that the writer thread counter is >= the counter in the value
  1187. ASSERT_OK(s);
  1188. int k, w, c;
  1189. ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value;
  1190. ASSERT_EQ(k, key);
  1191. ASSERT_GE(w, 0);
  1192. ASSERT_LT(w, kNumThreads);
  1193. ASSERT_LE(c, reinterpret_cast<uintptr_t>(
  1194. t->state->counter[w].Acquire_Load()));
  1195. }
  1196. }
  1197. counter++;
  1198. }
  1199. t->state->thread_done[id].Release_Store(t);
  1200. fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
  1201. }
  1202. } // namespace
  1203. TEST(DBTest, MultiThreaded) {
  1204. // Initialize state
  1205. MTState mt;
  1206. mt.test = this;
  1207. mt.stop.Release_Store(0);
  1208. for (int id = 0; id < kNumThreads; id++) {
  1209. mt.counter[id].Release_Store(0);
  1210. mt.thread_done[id].Release_Store(0);
  1211. }
  1212. // Start threads
  1213. MTThread thread[kNumThreads];
  1214. for (int id = 0; id < kNumThreads; id++) {
  1215. thread[id].state = &mt;
  1216. thread[id].id = id;
  1217. env_->StartThread(MTThreadBody, &thread[id]);
  1218. }
  1219. // Let them run for a while
  1220. env_->SleepForMicroseconds(kTestSeconds * 1000000);
  1221. // Stop the threads and wait for them to finish
  1222. mt.stop.Release_Store(&mt);
  1223. for (int id = 0; id < kNumThreads; id++) {
  1224. while (mt.thread_done[id].Acquire_Load() == NULL) {
  1225. env_->SleepForMicroseconds(100000);
  1226. }
  1227. }
  1228. }
  1229. namespace {
  1230. typedef std::map<std::string, std::string> KVMap;
  1231. }
  1232. class ModelDB: public DB {
  1233. public:
  1234. class ModelSnapshot : public Snapshot {
  1235. public:
  1236. KVMap map_;
  1237. };
  1238. explicit ModelDB(const Options& options): options_(options) { }
  1239. ~ModelDB() { }
  1240. virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
  1241. return DB::Put(o, k, v);
  1242. }
  1243. virtual Status Delete(const WriteOptions& o, const Slice& key) {
  1244. return DB::Delete(o, key);
  1245. }
  1246. virtual Status Get(const ReadOptions& options,
  1247. const Slice& key, std::string* value) {
  1248. assert(false); // Not implemented
  1249. return Status::NotFound(key);
  1250. }
  1251. virtual Iterator* NewIterator(const ReadOptions& options) {
  1252. if (options.snapshot == NULL) {
  1253. KVMap* saved = new KVMap;
  1254. *saved = map_;
  1255. return new ModelIter(saved, true);
  1256. } else {
  1257. const KVMap* snapshot_state =
  1258. &(reinterpret_cast<const ModelSnapshot*>(options.snapshot)->map_);
  1259. return new ModelIter(snapshot_state, false);
  1260. }
  1261. }
  1262. virtual const Snapshot* GetSnapshot() {
  1263. ModelSnapshot* snapshot = new ModelSnapshot;
  1264. snapshot->map_ = map_;
  1265. return snapshot;
  1266. }
  1267. virtual void ReleaseSnapshot(const Snapshot* snapshot) {
  1268. delete reinterpret_cast<const ModelSnapshot*>(snapshot);
  1269. }
  1270. virtual Status Write(const WriteOptions& options, WriteBatch* batch) {
  1271. class Handler : public WriteBatch::Handler {
  1272. public:
  1273. KVMap* map_;
  1274. virtual void Put(const Slice& key, const Slice& value) {
  1275. (*map_)[key.ToString()] = value.ToString();
  1276. }
  1277. virtual void Delete(const Slice& key) {
  1278. map_->erase(key.ToString());
  1279. }
  1280. };
  1281. Handler handler;
  1282. handler.map_ = &map_;
  1283. return batch->Iterate(&handler);
  1284. }
  1285. virtual bool GetProperty(const Slice& property, std::string* value) {
  1286. return false;
  1287. }
  1288. virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) {
  1289. for (int i = 0; i < n; i++) {
  1290. sizes[i] = 0;
  1291. }
  1292. }
  1293. virtual void CompactRange(const Slice* start, const Slice* end) {
  1294. }
  1295. private:
  1296. class ModelIter: public Iterator {
  1297. public:
  1298. ModelIter(const KVMap* map, bool owned)
  1299. : map_(map), owned_(owned), iter_(map_->end()) {
  1300. }
  1301. ~ModelIter() {
  1302. if (owned_) delete map_;
  1303. }
  1304. virtual bool Valid() const { return iter_ != map_->end(); }
  1305. virtual void SeekToFirst() { iter_ = map_->begin(); }
  1306. virtual void SeekToLast() {
  1307. if (map_->empty()) {
  1308. iter_ = map_->end();
  1309. } else {
  1310. iter_ = map_->find(map_->rbegin()->first);
  1311. }
  1312. }
  1313. virtual void Seek(const Slice& k) {
  1314. iter_ = map_->lower_bound(k.ToString());
  1315. }
  1316. virtual void Next() { ++iter_; }
  1317. virtual void Prev() { --iter_; }
  1318. virtual Slice key() const { return iter_->first; }
  1319. virtual Slice value() const { return iter_->second; }
  1320. virtual Status status() const { return Status::OK(); }
  1321. private:
  1322. const KVMap* const map_;
  1323. const bool owned_; // Do we own map_
  1324. KVMap::const_iterator iter_;
  1325. };
  1326. const Options options_;
  1327. KVMap map_;
  1328. };
  1329. static std::string RandomKey(Random* rnd) {
  1330. int len = (rnd->OneIn(3)
  1331. ? 1 // Short sometimes to encourage collisions
  1332. : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
  1333. return test::RandomKey(rnd, len);
  1334. }
  1335. static bool CompareIterators(int step,
  1336. DB* model,
  1337. DB* db,
  1338. const Snapshot* model_snap,
  1339. const Snapshot* db_snap) {
  1340. ReadOptions options;
  1341. options.snapshot = model_snap;
  1342. Iterator* miter = model->NewIterator(options);
  1343. options.snapshot = db_snap;
  1344. Iterator* dbiter = db->NewIterator(options);
  1345. bool ok = true;
  1346. int count = 0;
  1347. for (miter->SeekToFirst(), dbiter->SeekToFirst();
  1348. ok && miter->Valid() && dbiter->Valid();
  1349. miter->Next(), dbiter->Next()) {
  1350. count++;
  1351. if (miter->key().compare(dbiter->key()) != 0) {
  1352. fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
  1353. step,
  1354. EscapeString(miter->key()).c_str(),
  1355. EscapeString(dbiter->key()).c_str());
  1356. ok = false;
  1357. break;
  1358. }
  1359. if (miter->value().compare(dbiter->value()) != 0) {
  1360. fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
  1361. step,
  1362. EscapeString(miter->key()).c_str(),
  1363. EscapeString(miter->value()).c_str(),
  1364. EscapeString(miter->value()).c_str());
  1365. ok = false;
  1366. }
  1367. }
  1368. if (ok) {
  1369. if (miter->Valid() != dbiter->Valid()) {
  1370. fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n",
  1371. step, miter->Valid(), dbiter->Valid());
  1372. ok = false;
  1373. }
  1374. }
  1375. fprintf(stderr, "%d entries compared: ok=%d\n", count, ok);
  1376. delete miter;
  1377. delete dbiter;
  1378. return ok;
  1379. }
  1380. TEST(DBTest, Randomized) {
  1381. Random rnd(test::RandomSeed());
  1382. ModelDB model(last_options_);
  1383. const int N = 10000;
  1384. const Snapshot* model_snap = NULL;
  1385. const Snapshot* db_snap = NULL;
  1386. std::string k, v;
  1387. for (int step = 0; step < N; step++) {
  1388. if (step % 100 == 0) {
  1389. fprintf(stderr, "Step %d of %d\n", step, N);
  1390. }
  1391. int p = rnd.Uniform(100);
  1392. if (p < 45) { // Put
  1393. k = RandomKey(&rnd);
  1394. v = RandomString(&rnd,
  1395. rnd.OneIn(20)
  1396. ? 100 + rnd.Uniform(100)
  1397. : rnd.Uniform(8));
  1398. ASSERT_OK(model.Put(WriteOptions(), k, v));
  1399. ASSERT_OK(db_->Put(WriteOptions(), k, v));
  1400. } else if (p < 90) { // Delete
  1401. k = RandomKey(&rnd);
  1402. ASSERT_OK(model.Delete(WriteOptions(), k));
  1403. ASSERT_OK(db_->Delete(WriteOptions(), k));
  1404. } else { // Multi-element batch
  1405. WriteBatch b;
  1406. const int num = rnd.Uniform(8);
  1407. for (int i = 0; i < num; i++) {
  1408. if (i == 0 || !rnd.OneIn(10)) {
  1409. k = RandomKey(&rnd);
  1410. } else {
  1411. // Periodically re-use the same key from the previous iter, so
  1412. // we have multiple entries in the write batch for the same key
  1413. }
  1414. if (rnd.OneIn(2)) {
  1415. v = RandomString(&rnd, rnd.Uniform(10));
  1416. b.Put(k, v);
  1417. } else {
  1418. b.Delete(k);
  1419. }
  1420. }
  1421. ASSERT_OK(model.Write(WriteOptions(), &b));
  1422. ASSERT_OK(db_->Write(WriteOptions(), &b));
  1423. }
  1424. if ((step % 100) == 0) {
  1425. ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
  1426. ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
  1427. // Save a snapshot from each DB this time that we'll use next
  1428. // time we compare things, to make sure the current state is
  1429. // preserved with the snapshot
  1430. if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
  1431. if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
  1432. Reopen();
  1433. ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
  1434. model_snap = model.GetSnapshot();
  1435. db_snap = db_->GetSnapshot();
  1436. }
  1437. }
  1438. if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
  1439. if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
  1440. }
  1441. std::string MakeKey(unsigned int num) {
  1442. char buf[30];
  1443. snprintf(buf, sizeof(buf), "%016u", num);
  1444. return std::string(buf);
  1445. }
  1446. void BM_LogAndApply(int iters, int num_base_files) {
  1447. std::string dbname = test::TmpDir() + "/leveldb_test_benchmark";
  1448. DestroyDB(dbname, Options());
  1449. DB* db = NULL;
  1450. Options opts;
  1451. opts.create_if_missing = true;
  1452. Status s = DB::Open(opts, dbname, &db);
  1453. ASSERT_OK(s);
  1454. ASSERT_TRUE(db != NULL);
  1455. delete db;
  1456. db = NULL;
  1457. Env* env = Env::Default();
  1458. port::Mutex mu;
  1459. MutexLock l(&mu);
  1460. InternalKeyComparator cmp(BytewiseComparator());
  1461. Options options;
  1462. VersionSet vset(dbname, &options, NULL, &cmp);
  1463. ASSERT_OK(vset.Recover());
  1464. VersionEdit vbase;
  1465. uint64_t fnum = 1;
  1466. for (int i = 0; i < num_base_files; i++) {
  1467. InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
  1468. InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
  1469. vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
  1470. }
  1471. ASSERT_OK(vset.LogAndApply(&vbase, &mu));
  1472. uint64_t start_micros = env->NowMicros();
  1473. for (int i = 0; i < iters; i++) {
  1474. VersionEdit vedit;
  1475. vedit.DeleteFile(2, fnum);
  1476. InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
  1477. InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
  1478. vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
  1479. vset.LogAndApply(&vedit, &mu);
  1480. }
  1481. uint64_t stop_micros = env->NowMicros();
  1482. unsigned int us = stop_micros - start_micros;
  1483. char buf[16];
  1484. snprintf(buf, sizeof(buf), "%d", num_base_files);
  1485. fprintf(stderr,
  1486. "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n",
  1487. buf, iters, us, ((float)us) / iters);
  1488. }
  1489. } // namespace leveldb
  1490. int main(int argc, char** argv) {
  1491. if (argc > 1 && std::string(argv[1]) == "--benchmark") {
  1492. leveldb::BM_LogAndApply(1000, 1);
  1493. leveldb::BM_LogAndApply(1000, 100);
  1494. leveldb::BM_LogAndApply(1000, 10000);
  1495. leveldb::BM_LogAndApply(100, 100000);
  1496. return 0;
  1497. }
  1498. return leveldb::test::RunAllTests();
  1499. }