876 行
24 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "leveldb/table.h"
  5. #include <map>
  6. #include <string>
  7. #include "db/dbformat.h"
  8. #include "db/memtable.h"
  9. #include "db/write_batch_internal.h"
  10. #include "leveldb/db.h"
  11. #include "leveldb/env.h"
  12. #include "leveldb/iterator.h"
  13. #include "leveldb/table_builder.h"
  14. #include "table/block.h"
  15. #include "table/block_builder.h"
  16. #include "table/format.h"
  17. #include "util/random.h"
  18. #include "util/testharness.h"
  19. #include "util/testutil.h"
  20. namespace leveldb {
  21. // Return reverse of "key".
  22. // Used to test non-lexicographic comparators.
  23. static std::string Reverse(const Slice& key) {
  24. std::string str(key.ToString());
  25. std::string rev("");
  26. for (std::string::reverse_iterator rit = str.rbegin();
  27. rit != str.rend(); ++rit) {
  28. rev.push_back(*rit);
  29. }
  30. return rev;
  31. }
  32. namespace {
  33. class ReverseKeyComparator : public Comparator {
  34. public:
  35. virtual const char* Name() const {
  36. return "leveldb.ReverseBytewiseComparator";
  37. }
  38. virtual int Compare(const Slice& a, const Slice& b) const {
  39. return BytewiseComparator()->Compare(Reverse(a), Reverse(b));
  40. }
  41. virtual void FindShortestSeparator(
  42. std::string* start,
  43. const Slice& limit) const {
  44. std::string s = Reverse(*start);
  45. std::string l = Reverse(limit);
  46. BytewiseComparator()->FindShortestSeparator(&s, l);
  47. *start = Reverse(s);
  48. }
  49. virtual void FindShortSuccessor(std::string* key) const {
  50. std::string s = Reverse(*key);
  51. BytewiseComparator()->FindShortSuccessor(&s);
  52. *key = Reverse(s);
  53. }
  54. };
  55. } // namespace
  56. static ReverseKeyComparator reverse_key_comparator;
  57. static void Increment(const Comparator* cmp, std::string* key) {
  58. if (cmp == BytewiseComparator()) {
  59. key->push_back('\0');
  60. } else {
  61. assert(cmp == &reverse_key_comparator);
  62. std::string rev = Reverse(*key);
  63. rev.push_back('\0');
  64. *key = Reverse(rev);
  65. }
  66. }
  67. // An STL comparator that uses a Comparator
  68. namespace {
  69. struct STLLessThan {
  70. const Comparator* cmp;
  71. STLLessThan() : cmp(BytewiseComparator()) { }
  72. STLLessThan(const Comparator* c) : cmp(c) { }
  73. bool operator()(const std::string& a, const std::string& b) const {
  74. return cmp->Compare(Slice(a), Slice(b)) < 0;
  75. }
  76. };
  77. } // namespace
  78. class StringSink: public WritableFile {
  79. public:
  80. ~StringSink() { }
  81. const std::string& contents() const { return contents_; }
  82. virtual Status Close() { return Status::OK(); }
  83. virtual Status Flush() { return Status::OK(); }
  84. virtual Status Sync() { return Status::OK(); }
  85. virtual Status Append(const Slice& data) {
  86. contents_.append(data.data(), data.size());
  87. return Status::OK();
  88. }
  89. private:
  90. std::string contents_;
  91. };
  92. class StringSource: public RandomAccessFile {
  93. public:
  94. StringSource(const Slice& contents)
  95. : contents_(contents.data(), contents.size()) {
  96. }
  97. virtual ~StringSource() { }
  98. uint64_t Size() const { return contents_.size(); }
  99. virtual Status Read(uint64_t offset, size_t n, Slice* result,
  100. char* scratch) const {
  101. if (offset > contents_.size()) {
  102. return Status::InvalidArgument("invalid Read offset");
  103. }
  104. if (offset + n > contents_.size()) {
  105. n = contents_.size() - offset;
  106. }
  107. memcpy(scratch, &contents_[offset], n);
  108. *result = Slice(scratch, n);
  109. return Status::OK();
  110. }
  111. private:
  112. std::string contents_;
  113. };
  114. typedef std::map<std::string, std::string, STLLessThan> KVMap;
  115. // Helper class for tests to unify the interface between
  116. // BlockBuilder/TableBuilder and Block/Table.
  117. class Constructor {
  118. public:
  119. explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) { }
  120. virtual ~Constructor() { }
  121. void Add(const std::string& key, const Slice& value) {
  122. data_[key] = value.ToString();
  123. }
  124. // Finish constructing the data structure with all the keys that have
  125. // been added so far. Returns the keys in sorted order in "*keys"
  126. // and stores the key/value pairs in "*kvmap"
  127. void Finish(const Options& options,
  128. std::vector<std::string>* keys,
  129. KVMap* kvmap) {
  130. *kvmap = data_;
  131. keys->clear();
  132. for (KVMap::const_iterator it = data_.begin();
  133. it != data_.end();
  134. ++it) {
  135. keys->push_back(it->first);
  136. }
  137. data_.clear();
  138. Status s = FinishImpl(options, *kvmap);
  139. ASSERT_TRUE(s.ok()) << s.ToString();
  140. }
  141. // Construct the data structure from the data in "data"
  142. virtual Status FinishImpl(const Options& options, const KVMap& data) = 0;
  143. virtual Iterator* NewIterator() const = 0;
  144. virtual const KVMap& data() { return data_; }
  145. virtual DB* db() const { return NULL; } // Overridden in DBConstructor
  146. private:
  147. KVMap data_;
  148. };
  149. class BlockConstructor: public Constructor {
  150. public:
  151. explicit BlockConstructor(const Comparator* cmp)
  152. : Constructor(cmp),
  153. comparator_(cmp),
  154. block_(NULL) { }
  155. ~BlockConstructor() {
  156. delete block_;
  157. }
  158. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  159. delete block_;
  160. block_ = NULL;
  161. BlockBuilder builder(&options);
  162. for (KVMap::const_iterator it = data.begin();
  163. it != data.end();
  164. ++it) {
  165. builder.Add(it->first, it->second);
  166. }
  167. // Open the block
  168. data_ = builder.Finish().ToString();
  169. BlockContents contents;
  170. contents.data = data_;
  171. contents.cachable = false;
  172. contents.heap_allocated = false;
  173. block_ = new Block(contents);
  174. return Status::OK();
  175. }
  176. virtual Iterator* NewIterator() const {
  177. return block_->NewIterator(comparator_);
  178. }
  179. private:
  180. const Comparator* comparator_;
  181. std::string data_;
  182. Block* block_;
  183. BlockConstructor();
  184. };
  185. class TableConstructor: public Constructor {
  186. public:
  187. TableConstructor(const Comparator* cmp)
  188. : Constructor(cmp),
  189. source_(NULL), table_(NULL) {
  190. }
  191. ~TableConstructor() {
  192. Reset();
  193. }
  194. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  195. Reset();
  196. StringSink sink;
  197. TableBuilder builder(options, &sink);
  198. for (KVMap::const_iterator it = data.begin();
  199. it != data.end();
  200. ++it) {
  201. builder.Add(it->first, it->second);
  202. ASSERT_TRUE(builder.status().ok());
  203. }
  204. Status s = builder.Finish();
  205. ASSERT_TRUE(s.ok()) << s.ToString();
  206. ASSERT_EQ(sink.contents().size(), builder.FileSize());
  207. // Open the table
  208. source_ = new StringSource(sink.contents());
  209. Options table_options;
  210. table_options.comparator = options.comparator;
  211. return Table::Open(table_options, source_, sink.contents().size(), &table_);
  212. }
  213. virtual Iterator* NewIterator() const {
  214. return table_->NewIterator(ReadOptions());
  215. }
  216. uint64_t ApproximateOffsetOf(const Slice& key) const {
  217. return table_->ApproximateOffsetOf(key);
  218. }
  219. private:
  220. void Reset() {
  221. delete table_;
  222. delete source_;
  223. table_ = NULL;
  224. source_ = NULL;
  225. }
  226. StringSource* source_;
  227. Table* table_;
  228. TableConstructor();
  229. };
  230. // A helper class that converts internal format keys into user keys
  231. class KeyConvertingIterator: public Iterator {
  232. public:
  233. explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) { }
  234. virtual ~KeyConvertingIterator() { delete iter_; }
  235. virtual bool Valid() const { return iter_->Valid(); }
  236. virtual void Seek(const Slice& target) {
  237. ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue);
  238. std::string encoded;
  239. AppendInternalKey(&encoded, ikey);
  240. iter_->Seek(encoded);
  241. }
  242. virtual void SeekToFirst() { iter_->SeekToFirst(); }
  243. virtual void SeekToLast() { iter_->SeekToLast(); }
  244. virtual void Next() { iter_->Next(); }
  245. virtual void Prev() { iter_->Prev(); }
  246. virtual Slice key() const {
  247. assert(Valid());
  248. ParsedInternalKey key;
  249. if (!ParseInternalKey(iter_->key(), &key)) {
  250. status_ = Status::Corruption("malformed internal key");
  251. return Slice("corrupted key");
  252. }
  253. return key.user_key;
  254. }
  255. virtual Slice value() const { return iter_->value(); }
  256. virtual Status status() const {
  257. return status_.ok() ? iter_->status() : status_;
  258. }
  259. private:
  260. mutable Status status_;
  261. Iterator* iter_;
  262. // No copying allowed
  263. KeyConvertingIterator(const KeyConvertingIterator&);
  264. void operator=(const KeyConvertingIterator&);
  265. };
  266. class MemTableConstructor: public Constructor {
  267. public:
  268. explicit MemTableConstructor(const Comparator* cmp)
  269. : Constructor(cmp),
  270. internal_comparator_(cmp) {
  271. memtable_ = new MemTable(internal_comparator_);
  272. memtable_->Ref();
  273. }
  274. ~MemTableConstructor() {
  275. memtable_->Unref();
  276. }
  277. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  278. memtable_->Unref();
  279. memtable_ = new MemTable(internal_comparator_);
  280. memtable_->Ref();
  281. int seq = 1;
  282. for (KVMap::const_iterator it = data.begin();
  283. it != data.end();
  284. ++it) {
  285. memtable_->Add(seq, kTypeValue, it->first, it->second);
  286. seq++;
  287. }
  288. return Status::OK();
  289. }
  290. virtual Iterator* NewIterator() const {
  291. return new KeyConvertingIterator(memtable_->NewIterator());
  292. }
  293. private:
  294. InternalKeyComparator internal_comparator_;
  295. MemTable* memtable_;
  296. };
  297. class DBConstructor: public Constructor {
  298. public:
  299. explicit DBConstructor(const Comparator* cmp)
  300. : Constructor(cmp),
  301. comparator_(cmp) {
  302. db_ = NULL;
  303. NewDB();
  304. }
  305. ~DBConstructor() {
  306. delete db_;
  307. }
  308. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  309. delete db_;
  310. db_ = NULL;
  311. NewDB();
  312. for (KVMap::const_iterator it = data.begin();
  313. it != data.end();
  314. ++it) {
  315. WriteBatch batch;
  316. batch.Put(it->first, it->second);
  317. ASSERT_TRUE(db_->Write(WriteOptions(), &batch).ok());
  318. }
  319. return Status::OK();
  320. }
  321. virtual Iterator* NewIterator() const {
  322. return db_->NewIterator(ReadOptions());
  323. }
  324. virtual DB* db() const { return db_; }
  325. private:
  326. void NewDB() {
  327. std::string name = test::TmpDir() + "/table_testdb";
  328. Options options;
  329. options.comparator = comparator_;
  330. Status status = DestroyDB(name, options);
  331. ASSERT_TRUE(status.ok()) << status.ToString();
  332. options.create_if_missing = true;
  333. options.error_if_exists = true;
  334. options.write_buffer_size = 10000; // Something small to force merging
  335. status = DB::Open(options, name, &db_);
  336. ASSERT_TRUE(status.ok()) << status.ToString();
  337. }
  338. const Comparator* comparator_;
  339. DB* db_;
  340. };
  341. enum TestType {
  342. TABLE_TEST,
  343. BLOCK_TEST,
  344. MEMTABLE_TEST,
  345. DB_TEST
  346. };
  347. struct TestArgs {
  348. TestType type;
  349. bool reverse_compare;
  350. int restart_interval;
  351. };
  352. static const TestArgs kTestArgList[] = {
  353. { TABLE_TEST, false, 16 },
  354. { TABLE_TEST, false, 1 },
  355. { TABLE_TEST, false, 1024 },
  356. { TABLE_TEST, true, 16 },
  357. { TABLE_TEST, true, 1 },
  358. { TABLE_TEST, true, 1024 },
  359. { BLOCK_TEST, false, 16 },
  360. { BLOCK_TEST, false, 1 },
  361. { BLOCK_TEST, false, 1024 },
  362. { BLOCK_TEST, true, 16 },
  363. { BLOCK_TEST, true, 1 },
  364. { BLOCK_TEST, true, 1024 },
  365. // Restart interval does not matter for memtables
  366. { MEMTABLE_TEST, false, 16 },
  367. { MEMTABLE_TEST, true, 16 },
  368. // Do not bother with restart interval variations for DB
  369. { DB_TEST, false, 16 },
  370. { DB_TEST, true, 16 },
  371. };
  372. static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]);
  373. class Harness {
  374. public:
  375. Harness() : constructor_(NULL) { }
  376. void Init(const TestArgs& args) {
  377. delete constructor_;
  378. constructor_ = NULL;
  379. options_ = Options();
  380. options_.block_restart_interval = args.restart_interval;
  381. // Use shorter block size for tests to exercise block boundary
  382. // conditions more.
  383. options_.block_size = 256;
  384. if (args.reverse_compare) {
  385. options_.comparator = &reverse_key_comparator;
  386. }
  387. switch (args.type) {
  388. case TABLE_TEST:
  389. constructor_ = new TableConstructor(options_.comparator);
  390. break;
  391. case BLOCK_TEST:
  392. constructor_ = new BlockConstructor(options_.comparator);
  393. break;
  394. case MEMTABLE_TEST:
  395. constructor_ = new MemTableConstructor(options_.comparator);
  396. break;
  397. case DB_TEST:
  398. constructor_ = new DBConstructor(options_.comparator);
  399. break;
  400. }
  401. }
  402. ~Harness() {
  403. delete constructor_;
  404. }
  405. void Add(const std::string& key, const std::string& value) {
  406. constructor_->Add(key, value);
  407. }
  408. void Test(Random* rnd) {
  409. std::vector<std::string> keys;
  410. KVMap data;
  411. constructor_->Finish(options_, &keys, &data);
  412. TestForwardScan(keys, data);
  413. TestBackwardScan(keys, data);
  414. TestRandomAccess(rnd, keys, data);
  415. }
  416. void TestForwardScan(const std::vector<std::string>& keys,
  417. const KVMap& data) {
  418. Iterator* iter = constructor_->NewIterator();
  419. ASSERT_TRUE(!iter->Valid());
  420. iter->SeekToFirst();
  421. for (KVMap::const_iterator model_iter = data.begin();
  422. model_iter != data.end();
  423. ++model_iter) {
  424. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  425. iter->Next();
  426. }
  427. ASSERT_TRUE(!iter->Valid());
  428. delete iter;
  429. }
  430. void TestBackwardScan(const std::vector<std::string>& keys,
  431. const KVMap& data) {
  432. Iterator* iter = constructor_->NewIterator();
  433. ASSERT_TRUE(!iter->Valid());
  434. iter->SeekToLast();
  435. for (KVMap::const_reverse_iterator model_iter = data.rbegin();
  436. model_iter != data.rend();
  437. ++model_iter) {
  438. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  439. iter->Prev();
  440. }
  441. ASSERT_TRUE(!iter->Valid());
  442. delete iter;
  443. }
  444. void TestRandomAccess(Random* rnd,
  445. const std::vector<std::string>& keys,
  446. const KVMap& data) {
  447. static const bool kVerbose = false;
  448. Iterator* iter = constructor_->NewIterator();
  449. ASSERT_TRUE(!iter->Valid());
  450. KVMap::const_iterator model_iter = data.begin();
  451. if (kVerbose) fprintf(stderr, "---\n");
  452. for (int i = 0; i < 200; i++) {
  453. const int toss = rnd->Uniform(5);
  454. switch (toss) {
  455. case 0: {
  456. if (iter->Valid()) {
  457. if (kVerbose) fprintf(stderr, "Next\n");
  458. iter->Next();
  459. ++model_iter;
  460. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  461. }
  462. break;
  463. }
  464. case 1: {
  465. if (kVerbose) fprintf(stderr, "SeekToFirst\n");
  466. iter->SeekToFirst();
  467. model_iter = data.begin();
  468. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  469. break;
  470. }
  471. case 2: {
  472. std::string key = PickRandomKey(rnd, keys);
  473. model_iter = data.lower_bound(key);
  474. if (kVerbose) fprintf(stderr, "Seek '%s'\n",
  475. EscapeString(key).c_str());
  476. iter->Seek(Slice(key));
  477. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  478. break;
  479. }
  480. case 3: {
  481. if (iter->Valid()) {
  482. if (kVerbose) fprintf(stderr, "Prev\n");
  483. iter->Prev();
  484. if (model_iter == data.begin()) {
  485. model_iter = data.end(); // Wrap around to invalid value
  486. } else {
  487. --model_iter;
  488. }
  489. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  490. }
  491. break;
  492. }
  493. case 4: {
  494. if (kVerbose) fprintf(stderr, "SeekToLast\n");
  495. iter->SeekToLast();
  496. if (keys.empty()) {
  497. model_iter = data.end();
  498. } else {
  499. std::string last = data.rbegin()->first;
  500. model_iter = data.lower_bound(last);
  501. }
  502. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  503. break;
  504. }
  505. }
  506. }
  507. delete iter;
  508. }
  509. std::string ToString(const KVMap& data, const KVMap::const_iterator& it) {
  510. if (it == data.end()) {
  511. return "END";
  512. } else {
  513. return "'" + it->first + "->" + it->second + "'";
  514. }
  515. }
  516. std::string ToString(const KVMap& data,
  517. const KVMap::const_reverse_iterator& it) {
  518. if (it == data.rend()) {
  519. return "END";
  520. } else {
  521. return "'" + it->first + "->" + it->second + "'";
  522. }
  523. }
  524. std::string ToString(const Iterator* it) {
  525. if (!it->Valid()) {
  526. return "END";
  527. } else {
  528. return "'" + it->key().ToString() + "->" + it->value().ToString() + "'";
  529. }
  530. }
  531. std::string PickRandomKey(Random* rnd, const std::vector<std::string>& keys) {
  532. if (keys.empty()) {
  533. return "foo";
  534. } else {
  535. const int index = rnd->Uniform(keys.size());
  536. std::string result = keys[index];
  537. switch (rnd->Uniform(3)) {
  538. case 0:
  539. // Return an existing key
  540. break;
  541. case 1: {
  542. // Attempt to return something smaller than an existing key
  543. if (result.size() > 0 && result[result.size()-1] > '\0') {
  544. result[result.size()-1]--;
  545. }
  546. break;
  547. }
  548. case 2: {
  549. // Return something larger than an existing key
  550. Increment(options_.comparator, &result);
  551. break;
  552. }
  553. }
  554. return result;
  555. }
  556. }
  557. // Returns NULL if not running against a DB
  558. DB* db() const { return constructor_->db(); }
  559. private:
  560. Options options_;
  561. Constructor* constructor_;
  562. };
  563. // Test empty table/block.
  564. TEST(Harness, Empty) {
  565. for (int i = 0; i < kNumTestArgs; i++) {
  566. Init(kTestArgList[i]);
  567. Random rnd(test::RandomSeed() + 1);
  568. Test(&rnd);
  569. }
  570. }
  571. // Special test for a block with no restart entries. The C++ leveldb
  572. // code never generates such blocks, but the Java version of leveldb
  573. // seems to.
  574. TEST(Harness, ZeroRestartPointsInBlock) {
  575. char data[sizeof(uint32_t)];
  576. memset(data, 0, sizeof(data));
  577. BlockContents contents;
  578. contents.data = Slice(data, sizeof(data));
  579. contents.cachable = false;
  580. contents.heap_allocated = false;
  581. Block block(contents);
  582. Iterator* iter = block.NewIterator(BytewiseComparator());
  583. iter->SeekToFirst();
  584. ASSERT_TRUE(!iter->Valid());
  585. iter->SeekToLast();
  586. ASSERT_TRUE(!iter->Valid());
  587. iter->Seek("foo");
  588. ASSERT_TRUE(!iter->Valid());
  589. delete iter;
  590. }
  591. // Test the empty key
  592. TEST(Harness, SimpleEmptyKey) {
  593. for (int i = 0; i < kNumTestArgs; i++) {
  594. Init(kTestArgList[i]);
  595. Random rnd(test::RandomSeed() + 1);
  596. Add("", "v");
  597. Test(&rnd);
  598. }
  599. }
  600. TEST(Harness, SimpleSingle) {
  601. for (int i = 0; i < kNumTestArgs; i++) {
  602. Init(kTestArgList[i]);
  603. Random rnd(test::RandomSeed() + 2);
  604. Add("abc", "v");
  605. Test(&rnd);
  606. }
  607. }
  608. TEST(Harness, SimpleMulti) {
  609. for (int i = 0; i < kNumTestArgs; i++) {
  610. Init(kTestArgList[i]);
  611. Random rnd(test::RandomSeed() + 3);
  612. Add("abc", "v");
  613. Add("abcd", "v");
  614. Add("ac", "v2");
  615. Test(&rnd);
  616. }
  617. }
  618. TEST(Harness, SimpleSpecialKey) {
  619. for (int i = 0; i < kNumTestArgs; i++) {
  620. Init(kTestArgList[i]);
  621. Random rnd(test::RandomSeed() + 4);
  622. Add("\xff\xff", "v3");
  623. Test(&rnd);
  624. }
  625. }
  626. TEST(Harness, Randomized) {
  627. for (int i = 0; i < kNumTestArgs; i++) {
  628. Init(kTestArgList[i]);
  629. Random rnd(test::RandomSeed() + 5);
  630. for (int num_entries = 0; num_entries < 2000;
  631. num_entries += (num_entries < 50 ? 1 : 200)) {
  632. if ((num_entries % 10) == 0) {
  633. fprintf(stderr, "case %d of %d: num_entries = %d\n",
  634. (i + 1), int(kNumTestArgs), num_entries);
  635. }
  636. for (int e = 0; e < num_entries; e++) {
  637. std::string v;
  638. Add(test::RandomKey(&rnd, rnd.Skewed(4)),
  639. test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
  640. }
  641. Test(&rnd);
  642. }
  643. }
  644. }
  645. TEST(Harness, RandomizedLongDB) {
  646. Random rnd(test::RandomSeed());
  647. TestArgs args = { DB_TEST, false, 16 };
  648. Init(args);
  649. int num_entries = 100000;
  650. for (int e = 0; e < num_entries; e++) {
  651. std::string v;
  652. Add(test::RandomKey(&rnd, rnd.Skewed(4)),
  653. test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
  654. }
  655. Test(&rnd);
  656. // We must have created enough data to force merging
  657. int files = 0;
  658. for (int level = 0; level < config::kNumLevels; level++) {
  659. std::string value;
  660. char name[100];
  661. snprintf(name, sizeof(name), "leveldb.num-files-at-level%d", level);
  662. ASSERT_TRUE(db()->GetProperty(name, &value));
  663. files += atoi(value.c_str());
  664. }
  665. ASSERT_GT(files, 0);
  666. }
  667. class MemTableTest { };
  668. TEST(MemTableTest, Simple) {
  669. InternalKeyComparator cmp(BytewiseComparator());
  670. MemTable* memtable = new MemTable(cmp);
  671. memtable->Ref();
  672. WriteBatch batch;
  673. WriteBatchInternal::SetSequence(&batch, 100);
  674. batch.Put(std::string("k1"), std::string("v1"));
  675. batch.Put(std::string("k2"), std::string("v2"));
  676. batch.Put(std::string("k3"), std::string("v3"));
  677. batch.Put(std::string("largekey"), std::string("vlarge"));
  678. ASSERT_TRUE(WriteBatchInternal::InsertInto(&batch, memtable).ok());
  679. Iterator* iter = memtable->NewIterator();
  680. iter->SeekToFirst();
  681. while (iter->Valid()) {
  682. fprintf(stderr, "key: '%s' -> '%s'\n",
  683. iter->key().ToString().c_str(),
  684. iter->value().ToString().c_str());
  685. iter->Next();
  686. }
  687. delete iter;
  688. memtable->Unref();
  689. }
  690. static bool Between(uint64_t val, uint64_t low, uint64_t high) {
  691. bool result = (val >= low) && (val <= high);
  692. if (!result) {
  693. fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
  694. (unsigned long long)(val),
  695. (unsigned long long)(low),
  696. (unsigned long long)(high));
  697. }
  698. return result;
  699. }
  700. class TableTest { };
  701. TEST(TableTest, ApproximateOffsetOfPlain) {
  702. TableConstructor c(BytewiseComparator());
  703. c.Add("k01", "hello");
  704. c.Add("k02", "hello2");
  705. c.Add("k03", std::string(10000, 'x'));
  706. c.Add("k04", std::string(200000, 'x'));
  707. c.Add("k05", std::string(300000, 'x'));
  708. c.Add("k06", "hello3");
  709. c.Add("k07", std::string(100000, 'x'));
  710. std::vector<std::string> keys;
  711. KVMap kvmap;
  712. Options options;
  713. options.block_size = 1024;
  714. options.compression = kNoCompression;
  715. c.Finish(options, &keys, &kvmap);
  716. ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
  717. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
  718. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01a"), 0, 0));
  719. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
  720. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 0, 0));
  721. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 10000, 11000));
  722. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04a"), 210000, 211000));
  723. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k05"), 210000, 211000));
  724. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000));
  725. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000));
  726. ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 612000));
  727. }
  728. static bool SnappyCompressionSupported() {
  729. std::string out;
  730. Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
  731. return port::Snappy_Compress(in.data(), in.size(), &out);
  732. }
  733. TEST(TableTest, ApproximateOffsetOfCompressed) {
  734. if (!SnappyCompressionSupported()) {
  735. fprintf(stderr, "skipping compression tests\n");
  736. return;
  737. }
  738. Random rnd(301);
  739. TableConstructor c(BytewiseComparator());
  740. std::string tmp;
  741. c.Add("k01", "hello");
  742. c.Add("k02", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
  743. c.Add("k03", "hello3");
  744. c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
  745. std::vector<std::string> keys;
  746. KVMap kvmap;
  747. Options options;
  748. options.block_size = 1024;
  749. options.compression = kSnappyCompression;
  750. c.Finish(options, &keys, &kvmap);
  751. // Expected upper and lower bounds of space used by compressible strings.
  752. static const int kSlop = 1000; // Compressor effectiveness varies.
  753. const int expected = 2500; // 10000 * compression ratio (0.25)
  754. const int min_z = expected - kSlop;
  755. const int max_z = expected + kSlop;
  756. ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, kSlop));
  757. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, kSlop));
  758. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, kSlop));
  759. // Have now emitted a large compressible string, so adjust expected offset.
  760. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), min_z, max_z));
  761. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), min_z, max_z));
  762. // Have now emitted two large compressible strings, so adjust expected offset.
  763. ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 2 * min_z, 2 * max_z));
  764. }
  765. } // namespace leveldb
  766. int main(int argc, char** argv) {
  767. return leveldb::test::RunAllTests();
  768. }