You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

849 lines
23 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "leveldb/table.h"
  5. #include <map>
  6. #include "db/dbformat.h"
  7. #include "db/memtable.h"
  8. #include "db/write_batch_internal.h"
  9. #include "leveldb/db.h"
  10. #include "leveldb/env.h"
  11. #include "leveldb/iterator.h"
  12. #include "leveldb/table_builder.h"
  13. #include "table/block.h"
  14. #include "table/block_builder.h"
  15. #include "table/format.h"
  16. #include "util/random.h"
  17. #include "util/testharness.h"
  18. #include "util/testutil.h"
  19. namespace leveldb {
  20. // Return reverse of "key".
  21. // Used to test non-lexicographic comparators.
  22. static std::string Reverse(const Slice& key) {
  23. std::string str(key.ToString());
  24. std::string rev(str.rbegin(), str.rend());
  25. return rev;
  26. }
  27. namespace {
  28. class ReverseKeyComparator : public Comparator {
  29. public:
  30. virtual const char* Name() const {
  31. return "leveldb.ReverseBytewiseComparator";
  32. }
  33. virtual int Compare(const Slice& a, const Slice& b) const {
  34. return BytewiseComparator()->Compare(Reverse(a), Reverse(b));
  35. }
  36. virtual void FindShortestSeparator(
  37. std::string* start,
  38. const Slice& limit) const {
  39. std::string s = Reverse(*start);
  40. std::string l = Reverse(limit);
  41. BytewiseComparator()->FindShortestSeparator(&s, l);
  42. *start = Reverse(s);
  43. }
  44. virtual void FindShortSuccessor(std::string* key) const {
  45. std::string s = Reverse(*key);
  46. BytewiseComparator()->FindShortSuccessor(&s);
  47. *key = Reverse(s);
  48. }
  49. };
  50. }
  51. static ReverseKeyComparator reverse_key_comparator;
  52. static void Increment(const Comparator* cmp, std::string* key) {
  53. if (cmp == BytewiseComparator()) {
  54. key->push_back('\0');
  55. } else {
  56. assert(cmp == &reverse_key_comparator);
  57. std::string rev = Reverse(*key);
  58. rev.push_back('\0');
  59. *key = Reverse(rev);
  60. }
  61. }
  62. // An STL comparator that uses a Comparator
  63. namespace {
  64. struct STLLessThan {
  65. const Comparator* cmp;
  66. STLLessThan() : cmp(BytewiseComparator()) { }
  67. STLLessThan(const Comparator* c) : cmp(c) { }
  68. bool operator()(const std::string& a, const std::string& b) const {
  69. return cmp->Compare(Slice(a), Slice(b)) < 0;
  70. }
  71. };
  72. }
  73. class StringSink: public WritableFile {
  74. public:
  75. ~StringSink() { }
  76. const std::string& contents() const { return contents_; }
  77. virtual Status Close() { return Status::OK(); }
  78. virtual Status Flush() { return Status::OK(); }
  79. virtual Status Sync() { return Status::OK(); }
  80. virtual Status Append(const Slice& data) {
  81. contents_.append(data.data(), data.size());
  82. return Status::OK();
  83. }
  84. private:
  85. std::string contents_;
  86. };
  87. class StringSource: public RandomAccessFile {
  88. public:
  89. StringSource(const Slice& contents)
  90. : contents_(contents.data(), contents.size()) {
  91. }
  92. virtual ~StringSource() { }
  93. uint64_t Size() const { return contents_.size(); }
  94. virtual Status Read(uint64_t offset, size_t n, Slice* result,
  95. char* scratch) const {
  96. if (offset > contents_.size()) {
  97. return Status::InvalidArgument("invalid Read offset");
  98. }
  99. if (offset + n > contents_.size()) {
  100. n = contents_.size() - offset;
  101. }
  102. memcpy(scratch, &contents_[offset], n);
  103. *result = Slice(scratch, n);
  104. return Status::OK();
  105. }
  106. private:
  107. std::string contents_;
  108. };
  109. typedef std::map<std::string, std::string, STLLessThan> KVMap;
  110. // Helper class for tests to unify the interface between
  111. // BlockBuilder/TableBuilder and Block/Table.
  112. class Constructor {
  113. public:
  114. explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) { }
  115. virtual ~Constructor() { }
  116. void Add(const std::string& key, const Slice& value) {
  117. data_[key] = value.ToString();
  118. }
  119. // Finish constructing the data structure with all the keys that have
  120. // been added so far. Returns the keys in sorted order in "*keys"
  121. // and stores the key/value pairs in "*kvmap"
  122. void Finish(const Options& options,
  123. std::vector<std::string>* keys,
  124. KVMap* kvmap) {
  125. *kvmap = data_;
  126. keys->clear();
  127. for (KVMap::const_iterator it = data_.begin();
  128. it != data_.end();
  129. ++it) {
  130. keys->push_back(it->first);
  131. }
  132. data_.clear();
  133. Status s = FinishImpl(options, *kvmap);
  134. ASSERT_TRUE(s.ok()) << s.ToString();
  135. }
  136. // Construct the data structure from the data in "data"
  137. virtual Status FinishImpl(const Options& options, const KVMap& data) = 0;
  138. virtual size_t NumBytes() const = 0;
  139. virtual Iterator* NewIterator() const = 0;
  140. virtual const KVMap& data() { return data_; }
  141. virtual DB* db() const { return NULL; } // Overridden in DBConstructor
  142. private:
  143. KVMap data_;
  144. };
  145. class BlockConstructor: public Constructor {
  146. public:
  147. explicit BlockConstructor(const Comparator* cmp)
  148. : Constructor(cmp),
  149. comparator_(cmp),
  150. block_size_(-1),
  151. block_(NULL) { }
  152. ~BlockConstructor() {
  153. delete block_;
  154. }
  155. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  156. delete block_;
  157. block_ = NULL;
  158. BlockBuilder builder(&options);
  159. for (KVMap::const_iterator it = data.begin();
  160. it != data.end();
  161. ++it) {
  162. builder.Add(it->first, it->second);
  163. }
  164. // Open the block
  165. Slice block_data = builder.Finish();
  166. block_size_ = block_data.size();
  167. char* block_data_copy = new char[block_size_];
  168. memcpy(block_data_copy, block_data.data(), block_size_);
  169. block_ = new Block(block_data_copy, block_size_);
  170. return Status::OK();
  171. }
  172. virtual size_t NumBytes() const { return block_size_; }
  173. virtual Iterator* NewIterator() const {
  174. return block_->NewIterator(comparator_);
  175. }
  176. private:
  177. const Comparator* comparator_;
  178. int block_size_;
  179. Block* block_;
  180. BlockConstructor();
  181. };
  182. class TableConstructor: public Constructor {
  183. public:
  184. TableConstructor(const Comparator* cmp)
  185. : Constructor(cmp),
  186. source_(NULL), table_(NULL) {
  187. }
  188. ~TableConstructor() {
  189. Reset();
  190. }
  191. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  192. Reset();
  193. StringSink sink;
  194. TableBuilder builder(options, &sink);
  195. for (KVMap::const_iterator it = data.begin();
  196. it != data.end();
  197. ++it) {
  198. builder.Add(it->first, it->second);
  199. ASSERT_TRUE(builder.status().ok());
  200. }
  201. Status s = builder.Finish();
  202. ASSERT_TRUE(s.ok()) << s.ToString();
  203. ASSERT_EQ(sink.contents().size(), builder.FileSize());
  204. // Open the table
  205. source_ = new StringSource(sink.contents());
  206. Options table_options;
  207. table_options.comparator = options.comparator;
  208. return Table::Open(table_options, source_, sink.contents().size(), &table_);
  209. }
  210. virtual size_t NumBytes() const { return source_->Size(); }
  211. virtual Iterator* NewIterator() const {
  212. return table_->NewIterator(ReadOptions());
  213. }
  214. uint64_t ApproximateOffsetOf(const Slice& key) const {
  215. return table_->ApproximateOffsetOf(key);
  216. }
  217. private:
  218. void Reset() {
  219. delete table_;
  220. delete source_;
  221. table_ = NULL;
  222. source_ = NULL;
  223. }
  224. StringSource* source_;
  225. Table* table_;
  226. TableConstructor();
  227. };
  228. // A helper class that converts internal format keys into user keys
  229. class KeyConvertingIterator: public Iterator {
  230. public:
  231. explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) { }
  232. virtual ~KeyConvertingIterator() { delete iter_; }
  233. virtual bool Valid() const { return iter_->Valid(); }
  234. virtual void Seek(const Slice& target) {
  235. ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue);
  236. std::string encoded;
  237. AppendInternalKey(&encoded, ikey);
  238. iter_->Seek(encoded);
  239. }
  240. virtual void SeekToFirst() { iter_->SeekToFirst(); }
  241. virtual void SeekToLast() { iter_->SeekToLast(); }
  242. virtual void Next() { iter_->Next(); }
  243. virtual void Prev() { iter_->Prev(); }
  244. virtual Slice key() const {
  245. assert(Valid());
  246. ParsedInternalKey key;
  247. if (!ParseInternalKey(iter_->key(), &key)) {
  248. status_ = Status::Corruption("malformed internal key");
  249. return Slice("corrupted key");
  250. }
  251. return key.user_key;
  252. }
  253. virtual Slice value() const { return iter_->value(); }
  254. virtual Status status() const {
  255. return status_.ok() ? iter_->status() : status_;
  256. }
  257. private:
  258. mutable Status status_;
  259. Iterator* iter_;
  260. // No copying allowed
  261. KeyConvertingIterator(const KeyConvertingIterator&);
  262. void operator=(const KeyConvertingIterator&);
  263. };
  264. class MemTableConstructor: public Constructor {
  265. public:
  266. explicit MemTableConstructor(const Comparator* cmp)
  267. : Constructor(cmp),
  268. internal_comparator_(cmp) {
  269. memtable_ = new MemTable(internal_comparator_);
  270. memtable_->Ref();
  271. }
  272. ~MemTableConstructor() {
  273. memtable_->Unref();
  274. }
  275. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  276. memtable_->Unref();
  277. memtable_ = new MemTable(internal_comparator_);
  278. memtable_->Ref();
  279. int seq = 1;
  280. for (KVMap::const_iterator it = data.begin();
  281. it != data.end();
  282. ++it) {
  283. memtable_->Add(seq, kTypeValue, it->first, it->second);
  284. seq++;
  285. }
  286. return Status::OK();
  287. }
  288. virtual size_t NumBytes() const {
  289. return memtable_->ApproximateMemoryUsage();
  290. }
  291. virtual Iterator* NewIterator() const {
  292. return new KeyConvertingIterator(memtable_->NewIterator());
  293. }
  294. private:
  295. InternalKeyComparator internal_comparator_;
  296. MemTable* memtable_;
  297. };
  298. class DBConstructor: public Constructor {
  299. public:
  300. explicit DBConstructor(const Comparator* cmp)
  301. : Constructor(cmp),
  302. comparator_(cmp) {
  303. db_ = NULL;
  304. NewDB();
  305. }
  306. ~DBConstructor() {
  307. delete db_;
  308. }
  309. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  310. delete db_;
  311. db_ = NULL;
  312. NewDB();
  313. for (KVMap::const_iterator it = data.begin();
  314. it != data.end();
  315. ++it) {
  316. WriteBatch batch;
  317. batch.Put(it->first, it->second);
  318. ASSERT_TRUE(db_->Write(WriteOptions(), &batch).ok());
  319. }
  320. return Status::OK();
  321. }
  322. virtual size_t NumBytes() const {
  323. Range r("", "\xff\xff");
  324. uint64_t size;
  325. db_->GetApproximateSizes(&r, 1, &size);
  326. return size;
  327. }
  328. virtual Iterator* NewIterator() const {
  329. return db_->NewIterator(ReadOptions());
  330. }
  331. virtual DB* db() const { return db_; }
  332. private:
  333. void NewDB() {
  334. std::string name = test::TmpDir() + "/table_testdb";
  335. Options options;
  336. options.comparator = comparator_;
  337. Status status = DestroyDB(name, options);
  338. ASSERT_TRUE(status.ok()) << status.ToString();
  339. options.create_if_missing = true;
  340. options.error_if_exists = true;
  341. options.write_buffer_size = 10000; // Something small to force merging
  342. status = DB::Open(options, name, &db_);
  343. ASSERT_TRUE(status.ok()) << status.ToString();
  344. }
  345. const Comparator* comparator_;
  346. DB* db_;
  347. };
  348. enum TestType {
  349. TABLE_TEST,
  350. BLOCK_TEST,
  351. MEMTABLE_TEST,
  352. DB_TEST,
  353. };
  354. struct TestArgs {
  355. TestType type;
  356. bool reverse_compare;
  357. int restart_interval;
  358. };
  359. static const TestArgs kTestArgList[] = {
  360. { TABLE_TEST, false, 16 },
  361. { TABLE_TEST, false, 1 },
  362. { TABLE_TEST, false, 1024 },
  363. { TABLE_TEST, true, 16 },
  364. { TABLE_TEST, true, 1 },
  365. { TABLE_TEST, true, 1024 },
  366. { BLOCK_TEST, false, 16 },
  367. { BLOCK_TEST, false, 1 },
  368. { BLOCK_TEST, false, 1024 },
  369. { BLOCK_TEST, true, 16 },
  370. { BLOCK_TEST, true, 1 },
  371. { BLOCK_TEST, true, 1024 },
  372. // Restart interval does not matter for memtables
  373. { MEMTABLE_TEST, false, 16 },
  374. { MEMTABLE_TEST, true, 16 },
  375. // Do not bother with restart interval variations for DB
  376. { DB_TEST, false, 16 },
  377. { DB_TEST, true, 16 },
  378. };
  379. static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]);
  380. class Harness {
  381. public:
  382. Harness() : constructor_(NULL) { }
  383. void Init(const TestArgs& args) {
  384. delete constructor_;
  385. constructor_ = NULL;
  386. options_ = Options();
  387. options_.block_restart_interval = args.restart_interval;
  388. // Use shorter block size for tests to exercise block boundary
  389. // conditions more.
  390. options_.block_size = 256;
  391. if (args.reverse_compare) {
  392. options_.comparator = &reverse_key_comparator;
  393. }
  394. switch (args.type) {
  395. case TABLE_TEST:
  396. constructor_ = new TableConstructor(options_.comparator);
  397. break;
  398. case BLOCK_TEST:
  399. constructor_ = new BlockConstructor(options_.comparator);
  400. break;
  401. case MEMTABLE_TEST:
  402. constructor_ = new MemTableConstructor(options_.comparator);
  403. break;
  404. case DB_TEST:
  405. constructor_ = new DBConstructor(options_.comparator);
  406. break;
  407. }
  408. }
  409. ~Harness() {
  410. delete constructor_;
  411. }
  412. void Add(const std::string& key, const std::string& value) {
  413. constructor_->Add(key, value);
  414. }
  415. void Test(Random* rnd) {
  416. std::vector<std::string> keys;
  417. KVMap data;
  418. constructor_->Finish(options_, &keys, &data);
  419. TestForwardScan(keys, data);
  420. TestBackwardScan(keys, data);
  421. TestRandomAccess(rnd, keys, data);
  422. }
  423. void TestForwardScan(const std::vector<std::string>& keys,
  424. const KVMap& data) {
  425. Iterator* iter = constructor_->NewIterator();
  426. ASSERT_TRUE(!iter->Valid());
  427. iter->SeekToFirst();
  428. for (KVMap::const_iterator model_iter = data.begin();
  429. model_iter != data.end();
  430. ++model_iter) {
  431. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  432. iter->Next();
  433. }
  434. ASSERT_TRUE(!iter->Valid());
  435. delete iter;
  436. }
  437. void TestBackwardScan(const std::vector<std::string>& keys,
  438. const KVMap& data) {
  439. Iterator* iter = constructor_->NewIterator();
  440. ASSERT_TRUE(!iter->Valid());
  441. iter->SeekToLast();
  442. for (KVMap::const_reverse_iterator model_iter = data.rbegin();
  443. model_iter != data.rend();
  444. ++model_iter) {
  445. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  446. iter->Prev();
  447. }
  448. ASSERT_TRUE(!iter->Valid());
  449. delete iter;
  450. }
  451. void TestRandomAccess(Random* rnd,
  452. const std::vector<std::string>& keys,
  453. const KVMap& data) {
  454. static const bool kVerbose = false;
  455. Iterator* iter = constructor_->NewIterator();
  456. ASSERT_TRUE(!iter->Valid());
  457. KVMap::const_iterator model_iter = data.begin();
  458. if (kVerbose) fprintf(stderr, "---\n");
  459. for (int i = 0; i < 200; i++) {
  460. const int toss = rnd->Uniform(5);
  461. switch (toss) {
  462. case 0: {
  463. if (iter->Valid()) {
  464. if (kVerbose) fprintf(stderr, "Next\n");
  465. iter->Next();
  466. ++model_iter;
  467. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  468. }
  469. break;
  470. }
  471. case 1: {
  472. if (kVerbose) fprintf(stderr, "SeekToFirst\n");
  473. iter->SeekToFirst();
  474. model_iter = data.begin();
  475. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  476. break;
  477. }
  478. case 2: {
  479. std::string key = PickRandomKey(rnd, keys);
  480. model_iter = data.lower_bound(key);
  481. if (kVerbose) fprintf(stderr, "Seek '%s'\n",
  482. EscapeString(key).c_str());
  483. iter->Seek(Slice(key));
  484. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  485. break;
  486. }
  487. case 3: {
  488. if (iter->Valid()) {
  489. if (kVerbose) fprintf(stderr, "Prev\n");
  490. iter->Prev();
  491. if (model_iter == data.begin()) {
  492. model_iter = data.end(); // Wrap around to invalid value
  493. } else {
  494. --model_iter;
  495. }
  496. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  497. }
  498. break;
  499. }
  500. case 4: {
  501. if (kVerbose) fprintf(stderr, "SeekToLast\n");
  502. iter->SeekToLast();
  503. if (keys.empty()) {
  504. model_iter = data.end();
  505. } else {
  506. std::string last = data.rbegin()->first;
  507. model_iter = data.lower_bound(last);
  508. }
  509. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  510. break;
  511. }
  512. }
  513. }
  514. delete iter;
  515. }
  516. std::string ToString(const KVMap& data, const KVMap::const_iterator& it) {
  517. if (it == data.end()) {
  518. return "END";
  519. } else {
  520. return "'" + it->first + "->" + it->second + "'";
  521. }
  522. }
  523. std::string ToString(const KVMap& data,
  524. const KVMap::const_reverse_iterator& it) {
  525. if (it == data.rend()) {
  526. return "END";
  527. } else {
  528. return "'" + it->first + "->" + it->second + "'";
  529. }
  530. }
  531. std::string ToString(const Iterator* it) {
  532. if (!it->Valid()) {
  533. return "END";
  534. } else {
  535. return "'" + it->key().ToString() + "->" + it->value().ToString() + "'";
  536. }
  537. }
  538. std::string PickRandomKey(Random* rnd, const std::vector<std::string>& keys) {
  539. if (keys.empty()) {
  540. return "foo";
  541. } else {
  542. const int index = rnd->Uniform(keys.size());
  543. std::string result = keys[index];
  544. switch (rnd->Uniform(3)) {
  545. case 0:
  546. // Return an existing key
  547. break;
  548. case 1: {
  549. // Attempt to return something smaller than an existing key
  550. if (result.size() > 0 && result[result.size()-1] > '\0') {
  551. result[result.size()-1]--;
  552. }
  553. break;
  554. }
  555. case 2: {
  556. // Return something larger than an existing key
  557. Increment(options_.comparator, &result);
  558. break;
  559. }
  560. }
  561. return result;
  562. }
  563. }
  564. // Returns NULL if not running against a DB
  565. DB* db() const { return constructor_->db(); }
  566. private:
  567. Options options_;
  568. Constructor* constructor_;
  569. };
  570. // Test the empty key
  571. TEST(Harness, SimpleEmptyKey) {
  572. for (int i = 0; i < kNumTestArgs; i++) {
  573. Init(kTestArgList[i]);
  574. Random rnd(test::RandomSeed() + 1);
  575. Add("", "v");
  576. Test(&rnd);
  577. }
  578. }
  579. TEST(Harness, SimpleSingle) {
  580. for (int i = 0; i < kNumTestArgs; i++) {
  581. Init(kTestArgList[i]);
  582. Random rnd(test::RandomSeed() + 2);
  583. Add("abc", "v");
  584. Test(&rnd);
  585. }
  586. }
  587. TEST(Harness, SimpleMulti) {
  588. for (int i = 0; i < kNumTestArgs; i++) {
  589. Init(kTestArgList[i]);
  590. Random rnd(test::RandomSeed() + 3);
  591. Add("abc", "v");
  592. Add("abcd", "v");
  593. Add("ac", "v2");
  594. Test(&rnd);
  595. }
  596. }
  597. TEST(Harness, SimpleSpecialKey) {
  598. for (int i = 0; i < kNumTestArgs; i++) {
  599. Init(kTestArgList[i]);
  600. Random rnd(test::RandomSeed() + 4);
  601. Add("\xff\xff", "v3");
  602. Test(&rnd);
  603. }
  604. }
  605. TEST(Harness, Randomized) {
  606. for (int i = 0; i < kNumTestArgs; i++) {
  607. Init(kTestArgList[i]);
  608. Random rnd(test::RandomSeed() + 5);
  609. for (int num_entries = 0; num_entries < 2000;
  610. num_entries += (num_entries < 50 ? 1 : 200)) {
  611. if ((num_entries % 10) == 0) {
  612. fprintf(stderr, "case %d of %d: num_entries = %d\n",
  613. (i + 1), int(kNumTestArgs), num_entries);
  614. }
  615. for (int e = 0; e < num_entries; e++) {
  616. std::string v;
  617. Add(test::RandomKey(&rnd, rnd.Skewed(4)),
  618. test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
  619. }
  620. Test(&rnd);
  621. }
  622. }
  623. }
  624. TEST(Harness, RandomizedLongDB) {
  625. Random rnd(test::RandomSeed());
  626. TestArgs args = { DB_TEST, false, 16 };
  627. Init(args);
  628. int num_entries = 100000;
  629. for (int e = 0; e < num_entries; e++) {
  630. std::string v;
  631. Add(test::RandomKey(&rnd, rnd.Skewed(4)),
  632. test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
  633. }
  634. Test(&rnd);
  635. // We must have created enough data to force merging
  636. int files = 0;
  637. for (int level = 0; level < config::kNumLevels; level++) {
  638. std::string value;
  639. char name[100];
  640. snprintf(name, sizeof(name), "leveldb.num-files-at-level%d", level);
  641. ASSERT_TRUE(db()->GetProperty(name, &value));
  642. files += atoi(value.c_str());
  643. }
  644. ASSERT_GT(files, 0);
  645. }
  646. class MemTableTest { };
  647. TEST(MemTableTest, Simple) {
  648. InternalKeyComparator cmp(BytewiseComparator());
  649. MemTable* memtable = new MemTable(cmp);
  650. memtable->Ref();
  651. WriteBatch batch;
  652. WriteBatchInternal::SetSequence(&batch, 100);
  653. batch.Put(std::string("k1"), std::string("v1"));
  654. batch.Put(std::string("k2"), std::string("v2"));
  655. batch.Put(std::string("k3"), std::string("v3"));
  656. batch.Put(std::string("largekey"), std::string("vlarge"));
  657. ASSERT_TRUE(WriteBatchInternal::InsertInto(&batch, memtable).ok());
  658. Iterator* iter = memtable->NewIterator();
  659. iter->SeekToFirst();
  660. while (iter->Valid()) {
  661. fprintf(stderr, "key: '%s' -> '%s'\n",
  662. iter->key().ToString().c_str(),
  663. iter->value().ToString().c_str());
  664. iter->Next();
  665. }
  666. delete iter;
  667. memtable->Unref();
  668. }
  669. static bool Between(uint64_t val, uint64_t low, uint64_t high) {
  670. bool result = (val >= low) && (val <= high);
  671. if (!result) {
  672. fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
  673. (unsigned long long)(val),
  674. (unsigned long long)(low),
  675. (unsigned long long)(high));
  676. }
  677. return result;
  678. }
  679. class TableTest { };
  680. TEST(TableTest, ApproximateOffsetOfPlain) {
  681. TableConstructor c(BytewiseComparator());
  682. c.Add("k01", "hello");
  683. c.Add("k02", "hello2");
  684. c.Add("k03", std::string(10000, 'x'));
  685. c.Add("k04", std::string(200000, 'x'));
  686. c.Add("k05", std::string(300000, 'x'));
  687. c.Add("k06", "hello3");
  688. c.Add("k07", std::string(100000, 'x'));
  689. std::vector<std::string> keys;
  690. KVMap kvmap;
  691. Options options;
  692. options.block_size = 1024;
  693. options.compression = kNoCompression;
  694. c.Finish(options, &keys, &kvmap);
  695. ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
  696. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
  697. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01a"), 0, 0));
  698. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
  699. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 0, 0));
  700. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 10000, 11000));
  701. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04a"), 210000, 211000));
  702. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k05"), 210000, 211000));
  703. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000));
  704. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000));
  705. ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 611000));
  706. }
  707. static bool SnappyCompressionSupported() {
  708. std::string out;
  709. Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
  710. return port::Snappy_Compress(in.data(), in.size(), &out);
  711. }
  712. TEST(TableTest, ApproximateOffsetOfCompressed) {
  713. if (!SnappyCompressionSupported()) {
  714. fprintf(stderr, "skipping compression tests\n");
  715. return;
  716. }
  717. Random rnd(301);
  718. TableConstructor c(BytewiseComparator());
  719. std::string tmp;
  720. c.Add("k01", "hello");
  721. c.Add("k02", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
  722. c.Add("k03", "hello3");
  723. c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
  724. std::vector<std::string> keys;
  725. KVMap kvmap;
  726. Options options;
  727. options.block_size = 1024;
  728. options.compression = kSnappyCompression;
  729. c.Finish(options, &keys, &kvmap);
  730. ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
  731. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
  732. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
  733. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 2000, 3000));
  734. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 2000, 3000));
  735. ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 4000, 6000));
  736. }
  737. }
  738. int main(int argc, char** argv) {
  739. return leveldb::test::RunAllTests();
  740. }