You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

854 lines
23 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "leveldb/table.h"
  5. #include <map>
  6. #include <string>
  7. #include "db/dbformat.h"
  8. #include "db/memtable.h"
  9. #include "db/write_batch_internal.h"
  10. #include "leveldb/db.h"
  11. #include "leveldb/env.h"
  12. #include "leveldb/iterator.h"
  13. #include "leveldb/table_builder.h"
  14. #include "table/block.h"
  15. #include "table/block_builder.h"
  16. #include "table/format.h"
  17. #include "util/random.h"
  18. #include "util/testharness.h"
  19. #include "util/testutil.h"
  20. namespace leveldb {
  21. // Return reverse of "key".
  22. // Used to test non-lexicographic comparators.
  23. static std::string Reverse(const Slice& key) {
  24. std::string str(key.ToString());
  25. std::string rev("");
  26. for (std::string::reverse_iterator rit = str.rbegin();
  27. rit != str.rend(); ++rit) {
  28. rev.push_back(*rit);
  29. }
  30. return rev;
  31. }
  32. namespace {
  33. class ReverseKeyComparator : public Comparator {
  34. public:
  35. virtual const char* Name() const {
  36. return "leveldb.ReverseBytewiseComparator";
  37. }
  38. virtual int Compare(const Slice& a, const Slice& b) const {
  39. return BytewiseComparator()->Compare(Reverse(a), Reverse(b));
  40. }
  41. virtual void FindShortestSeparator(
  42. std::string* start,
  43. const Slice& limit) const {
  44. std::string s = Reverse(*start);
  45. std::string l = Reverse(limit);
  46. BytewiseComparator()->FindShortestSeparator(&s, l);
  47. *start = Reverse(s);
  48. }
  49. virtual void FindShortSuccessor(std::string* key) const {
  50. std::string s = Reverse(*key);
  51. BytewiseComparator()->FindShortSuccessor(&s);
  52. *key = Reverse(s);
  53. }
  54. };
  55. } // namespace
  56. static ReverseKeyComparator reverse_key_comparator;
  57. static void Increment(const Comparator* cmp, std::string* key) {
  58. if (cmp == BytewiseComparator()) {
  59. key->push_back('\0');
  60. } else {
  61. assert(cmp == &reverse_key_comparator);
  62. std::string rev = Reverse(*key);
  63. rev.push_back('\0');
  64. *key = Reverse(rev);
  65. }
  66. }
  67. // An STL comparator that uses a Comparator
  68. namespace {
  69. struct STLLessThan {
  70. const Comparator* cmp;
  71. STLLessThan() : cmp(BytewiseComparator()) { }
  72. STLLessThan(const Comparator* c) : cmp(c) { }
  73. bool operator()(const std::string& a, const std::string& b) const {
  74. return cmp->Compare(Slice(a), Slice(b)) < 0;
  75. }
  76. };
  77. } // namespace
  78. class StringSink: public WritableFile {
  79. public:
  80. ~StringSink() { }
  81. const std::string& contents() const { return contents_; }
  82. virtual Status Close() { return Status::OK(); }
  83. virtual Status Flush() { return Status::OK(); }
  84. virtual Status Sync() { return Status::OK(); }
  85. virtual Status Append(const Slice& data) {
  86. contents_.append(data.data(), data.size());
  87. return Status::OK();
  88. }
  89. private:
  90. std::string contents_;
  91. };
  92. class StringSource: public RandomAccessFile {
  93. public:
  94. StringSource(const Slice& contents)
  95. : contents_(contents.data(), contents.size()) {
  96. }
  97. virtual ~StringSource() { }
  98. uint64_t Size() const { return contents_.size(); }
  99. virtual Status Read(uint64_t offset, size_t n, Slice* result,
  100. char* scratch) const {
  101. if (offset > contents_.size()) {
  102. return Status::InvalidArgument("invalid Read offset");
  103. }
  104. if (offset + n > contents_.size()) {
  105. n = contents_.size() - offset;
  106. }
  107. memcpy(scratch, &contents_[offset], n);
  108. *result = Slice(scratch, n);
  109. return Status::OK();
  110. }
  111. private:
  112. std::string contents_;
  113. };
  114. typedef std::map<std::string, std::string, STLLessThan> KVMap;
  115. // Helper class for tests to unify the interface between
  116. // BlockBuilder/TableBuilder and Block/Table.
  117. class Constructor {
  118. public:
  119. explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) { }
  120. virtual ~Constructor() { }
  121. void Add(const std::string& key, const Slice& value) {
  122. data_[key] = value.ToString();
  123. }
  124. // Finish constructing the data structure with all the keys that have
  125. // been added so far. Returns the keys in sorted order in "*keys"
  126. // and stores the key/value pairs in "*kvmap"
  127. void Finish(const Options& options,
  128. std::vector<std::string>* keys,
  129. KVMap* kvmap) {
  130. *kvmap = data_;
  131. keys->clear();
  132. for (KVMap::const_iterator it = data_.begin();
  133. it != data_.end();
  134. ++it) {
  135. keys->push_back(it->first);
  136. }
  137. data_.clear();
  138. Status s = FinishImpl(options, *kvmap);
  139. ASSERT_TRUE(s.ok()) << s.ToString();
  140. }
  141. // Construct the data structure from the data in "data"
  142. virtual Status FinishImpl(const Options& options, const KVMap& data) = 0;
  143. virtual size_t NumBytes() const = 0;
  144. virtual Iterator* NewIterator() const = 0;
  145. virtual const KVMap& data() { return data_; }
  146. virtual DB* db() const { return NULL; } // Overridden in DBConstructor
  147. private:
  148. KVMap data_;
  149. };
  150. class BlockConstructor: public Constructor {
  151. public:
  152. explicit BlockConstructor(const Comparator* cmp)
  153. : Constructor(cmp),
  154. comparator_(cmp),
  155. block_size_(-1),
  156. block_(NULL) { }
  157. ~BlockConstructor() {
  158. delete block_;
  159. }
  160. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  161. delete block_;
  162. block_ = NULL;
  163. BlockBuilder builder(&options);
  164. for (KVMap::const_iterator it = data.begin();
  165. it != data.end();
  166. ++it) {
  167. builder.Add(it->first, it->second);
  168. }
  169. // Open the block
  170. Slice block_data = builder.Finish();
  171. block_size_ = block_data.size();
  172. char* block_data_copy = new char[block_size_];
  173. memcpy(block_data_copy, block_data.data(), block_size_);
  174. block_ = new Block(block_data_copy, block_size_, true /* take ownership */);
  175. return Status::OK();
  176. }
  177. virtual size_t NumBytes() const { return block_size_; }
  178. virtual Iterator* NewIterator() const {
  179. return block_->NewIterator(comparator_);
  180. }
  181. private:
  182. const Comparator* comparator_;
  183. int block_size_;
  184. Block* block_;
  185. BlockConstructor();
  186. };
  187. class TableConstructor: public Constructor {
  188. public:
  189. TableConstructor(const Comparator* cmp)
  190. : Constructor(cmp),
  191. source_(NULL), table_(NULL) {
  192. }
  193. ~TableConstructor() {
  194. Reset();
  195. }
  196. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  197. Reset();
  198. StringSink sink;
  199. TableBuilder builder(options, &sink);
  200. for (KVMap::const_iterator it = data.begin();
  201. it != data.end();
  202. ++it) {
  203. builder.Add(it->first, it->second);
  204. ASSERT_TRUE(builder.status().ok());
  205. }
  206. Status s = builder.Finish();
  207. ASSERT_TRUE(s.ok()) << s.ToString();
  208. ASSERT_EQ(sink.contents().size(), builder.FileSize());
  209. // Open the table
  210. source_ = new StringSource(sink.contents());
  211. Options table_options;
  212. table_options.comparator = options.comparator;
  213. return Table::Open(table_options, source_, sink.contents().size(), &table_);
  214. }
  215. virtual size_t NumBytes() const { return source_->Size(); }
  216. virtual Iterator* NewIterator() const {
  217. return table_->NewIterator(ReadOptions());
  218. }
  219. uint64_t ApproximateOffsetOf(const Slice& key) const {
  220. return table_->ApproximateOffsetOf(key);
  221. }
  222. private:
  223. void Reset() {
  224. delete table_;
  225. delete source_;
  226. table_ = NULL;
  227. source_ = NULL;
  228. }
  229. StringSource* source_;
  230. Table* table_;
  231. TableConstructor();
  232. };
  233. // A helper class that converts internal format keys into user keys
  234. class KeyConvertingIterator: public Iterator {
  235. public:
  236. explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) { }
  237. virtual ~KeyConvertingIterator() { delete iter_; }
  238. virtual bool Valid() const { return iter_->Valid(); }
  239. virtual void Seek(const Slice& target) {
  240. ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue);
  241. std::string encoded;
  242. AppendInternalKey(&encoded, ikey);
  243. iter_->Seek(encoded);
  244. }
  245. virtual void SeekToFirst() { iter_->SeekToFirst(); }
  246. virtual void SeekToLast() { iter_->SeekToLast(); }
  247. virtual void Next() { iter_->Next(); }
  248. virtual void Prev() { iter_->Prev(); }
  249. virtual Slice key() const {
  250. assert(Valid());
  251. ParsedInternalKey key;
  252. if (!ParseInternalKey(iter_->key(), &key)) {
  253. status_ = Status::Corruption("malformed internal key");
  254. return Slice("corrupted key");
  255. }
  256. return key.user_key;
  257. }
  258. virtual Slice value() const { return iter_->value(); }
  259. virtual Status status() const {
  260. return status_.ok() ? iter_->status() : status_;
  261. }
  262. private:
  263. mutable Status status_;
  264. Iterator* iter_;
  265. // No copying allowed
  266. KeyConvertingIterator(const KeyConvertingIterator&);
  267. void operator=(const KeyConvertingIterator&);
  268. };
  269. class MemTableConstructor: public Constructor {
  270. public:
  271. explicit MemTableConstructor(const Comparator* cmp)
  272. : Constructor(cmp),
  273. internal_comparator_(cmp) {
  274. memtable_ = new MemTable(internal_comparator_);
  275. memtable_->Ref();
  276. }
  277. ~MemTableConstructor() {
  278. memtable_->Unref();
  279. }
  280. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  281. memtable_->Unref();
  282. memtable_ = new MemTable(internal_comparator_);
  283. memtable_->Ref();
  284. int seq = 1;
  285. for (KVMap::const_iterator it = data.begin();
  286. it != data.end();
  287. ++it) {
  288. memtable_->Add(seq, kTypeValue, it->first, it->second);
  289. seq++;
  290. }
  291. return Status::OK();
  292. }
  293. virtual size_t NumBytes() const {
  294. return memtable_->ApproximateMemoryUsage();
  295. }
  296. virtual Iterator* NewIterator() const {
  297. return new KeyConvertingIterator(memtable_->NewIterator());
  298. }
  299. private:
  300. InternalKeyComparator internal_comparator_;
  301. MemTable* memtable_;
  302. };
  303. class DBConstructor: public Constructor {
  304. public:
  305. explicit DBConstructor(const Comparator* cmp)
  306. : Constructor(cmp),
  307. comparator_(cmp) {
  308. db_ = NULL;
  309. NewDB();
  310. }
  311. ~DBConstructor() {
  312. delete db_;
  313. }
  314. virtual Status FinishImpl(const Options& options, const KVMap& data) {
  315. delete db_;
  316. db_ = NULL;
  317. NewDB();
  318. for (KVMap::const_iterator it = data.begin();
  319. it != data.end();
  320. ++it) {
  321. WriteBatch batch;
  322. batch.Put(it->first, it->second);
  323. ASSERT_TRUE(db_->Write(WriteOptions(), &batch).ok());
  324. }
  325. return Status::OK();
  326. }
  327. virtual size_t NumBytes() const {
  328. Range r("", "\xff\xff");
  329. uint64_t size;
  330. db_->GetApproximateSizes(&r, 1, &size);
  331. return size;
  332. }
  333. virtual Iterator* NewIterator() const {
  334. return db_->NewIterator(ReadOptions());
  335. }
  336. virtual DB* db() const { return db_; }
  337. private:
  338. void NewDB() {
  339. std::string name = test::TmpDir() + "/table_testdb";
  340. Options options;
  341. options.comparator = comparator_;
  342. Status status = DestroyDB(name, options);
  343. ASSERT_TRUE(status.ok()) << status.ToString();
  344. options.create_if_missing = true;
  345. options.error_if_exists = true;
  346. options.write_buffer_size = 10000; // Something small to force merging
  347. status = DB::Open(options, name, &db_);
  348. ASSERT_TRUE(status.ok()) << status.ToString();
  349. }
  350. const Comparator* comparator_;
  351. DB* db_;
  352. };
  353. enum TestType {
  354. TABLE_TEST,
  355. BLOCK_TEST,
  356. MEMTABLE_TEST,
  357. DB_TEST
  358. };
  359. struct TestArgs {
  360. TestType type;
  361. bool reverse_compare;
  362. int restart_interval;
  363. };
  364. static const TestArgs kTestArgList[] = {
  365. { TABLE_TEST, false, 16 },
  366. { TABLE_TEST, false, 1 },
  367. { TABLE_TEST, false, 1024 },
  368. { TABLE_TEST, true, 16 },
  369. { TABLE_TEST, true, 1 },
  370. { TABLE_TEST, true, 1024 },
  371. { BLOCK_TEST, false, 16 },
  372. { BLOCK_TEST, false, 1 },
  373. { BLOCK_TEST, false, 1024 },
  374. { BLOCK_TEST, true, 16 },
  375. { BLOCK_TEST, true, 1 },
  376. { BLOCK_TEST, true, 1024 },
  377. // Restart interval does not matter for memtables
  378. { MEMTABLE_TEST, false, 16 },
  379. { MEMTABLE_TEST, true, 16 },
  380. // Do not bother with restart interval variations for DB
  381. { DB_TEST, false, 16 },
  382. { DB_TEST, true, 16 },
  383. };
  384. static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]);
  385. class Harness {
  386. public:
  387. Harness() : constructor_(NULL) { }
  388. void Init(const TestArgs& args) {
  389. delete constructor_;
  390. constructor_ = NULL;
  391. options_ = Options();
  392. options_.block_restart_interval = args.restart_interval;
  393. // Use shorter block size for tests to exercise block boundary
  394. // conditions more.
  395. options_.block_size = 256;
  396. if (args.reverse_compare) {
  397. options_.comparator = &reverse_key_comparator;
  398. }
  399. switch (args.type) {
  400. case TABLE_TEST:
  401. constructor_ = new TableConstructor(options_.comparator);
  402. break;
  403. case BLOCK_TEST:
  404. constructor_ = new BlockConstructor(options_.comparator);
  405. break;
  406. case MEMTABLE_TEST:
  407. constructor_ = new MemTableConstructor(options_.comparator);
  408. break;
  409. case DB_TEST:
  410. constructor_ = new DBConstructor(options_.comparator);
  411. break;
  412. }
  413. }
  414. ~Harness() {
  415. delete constructor_;
  416. }
  417. void Add(const std::string& key, const std::string& value) {
  418. constructor_->Add(key, value);
  419. }
  420. void Test(Random* rnd) {
  421. std::vector<std::string> keys;
  422. KVMap data;
  423. constructor_->Finish(options_, &keys, &data);
  424. TestForwardScan(keys, data);
  425. TestBackwardScan(keys, data);
  426. TestRandomAccess(rnd, keys, data);
  427. }
  428. void TestForwardScan(const std::vector<std::string>& keys,
  429. const KVMap& data) {
  430. Iterator* iter = constructor_->NewIterator();
  431. ASSERT_TRUE(!iter->Valid());
  432. iter->SeekToFirst();
  433. for (KVMap::const_iterator model_iter = data.begin();
  434. model_iter != data.end();
  435. ++model_iter) {
  436. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  437. iter->Next();
  438. }
  439. ASSERT_TRUE(!iter->Valid());
  440. delete iter;
  441. }
  442. void TestBackwardScan(const std::vector<std::string>& keys,
  443. const KVMap& data) {
  444. Iterator* iter = constructor_->NewIterator();
  445. ASSERT_TRUE(!iter->Valid());
  446. iter->SeekToLast();
  447. for (KVMap::const_reverse_iterator model_iter = data.rbegin();
  448. model_iter != data.rend();
  449. ++model_iter) {
  450. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  451. iter->Prev();
  452. }
  453. ASSERT_TRUE(!iter->Valid());
  454. delete iter;
  455. }
  456. void TestRandomAccess(Random* rnd,
  457. const std::vector<std::string>& keys,
  458. const KVMap& data) {
  459. static const bool kVerbose = false;
  460. Iterator* iter = constructor_->NewIterator();
  461. ASSERT_TRUE(!iter->Valid());
  462. KVMap::const_iterator model_iter = data.begin();
  463. if (kVerbose) fprintf(stderr, "---\n");
  464. for (int i = 0; i < 200; i++) {
  465. const int toss = rnd->Uniform(5);
  466. switch (toss) {
  467. case 0: {
  468. if (iter->Valid()) {
  469. if (kVerbose) fprintf(stderr, "Next\n");
  470. iter->Next();
  471. ++model_iter;
  472. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  473. }
  474. break;
  475. }
  476. case 1: {
  477. if (kVerbose) fprintf(stderr, "SeekToFirst\n");
  478. iter->SeekToFirst();
  479. model_iter = data.begin();
  480. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  481. break;
  482. }
  483. case 2: {
  484. std::string key = PickRandomKey(rnd, keys);
  485. model_iter = data.lower_bound(key);
  486. if (kVerbose) fprintf(stderr, "Seek '%s'\n",
  487. EscapeString(key).c_str());
  488. iter->Seek(Slice(key));
  489. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  490. break;
  491. }
  492. case 3: {
  493. if (iter->Valid()) {
  494. if (kVerbose) fprintf(stderr, "Prev\n");
  495. iter->Prev();
  496. if (model_iter == data.begin()) {
  497. model_iter = data.end(); // Wrap around to invalid value
  498. } else {
  499. --model_iter;
  500. }
  501. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  502. }
  503. break;
  504. }
  505. case 4: {
  506. if (kVerbose) fprintf(stderr, "SeekToLast\n");
  507. iter->SeekToLast();
  508. if (keys.empty()) {
  509. model_iter = data.end();
  510. } else {
  511. std::string last = data.rbegin()->first;
  512. model_iter = data.lower_bound(last);
  513. }
  514. ASSERT_EQ(ToString(data, model_iter), ToString(iter));
  515. break;
  516. }
  517. }
  518. }
  519. delete iter;
  520. }
  521. std::string ToString(const KVMap& data, const KVMap::const_iterator& it) {
  522. if (it == data.end()) {
  523. return "END";
  524. } else {
  525. return "'" + it->first + "->" + it->second + "'";
  526. }
  527. }
  528. std::string ToString(const KVMap& data,
  529. const KVMap::const_reverse_iterator& it) {
  530. if (it == data.rend()) {
  531. return "END";
  532. } else {
  533. return "'" + it->first + "->" + it->second + "'";
  534. }
  535. }
  536. std::string ToString(const Iterator* it) {
  537. if (!it->Valid()) {
  538. return "END";
  539. } else {
  540. return "'" + it->key().ToString() + "->" + it->value().ToString() + "'";
  541. }
  542. }
  543. std::string PickRandomKey(Random* rnd, const std::vector<std::string>& keys) {
  544. if (keys.empty()) {
  545. return "foo";
  546. } else {
  547. const int index = rnd->Uniform(keys.size());
  548. std::string result = keys[index];
  549. switch (rnd->Uniform(3)) {
  550. case 0:
  551. // Return an existing key
  552. break;
  553. case 1: {
  554. // Attempt to return something smaller than an existing key
  555. if (result.size() > 0 && result[result.size()-1] > '\0') {
  556. result[result.size()-1]--;
  557. }
  558. break;
  559. }
  560. case 2: {
  561. // Return something larger than an existing key
  562. Increment(options_.comparator, &result);
  563. break;
  564. }
  565. }
  566. return result;
  567. }
  568. }
  569. // Returns NULL if not running against a DB
  570. DB* db() const { return constructor_->db(); }
  571. private:
  572. Options options_;
  573. Constructor* constructor_;
  574. };
  575. // Test the empty key
  576. TEST(Harness, SimpleEmptyKey) {
  577. for (int i = 0; i < kNumTestArgs; i++) {
  578. Init(kTestArgList[i]);
  579. Random rnd(test::RandomSeed() + 1);
  580. Add("", "v");
  581. Test(&rnd);
  582. }
  583. }
  584. TEST(Harness, SimpleSingle) {
  585. for (int i = 0; i < kNumTestArgs; i++) {
  586. Init(kTestArgList[i]);
  587. Random rnd(test::RandomSeed() + 2);
  588. Add("abc", "v");
  589. Test(&rnd);
  590. }
  591. }
  592. TEST(Harness, SimpleMulti) {
  593. for (int i = 0; i < kNumTestArgs; i++) {
  594. Init(kTestArgList[i]);
  595. Random rnd(test::RandomSeed() + 3);
  596. Add("abc", "v");
  597. Add("abcd", "v");
  598. Add("ac", "v2");
  599. Test(&rnd);
  600. }
  601. }
  602. TEST(Harness, SimpleSpecialKey) {
  603. for (int i = 0; i < kNumTestArgs; i++) {
  604. Init(kTestArgList[i]);
  605. Random rnd(test::RandomSeed() + 4);
  606. Add("\xff\xff", "v3");
  607. Test(&rnd);
  608. }
  609. }
  610. TEST(Harness, Randomized) {
  611. for (int i = 0; i < kNumTestArgs; i++) {
  612. Init(kTestArgList[i]);
  613. Random rnd(test::RandomSeed() + 5);
  614. for (int num_entries = 0; num_entries < 2000;
  615. num_entries += (num_entries < 50 ? 1 : 200)) {
  616. if ((num_entries % 10) == 0) {
  617. fprintf(stderr, "case %d of %d: num_entries = %d\n",
  618. (i + 1), int(kNumTestArgs), num_entries);
  619. }
  620. for (int e = 0; e < num_entries; e++) {
  621. std::string v;
  622. Add(test::RandomKey(&rnd, rnd.Skewed(4)),
  623. test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
  624. }
  625. Test(&rnd);
  626. }
  627. }
  628. }
  629. TEST(Harness, RandomizedLongDB) {
  630. Random rnd(test::RandomSeed());
  631. TestArgs args = { DB_TEST, false, 16 };
  632. Init(args);
  633. int num_entries = 100000;
  634. for (int e = 0; e < num_entries; e++) {
  635. std::string v;
  636. Add(test::RandomKey(&rnd, rnd.Skewed(4)),
  637. test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
  638. }
  639. Test(&rnd);
  640. // We must have created enough data to force merging
  641. int files = 0;
  642. for (int level = 0; level < config::kNumLevels; level++) {
  643. std::string value;
  644. char name[100];
  645. snprintf(name, sizeof(name), "leveldb.num-files-at-level%d", level);
  646. ASSERT_TRUE(db()->GetProperty(name, &value));
  647. files += atoi(value.c_str());
  648. }
  649. ASSERT_GT(files, 0);
  650. }
  651. class MemTableTest { };
  652. TEST(MemTableTest, Simple) {
  653. InternalKeyComparator cmp(BytewiseComparator());
  654. MemTable* memtable = new MemTable(cmp);
  655. memtable->Ref();
  656. WriteBatch batch;
  657. WriteBatchInternal::SetSequence(&batch, 100);
  658. batch.Put(std::string("k1"), std::string("v1"));
  659. batch.Put(std::string("k2"), std::string("v2"));
  660. batch.Put(std::string("k3"), std::string("v3"));
  661. batch.Put(std::string("largekey"), std::string("vlarge"));
  662. ASSERT_TRUE(WriteBatchInternal::InsertInto(&batch, memtable).ok());
  663. Iterator* iter = memtable->NewIterator();
  664. iter->SeekToFirst();
  665. while (iter->Valid()) {
  666. fprintf(stderr, "key: '%s' -> '%s'\n",
  667. iter->key().ToString().c_str(),
  668. iter->value().ToString().c_str());
  669. iter->Next();
  670. }
  671. delete iter;
  672. memtable->Unref();
  673. }
  674. static bool Between(uint64_t val, uint64_t low, uint64_t high) {
  675. bool result = (val >= low) && (val <= high);
  676. if (!result) {
  677. fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
  678. (unsigned long long)(val),
  679. (unsigned long long)(low),
  680. (unsigned long long)(high));
  681. }
  682. return result;
  683. }
  684. class TableTest { };
  685. TEST(TableTest, ApproximateOffsetOfPlain) {
  686. TableConstructor c(BytewiseComparator());
  687. c.Add("k01", "hello");
  688. c.Add("k02", "hello2");
  689. c.Add("k03", std::string(10000, 'x'));
  690. c.Add("k04", std::string(200000, 'x'));
  691. c.Add("k05", std::string(300000, 'x'));
  692. c.Add("k06", "hello3");
  693. c.Add("k07", std::string(100000, 'x'));
  694. std::vector<std::string> keys;
  695. KVMap kvmap;
  696. Options options;
  697. options.block_size = 1024;
  698. options.compression = kNoCompression;
  699. c.Finish(options, &keys, &kvmap);
  700. ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
  701. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
  702. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01a"), 0, 0));
  703. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
  704. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 0, 0));
  705. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 10000, 11000));
  706. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04a"), 210000, 211000));
  707. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k05"), 210000, 211000));
  708. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000));
  709. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000));
  710. ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 611000));
  711. }
  712. static bool SnappyCompressionSupported() {
  713. std::string out;
  714. Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
  715. return port::Snappy_Compress(in.data(), in.size(), &out);
  716. }
  717. TEST(TableTest, ApproximateOffsetOfCompressed) {
  718. if (!SnappyCompressionSupported()) {
  719. fprintf(stderr, "skipping compression tests\n");
  720. return;
  721. }
  722. Random rnd(301);
  723. TableConstructor c(BytewiseComparator());
  724. std::string tmp;
  725. c.Add("k01", "hello");
  726. c.Add("k02", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
  727. c.Add("k03", "hello3");
  728. c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
  729. std::vector<std::string> keys;
  730. KVMap kvmap;
  731. Options options;
  732. options.block_size = 1024;
  733. options.compression = kSnappyCompression;
  734. c.Finish(options, &keys, &kvmap);
  735. ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
  736. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
  737. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
  738. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 2000, 3000));
  739. ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 2000, 3000));
  740. ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 4000, 6000));
  741. }
  742. } // namespace leveldb
  743. int main(int argc, char** argv) {
  744. return leveldb::test::RunAllTests();
  745. }