109 lines
3.5 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. //
  5. // BlockBuilder generates blocks where keys are prefix-compressed:
  6. //
  7. // When we store a key, we drop the prefix shared with the previous
  8. // string. This helps reduce the space requirement significantly.
  9. // Furthermore, once every K keys, we do not apply the prefix
  10. // compression and store the entire key. We call this a "restart
  11. // point". The tail end of the block stores the offsets of all of the
  12. // restart points, and can be used to do a binary search when looking
  13. // for a particular key. Values are stored as-is (without compression)
  14. // immediately following the corresponding key.
  15. //
  16. // An entry for a particular key-value pair has the form:
  17. // shared_bytes: varint32
  18. // unshared_bytes: varint32
  19. // value_length: varint32
  20. // key_delta: char[unshared_bytes]
  21. // value: char[value_length]
  22. // shared_bytes == 0 for restart points.
  23. //
  24. // The trailer of the block has the form:
  25. // restarts: uint32[num_restarts]
  26. // num_restarts: uint32
  27. // restarts[i] contains the offset within the block of the ith restart point.
  28. #include "table/block_builder.h"
  29. #include <algorithm>
  30. #include <assert.h>
  31. #include "leveldb/comparator.h"
  32. #include "leveldb/table_builder.h"
  33. #include "util/coding.h"
  34. namespace leveldb {
  35. BlockBuilder::BlockBuilder(const Options* options)
  36. : options_(options),
  37. restarts_(),
  38. counter_(0),
  39. finished_(false) {
  40. assert(options->block_restart_interval >= 1);
  41. restarts_.push_back(0); // First restart point is at offset 0
  42. }
  43. void BlockBuilder::Reset() {
  44. buffer_.clear();
  45. restarts_.clear();
  46. restarts_.push_back(0); // First restart point is at offset 0
  47. counter_ = 0;
  48. finished_ = false;
  49. last_key_.clear();
  50. }
  51. size_t BlockBuilder::CurrentSizeEstimate() const {
  52. return (buffer_.size() + // Raw data buffer
  53. restarts_.size() * sizeof(uint32_t) + // Restart array
  54. sizeof(uint32_t)); // Restart array length
  55. }
  56. Slice BlockBuilder::Finish() {
  57. // Append restart array
  58. for (size_t i = 0; i < restarts_.size(); i++) {
  59. PutFixed32(&buffer_, restarts_[i]);
  60. }
  61. PutFixed32(&buffer_, restarts_.size());
  62. finished_ = true;
  63. return Slice(buffer_);
  64. }
  65. void BlockBuilder::Add(const Slice& key, const Slice& value) {
  66. Slice last_key_piece(last_key_);
  67. assert(!finished_);
  68. assert(counter_ <= options_->block_restart_interval);
  69. assert(buffer_.empty() // No values yet?
  70. || options_->comparator->Compare(key, last_key_piece) > 0);
  71. size_t shared = 0;
  72. if (counter_ < options_->block_restart_interval) {
  73. // See how much sharing to do with previous string
  74. const size_t min_length = std::min(last_key_piece.size(), key.size());
  75. while ((shared < min_length) && (last_key_piece[shared] == key[shared])) {
  76. shared++;
  77. }
  78. } else {
  79. // Restart compression
  80. restarts_.push_back(buffer_.size());
  81. counter_ = 0;
  82. }
  83. const size_t non_shared = key.size() - shared;
  84. // Add "<shared><non_shared><value_size>" to buffer_
  85. PutVarint32(&buffer_, shared);
  86. PutVarint32(&buffer_, non_shared);
  87. PutVarint32(&buffer_, value.size());
  88. // Add string delta to buffer_ followed by value
  89. buffer_.append(key.data() + shared, non_shared);
  90. buffer_.append(value.data(), value.size());
  91. // Update state
  92. last_key_.resize(shared);
  93. last_key_.append(key.data() + shared, non_shared);
  94. assert(Slice(last_key_) == key);
  95. counter_++;
  96. }
  97. }