作者: 韩晨旭 10225101440 李畅 10225102463
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

227 lines
6.6 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "leveldb/table_builder.h"
  5. #include <assert.h>
  6. #include <stdio.h>
  7. #include "leveldb/comparator.h"
  8. #include "leveldb/env.h"
  9. #include "table/block_builder.h"
  10. #include "table/format.h"
  11. #include "util/coding.h"
  12. #include "util/crc32c.h"
  13. #include "util/logging.h"
  14. namespace leveldb {
  15. struct TableBuilder::Rep {
  16. Options options;
  17. Options index_block_options;
  18. WritableFile* file;
  19. uint64_t offset;
  20. Status status;
  21. BlockBuilder data_block;
  22. BlockBuilder index_block;
  23. std::string last_key;
  24. int64_t num_entries;
  25. bool closed; // Either Finish() or Abandon() has been called.
  26. // We do not emit the index entry for a block until we have seen the
  27. // first key for the next data block. This allows us to use shorter
  28. // keys in the index block. For example, consider a block boundary
  29. // between the keys "the quick brown fox" and "the who". We can use
  30. // "the r" as the key for the index block entry since it is >= all
  31. // entries in the first block and < all entries in subsequent
  32. // blocks.
  33. //
  34. // Invariant: r->pending_index_entry is true only if data_block is empty.
  35. bool pending_index_entry;
  36. BlockHandle pending_handle; // Handle to add to index block
  37. std::string compressed_output;
  38. Rep(const Options& opt, WritableFile* f)
  39. : options(opt),
  40. index_block_options(opt),
  41. file(f),
  42. offset(0),
  43. data_block(&options),
  44. index_block(&index_block_options),
  45. num_entries(0),
  46. closed(false),
  47. pending_index_entry(false) {
  48. index_block_options.block_restart_interval = 1;
  49. }
  50. };
  51. TableBuilder::TableBuilder(const Options& options, WritableFile* file)
  52. : rep_(new Rep(options, file)) {
  53. }
  54. TableBuilder::~TableBuilder() {
  55. assert(rep_->closed); // Catch errors where caller forgot to call Finish()
  56. delete rep_;
  57. }
  58. Status TableBuilder::ChangeOptions(const Options& options) {
  59. // Note: if more fields are added to Options, update
  60. // this function to catch changes that should not be allowed to
  61. // change in the middle of building a Table.
  62. if (options.comparator != rep_->options.comparator) {
  63. return Status::InvalidArgument("changing comparator while building table");
  64. }
  65. // Note that any live BlockBuilders point to rep_->options and therefore
  66. // will automatically pick up the updated options.
  67. rep_->options = options;
  68. rep_->index_block_options = options;
  69. rep_->index_block_options.block_restart_interval = 1;
  70. return Status::OK();
  71. }
  72. void TableBuilder::Add(const Slice& key, const Slice& value) {
  73. Rep* r = rep_;
  74. assert(!r->closed);
  75. if (!ok()) return;
  76. if (r->num_entries > 0) {
  77. assert(r->options.comparator->Compare(key, Slice(r->last_key)) > 0);
  78. }
  79. if (r->pending_index_entry) {
  80. assert(r->data_block.empty());
  81. r->options.comparator->FindShortestSeparator(&r->last_key, key);
  82. std::string handle_encoding;
  83. r->pending_handle.EncodeTo(&handle_encoding);
  84. r->index_block.Add(r->last_key, Slice(handle_encoding));
  85. r->pending_index_entry = false;
  86. }
  87. r->last_key.assign(key.data(), key.size());
  88. r->num_entries++;
  89. r->data_block.Add(key, value);
  90. const size_t estimated_block_size = r->data_block.CurrentSizeEstimate();
  91. if (estimated_block_size >= r->options.block_size) {
  92. Flush();
  93. }
  94. }
  95. void TableBuilder::Flush() {
  96. Rep* r = rep_;
  97. assert(!r->closed);
  98. if (!ok()) return;
  99. if (r->data_block.empty()) return;
  100. assert(!r->pending_index_entry);
  101. WriteBlock(&r->data_block, &r->pending_handle);
  102. if (ok()) {
  103. r->pending_index_entry = true;
  104. r->status = r->file->Flush();
  105. }
  106. }
  107. void TableBuilder::WriteBlock(BlockBuilder* block, BlockHandle* handle) {
  108. // File format contains a sequence of blocks where each block has:
  109. // block_data: uint8[n]
  110. // type: uint8
  111. // crc: uint32
  112. assert(ok());
  113. Rep* r = rep_;
  114. Slice raw = block->Finish();
  115. Slice block_contents;
  116. CompressionType type = r->options.compression;
  117. // TODO(postrelease): Support more compression options: zlib?
  118. switch (type) {
  119. case kNoCompression:
  120. block_contents = raw;
  121. break;
  122. case kSnappyCompression: {
  123. std::string* compressed = &r->compressed_output;
  124. if (port::Snappy_Compress(raw.data(), raw.size(), compressed) &&
  125. compressed->size() < raw.size() - (raw.size() / 8u)) {
  126. block_contents = *compressed;
  127. } else {
  128. // Snappy not supported, or compressed less than 12.5%, so just
  129. // store uncompressed form
  130. block_contents = raw;
  131. type = kNoCompression;
  132. }
  133. break;
  134. }
  135. }
  136. handle->set_offset(r->offset);
  137. handle->set_size(block_contents.size());
  138. r->status = r->file->Append(block_contents);
  139. if (r->status.ok()) {
  140. char trailer[kBlockTrailerSize];
  141. trailer[0] = type;
  142. uint32_t crc = crc32c::Value(block_contents.data(), block_contents.size());
  143. crc = crc32c::Extend(crc, trailer, 1); // Extend crc to cover block type
  144. EncodeFixed32(trailer+1, crc32c::Mask(crc));
  145. r->status = r->file->Append(Slice(trailer, kBlockTrailerSize));
  146. if (r->status.ok()) {
  147. r->offset += block_contents.size() + kBlockTrailerSize;
  148. }
  149. }
  150. r->compressed_output.clear();
  151. block->Reset();
  152. }
  153. Status TableBuilder::status() const {
  154. return rep_->status;
  155. }
  156. Status TableBuilder::Finish() {
  157. Rep* r = rep_;
  158. Flush();
  159. assert(!r->closed);
  160. r->closed = true;
  161. BlockHandle metaindex_block_handle;
  162. BlockHandle index_block_handle;
  163. if (ok()) {
  164. BlockBuilder meta_index_block(&r->options);
  165. // TODO(postrelease): Add stats and other meta blocks
  166. WriteBlock(&meta_index_block, &metaindex_block_handle);
  167. }
  168. if (ok()) {
  169. if (r->pending_index_entry) {
  170. r->options.comparator->FindShortSuccessor(&r->last_key);
  171. std::string handle_encoding;
  172. r->pending_handle.EncodeTo(&handle_encoding);
  173. r->index_block.Add(r->last_key, Slice(handle_encoding));
  174. r->pending_index_entry = false;
  175. }
  176. WriteBlock(&r->index_block, &index_block_handle);
  177. }
  178. if (ok()) {
  179. Footer footer;
  180. footer.set_metaindex_handle(metaindex_block_handle);
  181. footer.set_index_handle(index_block_handle);
  182. std::string footer_encoding;
  183. footer.EncodeTo(&footer_encoding);
  184. r->status = r->file->Append(footer_encoding);
  185. if (r->status.ok()) {
  186. r->offset += footer_encoding.size();
  187. }
  188. }
  189. return r->status;
  190. }
  191. void TableBuilder::Abandon() {
  192. Rep* r = rep_;
  193. assert(!r->closed);
  194. r->closed = true;
  195. }
  196. uint64_t TableBuilder::NumEntries() const {
  197. return rep_->num_entries;
  198. }
  199. uint64_t TableBuilder::FileSize() const {
  200. return rep_->offset;
  201. }
  202. } // namespace leveldb