first release of vTable builder and reader, pass test_vtable

.idea/misc.xml

@@ -1,12 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="BackendCodeEditorMiscSettings">
-    <option name="/Default/RiderDebugger/RiderRestoreDecompile/RestoreDecompileSetting/@EntryValue" value="false" type="bool" />
-    <option name="/Default/Housekeeping/GlobalSettingsUpgraded/IsUpgraded/@EntryValue" value="true" type="bool" />
-    <option name="/Default/Housekeeping/FeatureSuggestion/FeatureSuggestionManager/DisabledSuggesters/=SwitchToGoToActionSuggester/@EntryIndexedValue" value="true" type="bool" />
-    <option name="/Default/Housekeeping/FeatureSuggestion/FeatureSuggestionManager/DisabledSuggesters/=SwitchToGoToActionSuggester/@EntryIndexRemoved" />
-    <option name="/Default/Environment/Hierarchy/GeneratedFilesCacheKey/Timestamp/@EntryValue" value="5" type="long" />
-  </component>
   <component name="CMakePythonSetting">
     <option name="pythonIntegrationState" value="YES" />
   </component>

CMakeLists.txt

@@ -176,6 +176,8 @@ target_sources(leveldb
     "table/vtable_builder.h"
     "table/vtable_format.cc"
     "table/vtable_format.h"
+    "table/vtable_reader.cc"
+    "table/vtable_reader.h"
     "util/arena.cc"
     "util/arena.h"
     "util/bloom.cc"
@@ -528,3 +530,8 @@ add_executable(test_fields
         "${PROJECT_SOURCE_DIR}/test/test_fields.cc"
 )
 target_link_libraries(test_fields PRIVATE leveldb gtest)
+
+add_executable(test_vtable
+        "${PROJECT_SOURCE_DIR}/test/test_vtable.cc"
+)
+target_link_libraries(test_vtable PRIVATE leveldb gtest)

table/vtable_builder.cc

@@ -1,15 +1,36 @@
 #include "table/vtable_builder.h"
 
+#include "leveldb/env.h"
+
 namespace leveldb {
 
 VTableBuilder::VTableBuilder(const Options& options, WritableFile* file)
-  : file_(file) {}
+  : file_(file),
+    encoder_() {}
 
 void VTableBuilder::Add(const VTableRecord& record, VTableHandle* handle) {
   if (!ok()) return;
 
+  encoder_.Encode(record);
+  handle->offset = file_size_;
+  handle->size = encoder_.GetEncodedSize();
+  file_size_ += encoder_.GetEncodedSize();
+
+  status_ = file_->Append(encoder_.GetHeader().ToString() +
+                          encoder_.GetRecord().ToString());
+
+  assert(ok());
+  //TODO: meta info support in the future
+}
+
+Status VTableBuilder::Finish() {
+  if (!ok()) return status();
+
+  status_ = file_->Flush();
 
+  return status();
 }
 
+void VTableBuilder::Abandon() { }
 
 } // namespace leveldb

table/vtable_builder.h

@@ -27,8 +27,11 @@ class VTableBuilder {
     bool ok() const { return status().ok(); }
 
     WritableFile* file_;
+    uint64_t file_size_{0};
 
     Status status_;
+
+    RecordEncoder encoder_;
 };
 
 } // namespace leveldb

table/vtable_format.cc

@@ -39,13 +39,23 @@ void RecordEncoder::Encode(const VTableRecord& record) {
   record_ = Slice(record_buff_.data(), record_buff_.size());
 
   assert(record.size() < std::numeric_limits<uint32_t>::max());
-}
 
-Status RecordDecoder::Decode(Slice* input, VTableRecord* record) {
+  EncodeFixed32(header_, static_cast<uint32_t>(record_.size()));
+}
 
+Status RecordDecoder::DecodeHeader(Slice* input) {
+  if (!GetFixed32(input, &record_size_)) {
+    return Status::Corruption("Error decode record header");
+  }
+  return Status::OK();
 }
 
+Status RecordDecoder::DecodeRecord(Slice* input, VTableRecord* record) const {
+  Slice record_input(input->data(), record_size_);
+  input->remove_prefix(record_size_);
 
+  return DecodeSrcIntoObj(record_input, record);
+}
 
 void VTableHandle::Encode(std::string* target) const {
   PutVarint64(target, offset);

table/vtable_format.h

@@ -7,6 +7,8 @@
 
 namespace leveldb {
 
+const uint64_t kRecordHeaderSize = 4;
+
 struct VTableRecord {
   Slice key;
   Slice value;
@@ -30,8 +32,15 @@ class RecordEncoder {
     void Encode(const VTableRecord& record);
 
     // Get the size of encoded record
-    size_t GetEncodedSize() const { return record_.size(); }
+    size_t GetEncodedSize() const { return sizeof(header_) + record_.size(); }
+
+    // Get the header
+    Slice GetHeader() const { return {header_, sizeof(header_)}; }
+
+    // Get the encoded record
+    Slice GetRecord() const {return record_; }
   private:
+    char header_[kRecordHeaderSize];
     Slice record_;
 
     std::string record_buff_;
@@ -39,7 +48,9 @@ class RecordEncoder {
 
 class RecordDecoder {
   public:
-    Status Decode(Slice* input, VTableRecord* record);
+    Status DecodeHeader(Slice* input);
+
+    Status DecodeRecord(Slice* input, VTableRecord* record) const;
 
     size_t GetDecodedSize() const { return record_size_; }
 
@@ -75,5 +86,15 @@ struct VTableIndex {
   }
 };
 
+template <typename T>
+Status DecodeSrcIntoObj(const Slice& src, T* target) {
+  Slice input = src;
+  Status s = target->Decode(&input);
+  if (s.ok() && !input.empty()) {
+    s = Status::Corruption(Slice());
+  }
+  return s;
+}
+
 } // namespace leveldb
 #endif //VTABLE_FORMAT_H

table/vtable_reader.cc

@@ -0,0 +1,39 @@
+#include <string>
+
+#include "leveldb/env.h"
+
+#include <table/vtable_reader.h>
+
+namespace leveldb {
+  Status VTableReader::Open(const Options& options, std::string fname) {
+    options_ = options;
+    return options_.env->NewRandomAccessFile(fname, &file_);
+  }
+
+  Status VTableReader::Get(const VTableHandle& handle,
+                           VTableRecord* record) const {
+    auto buf = new char[handle.size];
+    Slice input;
+    Status s = file_->Read(handle.offset, handle.size, &input, buf);
+
+    if (!s.ok()) {
+      return s;
+    }
+    if (handle.size != static_cast<uint64_t>(input.size())) {
+      return Status::Corruption("Read input size not equal to record size: " +
+                                std::to_string(input.size()) + ":" +
+                                std::to_string(handle.size));
+    }
+
+    RecordDecoder decoder;
+    s = decoder.DecodeHeader(&input);
+    if (!s.ok()) {
+      return s;
+    }
+
+    s = decoder.DecodeRecord(&input, record);
+    return s;
+  }
+
+
+} // namespace leveldb

table/vtable_reader.h

@@ -0,0 +1,30 @@
+#ifndef VTABLE_READER_H
+#define VTABLE_READER_H
+
+#include <memory>
+
+#include "leveldb/env.h"
+#include "leveldb/options.h"
+#include "leveldb/slice.h"
+#include "leveldb/status.h"
+
+#include "util/coding.h"
+
+#include "vtable_format.h"
+
+namespace leveldb {
+
+class VTableReader {
+  public:
+    Status Open(const Options& options, std::string fname);
+
+    Status Get(const VTableHandle& handle,
+               VTableRecord* record) const ;
+  private:
+    Options options_;
+    RandomAccessFile* file_{nullptr};
+};
+
+} // namespace leveldb
+
+#endif //VTABLE_READER_H

test/test_vtable.cc

@@ -0,0 +1,43 @@
+#include <iostream>
+#include <gtest/gtest.h>
+
+#include "leveldb/db.h"
+#include "leveldb/env.h"
+#include "table/vtable_builder.h"
+#include "table/vtable_reader.h"
+#include "table/vtable_format.h"
+
+using namespace std;
+using namespace leveldb;
+
+int main() {
+  VTableHandle handle1;
+  VTableHandle handle2;
+  VTableRecord record1;
+  VTableRecord record2;
+  record1.key = "001";
+  record1.value = "value1";
+  record2.key = "002";
+  record2.value = "value2";
+
+  Options opt;
+  WritableFile *file;
+  opt.env->NewWritableFile("1.vtb", &file);
+  VTableBuilder builder(opt, file);
+
+  builder.Add(record1, &handle1);
+  builder.Add(record2, &handle2);
+  builder.Finish();
+
+  VTableReader reader;
+  reader.Open(opt, "1.vtb");
+
+  VTableRecord res_record;
+  reader.Get(handle2, &res_record);
+
+  cout << res_record.key.ToString() << " " << res_record.value.ToString() << endl;
+
+  reader.Get(handle1, &res_record);
+
+  cout << res_record.key.ToString() << " " << res_record.value.ToString() << endl;
+}

util/coding.cc

@@ -142,6 +142,20 @@ bool GetVarint64(Slice* input, uint64_t* value) {
   }
 }
 
+bool GetFixed32(Slice* input, uint32_t* value) {
+  const char* p = input->data();
+  const char* limit = p + input->size();
+  *value = DecodeFixed32(p);
+  const char* q = p + 4;
+  if (q > limit) {
+    return false;
+  } else {
+    *input = Slice(q, limit - q);
+    return true;
+  }
+}
+
+
 bool GetLengthPrefixedSlice(Slice* input, Slice* result) {
   uint32_t len;
   if (GetVarint32(input, &len) && input->size() >= len) {

util/coding.h

@@ -30,6 +30,7 @@ void PutLengthPrefixedSlice(std::string* dst, const Slice& value);
 // and advance the slice past the parsed value.
 bool GetVarint32(Slice* input, uint32_t* value);
 bool GetVarint64(Slice* input, uint64_t* value);
+bool GetFixed32(Slice* input, uint32_t* value);
 bool GetLengthPrefixedSlice(Slice* input, Slice* result);
 
 // Pointer-based variants of GetVarint...  These either store a value