From 7ec558c54b54b83a00431167d606a2cc652cdeb8 Mon Sep 17 00:00:00 2001 From: Arcueid <981354012@qq.com> Date: Wed, 11 Dec 2024 01:51:24 -0800 Subject: [PATCH] first release of vTable builder and reader, pass test_vtable --- .idea/misc.xml | 7 ------- CMakeLists.txt | 7 +++++++ table/vtable_builder.cc | 23 ++++++++++++++++++++++- table/vtable_builder.h | 3 +++ table/vtable_format.cc | 14 ++++++++++++-- table/vtable_format.h | 25 +++++++++++++++++++++++-- table/vtable_reader.cc | 39 +++++++++++++++++++++++++++++++++++++++ table/vtable_reader.h | 30 ++++++++++++++++++++++++++++++ test/test_vtable.cc | 43 +++++++++++++++++++++++++++++++++++++++++++ util/coding.cc | 14 ++++++++++++++ util/coding.h | 1 + 11 files changed, 194 insertions(+), 12 deletions(-) create mode 100644 table/vtable_reader.cc create mode 100644 table/vtable_reader.h create mode 100644 test/test_vtable.cc diff --git a/.idea/misc.xml b/.idea/misc.xml index 03a304b..0b76fe5 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,12 +1,5 @@ - - diff --git a/CMakeLists.txt b/CMakeLists.txt index d1e8f0c..6930e85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,6 +176,8 @@ target_sources(leveldb "table/vtable_builder.h" "table/vtable_format.cc" "table/vtable_format.h" + "table/vtable_reader.cc" + "table/vtable_reader.h" "util/arena.cc" "util/arena.h" "util/bloom.cc" @@ -528,3 +530,8 @@ add_executable(test_fields "${PROJECT_SOURCE_DIR}/test/test_fields.cc" ) target_link_libraries(test_fields PRIVATE leveldb gtest) + +add_executable(test_vtable + "${PROJECT_SOURCE_DIR}/test/test_vtable.cc" +) +target_link_libraries(test_vtable PRIVATE leveldb gtest) diff --git a/table/vtable_builder.cc b/table/vtable_builder.cc index d6d2eae..3833e57 100644 --- a/table/vtable_builder.cc +++ b/table/vtable_builder.cc @@ -1,15 +1,36 @@ #include "table/vtable_builder.h" +#include "leveldb/env.h" + namespace leveldb { VTableBuilder::VTableBuilder(const Options& options, WritableFile* file) - : file_(file) {} + : file_(file), + encoder_() {} void VTableBuilder::Add(const VTableRecord& record, VTableHandle* handle) { if (!ok()) return; + encoder_.Encode(record); + handle->offset = file_size_; + handle->size = encoder_.GetEncodedSize(); + file_size_ += encoder_.GetEncodedSize(); + + status_ = file_->Append(encoder_.GetHeader().ToString() + + encoder_.GetRecord().ToString()); + + assert(ok()); + //TODO: meta info support in the future +} + +Status VTableBuilder::Finish() { + if (!ok()) return status(); + + status_ = file_->Flush(); + return status(); } +void VTableBuilder::Abandon() { } } // namespace leveldb \ No newline at end of file diff --git a/table/vtable_builder.h b/table/vtable_builder.h index ee2085a..0eedb84 100644 --- a/table/vtable_builder.h +++ b/table/vtable_builder.h @@ -27,8 +27,11 @@ class VTableBuilder { bool ok() const { return status().ok(); } WritableFile* file_; + uint64_t file_size_{0}; Status status_; + + RecordEncoder encoder_; }; } // namespace leveldb diff --git a/table/vtable_format.cc b/table/vtable_format.cc index c0fea17..5aaa58b 100644 --- a/table/vtable_format.cc +++ b/table/vtable_format.cc @@ -39,13 +39,23 @@ void RecordEncoder::Encode(const VTableRecord& record) { record_ = Slice(record_buff_.data(), record_buff_.size()); assert(record.size() < std::numeric_limits::max()); -} -Status RecordDecoder::Decode(Slice* input, VTableRecord* record) { + EncodeFixed32(header_, static_cast(record_.size())); +} +Status RecordDecoder::DecodeHeader(Slice* input) { + if (!GetFixed32(input, &record_size_)) { + return Status::Corruption("Error decode record header"); + } + return Status::OK(); } +Status RecordDecoder::DecodeRecord(Slice* input, VTableRecord* record) const { + Slice record_input(input->data(), record_size_); + input->remove_prefix(record_size_); + return DecodeSrcIntoObj(record_input, record); +} void VTableHandle::Encode(std::string* target) const { PutVarint64(target, offset); diff --git a/table/vtable_format.h b/table/vtable_format.h index 4154ac1..a2f61a8 100644 --- a/table/vtable_format.h +++ b/table/vtable_format.h @@ -7,6 +7,8 @@ namespace leveldb { +const uint64_t kRecordHeaderSize = 4; + struct VTableRecord { Slice key; Slice value; @@ -30,8 +32,15 @@ class RecordEncoder { void Encode(const VTableRecord& record); // Get the size of encoded record - size_t GetEncodedSize() const { return record_.size(); } + size_t GetEncodedSize() const { return sizeof(header_) + record_.size(); } + + // Get the header + Slice GetHeader() const { return {header_, sizeof(header_)}; } + + // Get the encoded record + Slice GetRecord() const {return record_; } private: + char header_[kRecordHeaderSize]; Slice record_; std::string record_buff_; @@ -39,7 +48,9 @@ class RecordEncoder { class RecordDecoder { public: - Status Decode(Slice* input, VTableRecord* record); + Status DecodeHeader(Slice* input); + + Status DecodeRecord(Slice* input, VTableRecord* record) const; size_t GetDecodedSize() const { return record_size_; } @@ -75,5 +86,15 @@ struct VTableIndex { } }; +template +Status DecodeSrcIntoObj(const Slice& src, T* target) { + Slice input = src; + Status s = target->Decode(&input); + if (s.ok() && !input.empty()) { + s = Status::Corruption(Slice()); + } + return s; +} + } // namespace leveldb #endif //VTABLE_FORMAT_H diff --git a/table/vtable_reader.cc b/table/vtable_reader.cc new file mode 100644 index 0000000..cb56f12 --- /dev/null +++ b/table/vtable_reader.cc @@ -0,0 +1,39 @@ +#include + +#include "leveldb/env.h" + +#include + +namespace leveldb { + Status VTableReader::Open(const Options& options, std::string fname) { + options_ = options; + return options_.env->NewRandomAccessFile(fname, &file_); + } + + Status VTableReader::Get(const VTableHandle& handle, + VTableRecord* record) const { + auto buf = new char[handle.size]; + Slice input; + Status s = file_->Read(handle.offset, handle.size, &input, buf); + + if (!s.ok()) { + return s; + } + if (handle.size != static_cast(input.size())) { + return Status::Corruption("Read input size not equal to record size: " + + std::to_string(input.size()) + ":" + + std::to_string(handle.size)); + } + + RecordDecoder decoder; + s = decoder.DecodeHeader(&input); + if (!s.ok()) { + return s; + } + + s = decoder.DecodeRecord(&input, record); + return s; + } + + +} // namespace leveldb \ No newline at end of file diff --git a/table/vtable_reader.h b/table/vtable_reader.h new file mode 100644 index 0000000..bad18f5 --- /dev/null +++ b/table/vtable_reader.h @@ -0,0 +1,30 @@ +#ifndef VTABLE_READER_H +#define VTABLE_READER_H + +#include + +#include "leveldb/env.h" +#include "leveldb/options.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" + +#include "util/coding.h" + +#include "vtable_format.h" + +namespace leveldb { + +class VTableReader { + public: + Status Open(const Options& options, std::string fname); + + Status Get(const VTableHandle& handle, + VTableRecord* record) const ; + private: + Options options_; + RandomAccessFile* file_{nullptr}; +}; + +} // namespace leveldb + +#endif //VTABLE_READER_H diff --git a/test/test_vtable.cc b/test/test_vtable.cc new file mode 100644 index 0000000..7caf721 --- /dev/null +++ b/test/test_vtable.cc @@ -0,0 +1,43 @@ +#include +#include + +#include "leveldb/db.h" +#include "leveldb/env.h" +#include "table/vtable_builder.h" +#include "table/vtable_reader.h" +#include "table/vtable_format.h" + +using namespace std; +using namespace leveldb; + +int main() { + VTableHandle handle1; + VTableHandle handle2; + VTableRecord record1; + VTableRecord record2; + record1.key = "001"; + record1.value = "value1"; + record2.key = "002"; + record2.value = "value2"; + + Options opt; + WritableFile *file; + opt.env->NewWritableFile("1.vtb", &file); + VTableBuilder builder(opt, file); + + builder.Add(record1, &handle1); + builder.Add(record2, &handle2); + builder.Finish(); + + VTableReader reader; + reader.Open(opt, "1.vtb"); + + VTableRecord res_record; + reader.Get(handle2, &res_record); + + cout << res_record.key.ToString() << " " << res_record.value.ToString() << endl; + + reader.Get(handle1, &res_record); + + cout << res_record.key.ToString() << " " << res_record.value.ToString() << endl; +} \ No newline at end of file diff --git a/util/coding.cc b/util/coding.cc index a8f8af8..d48683f 100644 --- a/util/coding.cc +++ b/util/coding.cc @@ -142,6 +142,20 @@ bool GetVarint64(Slice* input, uint64_t* value) { } } +bool GetFixed32(Slice* input, uint32_t* value) { + const char* p = input->data(); + const char* limit = p + input->size(); + *value = DecodeFixed32(p); + const char* q = p + 4; + if (q > limit) { + return false; + } else { + *input = Slice(q, limit - q); + return true; + } +} + + bool GetLengthPrefixedSlice(Slice* input, Slice* result) { uint32_t len; if (GetVarint32(input, &len) && input->size() >= len) { diff --git a/util/coding.h b/util/coding.h index f0bb57b..c5cb955 100644 --- a/util/coding.h +++ b/util/coding.h @@ -30,6 +30,7 @@ void PutLengthPrefixedSlice(std::string* dst, const Slice& value); // and advance the slice past the parsed value. bool GetVarint32(Slice* input, uint32_t* value); bool GetVarint64(Slice* input, uint64_t* value); +bool GetFixed32(Slice* input, uint32_t* value); bool GetLengthPrefixedSlice(Slice* input, Slice* result); // Pointer-based variants of GetVarint... These either store a value