diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a8498b..54b14a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,14 +528,4 @@ target_link_libraries(db_test2 PRIVATE leveldb) add_executable(ttl_test "${PROJECT_SOURCE_DIR}/test/ttl_test.cc" ) -target_link_libraries(ttl_test PRIVATE leveldb gtest) - -add_executable(field_test - "${PROJECT_SOURCE_DIR}/test/field_test.cc" -) -target_link_libraries(field_test PRIVATE leveldb gtest) - -# add_executable(kv_seperate_test -# "${PROJECT_SOURCE_DIR}/test/kv_seperate_test.cc" -# ) -# target_link_libraries(kv_seperate_test PRIVATE leveldb gtest) \ No newline at end of file +target_link_libraries(ttl_test PRIVATE leveldb gtest) \ No newline at end of file diff --git a/db/db_impl.cc b/db/db_impl.cc index 292eb0f..f96d245 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include "db/builder.h" #include "db/db_iter.h" @@ -149,8 +148,6 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) manual_compaction_(nullptr), versions_(new VersionSet(dbname_, &options_, table_cache_, &internal_comparator_)) {} - bool static key_value_separated_; //朴,添加是否kv分离,12.07 - DBImpl::~DBImpl() { // Wait for background work to finish. @@ -1196,27 +1193,11 @@ void DBImpl::ReleaseSnapshot(const Snapshot* snapshot) { snapshots_.Delete(static_cast(snapshot)); } - // Convenience methods -// Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val) { -// return DB::Put(o, key, val); -// } - Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val) { - if (key_value_separated_) { - // 分离key和value的逻辑,朴,12.07 - //... - } else { - // 不分离key和value的逻辑 - return DB::Put(o, key, val); - } + return DB::Put(o, key, val); } - -Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val, uint64_t ttl) { - return DB::Put(o, key, val, ttl); -} // 实现新的put接口,心 - Status DBImpl::Delete(const WriteOptions& options, const Slice& key) { return DB::Delete(options, key); } @@ -1504,32 +1485,18 @@ void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) { // Default implementations of convenience methods that subclasses of DB // can call if they wish -Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) { //朴 +Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) { WriteBatch batch; batch.Put(key, value); return Write(opt, &batch); } -// 假设增加一个新的Put接口,包含TTL参数, 单位(秒) -Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value, uint64_t ttl){ - WriteBatch batch; - batch.Put(key, value, ttl); - return Write(opt, &batch); - } // 这里应该是新的PUT接口的真正实现的地方,还是由本来的DB类实现,怪?心 - - - - - Status DB::Delete(const WriteOptions& opt, const Slice& key) { WriteBatch batch; batch.Delete(key); return Write(opt, &batch); } - - - DB::~DB() = default; Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) { diff --git a/db/db_impl.h b/db/db_impl.h index 14189d5..c7b0172 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -38,8 +38,6 @@ class DBImpl : public DB { // Implementations of the DB interface Status Put(const WriteOptions&, const Slice& key, const Slice& value) override; - Status Put(const WriteOptions&, const Slice& key, - const Slice& value, uint64_t ttl) override; //实现新的put接口,心 Status Delete(const WriteOptions&, const Slice& key) override; Status Write(const WriteOptions& options, WriteBatch* updates) override; Status Get(const ReadOptions& options, const Slice& key, @@ -50,11 +48,8 @@ class DBImpl : public DB { bool GetProperty(const Slice& property, std::string* value) override; void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) override; void CompactRange(const Slice* begin, const Slice* end) override; - // 朴,添加是否kv分离接口,12.07 - bool static key_value_separated_; // Extra methods (for testing) that are not in the public DB interface - // Compact any files in the named level that overlap [*begin,*end] void TEST_CompactRange(int level, const Slice* begin, const Slice* end); @@ -81,8 +76,6 @@ class DBImpl : public DB { struct CompactionState; struct Writer; - - // Information for a manual compaction struct ManualCompaction { int level; @@ -219,8 +212,6 @@ Options SanitizeOptions(const std::string& db, const InternalFilterPolicy* ipolicy, const Options& src); - - } // namespace leveldb #endif // STORAGE_LEVELDB_DB_DB_IMPL_H_ diff --git a/db/db_test.cc b/db/db_test.cc index 01abf71..a4a84cd 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2117,10 +2117,6 @@ class ModelDB : public DB { Status Put(const WriteOptions& o, const Slice& k, const Slice& v) override { return DB::Put(o, k, v); } - Status Put(const WriteOptions& o, const Slice& k, - const Slice& v, uint64_t ttl) override { - return DB::Put(o, k, v); - }// 实现的是DB里的新put接口,心 Status Delete(const WriteOptions& o, const Slice& key) override { return DB::Delete(o, key); } diff --git a/db/write_batch.cc b/db/write_batch.cc index e5454f8..b54313c 100644 --- a/db/write_batch.cc +++ b/db/write_batch.cc @@ -19,13 +19,8 @@ #include "db/memtable.h" #include "db/write_batch_internal.h" #include "leveldb/db.h" -#include "db/db_impl.h" //朴 #include "util/coding.h" -#include // For std::ostringstream 心 -#include -#include - namespace leveldb { // WriteBatch header has an 8-byte sequence number followed by a 4-byte count. @@ -107,60 +102,6 @@ void WriteBatch::Put(const Slice& key, const Slice& value) { PutLengthPrefixedSlice(&rep_, value); } -// void WriteBatch::Put(const Slice& key, const Slice& value) { // 朴,kv分离,12.07 -// if (DBImpl::key_value_separated_) { -// // 分离key和value的逻辑 -// // 例如,你可以将key和value分别存储在不同的容器中 -// // 这里需要根据你的具体需求来实现 -// //... -// if (value.size() > max_value_size_) { -// // 分离key和value的逻辑 -// // 将value存进新的数据结构blobfile -// //... -// // 例如,你可以使用以下代码将value写入blobfile -// std::ofstream blobfile("blobfile.dat", std::ios::binary | std::ios::app); -// blobfile.write(value.data(), value.size()); -// blobfile.close(); -// } -// } -// else { -// // 不分离key和value的逻辑 -// WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); -// rep_.push_back(static_cast(kTypeValue)); -// PutLengthPrefixedSlice(&rep_, key); -// PutLengthPrefixedSlice(&rep_, value); -// } -// } - - -void WriteBatch::Put(const Slice& key, const Slice& value, uint64_t ttl) { - WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); - rep_.push_back(static_cast(kTypeValue)); - PutLengthPrefixedSlice(&rep_, key); - - // 获取当前时间 - auto now = std::chrono::system_clock::now(); - - // 加上ttl - auto future_time = now + std::chrono::seconds(ttl); - - // 转换为 time_t - std::time_t future_time_t = std::chrono::system_clock::to_time_t(future_time); - - // 将 time_t 转换为 tm 结构 - std::tm* local_tm = std::localtime(&future_time_t); - - // 格式化为字符串 - char buffer[20]; // 格式化字符串的缓冲区 - std::strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", local_tm); - std::string future_time_str(buffer); - - // 拼接原本的值和时间字符串 - std::string combined_str = value.ToString() + future_time_str; - PutLengthPrefixedSlice(&rep_, Slice(combined_str)); - -} // 心 - void WriteBatch::Delete(const Slice& key) { WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); rep_.push_back(static_cast(kTypeDeletion)); diff --git a/include/leveldb/db.h b/include/leveldb/db.h index 3d34cca..bf4eec5 100644 --- a/include/leveldb/db.h +++ b/include/leveldb/db.h @@ -11,16 +11,8 @@ #include "leveldb/export.h" #include "leveldb/iterator.h" #include "leveldb/options.h" -#include "util/coding.h" - -#include namespace leveldb { -// 用于表示一个包含多个字段的值 - - -using Field = std::pair; // field_name:field_value -using FieldArray = std::vector>; // Update CMakeLists.txt if you change these static const int kMajorVersion = 1; diff --git a/include/leveldb/options.h b/include/leveldb/options.h index 18520e2..d755f46 100644 --- a/include/leveldb/options.h +++ b/include/leveldb/options.h @@ -145,14 +145,6 @@ struct LEVELDB_EXPORT Options { // Many applications will benefit from passing the result of // NewBloomFilterPolicy() here. const FilterPolicy* filter_policy = nullptr; - - // 添加新的 KV 分离选项,朴 - bool key_value_separated = false; - - Options() { - // 初始化选项值,朴 - key_value_separated = false; - } }; // Options that control read operations diff --git a/include/leveldb/write_batch.h b/include/leveldb/write_batch.h index 015fcd8..94d4115 100644 --- a/include/leveldb/write_batch.h +++ b/include/leveldb/write_batch.h @@ -25,7 +25,6 @@ #include "leveldb/export.h" #include "leveldb/status.h" -#include namespace leveldb { @@ -51,8 +50,6 @@ class LEVELDB_EXPORT WriteBatch { // Store the mapping "key->value" in the database. void Put(const Slice& key, const Slice& value); - void Put(const Slice& key, const Slice& value, uint64_t ttl); //心 - // If the database contains a mapping for "key", erase it. Else do nothing. void Delete(const Slice& key); diff --git a/table/blob_file.cc b/table/blob_file.cc deleted file mode 100644 index a23c92c..0000000 --- a/table/blob_file.cc +++ /dev/null @@ -1,26 +0,0 @@ -#include "blob_file.h" -#include - -namespace leveldb { - -BlobFile::BlobFile(const std::string& filename) : filename_(filename) { - // 初始化 BlobFile,例如打开文件 -} - -BlobFile::~BlobFile() { - // 关闭文件 -} - -Status BlobFile::Put(const Slice& key, const Slice& value) { - std::ofstream file(filename_, std::ios::app | std::ios::binary); - if (!file.is_open()) { - return Status::IOError("Failed to open blob file"); - } - // 简单实现,将 key 和 value 写入文件 - file.write(key.data(), key.size()); - file.write(value.data(), value.size()); - file.close(); - return Status::OK(); -} - -} // namespace leveldb diff --git a/table/blob_file.h b/table/blob_file.h deleted file mode 100644 index 45fa3ef..0000000 --- a/table/blob_file.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef LEVELDB_BLOB_FILE_H_ -#define LEVELDB_BLOB_FILE_H_ - -#include -#include "leveldb/status.h" -#include "leveldb/slice.h" - -namespace leveldb { - -class BlobFile { -public: - BlobFile(const std::string& filename); - ~BlobFile(); - - // 写入键值对 - Status Put(const Slice& key, const Slice& value); - -private: - std::string filename_; - // 内部实现,例如文件指针或缓冲区 -}; - -} // namespace leveldb - -#endif // LEVELDB_BLOB_FILE_H_ diff --git a/table/table_builder.cc b/table/table_builder.cc index 7eafa6a..0932c94 100644 --- a/table/table_builder.cc +++ b/table/table_builder.cc @@ -15,23 +15,9 @@ #include "table/format.h" #include "util/coding.h" #include "util/crc32c.h" -#include "db/db_impl.h" //朴 -#include "table/blob_file.h" //朴 -#include "table/block.h" //朴 - -const size_t min_blob_size = 1024; // 设定值大小阈值为 1KB,朴 namespace leveldb { - BlobFile* blobfile = new BlobFile("blob_data"); // 初始化全局 blobfile 对象,朴 - class BlobFileManager { - public: - static BlobFile* GetInstance() { - static BlobFile instance("blob_data"); - return &instance; - } - }; - struct TableBuilder::Rep { Rep(const Options& opt, WritableFile* f) : options(opt), @@ -140,41 +126,12 @@ void TableBuilder::Flush() { Rep* r = rep_; assert(!r->closed); if (!ok()) return; - if (r->data_block.empty()) return; //朴,正常判断 + if (r->data_block.empty()) return; assert(!r->pending_index_entry); - - if (DBImpl::key_value_separated_) { - // 这里获取数据块内容并初始化 Block 对象,朴 - Slice block_content = r->data_block.Finish(); - BlockContents contents; - contents.data = block_content; - contents.heap_allocated = false; - contents.cachable = false; - - // 初始化 Block - Block data_block(contents); - std::unique_ptr iter(data_block.NewIterator(Options().comparator)); - - // 遍历数据块中的键值对 - for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { - const Slice& key = iter->key(); - const Slice& value = iter->value(); - - // 检查值是否大于阈值 - if (value.size() > min_blob_size) { - // 将值存储到 blobfile 中 - Status status = blobfile->Put(key, value); - if (!status.ok()) { - r->status = status; - } - } - } - } - - WriteBlock(&r->data_block, &r->pending_handle); //将数据块写入文件,并获取数据块的句柄。 + WriteBlock(&r->data_block, &r->pending_handle); if (ok()) { r->pending_index_entry = true; - r->status = r->file->Flush(); //刷新 + r->status = r->file->Flush(); } if (r->filter_block != nullptr) { r->filter_block->StartBlock(r->offset); diff --git a/test/field_test.cc b/test/field_test.cc deleted file mode 100644 index 252039f..0000000 --- a/test/field_test.cc +++ /dev/null @@ -1,122 +0,0 @@ - - -#include "gtest/gtest.h" - -#include "leveldb/env.h" -#include "leveldb/db.h" -#include "util/coding.h" - -#include - -using namespace leveldb; -constexpr int value_size = 2048; -constexpr int data_size = 128 << 20; - -// 根据字段值查找所有包含该字段的 key -std::vector FindKeysByField(leveldb::DB* db, Field &field) { - Iterator* iter = db->NewIterator(ReadOptions()); - std::vector ret_keys; - int64_t bytes = 0; - for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { - auto fields_ret = ParseValue(iter->value().data()); - for (Field each_field : fields_ret) - { - std::cout << each_field.first << " " << each_field.second << std::endl; - if (field.first.compare(each_field.first) == 0) { - if (field.second.compare(each_field.second)==0) - { - ret_keys.push_back(iter->key().data()); - } - else - break; - } - } - } - delete iter; - return ret_keys; -} - - -Status OpenDB(std::string dbName, DB **db) { - Options options; - options.create_if_missing = true; - return DB::Open(options, dbName, db); -} - -TEST(TestField, Read) { - DB *db; - if(OpenDB("testdb", &db).ok() == false) { - std::cerr << "open db failed" << std::endl; - abort(); - } - - std::string key = "k_1"; - - FieldArray fields = { - {"name", "Customer#000000001"}, - {"address", "IVhzIApeRb"}, - {"phone", "25-989-741-2988"} - }; - - // 序列化并插入 - std::string value = SerializeValue(fields); - db->Put(WriteOptions(), key, value); - - // 读取并反序列化 - std::string value_ret; - db->Get(ReadOptions(), key, &value_ret); - auto fields_ret = ParseValue(value_ret); - std::cout << "第一个字段名:"<< fields_ret[0].first << "第一个字段值" << fields_ret[0].second<< std::endl; - delete db; -} - -TEST(TestField, Find) { - DB *db; - if(OpenDB("testdb", &db).ok() == false) { - std::cerr << "open db failed" << std::endl; - abort(); - } - std::vector keys = {"s_1", "s_2", "s_3", "s_4"}; - - // 构造一组字段数组 - std::vector FieldArrays = { - { - {"name", "Sarah"},{"sex", "f"},{"age", "20"} - }, - { - {"name", "Mike"},{"sex", "m"},{"age", "19"},{"hobby", "badminton"} - }, - { - {"name", "Amy"},{"sex", "f"},{"age", "21"},{"talent", "sing"} - }, - { - {"name", "John"}, {"sex", "m"},{"age", "20"} - } - }; - - - // 序列化并插入 - for (int i=0; iPut(WriteOptions(), key, value); - } - - // 构建目标字段 - Field field = {"sex", "f"}; - std::vector key_ret; - // 查询得到对应的key - key_ret = FindKeysByField(db, field); - for (int i = 0; i < key_ret.size(); i++) { - std::cout << "找到的键:" << key_ret[i] << std::endl; - } - delete db; -} - -int main(int argc, char** argv) { - // All tests currently run with the same read-only file limits. - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/test/kv_seperate_test.cc b/test/kv_seperate_test.cc deleted file mode 100644 index ef69179..0000000 --- a/test/kv_seperate_test.cc +++ /dev/null @@ -1,119 +0,0 @@ -#include "gtest/gtest.h" -#include "leveldb/env.h" -#include "leveldb/db.h" -#include "table/blob_file.h" // 假设 BlobFile 的头文件 - -using namespace leveldb; - -constexpr int value_size = 2048; // 单个值的大小 -constexpr int data_size = 128 << 20; // 总数据大小 -constexpr int min_blob_size = 1024; // KV 分离的阈值 - -Status OpenDB(std::string dbName, DB** db) { - Options options; - options.create_if_missing = true; - options.key_value_separated = true; // 启用 KV 分离 - return DB::Open(options, dbName, db); -} - -// 插入数据,模拟 KV 分离 -void InsertData(DB* db) { - WriteOptions writeOptions; - int key_num = data_size / value_size; - srand(static_cast(time(0))); - - for (int i = 0; i < key_num; i++) { - int key_ = rand() % key_num + 1; - std::string key = std::to_string(key_); - std::string value(value_size, 'a'); // 大 value - db->Put(writeOptions, key, value); // 使用标准 Put 接口插入 - } -} - -// 检查数据是否被正确存入 BlobFile -void VerifyBlobFile(const std::string& blob_file_path, int expected_entries) { - BlobFile blobfile(blob_file_path, BlobFile::kReadMode); - Status status = blobfile.Open(); - ASSERT_TRUE(status.ok()); - - int entry_count = 0; - BlobFile::Iterator it = blobfile.NewIterator(); - for (it.SeekToFirst(); it.Valid(); it.Next()) { - ++entry_count; - const Slice& key = it.key(); - const Slice& value = it.value(); - ASSERT_GT(value.size(), min_blob_size); // 确认 value 大于阈值 - } - - ASSERT_EQ(entry_count, expected_entries); // 确认条目数是否正确 - blobfile.Close(); -} - -// KV 分离读写测试 -TEST(TestKVSeparation, WriteAndRead) { - DB* db; - if (OpenDB("testdb", &db).ok() == false) { - std::cerr << "open db failed" << std::endl; - abort(); - } - - // 插入数据 - InsertData(db); - - // 验证 BlobFile 内容 - VerifyBlobFile("blob_data", data_size / value_size); - - // 随机点查数据 - ReadOptions readOptions; - srand(static_cast(time(0))); - int key_num = data_size / value_size; - for (int i = 0; i < 100; i++) { - int key_ = rand() % key_num + 1; - std::string key = std::to_string(key_); - std::string value; - Status status = db->Get(readOptions, key, &value); - ASSERT_TRUE(status.ok()); // 验证是否成功读取 - if (value.size() > min_blob_size) { - ASSERT_TRUE(value == std::string(value_size, 'a')); // 验证大 value 的内容 - } - } - - delete db; -} - -// KV 分离压缩测试 -TEST(TestKVSeparation, Compaction) { - DB* db; - - if (OpenDB("testdb", &db).ok() == false) { - std::cerr << "open db failed" << std::endl; - abort(); - } - - // 插入数据 - InsertData(db); - - leveldb::Range ranges[1]; - ranges[0] = leveldb::Range("-", "A"); - uint64_t sizes[1]; - db->GetApproximateSizes(ranges, 1, sizes); - ASSERT_GT(sizes[0], 0); - - // 执行压缩 - db->CompactRange(nullptr, nullptr); - - // 验证压缩后主数据区的大小 - ranges[0] = leveldb::Range("-", "A"); - db->GetApproximateSizes(ranges, 1, sizes); - ASSERT_EQ(sizes[0], 0); - - // 验证 BlobFile 内容仍然有效 - VerifyBlobFile("blob_data", data_size / value_size); - - delete db; -} - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/test/ttl_test.cc b/test/ttl_test.cc index 8a9641c..06f4cda 100644 --- a/test/ttl_test.cc +++ b/test/ttl_test.cc @@ -74,7 +74,7 @@ TEST(TestTTL, ReadTTL) { std::string key = std::to_string(key_); std::string value; status = db->Get(readOptions, key, &value); - ASSERT_FALSE(status.ok()); // 经过超长时间之后所有的键值对应该都过期了,心 + ASSERT_FALSE(status.ok()); } } @@ -99,9 +99,9 @@ TEST(TestTTL, CompactionTTL) { db->CompactRange(nullptr, nullptr); - // leveldb::Range ranges[1]; // 这里为什么要重复定义?心 + leveldb::Range ranges[1]; ranges[0] = leveldb::Range("-", "A"); - // uint64_t sizes[1]; // 心 + uint64_t sizes[1]; db->GetApproximateSizes(ranges, 1, sizes); ASSERT_EQ(sizes[0], 0); } diff --git a/util/coding.cc b/util/coding.cc index 8ee6664..a8f8af8 100644 --- a/util/coding.cc +++ b/util/coding.cc @@ -3,10 +3,6 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "util/coding.h" -#include "leveldb/db.h" -#include - -#include namespace leveldb { @@ -53,9 +49,7 @@ char* EncodeVarint32(char* dst, uint32_t v) { void PutVarint32(std::string* dst, uint32_t v) { char buf[5]; char* ptr = EncodeVarint32(buf, v); - // printf("加入的长度%d\n", ptr - buf); dst->append(buf, ptr - buf); - // printf("插入之后的长度为:%d", dst->length()); } char* EncodeVarint64(char* dst, uint64_t v) { @@ -158,42 +152,5 @@ bool GetLengthPrefixedSlice(Slice* input, Slice* result) { return false; } } -// 序列化为字符串 -std::string SerializeValue(const FieldArray& fields) { - std::string serialized_value = ""; - uint32_t num = fields.size(); - // std::cout<< "加入的值中包含的字段数:"< #include #include -#include -#include "util/coding.h" -#include "leveldb/db.h" -#include #include "leveldb/slice.h" #include "port/port.h" namespace leveldb { -using Field = std::pair; // field_name:field_value -using FieldArray = std::vector>; // Standard Put... routines append to a string void PutFixed32(std::string* dst, uint32_t value); @@ -122,8 +116,7 @@ inline const char* GetVarint32Ptr(const char* p, const char* limit, } return GetVarint32PtrFallback(p, limit, value); } -std::string SerializeValue(const FieldArray& fields); -FieldArray ParseValue(const std::string& value_str); + } // namespace leveldb #endif // STORAGE_LEVELDB_UTIL_CODING_H_