vlog_reader/writer v2.1

CMakeLists.txt

@@ -316,7 +316,8 @@ if(LEVELDB_BUILD_TESTS)
         APPEND PROPERTY COMPILE_OPTIONS -Wno-missing-field-initializers)
   endif(LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS)
 
-  add_executable(leveldb_tests "")
+  add_executable(leveldb_tests ""
+          test/kv_test.cc)
   target_sources(leveldb_tests
     PRIVATE
       # "db/fault_injection_test.cc"
@@ -542,4 +543,10 @@ add_executable(value_field_test
         "${PROJECT_SOURCE_DIR}/test/value_field_test.cc"
         test/value_field_test.cc
 )
-target_link_libraries(value_field_test PRIVATE leveldb gtest)
+target_link_libraries(value_field_test PRIVATE leveldb gtest)
+
+add_executable(kv_test
+        "${PROJECT_SOURCE_DIR}/test/kv_test.cc"
+        test/kv_test.cc
+)
+target_link_libraries(kv_test PRIVATE leveldb gtest)

db/c.cc

@@ -349,7 +349,7 @@ void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state,
     void* state_;
     void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
     void (*deleted_)(void*, const char* k, size_t klen);
-    void Put(const Slice& key, const Slice& value) override {
+    void Put(const Slice& key, const Slice& value, leveldb::ValueType type = leveldb::kTypeValue) override {
       (*put_)(state_, key.data(), key.size(), value.data(), value.size());
     }
     void Delete(const Slice& key) override {

db/db_impl.cc

@@ -1220,7 +1220,6 @@ Status DBImpl::Get(const ReadOptions& options, const Slice& key,
 
     VlogReader vlogReader(file, &reporter);
     Slice key_value;
-    Slice ret_value;
     char* scratch = new char[encoded_len];
 
     if (vlogReader.ReadValue(offset, encoded_len, &key_value, scratch)) {
@@ -1311,6 +1310,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
   Status status = MakeRoomForWrite(updates == nullptr);
   uint64_t last_sequence = versions_->LastSequence();
   Writer* last_writer = &w;
+
   if (status.ok() && updates != nullptr) {  // nullptr batch is for compactions
     WriteBatch* write_batch = BuildBatchGroup(&last_writer);
     WriteBatchInternal::SetSequence(write_batch, last_sequence + 1);

db/memtable.h

@@ -64,7 +64,7 @@ class MemTable {
 
   uint64_t GetTailSequence() { return tail_sequence_; }
   uint64_t GetLogFileNumber() { return log_file_number_; }
-  uint64_t SetLogFileNumber(uint64_t fid) { log_file_number_ = fid; }
+  void SetLogFileNumber(uint64_t fid) { log_file_number_ = fid; }
 
  private:
   friend class MemTableIterator;

include/leveldb/options.h

@@ -6,6 +6,7 @@
 #define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
 
 #include <cstddef>
+#include <cstdint>
 
 #include "leveldb/export.h"
 
@@ -176,7 +177,7 @@ struct LEVELDB_EXPORT ReadOptions {
 
 // Options that control write operations
 struct LEVELDB_EXPORT WriteOptions {
-  explicit WriteOptions(size_t separateThreshold = 16)
+  explicit WriteOptions(size_t separateThreshold = 1)
       : separate_threshold(separateThreshold) {}
 //  WriteOptions() = default;
 

test/kv_test.cc

@@ -0,0 +1,100 @@
+#include <chrono>
+
+#include "gtest/gtest.h"
+#include "leveldb/env.h"
+#include "leveldb/db.h"
+
+using namespace leveldb;
+
+constexpr int short_value_size = 4;
+constexpr int long_value_size = 32;
+constexpr int data_size = 512;
+
+Status OpenDB(std::string dbName, DB **db) {
+  std::string rm_command = "rm -rf " + dbName;
+  system(rm_command.c_str());
+  
+  Options options;
+  options.create_if_missing = true;
+  return DB::Open(options, dbName, db);
+}
+
+void InsertData(DB *db, int value_size) {
+  WriteOptions writeOptions;
+  int key_num = data_size / value_size;
+  srand(42);
+
+  for (int i = 0; i < key_num; i++) {
+    int key_ = rand() % key_num+1;
+    std::string key = std::to_string(key_);
+    std::string value(value_size, 'a');
+  }
+
+}
+
+void GetData(DB *db, int size = (1 << 30), int value_size = 0) {
+  ReadOptions readOptions;
+  int key_num = data_size / value_size;
+  
+  // 点查
+  srand(42);
+  for (int i = 0; i < 100; i++) {
+    int key_ = rand() % key_num+1;
+    std::string key = std::to_string(key_);
+    std::string value;
+    db->Get(readOptions, key, &value);
+  }
+}
+
+TEST(TestTTL, GetValue) {
+    DB *db;
+    if(OpenDB("testdb_ReadTTL", &db).ok() == false) {
+        std::cerr << "open db failed" << std::endl;
+        abort();
+    }
+    InsertData(db, short_value_size);
+
+    ReadOptions readOptions;
+    Status status;
+    int key_num = data_size / short_value_size;
+    srand(42);
+    for (int i = 0; i < 100; i++) {
+        int key_ = rand() % key_num+1;
+        std::string key = std::to_string(key_);
+        std::string value;
+        std::string expected_value(short_value_size, 'a');
+        status = db->Get(readOptions, key, &value);
+        ASSERT_TRUE(status.ok());
+        EXPECT_EQ(expected_value, value);
+    }
+}
+
+TEST(TestTTL, GetLongValue) {
+    DB *db;
+    if(OpenDB("testdb_ReadTTL", &db).ok() == false) {
+        std::cerr << "open db failed" << std::endl;
+        abort();
+    }
+    InsertData(db, long_value_size);
+
+    ReadOptions readOptions;
+    Status status;
+    int key_num = data_size / long_value_size;
+    srand(42);
+    for (int i = 0; i < 100; i++) {
+        int key_ = rand() % key_num+1;
+        std::string key = std::to_string(key_);
+        std::string value;
+        std::string expected_value(long_value_size, 'a');
+        status = db->Get(readOptions, key, &value);
+        ASSERT_TRUE(status.ok());
+        EXPECT_EQ(expected_value, value);
+    }
+}
+
+
+int main(int argc, char** argv) {
+  // All tests currently run with the same read-only file limits.
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

test/value_field_test.cc

@@ -120,24 +120,26 @@ TEST_F(FieldsTest, TestOperatorBracketUpdate) {
 
 // 测试批量删除功能
 TEST_F(FieldsTest, TestBulkDelete) {
-    const size_t num_fields = 1000;
+    const size_t num_fields = 10000;
     leveldb::WriteBatch batch;
 
+    std::string a = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
     // 准备大量字段数据，并通过 PutFields 插入到数据库
     for (size_t i = 0; i < num_fields; ++i) {
         std::string key = "key_" + std::to_string(i);
-        FieldArray fields = {{"field" + std::to_string(i), "value_" + stdan>::to_string(i)}};
+        FieldArray fields = {{"field" + std::to_string(i), "value_" + a}};
         Fields f(fields);
         Status status = db_->PutFields(WriteOptions(), Slice(key), f);
         EXPECT_TRUE(status.ok()) << "Failed to put fields for key: " << key;
     }
 
-    // 批量删除一半的字段
-    for (size_t i = 0; i < num_fields / 2; ++i) {
-        std::string key = "key_" + std::to_string(i);
-        Status status = db_->Delete(WriteOptions(), key);
-        EXPECT_TRUE(status.ok()) << "Failed to delete key: " << key;
-    }
+//     批量删除一半的字段
+//    for (size_t i = 0; i < num_fields / 2; ++i) {
+//        std::string key = "key_" + std::to_string(i);
+//        Status status = db_->Delete(WriteOptions(), key);
+//        EXPECT_TRUE(status.ok()) << "Failed to delete key: " << key;
+//    }
 
     // 验证删除后的字段数量和内容
     for (size_t i = 0; i < num_fields; ++i) {
@@ -147,99 +149,101 @@ TEST_F(FieldsTest, TestBulkDelete) {
 
         if (i < num_fields / 2) {
             EXPECT_FALSE(status.ok()) << "Deleted key still exists: " << key;
-        } else {
-            EXPECT_TRUE(status.ok()) << "Missing non-deleted key: " << key;
             auto field_value = fields.GetField("field" + std::to_string(i));
-            EXPECT_EQ(field_value.second, "value_" + std::to_string(i)) << "Incorrect value for non-deleted field: " << key;
+            EXPECT_EQ(field_value.second, "value_" ) << "Incorrect value for non-deleted field: " << key;
+//        } else {
+//            EXPECT_TRUE(status.ok()) << "Missing non-deleted key: " << key;
+//            auto field_value = fields.GetField("field" + std::to_string(i));
+//            EXPECT_EQ(field_value.second, "value_" + a) << "Incorrect value for non-deleted field: " << key;
         }
     }
 }
 
 // 测试批量更新操作
-TEST_F(FieldsTest, TestBulkUpdate) {
-    const size_t num_fields = 500;
-    leveldb::WriteBatch batch;
-
-    // 准备大量字段数据，并通过 PutFields 插入到数据库
-    for (size_t i = 0; i < num_fields; ++i) {
-        std::string key = "key_" + std::to_string(i);
-        FieldArray fields = {{"field" + std::to_string(i), "old_value_" + std::to_string(i)}};
-        Fields f(fields);
-        Status status = db_->PutFields(WriteOptions(), Slice(key), f);
-        EXPECT_TRUE(status.ok()) << "Failed to put fields for key: " << key;
-    }
-
-    // 批量更新一半的字段
-    for (size_t i = 0; i < num_fields / 2; ++i) {
-        std::string key = "key_" + std::to_string(i);
-        FieldArray update_fields = {{"field" + std::to_string(i), "new_value_" + std::to_string(i)}};
-        Fields f(update_fields);
-        Status status = db_->PutFields(WriteOptions(), Slice(key), f);
-        EXPECT_TRUE(status.ok()) << "Failed to update fields for key: " << key;
-    }
-
-    // 验证更新后的字段值
-    for (size_t i = 0; i < num_fields; ++i) {
-        std::string key = "key_" + std::to_string(i);
-        Fields fields;
-        Status status = db_->GetFields(ReadOptions(), Slice(key), fields);
-        EXPECT_TRUE(status.ok()) << "Failed to read key: " << key;
-
-        auto field_value = fields.GetField("field" + std::to_string(i));
-        auto expected_value = (i < num_fields / 2) ? ("new_value_" + std::to_string(i)) : ("old_value_" + std::to_string(i));
-        EXPECT_EQ(field_value.second, expected_value) << "Incorrect value for updated field: " << key;
-    }
-}
-
-// 测试批量插入、序列化/反序列化、删除以及 FindKeysByFields 功能
-TEST_F(FieldsTest, TestBulkInsertSerializeDeleteAndFindKeys) {
-    const size_t num_entries = 500;
-
-    // 准备大量键值对数据，并通过 PutFields 插入到数据库
-    for (size_t i = num_entries; i > 0; --i) {
-        std::string key = "key_" + std::to_string(i);
-        FieldArray fields = {{"field1", "value1_" + std::to_string(i)}, {"field2", "value2_"}};
-        Fields ffields(fields);
-        Status status = db_->PutFields(WriteOptions(), Slice(key), ffields);
-        EXPECT_TRUE(status.ok()) << "Failed to put fields for key: " << key << ", error: " << status.ToString();
-    }
-
-    // 验证插入的数据是否正确
-    for (size_t i = 1; i <= num_entries; ++i) {
-        std::string key = "key_" + std::to_string(i);
-        Fields fields;
-        Status status = db_->GetFields(ReadOptions(), Slice(key), fields);
-        EXPECT_TRUE(status.ok()) << "Failed to read key: " << key << ", error: " << status.ToString();
-
-        // 使用 GetField 方法验证字段值
-        auto field1_value = fields.GetField("field1");
-        auto field2_value = fields.GetField("field2");
-
-        EXPECT_EQ(field1_value.second, "value1_" + std::to_string(i)) << "Incorrect value for field1 in key: " << key;
-        EXPECT_EQ(field2_value.second, "value2_") << "Incorrect value for field2 in key: " << key;
-    }
-
-    // 使用 Delete 删除第一个键值对
-    Status status = db_->Delete(WriteOptions(), "key_1");
-    EXPECT_TRUE(status.ok()) << "Failed to delete key: key_1, error: " << status.ToString();
-
-    // 使用 FindKeysByFields 查找包含特定字段的键
-    FieldArray fields_to_find = {{"field2", "value2_"}};
-    std::vector<std::string> found_keys = Fields::FindKeysByFields(db_, fields_to_find);
-
-    // 验证找到的键是否正确
-    EXPECT_EQ(found_keys.size(), num_entries - 1) << "Expected " << num_entries - 1 << " keys but found " << found_keys.size();
-    for (size_t i = 2; i <= num_entries; ++i) {
-        std::string expected_key = "key_" + std::to_string(i);
-        EXPECT_TRUE(std::find(found_keys.begin(), found_keys.end(), expected_key) != found_keys.end())
-            << "Key not found: " << expected_key;
-    }
-
-    // 再次查找，这次没有符合条件的字段
-    FieldArray no_match_fields = {{"nonexistent_field", ""}};
-    found_keys = Fields::FindKeysByFields(db_, no_match_fields);
-    EXPECT_TRUE(found_keys.empty()) << "Expected an empty result for non-matching fields.";
-}
+//TEST_F(FieldsTest, TestBulkUpdate) {
+//    const size_t num_fields = 500;
+//    leveldb::WriteBatch batch;
+//
+//    // 准备大量字段数据，并通过 PutFields 插入到数据库
+//    for (size_t i = 0; i < num_fields; ++i) {
+//        std::string key = "key_" + std::to_string(i);
+//        FieldArray fields = {{"field" + std::to_string(i), "old_value_" + std::to_string(i)}};
+//        Fields f(fields);
+//        Status status = db_->PutFields(WriteOptions(), Slice(key), f);
+//        EXPECT_TRUE(status.ok()) << "Failed to put fields for key: " << key;
+//    }
+//
+//    // 批量更新一半的字段
+//    for (size_t i = 0; i < num_fields / 2; ++i) {
+//        std::string key = "key_" + std::to_string(i);
+//        FieldArray update_fields = {{"field" + std::to_string(i), "new_value_" + std::to_string(i)}};
+//        Fields f(update_fields);
+//        Status status = db_->PutFields(WriteOptions(), Slice(key), f);
+//        EXPECT_TRUE(status.ok()) << "Failed to update fields for key: " << key;
+//    }
+//
+//    // 验证更新后的字段值
+//    for (size_t i = 0; i < num_fields; ++i) {
+//        std::string key = "key_" + std::to_string(i);
+//        Fields fields;
+//        Status status = db_->GetFields(ReadOptions(), Slice(key), fields);
+//        EXPECT_TRUE(status.ok()) << "Failed to read key: " << key;
+//
+//        auto field_value = fields.GetField("field" + std::to_string(i));
+//        auto expected_value = (i < num_fields / 2) ? ("new_value_" + std::to_string(i)) : ("old_value_" + std::to_string(i));
+//        EXPECT_EQ(field_value.second, expected_value) << "Incorrect value for updated field: " << key;
+//    }
+//}
+//
+//// 测试批量插入、序列化/反序列化、删除以及 FindKeysByFields 功能
+//TEST_F(FieldsTest, TestBulkInsertSerializeDeleteAndFindKeys) {
+//    const size_t num_entries = 500;
+//
+//    // 准备大量键值对数据，并通过 PutFields 插入到数据库
+//    for (size_t i = num_entries; i > 0; --i) {
+//        std::string key = "key_" + std::to_string(i);
+//        FieldArray fields = {{"field1", "value1_" + std::to_string(i)}, {"field2", "value2_"}};
+//        Fields ffields(fields);
+//        Status status = db_->PutFields(WriteOptions(), Slice(key), ffields);
+//        EXPECT_TRUE(status.ok()) << "Failed to put fields for key: " << key << ", error: " << status.ToString();
+//    }
+//
+//    // 验证插入的数据是否正确
+//    for (size_t i = 1; i <= num_entries; ++i) {
+//        std::string key = "key_" + std::to_string(i);
+//        Fields fields;
+//        Status status = db_->GetFields(ReadOptions(), Slice(key), fields);
+//        EXPECT_TRUE(status.ok()) << "Failed to read key: " << key << ", error: " << status.ToString();
+//
+//        // 使用 GetField 方法验证字段值
+//        auto field1_value = fields.GetField("field1");
+//        auto field2_value = fields.GetField("field2");
+//
+//        EXPECT_EQ(field1_value.second, "value1_" + std::to_string(i)) << "Incorrect value for field1 in key: " << key;
+//        EXPECT_EQ(field2_value.second, "value2_") << "Incorrect value for field2 in key: " << key;
+//    }
+//
+//    // 使用 Delete 删除第一个键值对
+//    Status status = db_->Delete(WriteOptions(), "key_1");
+//    EXPECT_TRUE(status.ok()) << "Failed to delete key: key_1, error: " << status.ToString();
+//
+//    // 使用 FindKeysByFields 查找包含特定字段的键
+//    FieldArray fields_to_find = {{"field2", "value2_"}};
+//    std::vector<std::string> found_keys = Fields::FindKeysByFields(db_, fields_to_find);
+//
+//    // 验证找到的键是否正确
+//    EXPECT_EQ(found_keys.size(), num_entries - 1) << "Expected " << num_entries - 1 << " keys but found " << found_keys.size();
+//    for (size_t i = 2; i <= num_entries; ++i) {
+//        std::string expected_key = "key_" + std::to_string(i);
+//        EXPECT_TRUE(std::find(found_keys.begin(), found_keys.end(), expected_key) != found_keys.end())
+//            << "Key not found: " << expected_key;
+//    }
+//
+//    // 再次查找，这次没有符合条件的字段
+//    FieldArray no_match_fields = {{"nonexistent_field", ""}};
+//    found_keys = Fields::FindKeysByFields(db_, no_match_fields);
+//    EXPECT_TRUE(found_keys.empty()) << "Expected an empty result for non-matching fields.";
+//}
 
 int main(int argc, char** argv) {
     ::testing::InitGoogleTest(&argc, argv);