get delay

CMakeLists.txt

@@ -522,6 +522,10 @@ add_executable(db_test1
 "${PROJECT_SOURCE_DIR}/test/test.cpp"
       )
 target_link_libraries(db_test1 PRIVATE leveldb gtest)
+add_executable(db_test2
+"${PROJECT_SOURCE_DIR}/test/test2.cpp"
+      )
+target_link_libraries(db_test2 PRIVATE leveldb gtest)
 add_executable(db_test_bench
 "${PROJECT_SOURCE_DIR}/test/benchmark_4leveldb.cpp"
       )

db/db_impl.cc

@@ -1829,24 +1829,23 @@ void DBImpl::GarbageCollect() {
       // 更新当前偏移
       current_offset += sizeof(uint64_t);
 
-      // Now seek to the actual data position and read the value
-      cur_valuelog.seekg(current_offset);
-      char* value_buf = new char[val_len];
-      cur_valuelog.read(value_buf, val_len);
-      if (!cur_valuelog.good()) {
-        delete[] key_buf;
-        delete[] key_buf_len;
-        delete[] value_buf_len;
-        delete[] value_buf;
-        cur_valuelog.close();
-        std::cerr << "Failed to read file: " << valuelog_name << std::endl;
-        break;
-      }
+      // // Now seek to the actual data position and read the value
+      // cur_valuelog.seekg(current_offset);
+      // char* value_buf = new char[val_len];
+      // cur_valuelog.read(value_buf, val_len);
+      // if (!cur_valuelog.good()) {
+      //   delete[] key_buf;
+      //   delete[] key_buf_len;
+      //   delete[] value_buf_len;
+      //   delete[] value_buf;
+      //   cur_valuelog.close();
+      //   std::cerr << "Failed to read file: " << valuelog_name << std::endl;
+      //   break;
+      // }
       current_offset += val_len;
 
-      // Assign the read value data to the Slice
-      value = Slice(value_buf, val_len);
-      // std::cout<<val_len<<std::endl;
+      // // Assign the read value data to the Slice
+      // value = Slice(value_buf, val_len);
 
       // 检查 key 是否在 sstable 中存在
       std::string stored_value;
@@ -1894,6 +1893,23 @@ void DBImpl::GarbageCollect() {
         // 记录无效，跳过
         continue;
       }
+
+      // Now seek to the actual data position and read the value
+      cur_valuelog.seekg(current_offset-val_len);
+      char* value_buf = new char[val_len];
+      cur_valuelog.read(value_buf, val_len);
+      if (!cur_valuelog.good()) {
+        delete[] key_buf;
+        delete[] key_buf_len;
+        delete[] value_buf_len;
+        delete[] value_buf;
+        cur_valuelog.close();
+        std::cerr << "Failed to read file: " << valuelog_name << std::endl;
+        break;
+      }
+
+      // Assign the read value data to the Slice
+      value = Slice(value_buf, val_len);
   
       auto write_op=leveldb::WriteOptions();
       write_op.valuelog_write=true;

db/dbformat.h

@@ -45,7 +45,7 @@ static const int kMaxMemCompactLevel = 2;
 static const int kReadBytesPeriod = 1048576;
 
 // maximum size of value_log file
-static const int value_log_size=4<<14;
+static const int value_log_size=4<<24;
 
 }  // namespace config
 

test/test.cpp

@@ -16,75 +16,6 @@ Status OpenDB(std::string dbName, DB **db) {
   return DB::Open(options, dbName, db);
 }
 
-bool CompareFieldArray(const FieldArray &a, const FieldArray &b) {
-  if (a.size() != b.size()) return false;
-  for (size_t i = 0; i < a.size(); ++i) {
-    if (a[i].first != b[i].first || a[i].second != b[i].second) return false;
-  }
-  return true;
-}
-
-bool CompareKey(const std::vector<std::string> a, std::vector<std::string> b) {
-  if (a.size() != b.size()){
-     return false;
-  }
-  for (size_t i = 0; i < a.size(); ++i) {
-    if (a[i] != b[i]){
-        return false;
-    }
-  }
-  return true;
-}
-
-std::string SerializeValue(const FieldArray& fields){
-  std::string res_="";
-  PutVarint64(&res_,(uint64_t)fields.size());
-  for(auto pr:fields){
-    PutLengthPrefixedSlice(&res_, pr.first);
-    PutLengthPrefixedSlice(&res_, pr.second);
-  }
-  return res_;
-}
-
-  // 鍙嶅簭鍒楀寲涓哄瓧娈垫暟缁?
-void DeserializeValue(const std::string& value_str,FieldArray* res){
-  Slice slice=Slice(value_str.c_str());
-  uint64_t siz;
-  bool tmpres=GetVarint64(&slice,&siz);
-  assert(tmpres);
-  res->clear();
-  for(int i=0;i<siz;i++){
-    Slice value_name;
-    Slice value;
-    tmpres=GetLengthPrefixedSlice(&slice,&value_name);
-    assert(tmpres);
-    tmpres=GetLengthPrefixedSlice(&slice,&value);
-    assert(tmpres);
-    res->emplace_back(value_name,value);
-  }
-}
-
-Status Get_keys_by_field(DB *db,const ReadOptions& options, const Field field,std::vector<std::string> *keys){
-  auto it=db->NewIterator(options);
-  it->SeekToFirst();
-  keys->clear();
-  while(it->Valid()){
-    auto val=it->value();
-    FieldArray arr;
-    auto str_val=std::string(val.data(),val.size());
-    DeserializeValue(str_val,&arr);
-    for(auto pr:arr){
-      if(pr.first==field.first&&pr.second==field.second){
-        Slice key=it->key();
-        keys->push_back(std::string(key.data(),key.size()));
-        break;
-      }
-    }
-    it->Next();
-  }
-  delete it;
-  return Status::OK();
-}
 
 TEST(Test, CheckGetFields) {
     DB *db;
@@ -158,37 +89,37 @@ TEST(Test, CheckSearchKey) {
     delete db;
 }
 
-// TEST(Test, LARGE_DATA_COMPACT_TEST) {
-//     DB *db;
-//     WriteOptions writeOptions;
-//     ReadOptions readOptions;
-//     if(OpenDB("testdb_for_XOY_large", &db).ok() == false) {
-//         std::cerr << "open db failed" << std::endl;
-//         abort();
-//     }
-//     std::vector<std::string> values;
-//     for(int i=0;i<500000;i++){
-//         std::string key=std::to_string(i);
-//         std::string value;
-//         for(int j=0;j<1000;j++){
-//             value+=std::to_string(i);
-//         }
-//         values.push_back(value);
-//         db->Put(writeOptions,key,value);
-//     }
-//     for(int i=0;i<500000;i++){
-//         std::string key=std::to_string(i);
-//         std::string value;
-//         Status s=db->Get(readOptions,key,&value);
-//         assert(s.ok());
-//         if(values[i]!=value){
-//             std::cout<<value.size()<<std::endl;
-//             assert(0);
-//         }
-//         ASSERT_TRUE(values[i]==value);
-//     }
-//     delete db;
-// }
+TEST(Test, LARGE_DATA_COMPACT_TEST) {
+    DB *db;
+    WriteOptions writeOptions;
+    ReadOptions readOptions;
+    if(OpenDB("testdb_for_XOY_large", &db).ok() == false) {
+        std::cerr << "open db failed" << std::endl;
+        abort();
+    }
+    std::vector<std::string> values;
+    for(int i=0;i<500000;i++){
+        std::string key=std::to_string(i);
+        std::string value;
+        for(int j=0;j<1000;j++){
+            value+=std::to_string(i);
+        }
+        values.push_back(value);
+        db->Put(writeOptions,key,value);
+    }
+    // for(int i=0;i<500000;i++){
+    //     std::string key=std::to_string(i);
+    //     std::string value;
+    //     Status s=db->Get(readOptions,key,&value);
+    //     assert(s.ok());
+    //     if(values[i]!=value){
+    //         std::cout<<value.size()<<std::endl;
+    //         assert(0);
+    //     }
+    //     ASSERT_TRUE(values[i]==value);
+    // }
+    delete db;
+}
 
 TEST(Test, Garbage_Collect_TEST) {
     DB *db;

test/test2.cpp

@@ -0,0 +1,81 @@
+#include "gtest/gtest.h"
+#include "leveldb/env.h"
+#include "leveldb/db.h"
+#include "util/coding.h"
+#include <iostream>
+#include <chrono>
+
+using namespace std::chrono;
+using namespace leveldb;
+
+using Field=std::pair<Slice,Slice>;
+using FieldArray=std::vector<std::pair<Slice, Slice>>;
+
+int data_number=100000;
+
+Status OpenDB(std::string dbName, DB **db) {
+  Options options;
+  options.max_file_size=16*1024;
+  options.write_buffer_size=32*1024;
+  options.create_if_missing = true;
+  return DB::Open(options, dbName, db);
+}
+
+
+TEST(Test, Garbage_Collect_TEST) {
+    DB *db;
+    WriteOptions writeOptions;
+    ReadOptions readOptions;
+    if(OpenDB("testdb_for_XOY_large", &db).ok() == false) {
+        std::cerr << "open db failed" << std::endl;
+        abort();
+    }
+    std::vector<std::string> values;
+    for(int i=0;i<data_number;i++){
+        std::string key=std::to_string(i);
+        std::string value;
+        for(int j=0;j<1000;j++){
+            value+=std::to_string(i);
+        }
+        values.push_back(value);
+        db->Put(writeOptions,key,value);
+    }
+    // for(int i=0;i<data_number;i++){
+    //     std::string key=std::to_string(i);
+    //     std::string value;
+    //     for(int j=0;j<1000;j++){
+    //         value+=std::to_string(i);
+    //     }
+    //     values.push_back(value);
+    //     db->Put(writeOptions,key,value);
+    // }
+    // Measure GC time
+    auto start_time = high_resolution_clock::now();
+    db->TEST_GarbageCollect();
+    auto end_time = high_resolution_clock::now();
+
+    auto duration = duration_cast<milliseconds>(end_time - start_time);
+    std::cout << "GC finished. Time taken: " << duration.count() << " ms" << std::endl;
+
+
+    for(int i=0;i<data_number;i++){
+        // std::cout<<i<<std::endl;
+        std::string key=std::to_string(i);
+        std::string value;
+        Status s=db->Get(readOptions,key,&value);
+        assert(s.ok());
+        if(values[i]!=value){
+            std::cout<<value.size()<<std::endl;
+            assert(0);
+        }
+        ASSERT_TRUE(values[i]==value);
+    }
+    delete db;
+}
+
+
+int main(int argc, char** argv) {
+  // All tests currently run with the same read-only file limits.
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

util/coding.h

@@ -128,6 +128,13 @@ uint64_t GetValueLogID(const std::string& valuelog_name);
 void SplitIntoChunks(const std::set<std::string>& files, int num_workers,
                              std::vector<std::vector<std::string>>* chunks);
 
+bool CompareFieldArray(const FieldArray &a, const FieldArray &b);
+bool CompareKey(const std::vector<std::string> a, std::vector<std::string> b);
+std::string SerializeValue(const FieldArray& fields);
+void DeserializeValue(const std::string& value_str,FieldArray* res);
+Status Get_keys_by_field(DB *db,const ReadOptions& options, const Field field,std::vector<std::string> *keys);
+
+
 }  // namespace leveldb
 
 #endif  // STORAGE_LEVELDB_UTIL_CODING_H_