test performance

CMakeLists.txt

@@ -557,4 +557,9 @@ target_link_libraries(db_test5 PRIVATE leveldb)
 add_executable(db_test6
         "${PROJECT_SOURCE_DIR}/test/db_test6.cc"
 )
-target_link_libraries(db_test6 PRIVATE leveldb gtest)
+target_link_libraries(db_test6 PRIVATE leveldb gtest)
+
+add_executable(db_test_latent
+        "${PROJECT_SOURCE_DIR}/test/db_test_latent.cc"
+)
+target_link_libraries(db_test_latent PRIVATE leveldb gtest)

include/leveldb/db.h

@@ -157,6 +157,8 @@ class LEVELDB_EXPORT DB {
   // Todo(begin)
   virtual Status Put_Fields(const leveldb::WriteOptions& opt, const leveldb::Slice& key, const FieldArray& fields) = 0;
   virtual Status Get_Fields(const leveldb::ReadOptions& options, const leveldb::Slice& key, FieldArray& fields) = 0;
+  virtual Status get_slot_num(const ReadOptions& options, const Slice& key,
+                              size_t *slot_num) = 0;
   //  // Todo(end)
 };
 

report.md

@@ -403,6 +403,8 @@ int main(int argc, char **argv) {
 2. 测试在只有删除的情况下，GC的效率
 3. 测试在插入和删除不同比重的负载下，系统的吞吐情况
 
+吞吐率下降很多
+写放大下降很多
 #### 6. 可能遇到的挑战与解决方案
 列出实现过程中可能遇到的技术难题及其解决思路，如如何处理GC开销、数据同步、索引原子更新等问题。
 各种参数的设置，比如vlog的容量上限，以及slot_page的bitmap管理方式是否足够高效？以及在GC过程中如果对被GC中的vlog进行写入该让用户线程和后台线程以什么样的方式进行同步？slot_page的读写放大也是一个重要的问题。

test/db_test1.cc

@@ -12,10 +12,10 @@ int test1() {
   Status status = DB::Open(op, "testdb1", &db);
   assert(status.ok());
   db->Put(WriteOptions(), "001", "leveldb");
-  string s;
-  db->Get(ReadOptions(), "001", &s);
-  cout<<s<<endl;
-  cout << s.size() << endl;
+  size_t s;
+  db->get_slot_num(ReadOptions(), "001", &s);
+  cout<<to_string(s)<<endl;
+  cout << to_string(s).size() << endl;
 
   db->Put(WriteOptions(), "002", "world");
   string s1;

test/db_test2.cc

@@ -6,7 +6,7 @@
 using namespace leveldb;
 
 constexpr int value_size = 2048;
-constexpr int data_size = 512 << 20;
+constexpr int data_size = 512 << 19;
 
 // 3. 数据管理（Manifest/创建/恢复数据库）
 Status OpenDB(std::string dbName, DB **db) {
@@ -56,12 +56,12 @@ void GetData(DB *db, int size = (1 << 30)) {
 int main() {
 
   DB *db;
-  if(OpenDB("testdb", &db).ok()) {
+  if(OpenDB("testdb_19", &db).ok()) {
     InsertData(db);
     delete db;
   }
 
-  if(OpenDB("testdb", &db).ok()) {
+  if(OpenDB("testdb_19", &db).ok()) {
     GetData(db);
     delete db;
   }

test/db_test3.cc

@@ -40,55 +40,6 @@ Status OpenDB(std::string dbName, DB** db) {
   return DB::Open(options, dbName, db);
 }
 
-// 吞吐量测试函数
-void TestThroughput(leveldb::DB* db, int num_operations) {
-  WriteOptions writeOptions;
-  auto start_time = std::chrono::steady_clock::now();
-
-  for (int i = 0; i < num_operations; ++i) {
-    std::string key = "key_" + std::to_string(i);
-    FieldArray fields = {
-        {"name", "Customer" + std::to_string(i)},
-        {"address", "Address" + std::to_string(i)},
-        {"phone", "1234567890"}};
-    db->Put_Fields(writeOptions, key, fields);
-  }
-
-  auto end_time = std::chrono::steady_clock::now();
-  auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
-  std::cout << "Throughput: " << num_operations * 1000 / duration << " OPS" << std::endl;
-}
-
-// 延迟测试函数
-void TestLatency(leveldb::DB* db, int num_operations, std::vector<int64_t>& lat_res) {
-  WriteOptions writeOptions;
-  int64_t latency = 0;
-  auto end_time = std::chrono::steady_clock::now();
-  auto last_time = end_time;
-  for (int i = 0; i < num_operations; ++i) {
-    // 执行写入操作
-    std::string key = "key_" + std::to_string(i);
-    FieldArray fields = {
-        {"name", "Customer" + std::to_string(i)},
-        {"address", "Address" + std::to_string(i)},
-        {"phone", "1234567890"}};
-    db->Get_Fields(leveldb::ReadOptions(), key, fields);
-
-    end_time = std::chrono::steady_clock::now();
-    latency = std::chrono::duration_cast<std::chrono::milliseconds>(
-                  end_time - last_time).count();
-    last_time = end_time;
-
-    lat_res.emplace_back(latency);
-  }
-
-  // 输出延迟统计信息（可选）
-  double avg_latency = std::accumulate(lat_res.begin(), lat_res.end(), 0.0) / lat_res.size();
-  std::cout << "Average Latency: " << std::fixed << std::setprecision(2) << avg_latency << " ms" << std::endl;
-  std::cout << "Max Latency: " << *std::max_element(lat_res.begin(), lat_res.end()) << " ms" << std::endl;
-  std::cout << "Min Latency: " << *std::min_element(lat_res.begin(), lat_res.end()) << " ms" << std::endl;
-}
-
 TEST(TestSchema, Basic) {
   DB* db;
   WriteOptions writeOptions;
@@ -153,12 +104,6 @@ TEST(TestSchema, Basic) {
   Field query_field = {"name", "Customer1"};
   std::vector<std::string> found_keys = FindKeysByField(db, query_field);
   std::cout << "找到的key有：" << found_keys.size() << "个" << std::endl;
-  // 吞吐量测试
-    TestThroughput(db, 10000);
-
-    // 延迟测试
-    std::vector<int64_t> latency_results;
-    TestLatency(db, 1000, latency_results);
   // 关闭数据库
   delete db;
 }

test/db_test6.cc

@@ -46,7 +46,7 @@ TEST(ConcurrentTest, Basic) {
   DB* db;
   WriteOptions writeOptions;
   ReadOptions readOptions;
-  if (!OpenDB("testdb", &db).ok()) {
+  if (!OpenDB("testdb_current", &db).ok()) {
     std::cerr << "open db failed" << std::endl;
     abort();
   }

test/db_test_latent.cc

@@ -0,0 +1,122 @@
+#include <chrono>
+#include <iomanip>
+#include <iostream>
+#include <leveldb/db.h>
+#include <leveldb/options.h>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "leveldb/env.h"
+
+#include "gtest/gtest.h"
+
+using namespace leveldb;
+// 根据字段值查找所有包含该字段的 key，遍历
+std::vector<std::string> FindKeysByField(leveldb::DB* db, const Field& field) {
+  std::vector<std::string> keys;
+  leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions());
+
+  for (it->SeekToFirst(); it->Valid()   ; it->Next()) {
+    std::string key = it->key().ToString();
+    FieldArray fields;
+    db->Get_Fields(leveldb::ReadOptions(), key, fields);
+    for (const auto& f : fields) {
+      if (f.name == field.name && f.value == field.value) {
+        keys.push_back(key);
+        break; // 假设每个key中每个字段值唯一
+      }
+    }
+  }
+  std::cout << "Found " << keys.size() << " keys" << std::endl;
+  delete it;
+  return keys;
+}
+
+Status OpenDB(std::string dbName, DB** db) {
+  Options options;
+  options.create_if_missing = true;
+  return DB::Open(options, dbName, db);
+}
+
+// 吞吐量测试函数
+void TestThroughput(leveldb::DB* db, int num_operations) {
+  WriteOptions writeOptions;
+  auto start_time = std::chrono::steady_clock::now();
+
+  for (int i = 0; i < num_operations; ++i) {
+    std::string key = "key_" + std::to_string(i);
+    std::string value = "value_" + std::to_string(i);
+    db->Put(writeOptions, key, value);
+    //    FieldArray fields = {
+    //        {"name", "Customer" + std::to_string(i)},
+    //        {"address", "Address" + std::to_string(i)},
+    //        {"phone", "1234567890"}};
+    //    db->Put_Fields(writeOptions, key, fields);
+  }
+  for (int i = 0; i < num_operations; ++i) {
+    std::string key = "key_" + std::to_string(i);
+    std::string value;
+    db->Get(ReadOptions(), key, &value);
+  }
+
+  auto end_time = std::chrono::steady_clock::now();
+  auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
+  std::cout << "Throughput: " << num_operations * 1000 / duration << " OPS" << std::endl;
+}
+
+// 延迟测试函数
+void TestLatency(leveldb::DB* db, int num_operations, std::vector<int64_t>& lat_res) {
+  WriteOptions writeOptions;
+  int64_t latency = 0;
+  auto end_time = std::chrono::steady_clock::now();
+  auto last_time = end_time;
+  for (int i = 0; i < num_operations; ++i) {
+    // 执行写入操作
+    std::string key = "key_" + std::to_string(i);
+    //    FieldArray fields = {
+    //        {"name", "Customer" + std::to_string(i)},
+    //        {"address", "Address" + std::to_string(i)},
+    //        {"phone", "1234567890"}};
+    //    db->Get_Fields(leveldb::ReadOptions(), key, fields);
+    std::string value = "value_" + std::to_string(i);
+    db->Put(writeOptions, key, value);
+
+    end_time = std::chrono::steady_clock::now();
+    latency = std::chrono::duration_cast<std::chrono::milliseconds>(
+                  end_time - last_time).count();
+    last_time = end_time;
+
+    lat_res.emplace_back(latency);
+  }
+
+  // 输出延迟统计信息（可选）
+  double avg_latency = std::accumulate(lat_res.begin(), lat_res.end(), 0.0) / lat_res.size();
+  std::cout << "Average Latency: " << std::fixed << std::setprecision(2) << avg_latency << " ms" << std::endl;
+  std::cout << "Max Latency: " << *std::max_element(lat_res.begin(), lat_res.end()) << " ms" << std::endl;
+  std::cout << "Min Latency: " << *std::min_element(lat_res.begin(), lat_res.end()) << " ms" << std::endl;
+}
+
+TEST(TestSchema, Basic) {
+  DB* db;
+  WriteOptions writeOptions;
+  ReadOptions readOptions;
+  if (!OpenDB("testdb", &db).ok()) {
+    std::cerr << "open db failed" << std::endl;
+    abort();
+  }
+  // 吞吐量测试
+  TestThroughput(db, 1000);
+
+  // 延迟测试
+  std::vector<int64_t> latency_results;
+  TestLatency(db, 1000, latency_results);
+  // 关闭数据库
+  delete db;
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}