diff --git a/CMakeLists.txt b/CMakeLists.txt index fc0a5e8..eabc748 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -557,4 +557,9 @@ target_link_libraries(db_test5 PRIVATE leveldb) add_executable(db_test6 "${PROJECT_SOURCE_DIR}/test/db_test6.cc" ) -target_link_libraries(db_test6 PRIVATE leveldb gtest) \ No newline at end of file +target_link_libraries(db_test6 PRIVATE leveldb gtest) + +add_executable(db_test_latent + "${PROJECT_SOURCE_DIR}/test/db_test_latent.cc" +) +target_link_libraries(db_test_latent PRIVATE leveldb gtest) \ No newline at end of file diff --git a/include/leveldb/db.h b/include/leveldb/db.h index 6d739b4..6ac10f9 100644 --- a/include/leveldb/db.h +++ b/include/leveldb/db.h @@ -157,6 +157,8 @@ class LEVELDB_EXPORT DB { // Todo(begin) virtual Status Put_Fields(const leveldb::WriteOptions& opt, const leveldb::Slice& key, const FieldArray& fields) = 0; virtual Status Get_Fields(const leveldb::ReadOptions& options, const leveldb::Slice& key, FieldArray& fields) = 0; + virtual Status get_slot_num(const ReadOptions& options, const Slice& key, + size_t *slot_num) = 0; // // Todo(end) }; diff --git a/report.md b/report.md index 6d9d75b..e92043d 100644 --- a/report.md +++ b/report.md @@ -403,6 +403,8 @@ int main(int argc, char **argv) { 2. 测试在只有删除的情况下，GC的效率 3. 测试在插入和删除不同比重的负载下，系统的吞吐情况 +吞吐率下降很多 +写放大下降很多 #### 6. 可能遇到的挑战与解决方案 列出实现过程中可能遇到的技术难题及其解决思路，如如何处理GC开销、数据同步、索引原子更新等问题。 各种参数的设置，比如vlog的容量上限，以及slot_page的bitmap管理方式是否足够高效？以及在GC过程中如果对被GC中的vlog进行写入该让用户线程和后台线程以什么样的方式进行同步？slot_page的读写放大也是一个重要的问题。 diff --git a/test/db_test1.cc b/test/db_test1.cc index a1039b9..88fd797 100644 --- a/test/db_test1.cc +++ b/test/db_test1.cc @@ -12,10 +12,10 @@ int test1() { Status status = DB::Open(op, "testdb1", &db); assert(status.ok()); db->Put(WriteOptions(), "001", "leveldb"); - string s; - db->Get(ReadOptions(), "001", &s); - cout<get_slot_num(ReadOptions(), "001", &s); + cout<Put(WriteOptions(), "002", "world"); string s1; diff --git a/test/db_test2.cc b/test/db_test2.cc index cd266e6..51b43f9 100644 --- a/test/db_test2.cc +++ b/test/db_test2.cc @@ -6,7 +6,7 @@ using namespace leveldb; constexpr int value_size = 2048; -constexpr int data_size = 512 << 20; +constexpr int data_size = 512 << 19; // 3. 数据管理（Manifest/创建/恢复数据库） Status OpenDB(std::string dbName, DB **db) { @@ -56,12 +56,12 @@ void GetData(DB *db, int size = (1 << 30)) { int main() { DB *db; - if(OpenDB("testdb", &db).ok()) { + if(OpenDB("testdb_19", &db).ok()) { InsertData(db); delete db; } - if(OpenDB("testdb", &db).ok()) { + if(OpenDB("testdb_19", &db).ok()) { GetData(db); delete db; } diff --git a/test/db_test3.cc b/test/db_test3.cc index 3861ec1..1cdafe4 100644 --- a/test/db_test3.cc +++ b/test/db_test3.cc @@ -40,55 +40,6 @@ Status OpenDB(std::string dbName, DB** db) { return DB::Open(options, dbName, db); } -// 吞吐量测试函数 -void TestThroughput(leveldb::DB* db, int num_operations) { - WriteOptions writeOptions; - auto start_time = std::chrono::steady_clock::now(); - - for (int i = 0; i < num_operations; ++i) { - std::string key = "key_" + std::to_string(i); - FieldArray fields = { - {"name", "Customer" + std::to_string(i)}, - {"address", "Address" + std::to_string(i)}, - {"phone", "1234567890"}}; - db->Put_Fields(writeOptions, key, fields); - } - - auto end_time = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast(end_time - start_time).count(); - std::cout << "Throughput: " << num_operations * 1000 / duration << " OPS" << std::endl; -} - -// 延迟测试函数 -void TestLatency(leveldb::DB* db, int num_operations, std::vector& lat_res) { - WriteOptions writeOptions; - int64_t latency = 0; - auto end_time = std::chrono::steady_clock::now(); - auto last_time = end_time; - for (int i = 0; i < num_operations; ++i) { - // 执行写入操作 - std::string key = "key_" + std::to_string(i); - FieldArray fields = { - {"name", "Customer" + std::to_string(i)}, - {"address", "Address" + std::to_string(i)}, - {"phone", "1234567890"}}; - db->Get_Fields(leveldb::ReadOptions(), key, fields); - - end_time = std::chrono::steady_clock::now(); - latency = std::chrono::duration_cast( - end_time - last_time).count(); - last_time = end_time; - - lat_res.emplace_back(latency); - } - - // 输出延迟统计信息（可选） - double avg_latency = std::accumulate(lat_res.begin(), lat_res.end(), 0.0) / lat_res.size(); - std::cout << "Average Latency: " << std::fixed << std::setprecision(2) << avg_latency << " ms" << std::endl; - std::cout << "Max Latency: " << *std::max_element(lat_res.begin(), lat_res.end()) << " ms" << std::endl; - std::cout << "Min Latency: " << *std::min_element(lat_res.begin(), lat_res.end()) << " ms" << std::endl; -} - TEST(TestSchema, Basic) { DB* db; WriteOptions writeOptions; @@ -153,12 +104,6 @@ TEST(TestSchema, Basic) { Field query_field = {"name", "Customer1"}; std::vector found_keys = FindKeysByField(db, query_field); std::cout << "找到的key有：" << found_keys.size() << "个" << std::endl; - // 吞吐量测试 - TestThroughput(db, 10000); - - // 延迟测试 - std::vector latency_results; - TestLatency(db, 1000, latency_results); // 关闭数据库 delete db; } diff --git a/test/db_test6.cc b/test/db_test6.cc index a90d5e8..e401059 100644 --- a/test/db_test6.cc +++ b/test/db_test6.cc @@ -46,7 +46,7 @@ TEST(ConcurrentTest, Basic) { DB* db; WriteOptions writeOptions; ReadOptions readOptions; - if (!OpenDB("testdb", &db).ok()) { + if (!OpenDB("testdb_current", &db).ok()) { std::cerr << "open db failed" << std::endl; abort(); } diff --git a/test/db_test_latent.cc b/test/db_test_latent.cc new file mode 100644 index 0000000..6ef7373 --- /dev/null +++ b/test/db_test_latent.cc @@ -0,0 +1,122 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "leveldb/env.h" + +#include "gtest/gtest.h" + +using namespace leveldb; +// 根据字段值查找所有包含该字段的 key，遍历 +std::vector FindKeysByField(leveldb::DB* db, const Field& field) { + std::vector keys; + leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions()); + + for (it->SeekToFirst(); it->Valid() ; it->Next()) { + std::string key = it->key().ToString(); + FieldArray fields; + db->Get_Fields(leveldb::ReadOptions(), key, fields); + for (const auto& f : fields) { + if (f.name == field.name && f.value == field.value) { + keys.push_back(key); + break; // 假设每个key中每个字段值唯一 + } + } + } + std::cout << "Found " << keys.size() << " keys" << std::endl; + delete it; + return keys; +} + +Status OpenDB(std::string dbName, DB** db) { + Options options; + options.create_if_missing = true; + return DB::Open(options, dbName, db); +} + +// 吞吐量测试函数 +void TestThroughput(leveldb::DB* db, int num_operations) { + WriteOptions writeOptions; + auto start_time = std::chrono::steady_clock::now(); + + for (int i = 0; i < num_operations; ++i) { + std::string key = "key_" + std::to_string(i); + std::string value = "value_" + std::to_string(i); + db->Put(writeOptions, key, value); + // FieldArray fields = { + // {"name", "Customer" + std::to_string(i)}, + // {"address", "Address" + std::to_string(i)}, + // {"phone", "1234567890"}}; + // db->Put_Fields(writeOptions, key, fields); + } + for (int i = 0; i < num_operations; ++i) { + std::string key = "key_" + std::to_string(i); + std::string value; + db->Get(ReadOptions(), key, &value); + } + + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(end_time - start_time).count(); + std::cout << "Throughput: " << num_operations * 1000 / duration << " OPS" << std::endl; +} + +// 延迟测试函数 +void TestLatency(leveldb::DB* db, int num_operations, std::vector& lat_res) { + WriteOptions writeOptions; + int64_t latency = 0; + auto end_time = std::chrono::steady_clock::now(); + auto last_time = end_time; + for (int i = 0; i < num_operations; ++i) { + // 执行写入操作 + std::string key = "key_" + std::to_string(i); + // FieldArray fields = { + // {"name", "Customer" + std::to_string(i)}, + // {"address", "Address" + std::to_string(i)}, + // {"phone", "1234567890"}}; + // db->Get_Fields(leveldb::ReadOptions(), key, fields); + std::string value = "value_" + std::to_string(i); + db->Put(writeOptions, key, value); + + end_time = std::chrono::steady_clock::now(); + latency = std::chrono::duration_cast( + end_time - last_time).count(); + last_time = end_time; + + lat_res.emplace_back(latency); + } + + // 输出延迟统计信息（可选） + double avg_latency = std::accumulate(lat_res.begin(), lat_res.end(), 0.0) / lat_res.size(); + std::cout << "Average Latency: " << std::fixed << std::setprecision(2) << avg_latency << " ms" << std::endl; + std::cout << "Max Latency: " << *std::max_element(lat_res.begin(), lat_res.end()) << " ms" << std::endl; + std::cout << "Min Latency: " << *std::min_element(lat_res.begin(), lat_res.end()) << " ms" << std::endl; +} + +TEST(TestSchema, Basic) { + DB* db; + WriteOptions writeOptions; + ReadOptions readOptions; + if (!OpenDB("testdb", &db).ok()) { + std::cerr << "open db failed" << std::endl; + abort(); + } + // 吞吐量测试 + TestThroughput(db, 1000); + + // 延迟测试 + std::vector latency_results; + TestLatency(db, 1000, latency_results); + // 关闭数据库 + delete db; +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file