From 20cf51eeb00d0ae6a86851877e2a947480ba3311 Mon Sep 17 00:00:00 2001 From: Estella <10225101469@stu.ecnu.edu.cn> Date: Sun, 5 Jan 2025 21:45:56 +0800 Subject: [PATCH] test-zcy --- CMakeLists.txt | 39 ++- test/concurrency_performance_test.cc | 254 ++++++++++++++++++ test/concurrency_performance_test_put.cc | 198 ++++++++++++++ test/index_test.cc | 236 +++++++++++++++++ test/performance_test.cc | 436 +++++++++++++++++++++++++++++++ test/throughput_test.cc | 107 ++++++++ 6 files changed, 1267 insertions(+), 3 deletions(-) create mode 100644 test/concurrency_performance_test.cc create mode 100644 test/concurrency_performance_test_put.cc create mode 100644 test/index_test.cc create mode 100644 test/performance_test.cc create mode 100644 test/throughput_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index dd4fbee..bf5843b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,6 +190,8 @@ target_sources(leveldb "util/options.cc" "util/random.h" "util/status.cc" + "db/NewDB.cc" + "db/NewDB.h" # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install". $<$:PUBLIC> @@ -518,16 +520,27 @@ if(LEVELDB_INSTALL) ) endif(LEVELDB_INSTALL) -add_executable(kv_index_test - "${PROJECT_SOURCE_DIR}/test/kv_index_test.cc" +add_executable(db_test1 +"${PROJECT_SOURCE_DIR}/test/db_test1.cc" + ) +target_link_libraries(db_test1 leveldb) + + +add_executable(field_test +"${PROJECT_SOURCE_DIR}/test/field_test.cc" ) -target_link_libraries(kv_index_test PRIVATE leveldb gtest) +target_link_libraries(field_test PRIVATE leveldb gtest) add_executable(index_test "${PROJECT_SOURCE_DIR}/test/index_test.cc" ) target_link_libraries(index_test PRIVATE leveldb gtest) +add_executable(throughput_test +"${PROJECT_SOURCE_DIR}/test/throughput_test.cc" +) +target_link_libraries(throughput_test PRIVATE leveldb gtest) + add_executable(index_test_random "${PROJECT_SOURCE_DIR}/test/index_test_random.cc" ) @@ -542,3 +555,23 @@ add_executable(concurrency_test "${PROJECT_SOURCE_DIR}/test/concurrency_test.cc" ) target_link_libraries(concurrency_test PRIVATE leveldb gtest) + +add_executable(constancy_test +"${PROJECT_SOURCE_DIR}/test/constancy_test.cc" +) +target_link_libraries(constancy_test PRIVATE leveldb gtest) + +add_executable(performance_test +"${PROJECT_SOURCE_DIR}/test/performance_test.cc" +) +target_link_libraries(performance_test PRIVATE leveldb gtest) + +add_executable(concurrency_performance_test +"${PROJECT_SOURCE_DIR}/test/concurrency_performance_test.cc" +) +target_link_libraries(concurrency_performance_test PRIVATE leveldb gtest) + +add_executable(concurrency_performance_test_put +"${PROJECT_SOURCE_DIR}/test/concurrency_performance_test_put.cc" +) +target_link_libraries(concurrency_performance_test_put PRIVATE leveldb gtest) \ No newline at end of file diff --git a/test/concurrency_performance_test.cc b/test/concurrency_performance_test.cc new file mode 100644 index 0000000..44caf2f --- /dev/null +++ b/test/concurrency_performance_test.cc @@ -0,0 +1,254 @@ +#include "gtest/gtest.h" +#include "db/NewDB.h" // NewDB 的头文件 +#include "leveldb/env.h" +#include "leveldb/db.h" +#include "db/write_batch_internal.h" +#include +#include +#include +#include +#include +#include +#include +using namespace leveldb; + +// 全局计数器 +std::atomic put_count(0); +std::atomic delete_count(0); +std::atomic create_index_count(0); +std::atomic delete_index_count(0); +std::atomic total_operations(0); + +std::mutex latency_mutex; +std::vector latencies; + +thread_local std::vector put_latencies_local; +thread_local std::vector delete_latencies_local; +thread_local std::vector update_latencies_local; + +std::mutex put_latency_mutex; +std::vector put_latencies; + +std::mutex delete_latency_mutex; +std::vector delete_latencies; + +std::mutex update_latency_mutex; +std::vector update_latencies; + + +// void RecordLatency(long long duration) { +// std::unique_lock lock(latency_mutex); +// latencies.push_back(duration); +// } + +void RecordLatency(long long duration, const std::string& op_type) { + if (op_type == "PUT") { + put_latencies_local.push_back(duration); + } else if (op_type == "DELETE") { + delete_latencies_local.push_back(duration); + } else if (op_type == "UPDATE") { + update_latencies_local.push_back(duration); + } +} +void MergeLatencies() { + { + std::unique_lock lock(put_latency_mutex); + put_latencies.insert(put_latencies.end(), + put_latencies_local.begin(), + put_latencies_local.end()); + } + + { + std::unique_lock lock(delete_latency_mutex); + delete_latencies.insert(delete_latencies.end(), + delete_latencies_local.begin(), + delete_latencies_local.end()); + } + + { + std::unique_lock lock(update_latency_mutex); + update_latencies.insert(update_latencies.end(), + update_latencies_local.begin(), + update_latencies_local.end()); + } +} + + +Status OpenNewDB(std::string dbName, NewDB** db) { + Options options = Options(); + options.create_if_missing = true; + return NewDB::Open(options, dbName, db); +} + +// 全局的随机数引擎 +std::default_random_engine rng; + +// 设置随机种子 +void SetGlobalSeed(unsigned seed) { + rng.seed(seed); +} + +// 生成随机字符串 +std::string GenerateRandomString(size_t length) { + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + std::uniform_int_distribution dist(0, sizeof(alphanum) - 2); + + std::string str(length, 0); + for (size_t i = 0; i < length; ++i) { + str[i] = alphanum[dist(rng)]; + } + return str; +} + +int getRandomInRange(int min, int max) { + return min + std::rand() % ((max + 1) - min); +} + +std::mutex results_mutex; +std::vector results; + +void InsertResult(bool TRUEorFALSE){ + std::unique_lock lock(results_mutex, std::defer_lock); + lock.lock(); + results.emplace_back(TRUEorFALSE); + lock.unlock(); +} + +std::string testdbname = "dbtest36"; + + + + +void thread_task_put(NewDB* db, int thread_id, int num_operations) { + for (int i = 0; i < num_operations; ++i) { + auto start = std::chrono::high_resolution_clock::now(); + std::string key = "k_" + std::to_string(thread_id) + "_" + std::to_string(i); + FieldArray fields = { + {"name", "User" + std::to_string(i)}, + {"email", "user" + std::to_string(i) + "@test.com"} + }; + db->Put_fields(WriteOptions(), key, fields); + total_operations++; + + auto end = std::chrono::high_resolution_clock::now(); + long long duration = std::chrono::duration_cast(end - start).count(); + RecordLatency(duration, "PUT"); + } + // 合并本线程的延迟数据 + MergeLatencies(); +} +void thread_task_delete(NewDB* db, int thread_id, int num_operations) { + for (int i = 0; i < num_operations; ++i) { + int key_index = getRandomInRange(0, 9999); + std::string key = "k_" + std::to_string(key_index); + + auto start = std::chrono::high_resolution_clock::now(); + db->Delete(WriteOptions(), key); + delete_count++; + + auto end = std::chrono::high_resolution_clock::now(); + long long duration = std::chrono::duration_cast(end - start).count(); + RecordLatency(duration, "DELETE"); + total_operations++; + } + // 合并本线程的延迟数据 + MergeLatencies(); +} +void thread_task_update(NewDB* db, int thread_id, int num_operations) { + for (int i = 0; i < num_operations; ++i) { + std::string key = "k_" + std::to_string(thread_id) + "_" + std::to_string(i); + FieldArray fields = { + {"email", "updated_user" + std::to_string(i) + "@test.com"} + }; + + auto start = std::chrono::high_resolution_clock::now(); + db->Put_fields(WriteOptions(), key, fields); // 简单地更新一个字段 + total_operations++; + + auto end = std::chrono::high_resolution_clock::now(); + long long duration = std::chrono::duration_cast(end - start).count(); + RecordLatency(duration, "UPDATE"); + } + // 合并本线程的延迟数据 + MergeLatencies(); +} + + +// 并发测试：包括 put, delete, update +TEST(NewDBTest, ConcurrencyTest_mul) { + NewDB* db; + ASSERT_TRUE(OpenNewDB(testdbname, &db).ok()); + + const int thread_num_put = 10; + const int thread_num_delete = 1; + const int thread_num_update = 0; + const int operations_per_thread = 10; + const int operations_per_thread_delete = 50; + const int operations_per_thread_update = 50; + + std::vector threads; + + auto start = std::chrono::high_resolution_clock::now(); + // 启动 PUT 线程 + for (int i = 0; i < thread_num_put; ++i) { + threads.emplace_back(thread_task_put, db, i, operations_per_thread); + } + // 启动 DELETE 线程 + for (int i = 0; i < thread_num_delete; ++i) { + threads.emplace_back(thread_task_delete, db, i, operations_per_thread_delete); + } + + // 启动 UPDATE 线程 + for (int i = 0; i < thread_num_update; ++i) { + threads.emplace_back(thread_task_update, db, i, operations_per_thread_update); + } + + for (auto& th : threads) { + th.join(); + } + + auto end = std::chrono::high_resolution_clock::now(); + long long total_duration = std::chrono::duration_cast(end - start).count(); + + auto calculate_latency = [](const std::vector& latencies) { + long long sum = 0; + for (auto& lat : latencies) { + sum += lat; + } + return std::make_pair(sum, latencies.size() > 0 ? static_cast(sum) / latencies.size() : 0); + }; + + auto [put_sum, put_avg] = calculate_latency(put_latencies); + auto [delete_sum, delete_avg] = calculate_latency(delete_latencies); + auto [update_sum, update_avg] = calculate_latency(update_latencies); + + std::cout << "Total Operations: " << total_operations.load() << std::endl; + std::cout << "Total Duration: " << total_duration << " ms" << std::endl; + std::cout << "Throughput (ops/sec): " << static_cast(total_operations.load()) / (total_duration / 1000.0) << std::endl; + + std::cout << "\nPUT Operations:" << std::endl; + std::cout << " Total Latency (ms): " << put_sum / 1000.0 << std::endl; + std::cout << " Average Latency (us): " << put_avg << std::endl; + + std::cout << "\nDELETE Operations:" << std::endl; + std::cout << " Total Latency (ms): " << delete_sum / 1000.0 << std::endl; + std::cout << " Average Latency (us): " << delete_avg << std::endl; + + std::cout << "\nUPDATE Operations:" << std::endl; + std::cout << " Total Latency (ms): " << update_sum / 1000.0 << std::endl; + std::cout << " Average Latency (us): " << update_avg << std::endl; + + delete db; +} + + +int main(int argc, char** argv) { + // 设置全局随机种子 + SetGlobalSeed(static_cast(time(nullptr))); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/concurrency_performance_test_put.cc b/test/concurrency_performance_test_put.cc new file mode 100644 index 0000000..0da7e61 --- /dev/null +++ b/test/concurrency_performance_test_put.cc @@ -0,0 +1,198 @@ +#include "gtest/gtest.h" +#include "db/NewDB.h" // NewDB 的头文件 +#include "leveldb/env.h" +#include "leveldb/db.h" +#include "db/write_batch_internal.h" +#include +#include +#include +#include +#include +#include +#include +using namespace leveldb; + +// 全局计数器 +std::atomic put_count(0); +std::atomic delete_count(0); +std::atomic create_index_count(0); +std::atomic delete_index_count(0); +std::atomic total_operations(0); + +std::mutex latency_mutex; +std::vector latencies; + + + + +void RecordLatency(long long duration) { + std::unique_lock lock(latency_mutex); + latencies.push_back(duration); +} + + +Status OpenNewDB(std::string dbName, NewDB** db) { + Options options = Options(); + options.create_if_missing = true; + return NewDB::Open(options, dbName, db); +} + +// 全局的随机数引擎 +std::default_random_engine rng; + +// 设置随机种子 +void SetGlobalSeed(unsigned seed) { + rng.seed(seed); +} + +// 生成随机字符串 +std::string GenerateRandomString(size_t length) { + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + std::uniform_int_distribution dist(0, sizeof(alphanum) - 2); + + std::string str(length, 0); + for (size_t i = 0; i < length; ++i) { + str[i] = alphanum[dist(rng)]; + } + return str; +} + +int getRandomInRange(int min, int max) { + return min + std::rand() % ((max + 1) - min); +} + +std::mutex results_mutex; +std::vector results; + +void InsertResult(bool TRUEorFALSE){ + std::unique_lock lock(results_mutex, std::defer_lock); + lock.lock(); + results.emplace_back(TRUEorFALSE); + lock.unlock(); +} + +std::string testdbname = "dbtest36"; + + + +// 多线程插入 +void thread_task_put(NewDB* db, int thread_id, int num_operations) { + for (int i = 0; i < num_operations; ++i) { + auto start = std::chrono::high_resolution_clock::now(); + std::string key = "k_" + std::to_string(thread_id) + "_" + std::to_string(i); + FieldArray fields = { + {"name", "User" + std::to_string(i)}, + {"email", "user" + std::to_string(i) + "@test.com"} + }; + db->Put_fields(WriteOptions(), key, fields); + total_operations++; + + auto end = std::chrono::high_resolution_clock::now(); + long long duration = std::chrono::duration_cast(end - start).count(); + RecordLatency(duration); + } +} + + +// // 测试吞吐量和延迟 +// TEST(TestNewDB, PerformanceTest) { +// NewDB* db; +// ASSERT_TRUE(OpenNewDB(testdbname, &db).ok()); + +// const int thread_num = 100; +// std::vector threads; + +// auto test_start = std::chrono::high_resolution_clock::now(); + +// for (int i = 0; i < thread_num; ++i) { +// threads.emplace_back([=]() { thread_task_put(db, i); }); +// threads.emplace_back([=]() { thread_task_delete(db, i); }); +// threads.emplace_back([=]() { thread_task_createindex(db, i); }); +// threads.emplace_back([=]() { thread_task_deleteindex(db, i); }); +// } + +// for (auto& th : threads) { +// if (th.joinable()) { +// th.join(); +// } +// } + +// auto test_end = std::chrono::high_resolution_clock::now(); +// long long total_duration = std::chrono::duration_cast(test_end - test_start).count(); + +// // 计算吞吐量 +// double throughput = static_cast(total_operations) / (total_duration / 1000.0); + +// // 计算平均延迟 +// long long sum_latency = 0; +// for (auto& lat : latencies) { +// sum_latency += lat; +// } +// double avg_latency = static_cast(sum_latency) / latencies.size(); + +// std::cout << "Total Operations: " << total_operations.load() << std::endl; +// std::cout << "Throughput (ops/sec): " << throughput << std::endl; +// std::cout << "Average Latency (us): " << avg_latency << std::endl; +// std::cout << "PUT operations: " << put_count.load() << std::endl; +// std::cout << "DELETE operations: " << delete_count.load() << std::endl; +// std::cout << "CREATE INDEX operations: " << create_index_count.load() << std::endl; +// std::cout << "DELETE INDEX operations: " << delete_index_count.load() << std::endl; + +// delete db; +// } +//并发测试 +TEST(NewDBTest, ConcurrencyTest) { + NewDB* db; + ASSERT_TRUE(OpenNewDB(testdbname, &db).ok()); + + const int thread_num = 100; + const int operations_per_thread = 10; + std::vector threads; + + auto start = std::chrono::high_resolution_clock::now(); + + // 启动线程 + for (int i = 0; i < thread_num; ++i) { + threads.emplace_back(thread_task_put, db, i, operations_per_thread); + } + + // 等待线程完成 + for (auto& th : threads) { + th.join(); + } + + auto end = std::chrono::high_resolution_clock::now(); + long long total_duration = std::chrono::duration_cast(end - start).count(); + + // 计算吞吐量 + double throughput = static_cast(total_operations.load()) / (total_duration / 1000.0); + + // 计算平均延迟 + long long sum_latency = 0; + for (auto& lat : latencies) { + sum_latency += lat; + } + double avg_latency = static_cast(sum_latency) / latencies.size(); + //double avg_latency = static_cast(total_duration) * 1000.0 / total_operations.load(); + // 转换为毫秒 + double total_latency_ms = static_cast(sum_latency) / 1000.0; + + std::cout << "Total Operations: " << total_operations.load() << std::endl; + std::cout << "Total Duration: " << total_duration << std::endl; + std::cout << "Throughput (ops/sec): " << throughput << std::endl; + std::cout << "Average Latency (us): " << avg_latency << std::endl; + std::cout << "Total Latency (ms): " << total_latency_ms << std::endl; + delete db; +} + + +int main(int argc, char** argv) { + // 设置全局随机种子 + SetGlobalSeed(static_cast(time(nullptr))); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/index_test.cc b/test/index_test.cc new file mode 100644 index 0000000..18bbe00 --- /dev/null +++ b/test/index_test.cc @@ -0,0 +1,236 @@ +#include "gtest/gtest.h" +#include "db/NewDB.h" // NewDB 的头文件 +#include "leveldb/env.h" +#include "leveldb/db.h" +#include + +using namespace leveldb; + +// 打开数据库的辅助函数 +Status OpenDB(std::string dbName, NewDB** db) { + Options options = Options(); + options.create_if_missing = true; + return NewDB::Open(options, dbName, db); +} + +std::string testdbname = "testdb1"; + +// 测试 NewDB::Put_fields 方法和二级索引的创建 +TEST(TestNewDB, PutFieldsWithIndexTest) { + // 创建 NewDB 实例 + NewDB* db; + if (OpenDB(testdbname, &db).ok() == false) { + std::cerr << "open db failed" << std::endl; + abort(); + } + + // 创建索引字段 + db->CreateIndexOnField("address"); + + // 定义插入的字段数据 + std::string key1 = "k_1"; + FieldArray fields1 = { + {"name", "Customer#000000001"}, + {"address", "IVhzIApeRb"}, + {"phone", "25-989-741-2988"} + }; + + // 定义字段数组，用于获取数据 + FieldArray retrieved_fields; + + // 使用 NewDB::Put_fields 插入数据 + db->Put_fields(WriteOptions(), key1, fields1); + + // // 检查索引是否成功创建 + // EXPECT_TRUE(db->IsFieldIndexed("address")); // 检查字段是否已索引 + + // 获取并验证字段数据 + db->Get_fields(ReadOptions(), key1, &retrieved_fields); + EXPECT_EQ(retrieved_fields.size(), fields1.size()); + + // 验证插入的每个字段 + for (size_t i = 0; i < fields1.size(); ++i) { + EXPECT_EQ(fields1[i].first, retrieved_fields[i].first); // 字段名 + EXPECT_EQ(fields1[i].second, retrieved_fields[i].second); // 字段值 + } + + // 插入第二条数据 + std::string key2 = "k_2"; + FieldArray fields2 = { + {"name", "Customer#000000002"}, + {"address", "TXkjZEdIrZ"}, + {"phone", "25-123-456-7890"} + }; + db->Put_fields(WriteOptions(), key2, fields2); + + // 获取并验证第二条数据 + FieldArray retrieved_fields2; + db->Get_fields(ReadOptions(), key2, &retrieved_fields2); + EXPECT_EQ(retrieved_fields2.size(), fields2.size()); + + for (size_t i = 0; i < fields2.size(); ++i) { + EXPECT_EQ(fields2[i].first, retrieved_fields2[i].first); // 字段名 + EXPECT_EQ(fields2[i].second, retrieved_fields2[i].second); // 字段值 + } + + // 清理数据库 + delete db; +} + +TEST(TestNewDB, DeleteDataTest) { + // 创建 NewDB 实例 + NewDB* db; + ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + + // 插入测试数据 + std::string key = "k_3"; + FieldArray fields = { + {"name", "Customer#000000003"}, + {"address", "AddressToDelete"}, + {"phone", "25-789-123-4567"} + }; + ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok()); + + // 删除数据 + ASSERT_TRUE(db->Delete(WriteOptions(), key)); + + // 验证数据已被删除 + FieldArray retrieved_fields; + Status s = db->Get_fields(ReadOptions(), key, &retrieved_fields); + ASSERT_FALSE(s.ok()); // 数据应该不存在 + + // 清理数据库 + delete db; +} + +TEST(TestNewDB, QueryByIndexTest) { + // 创建 NewDB 实例 + NewDB* db; + ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + + // 创建索引字段 + db->CreateIndexOnField("address"); + + // 插入测试数据 + std::string key1 = "k_4"; + FieldArray fields1 = { + {"name", "Customer#000000004"}, + {"address", "AddressToFind"}, + {"phone", "25-987-654-3210"} + }; + ASSERT_TRUE(db->Put_fields(WriteOptions(), key1, fields1).ok()); + + // 查询索引 + Field field_to_query{"address", "AddressToFind"}; + std::vector matching_keys = db->QueryByIndex(field_to_query); + + // 验证查询结果 + ASSERT_EQ(matching_keys.size(), 1); + EXPECT_EQ(matching_keys[0], key1); + + // 清理数据库 + delete db; +} + +TEST(TestNewDB, DeleteIndexTest) { + // 创建 NewDB 实例 + NewDB* db; + ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + + // 创建索引字段 + db->CreateIndexOnField("address"); + + // 插入测试数据 + std::string key1 = "k_5"; + FieldArray fields1 = { + {"name", "Customer#000000005"}, + {"address", "AddressToDeleteIndex"}, + {"phone", "25-123-456-7890"} + }; + ASSERT_TRUE(db->Put_fields(WriteOptions(), key1, fields1).ok()); + + // 删除索引 + ASSERT_TRUE(db->DeleteIndex("address:AddressToDeleteIndex_k_5")); + + // 尝试查询已删除的索引 + Field field_to_query{"address", "AddressToDeleteIndex"}; + std::vector matching_keys = db->QueryByIndex(field_to_query); + + // 验证查询结果为空 + EXPECT_TRUE(matching_keys.empty()); + + // 清理数据库 + delete db; +} + +TEST(TestNewDB, UpdateFieldsAndQueryIndexTest) { + // 创建 NewDB 实例 + NewDB* db; + ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + + // 创建索引字段 + db->CreateIndexOnField("address"); + + // 插入初始数据 + std::string key = "k_6"; + FieldArray initial_fields = { + {"name", "Customer#000000006"}, + {"address", "InitialAddress"}, + {"phone", "25-987-654-3210"} + }; + ASSERT_TRUE(db->Put_fields(WriteOptions(), key, initial_fields).ok()); + + // 验证初始数据的索引查询 + Field query_field1{"address", "InitialAddress"}; + std::vector matching_keys = db->QueryByIndex(query_field1); + ASSERT_EQ(matching_keys.size(), 1); + EXPECT_EQ(matching_keys[0], key); + + // 更新数据 + FieldArray updated_fields = { + {"name", "Customer#000000006"}, + {"address", "UpdatedAddress"}, + {"phone", "25-987-654-3210"} + }; + ASSERT_TRUE(db->Put_fields(WriteOptions(), key, updated_fields).ok()); + + // 获取并验证字段数据 + FieldArray retrieved_fields; + + db->Get_fields(ReadOptions(), key, &retrieved_fields); + EXPECT_EQ(retrieved_fields.size(), updated_fields.size()); + + // 验证插入的每个字段 + for (size_t i = 0; i < updated_fields.size(); ++i) { + EXPECT_EQ(updated_fields[i].first, retrieved_fields[i].first); // 字段名 + EXPECT_EQ(updated_fields[i].second, retrieved_fields[i].second); // 字段值 + } + + // 验证新的 address 字段值存在于索引中 + Field query_field2{"address", "UpdatedAddress"}; + matching_keys = db->QueryByIndex(query_field2); + ASSERT_EQ(matching_keys.size(), 1); + EXPECT_EQ(matching_keys[0], key); + + // 验证旧的 address 字段值不再存在于索引中 + matching_keys = db->QueryByIndex(query_field1); + ASSERT_EQ(matching_keys.size(), 0); + EXPECT_TRUE(matching_keys.empty()); + + + // 清理数据库 + // db->Delete(WriteOptions(), "k_1"); + // db->Delete(WriteOptions(), "k_2"); + // db->Delete(WriteOptions(), "k_3"); + // db->Delete(WriteOptions(), "k_4"); + // db->Delete(WriteOptions(), "k_5"); + // db->Delete(WriteOptions(), "k_6"); + + delete db; +} + +// 主函数运行所有测试 +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/performance_test.cc b/test/performance_test.cc new file mode 100644 index 0000000..6e6e8ff --- /dev/null +++ b/test/performance_test.cc @@ -0,0 +1,436 @@ +#include "gtest/gtest.h" +#include "db/NewDB.h" // NewDB 的头文件 +#include "leveldb/env.h" +#include "leveldb/db.h" +#include +#include +#include +#include // 用于计时 +#include // 用于存储延迟 +using namespace std; +using namespace leveldb; + +Status OpenDB(std::string dbName, NewDB** db) { + Options options = Options(); + options.create_if_missing = true; + return NewDB::Open(options, dbName, db); +} + +// 全局的随机数引擎 +std::default_random_engine rng; + +// 设置随机种子 +void SetGlobalSeed(unsigned seed) { + rng.seed(seed); +} + +// 生成随机字符串 +std::string GenerateRandomString(size_t length) { + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + std::uniform_int_distribution dist(0, sizeof(alphanum) - 2); + + std::string str(length, 0); + for (size_t i = 0; i < length; ++i) { + str[i] = alphanum[dist(rng)]; + } + return str; +} + +std::string testdbname = "testdb2"; + +// TEST(TestNewDB, PersistentIndexTest) { +// // 创建 NewDB 实例 +// NewDB* db; +// ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + +// // 批量插入数据 +// std::vector> bulk_data; +// const int num_records = 1000; // 适当减少记录数，以便测试 + +// for (int i = 0; i < num_records; ++i) { +// std::string key = "k_" + GenerateRandomString(10); +// FieldArray fields = { +// {"name", GenerateRandomString(10)}, +// {"address", GenerateRandomString(25)}, +// {"phone", GenerateRandomString(11)} +// }; +// bulk_data.emplace_back(key, fields); + +// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok()); +// } + +// // 创建索引字段 +// db->CreateIndexOnField("address"); + +// // 关闭数据库 +// delete db; + +// // 重新打开数据库 +// ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + +// // 验证索引字段是否仍然存在 +// for (const auto& [key, fields] : bulk_data) { +// // 获取并验证字段数据 +// FieldArray retrieved_fields; +// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok()); +// EXPECT_EQ(retrieved_fields.size(), fields.size()); + +// // 验证索引是否能正确找到对应的键 +// Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询 +// std::vector matching_keys = db->QueryByIndex(field_to_query); + +// EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键 +// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end()); +// } + +// // 关闭数据库 +// delete db; +// } + +// TEST(TestNewDB, BulkCreateIndexAndQueryTest) { +// // 创建 NewDB 实例 +// NewDB* db; +// ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + +// // 批量插入数据 +// std::vector> bulk_data; +// const int num_records = 10000; + +// for (int i = 0; i < num_records; ++i) { +// std::string key = "k_" + GenerateRandomString(10); +// FieldArray fields = { +// {"name", GenerateRandomString(10)}, +// {"address", GenerateRandomString(25)}, +// {"phone", GenerateRandomString(11)} +// }; +// bulk_data.emplace_back(key, fields); + +// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok()); +// } + +// // 创建索引字段 +// db->CreateIndexOnField("address"); + +// // 验证插入的数据和索引 +// for (const auto& [key, fields] : bulk_data) { +// // 获取并验证字段数据 +// FieldArray retrieved_fields; +// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok()); +// EXPECT_EQ(retrieved_fields.size(), fields.size()); + +// // 验证插入的每个字段 +// for (size_t i = 0; i < fields.size(); ++i) { +// // EXPECT_EQ(fields[i].first, retrieved_fields[i].first); // 字段名 +// // EXPECT_EQ(fields[i].second, retrieved_fields[i].second); // 字段值 +// if (fields[i].second != retrieved_fields[i].second) { +// std::cerr << "Mismatch at index: " << i +// << ", Key: " << key +// << ", Field Name: " << fields[i].first +// << ", Expected: " << fields[i].second +// << ", Retrieved: " << retrieved_fields[i].second +// << std::endl; +// } +// } + +// // 验证索引是否能正确找到对应的键 +// Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询 +// std::vector matching_keys = db->QueryByIndex(field_to_query); +// EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键 +// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end()); +// } + +// delete db; +// } + +TEST(TestNewDB, QueryByIndexPerformance) { + // 创建 NewDB 实例 + NewDB* db; + ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + + // 批量插入数据 + std::vector> bulk_data; + const int num_records = 10000; + + for (int i = 0; i < num_records; ++i) { + std::string key = "k_" + GenerateRandomString(10); + FieldArray fields = { + {"name", GenerateRandomString(10)}, + {"address", GenerateRandomString(25)}, + {"phone", GenerateRandomString(11)} + }; + bulk_data.emplace_back(key, fields); + + ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok()); + } + + // 创建索引字段 + db->CreateIndexOnField("address"); + // 测试查询延迟 + auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 + // 验证插入的数据和索引 + for (const auto& [key, fields] : bulk_data) { + // 获取并验证字段数据 + FieldArray retrieved_fields; + ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok()); + EXPECT_EQ(retrieved_fields.size(), fields.size()); + + // 验证插入的每个字段 + // for (size_t i = 0; i < fields.size(); ++i) { + // // EXPECT_EQ(fields[i].first, retrieved_fields[i].first); // 字段名 + // // EXPECT_EQ(fields[i].second, retrieved_fields[i].second); // 字段值 + // if (fields[i].second != retrieved_fields[i].second) { + // std::cerr << "Mismatch at index: " << i + // << ", Key: " << key + // << ", Field Name: " << fields[i].first + // << ", Expected: " << fields[i].second + // << ", Retrieved: " << retrieved_fields[i].second + // << std::endl; + // } + // } + + // 验证索引是否能正确找到对应的键 + Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询 + std::vector matching_keys = db->QueryByIndex(field_to_query); + + EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键 + EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end()); + } + auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 + auto duration = std::chrono::duration_cast(end_time - start_time).count(); + cout << "Throughput: " << num_records * 1000 / duration << " OPS" << endl; + cout << "Total Latency " << duration << " ms" << endl; + delete db; +} + +// TEST(TestNewDB, FindKeysByFieldPerformance) { +// // 创建 NewDB 实例 +// NewDB* db; +// ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + +// // 批量插入数据 +// std::vector> bulk_data; +// const int num_records = 100000; + +// for (int i = 0; i < num_records; ++i) { +// std::string key = "k_" + GenerateRandomString(10); +// FieldArray fields = { +// {"name", GenerateRandomString(10)}, +// {"address", GenerateRandomString(25)}, +// {"phone", GenerateRandomString(11)} +// }; +// bulk_data.emplace_back(key, fields); + +// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok()); +// } + +// // 创建索引字段 +// db->CreateIndexOnField("address"); +// // 测试查询延迟 +// auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 +// // 验证插入的数据和索引 +// for (const auto& [key, fields] : bulk_data) { +// // 获取并验证字段数据 +// FieldArray retrieved_fields; +// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok()); +// EXPECT_EQ(retrieved_fields.size(), fields.size()); + +// // 验证插入的每个字段 +// // for (size_t i = 0; i < fields.size(); ++i) { +// // // EXPECT_EQ(fields[i].first, retrieved_fields[i].first); // 字段名 +// // // EXPECT_EQ(fields[i].second, retrieved_fields[i].second); // 字段值 +// // if (fields[i].second != retrieved_fields[i].second) { +// // std::cerr << "Mismatch at index: " << i +// // << ", Key: " << key +// // << ", Field Name: " << fields[i].first +// // << ", Expected: " << fields[i].second +// // << ", Retrieved: " << retrieved_fields[i].second +// // << std::endl; +// // } +// // } + +// // 验证索引是否能正确找到对应的键 +// Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询 +// std::vector matching_keys = db->FindKeysByField(field_to_query); + +// EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键 +// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end()); +// } +// auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 +// auto duration = std::chrono::duration_cast(end_time - start_time).count(); +// cout << "Throughput: " << num_records * 1000 / duration << " OPS" << endl; +// cout << "Total Latency " << duration << " ms" << endl; +// delete db; +// } + +//测试创建二级索引后批量插入、以及删除索引后的查询 +// TEST(TestNewDB, BulkPutFieldsAndDeleteIndexPerformance) { +// // 创建 NewDB 实例 +// NewDB* db; +// ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + +// db->CreateIndexOnField("address"); + +// // 批量插入数据 +// std::vector> bulk_data; +// const int num_records = 150000; + +// for (int i = 0; i < num_records; ++i) { +// std::string key = "k_" + GenerateRandomString(10); +// FieldArray fields = { +// {"name", GenerateRandomString(10)}, +// {"address", GenerateRandomString(25)}, +// {"phone", GenerateRandomString(11)} +// }; +// bulk_data.emplace_back(key, fields); + +// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok()); +// } + +// // 验证插入的数据和索引 +// for (const auto& [key, fields] : bulk_data) { +// // 获取并验证字段数据 +// FieldArray retrieved_fields; +// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok()); +// EXPECT_EQ(retrieved_fields.size(), fields.size()); + +// // 验证插入的每个字段 +// for (size_t i = 0; i < fields.size(); ++i) { +// EXPECT_EQ(fields[i].first, retrieved_fields[i].first); // 字段名 +// EXPECT_EQ(fields[i].second, retrieved_fields[i].second); // 字段值 +// } + +// // 验证索引是否能正确找到对应的键 +// Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询 +// std::vector matching_keys = db->QueryByIndex(field_to_query); +// EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键 +// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end()); +// } +// // 测试插入吞吐量 +// auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 +// // 删除部分数据 +// const int num_updates = 100000; + +// for (int i = 0; i < num_updates; ++i) { +// auto& [key, fields] = bulk_data[i]; +// ASSERT_TRUE(db->Delete(WriteOptions(), key)); +// } + +// // for (int i = 0; i < num_updates; ++i) { +// // Field query_field{"address", bulk_data[i].second[1].second}; +// // std::vector matching_keys = db->QueryByIndex(query_field); +// // EXPECT_TRUE(matching_keys.empty()); +// // } + +// // 删除索引 +// ASSERT_TRUE(db->DeleteIndex("address")); +// auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 +// auto duration = std::chrono::duration_cast(end_time - start_time).count(); + +// // 计算吞吐量并输出 +// cout << "Throughput: " << num_updates * 1000 / duration << " OPS" << endl; +// cout << "Total Latency " << duration << " ms" << endl; +// // // 验证删除索引后无法再通过索引查询到数据 +// // for (const auto& [key, fields] : bulk_data) { +// // Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询 +// // std::vector matching_keys = db->QueryByIndex(field_to_query); +// // EXPECT_TRUE(matching_keys.empty()); // 索引已被删除，应该找不到任何匹配的键 +// // } + +// delete db; +// } + + +// TEST(TestNewDB, UpdateFieldsAndQueryIndexPerformance) { +// // 创建 NewDB 实例 +// NewDB* db; +// ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + +// // 创建索引字段 +// ASSERT_TRUE(db->CreateIndexOnField("address")); + +// // 批量插入数据 +// const int num_records = 200000; +// std::vector> initial_bulk_data; + +// for (int i = 0; i < num_records; ++i) { +// std::string key = "k_" + GenerateRandomString(10); +// FieldArray fields = { +// {"name", GenerateRandomString(10)}, +// {"address", GenerateRandomString(25)}, +// {"phone", GenerateRandomString(11)} +// }; +// initial_bulk_data.emplace_back(key, fields); + +// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok()); +// } + +// //验证初始数据的索引查询 +// for (const auto& [key, fields] : initial_bulk_data) { +// Field query_field{"address", fields[1].second}; // 使用 address 字段进行查询 +// std::vector matching_keys = db->QueryByIndex(query_field); +// ASSERT_FALSE(matching_keys.empty()); // 应该能找到对应的键 +// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end()); +// } + +// // 更新部分数据 +// const int num_updates = 100000; +// std::vector> updated_bulk_data; +// // 测试插入吞吐量 +// auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 +// for (int i = 0; i < num_updates; ++i) { +// auto& [key, fields] = initial_bulk_data[i]; +// FieldArray updated_fields = { +// {"name", fields[0].second}, +// {"address", GenerateRandomString(15)}, // 更新 address +// {"phone", fields[2].second} +// }; +// updated_bulk_data.emplace_back(key, updated_fields); + +// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, updated_fields).ok()); +// } +// auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 +// auto duration = std::chrono::duration_cast(end_time - start_time).count(); + +// // 计算吞吐量并输出 +// cout << "Throughput: " << num_updates * 1000 / duration << " OPS" << endl; +// cout << "Total Latency " << duration << " ms" << endl; +// // 验证更新后的数据和索引 +// for (const auto& [key, updated_fields] : updated_bulk_data) { +// // 获取并验证字段数据 +// FieldArray retrieved_fields; +// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok()); +// EXPECT_EQ(retrieved_fields.size(), updated_fields.size()); + +// // 验证插入的每个字段 +// for (size_t i = 0; i < updated_fields.size(); ++i) { +// EXPECT_EQ(updated_fields[i].first, retrieved_fields[i].first); // 字段名 +// EXPECT_EQ(updated_fields[i].second, retrieved_fields[i].second); // 字段值 +// } + +// // 验证新的 address 字段值存在于索引中 +// Field query_field_new{"address", updated_fields[1].second}; +// std::vector matching_keys_new = db->QueryByIndex(query_field_new); +// ASSERT_FALSE(matching_keys_new.empty()); // 新地址应该能找到对应的键 +// EXPECT_NE(std::find(matching_keys_new.begin(), matching_keys_new.end(), key), matching_keys_new.end()); +// } +// for (int i = 0; i < num_updates; ++i) { +// // 验证旧的 address 字段值不再存在于索引中 +// Field query_field_old{"address", initial_bulk_data[i].second[1].second}; +// std::vector matching_keys_old = db->QueryByIndex(query_field_old); +// EXPECT_TRUE(matching_keys_old.empty()); // 旧地址不应该再找到对应的键 +// } + +// delete db; +// } + + +int main(int argc, char** argv) { + // 设置全局随机种子 + SetGlobalSeed(static_cast(time(nullptr))); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/throughput_test.cc b/test/throughput_test.cc new file mode 100644 index 0000000..3f46376 --- /dev/null +++ b/test/throughput_test.cc @@ -0,0 +1,107 @@ +#include "gtest/gtest.h" +#include "db/NewDB.h" // NewDB 的头文件 +#include "leveldb/env.h" +#include "leveldb/db.h" +#include // 用于计时 +#include // 用于生成随机数 +#include // 用于存储延迟 + +using namespace std; +using namespace leveldb; + +// 随机生成字符串的辅助函数 +std::string GenerateRandomString(size_t length) { + const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + const size_t max_index = sizeof(charset) - 1; + std::random_device rd; + std::mt19937 generator(rd()); + std::uniform_int_distribution distribution(0, max_index); + + std::string random_str; + for (size_t i = 0; i < length; ++i) { + random_str += charset[distribution(generator)]; + } + return random_str; +} + +// 打开数据库的辅助函数 +Status OpenDB(std::string dbName, NewDB** db) { + Options options; + options.create_if_missing = true; + Status s = NewDB::Open(options, dbName, db); + if (!s.ok()) { + cerr << "Error opening database: " << s.ToString() << endl; + } + return s; +} + +std::string testdbname = "testdb_for_performance"; + +// 延迟测试 +void TestLatency(int num_operations, std::vector& lat_res) { + int64_t latency = 0; + auto end_time = std::chrono::steady_clock::now(); + auto last_time = end_time; + + for (int i = 0; i < num_operations; ++i) { + // 记录操作的延迟 + end_time = std::chrono::steady_clock::now(); + latency = std::chrono::duration_cast( + end_time - last_time).count(); + last_time = end_time; + + lat_res.emplace_back(latency); + } +} + +// 吞吐量测试 +TEST(TestNewDB, PutFieldsThroughputTest) { + // 创建 NewDB 实例 + NewDB* db; + ASSERT_TRUE(OpenDB(testdbname, &db).ok()); + + // 插入索引字段 + db->CreateIndexOnField("address"); + + // 延迟测试 + vector latencies; // 存储每次插入的延迟 + int num_inserts = 1000; // 插入的条数 + + // 测试插入吞吐量 + auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 + for (int i = 0; i < num_inserts; ++i) { + std::string key = "k_" + std::to_string(i); + + // 随机生成字段内容 + FieldArray fields = { + {"name", "Customer#" + std::to_string(i)}, + {"address", GenerateRandomString(10)}, // 随机生成地址 + {"phone", GenerateRandomString(12)} // 随机生成电话号码 + }; + + db->Put_fields(WriteOptions(), key, fields); + } + + // 延迟测试 + TestLatency(num_inserts, latencies); + + auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 + auto duration = std::chrono::duration_cast(end_time - start_time).count(); + + // 计算吞吐量并输出 + cout << "Throughput: " << num_inserts * 1000 / duration << " OPS" << endl; + + // 输出延迟 + // for (size_t i = 0; i < latencies.size(); ++i) { + // cout << "Latency for operation " << i + 1 << ": " << latencies[i] << " ms" << endl; + // } + cout << "Latency for operation " << latencies.size()-1 << ": " << latencies[latencies.size()-1] << " ms" << endl; + // 清理数据库 + delete db; +} + +// 主函数运行所有测试 +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}