building_data_management_systems.Xuanzhou.2024Fall.DaSE
/
leveldb-project2
23
0
派生 0
代码 话题 0 合并请求 0 版本发布 0 百科 动态
浏览代码

test-zcy

master
朱陈媛 8 个月前
父节点
e20d55ebea
当前提交
20cf51eeb0
共有 6 个文件被更改，包括 1267 次插入 和 3 次删除
合并视图
Diff 选项
显示统计 下载 Patch 文件 下载 Diff 文件
  1. +36
    -3
      CMakeLists.txt
  2. +254
    -0
      test/concurrency_performance_test.cc
  3. +198
    -0
      test/concurrency_performance_test_put.cc
  4. +236
    -0
      test/index_test.cc
  5. +436
    -0
      test/performance_test.cc
  6. +107
    -0
      test/throughput_test.cc

+ 36
- 3
CMakeLists.txt 查看文件

@ -190,6 +190,8 @@ target_sources(leveldb
"util/options.cc" "util/options.cc"
"util/random.h" "util/random.h"
"util/status.cc" "util/status.cc"
"db/NewDB.cc"
"db/NewDB.h"
# Only CMake 3.3+ supports PUBLIC sources in targets exported by "install". # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
$<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC> $<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC>
@ -518,16 +520,27 @@ if(LEVELDB_INSTALL)
) )
endif(LEVELDB_INSTALL) endif(LEVELDB_INSTALL)
add_executable(kv_index_test
"${PROJECT_SOURCE_DIR}/test/kv_index_test.cc"
add_executable(db_test1
"${PROJECT_SOURCE_DIR}/test/db_test1.cc"
)
target_link_libraries(db_test1 leveldb)
add_executable(field_test
"${PROJECT_SOURCE_DIR}/test/field_test.cc"
) )
target_link_libraries(kv_index_test PRIVATE leveldb gtest)
target_link_libraries(field_test PRIVATE leveldb gtest)
add_executable(index_test add_executable(index_test
"${PROJECT_SOURCE_DIR}/test/index_test.cc" "${PROJECT_SOURCE_DIR}/test/index_test.cc"
) )
target_link_libraries(index_test PRIVATE leveldb gtest) target_link_libraries(index_test PRIVATE leveldb gtest)
add_executable(throughput_test
"${PROJECT_SOURCE_DIR}/test/throughput_test.cc"
)
target_link_libraries(throughput_test PRIVATE leveldb gtest)
add_executable(index_test_random add_executable(index_test_random
"${PROJECT_SOURCE_DIR}/test/index_test_random.cc" "${PROJECT_SOURCE_DIR}/test/index_test_random.cc"
) )
@ -542,3 +555,23 @@ add_executable(concurrency_test
"${PROJECT_SOURCE_DIR}/test/concurrency_test.cc" "${PROJECT_SOURCE_DIR}/test/concurrency_test.cc"
) )
target_link_libraries(concurrency_test PRIVATE leveldb gtest) target_link_libraries(concurrency_test PRIVATE leveldb gtest)
add_executable(constancy_test
"${PROJECT_SOURCE_DIR}/test/constancy_test.cc"
)
target_link_libraries(constancy_test PRIVATE leveldb gtest)
add_executable(performance_test
"${PROJECT_SOURCE_DIR}/test/performance_test.cc"
)
target_link_libraries(performance_test PRIVATE leveldb gtest)
add_executable(concurrency_performance_test
"${PROJECT_SOURCE_DIR}/test/concurrency_performance_test.cc"
)
target_link_libraries(concurrency_performance_test PRIVATE leveldb gtest)
add_executable(concurrency_performance_test_put
"${PROJECT_SOURCE_DIR}/test/concurrency_performance_test_put.cc"
)
target_link_libraries(concurrency_performance_test_put PRIVATE leveldb gtest)

+ 254
- 0
test/concurrency_performance_test.cc 查看文件

@ -0,0 +1,254 @@
#include "gtest/gtest.h"
#include "db/NewDB.h" // NewDB 的头文件
#include "leveldb/env.h"
#include "leveldb/db.h"
#include "db/write_batch_internal.h"
#include <iostream>
#include <random>
#include <ctime>
#include <thread>
#include <mutex>
#include <chrono>
#include <atomic>
using namespace leveldb;
// 全局计数器
std::atomic<int> put_count(0);
std::atomic<int> delete_count(0);
std::atomic<int> create_index_count(0);
std::atomic<int> delete_index_count(0);
std::atomic<int> total_operations(0);
std::mutex latency_mutex;
std::vector<long long> latencies;
thread_local std::vector<long long> put_latencies_local;
thread_local std::vector<long long> delete_latencies_local;
thread_local std::vector<long long> update_latencies_local;
std::mutex put_latency_mutex;
std::vector<long long> put_latencies;
std::mutex delete_latency_mutex;
std::vector<long long> delete_latencies;
std::mutex update_latency_mutex;
std::vector<long long> update_latencies;
// void RecordLatency(long long duration) {
// std::unique_lock<std::mutex> lock(latency_mutex);
// latencies.push_back(duration);
// }
void RecordLatency(long long duration, const std::string& op_type) {
if (op_type == "PUT") {
put_latencies_local.push_back(duration);
} else if (op_type == "DELETE") {
delete_latencies_local.push_back(duration);
} else if (op_type == "UPDATE") {
update_latencies_local.push_back(duration);
}
}
void MergeLatencies() {
{
std::unique_lock<std::mutex> lock(put_latency_mutex);
put_latencies.insert(put_latencies.end(),
put_latencies_local.begin(),
put_latencies_local.end());
}
{
std::unique_lock<std::mutex> lock(delete_latency_mutex);
delete_latencies.insert(delete_latencies.end(),
delete_latencies_local.begin(),
delete_latencies_local.end());
}
{
std::unique_lock<std::mutex> lock(update_latency_mutex);
update_latencies.insert(update_latencies.end(),
update_latencies_local.begin(),
update_latencies_local.end());
}
}
Status OpenNewDB(std::string dbName, NewDB** db) {
Options options = Options();
options.create_if_missing = true;
return NewDB::Open(options, dbName, db);
}
// 全局的随机数引擎
std::default_random_engine rng;
// 设置随机种子
void SetGlobalSeed(unsigned seed) {
rng.seed(seed);
}
// 生成随机字符串
std::string GenerateRandomString(size_t length) {
static const char alphanum[] =
"0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
std::uniform_int_distribution<int> dist(0, sizeof(alphanum) - 2);
std::string str(length, 0);
for (size_t i = 0; i < length; ++i) {
str[i] = alphanum[dist(rng)];
}
return str;
}
int getRandomInRange(int min, int max) {
return min + std::rand() % ((max + 1) - min);
}
std::mutex results_mutex;
std::vector<bool> results;
void InsertResult(bool TRUEorFALSE){
std::unique_lock<std::mutex> lock(results_mutex, std::defer_lock);
lock.lock();
results.emplace_back(TRUEorFALSE);
lock.unlock();
}
std::string testdbname = "dbtest36";
void thread_task_put(NewDB* db, int thread_id, int num_operations) {
for (int i = 0; i < num_operations; ++i) {
auto start = std::chrono::high_resolution_clock::now();
std::string key = "k_" + std::to_string(thread_id) + "_" + std::to_string(i);
FieldArray fields = {
{"name", "User" + std::to_string(i)},
{"email", "user" + std::to_string(i) + "@test.com"}
};
db->Put_fields(WriteOptions(), key, fields);
total_operations++;
auto end = std::chrono::high_resolution_clock::now();
long long duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
RecordLatency(duration, "PUT");
}
// 合并本线程的延迟数据
MergeLatencies();
}
void thread_task_delete(NewDB* db, int thread_id, int num_operations) {
for (int i = 0; i < num_operations; ++i) {
int key_index = getRandomInRange(0, 9999);
std::string key = "k_" + std::to_string(key_index);
auto start = std::chrono::high_resolution_clock::now();
db->Delete(WriteOptions(), key);
delete_count++;
auto end = std::chrono::high_resolution_clock::now();
long long duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
RecordLatency(duration, "DELETE");
total_operations++;
}
// 合并本线程的延迟数据
MergeLatencies();
}
void thread_task_update(NewDB* db, int thread_id, int num_operations) {
for (int i = 0; i < num_operations; ++i) {
std::string key = "k_" + std::to_string(thread_id) + "_" + std::to_string(i);
FieldArray fields = {
{"email", "updated_user" + std::to_string(i) + "@test.com"}
};
auto start = std::chrono::high_resolution_clock::now();
db->Put_fields(WriteOptions(), key, fields); // 简单地更新一个字段
total_operations++;
auto end = std::chrono::high_resolution_clock::now();
long long duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
RecordLatency(duration, "UPDATE");
}
// 合并本线程的延迟数据
MergeLatencies();
}
// 并发测试：包括 put, delete, update
TEST(NewDBTest, ConcurrencyTest_mul) {
NewDB* db;
ASSERT_TRUE(OpenNewDB(testdbname, &db).ok());
const int thread_num_put = 10;
const int thread_num_delete = 1;
const int thread_num_update = 0;
const int operations_per_thread = 10;
const int operations_per_thread_delete = 50;
const int operations_per_thread_update = 50;
std::vector<std::thread> threads;
auto start = std::chrono::high_resolution_clock::now();
// 启动 PUT 线程
for (int i = 0; i < thread_num_put; ++i) {
threads.emplace_back(thread_task_put, db, i, operations_per_thread);
}
// 启动 DELETE 线程
for (int i = 0; i < thread_num_delete; ++i) {
threads.emplace_back(thread_task_delete, db, i, operations_per_thread_delete);
}
// 启动 UPDATE 线程
for (int i = 0; i < thread_num_update; ++i) {
threads.emplace_back(thread_task_update, db, i, operations_per_thread_update);
}
for (auto& th : threads) {
th.join();
}
auto end = std::chrono::high_resolution_clock::now();
long long total_duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
auto calculate_latency = [](const std::vector<long long>& latencies) {
long long sum = 0;
for (auto& lat : latencies) {
sum += lat;
}
return std::make_pair(sum, latencies.size() > 0 ? static_cast<double>(sum) / latencies.size() : 0);
};
auto [put_sum, put_avg] = calculate_latency(put_latencies);
auto [delete_sum, delete_avg] = calculate_latency(delete_latencies);
auto [update_sum, update_avg] = calculate_latency(update_latencies);
std::cout << "Total Operations: " << total_operations.load() << std::endl;
std::cout << "Total Duration: " << total_duration << " ms" << std::endl;
std::cout << "Throughput (ops/sec): " << static_cast<double>(total_operations.load()) / (total_duration / 1000.0) << std::endl;
std::cout << "\nPUT Operations:" << std::endl;
std::cout << " Total Latency (ms): " << put_sum / 1000.0 << std::endl;
std::cout << " Average Latency (us): " << put_avg << std::endl;
std::cout << "\nDELETE Operations:" << std::endl;
std::cout << " Total Latency (ms): " << delete_sum / 1000.0 << std::endl;
std::cout << " Average Latency (us): " << delete_avg << std::endl;
std::cout << "\nUPDATE Operations:" << std::endl;
std::cout << " Total Latency (ms): " << update_sum / 1000.0 << std::endl;
std::cout << " Average Latency (us): " << update_avg << std::endl;
delete db;
}
int main(int argc, char** argv) {
// 设置全局随机种子
SetGlobalSeed(static_cast<unsigned>(time(nullptr)));
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

+ 198
- 0
test/concurrency_performance_test_put.cc 查看文件

@ -0,0 +1,198 @@
#include "gtest/gtest.h"
#include "db/NewDB.h" // NewDB 的头文件
#include "leveldb/env.h"
#include "leveldb/db.h"
#include "db/write_batch_internal.h"
#include <iostream>
#include <random>
#include <ctime>
#include <thread>
#include <mutex>
#include <chrono>
#include <atomic>
using namespace leveldb;
// 全局计数器
std::atomic<int> put_count(0);
std::atomic<int> delete_count(0);
std::atomic<int> create_index_count(0);
std::atomic<int> delete_index_count(0);
std::atomic<int> total_operations(0);
std::mutex latency_mutex;
std::vector<long long> latencies;
void RecordLatency(long long duration) {
std::unique_lock<std::mutex> lock(latency_mutex);
latencies.push_back(duration);
}
Status OpenNewDB(std::string dbName, NewDB** db) {
Options options = Options();
options.create_if_missing = true;
return NewDB::Open(options, dbName, db);
}
// 全局的随机数引擎
std::default_random_engine rng;
// 设置随机种子
void SetGlobalSeed(unsigned seed) {
rng.seed(seed);
}
// 生成随机字符串
std::string GenerateRandomString(size_t length) {
static const char alphanum[] =
"0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
std::uniform_int_distribution<int> dist(0, sizeof(alphanum) - 2);
std::string str(length, 0);
for (size_t i = 0; i < length; ++i) {
str[i] = alphanum[dist(rng)];
}
return str;
}
int getRandomInRange(int min, int max) {
return min + std::rand() % ((max + 1) - min);
}
std::mutex results_mutex;
std::vector<bool> results;
void InsertResult(bool TRUEorFALSE){
std::unique_lock<std::mutex> lock(results_mutex, std::defer_lock);
lock.lock();
results.emplace_back(TRUEorFALSE);
lock.unlock();
}
std::string testdbname = "dbtest36";
// 多线程插入
void thread_task_put(NewDB* db, int thread_id, int num_operations) {
for (int i = 0; i < num_operations; ++i) {
auto start = std::chrono::high_resolution_clock::now();
std::string key = "k_" + std::to_string(thread_id) + "_" + std::to_string(i);
FieldArray fields = {
{"name", "User" + std::to_string(i)},
{"email", "user" + std::to_string(i) + "@test.com"}
};
db->Put_fields(WriteOptions(), key, fields);
total_operations++;
auto end = std::chrono::high_resolution_clock::now();
long long duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
RecordLatency(duration);
}
}
// // 测试吞吐量和延迟
// TEST(TestNewDB, PerformanceTest) {
// NewDB* db;
// ASSERT_TRUE(OpenNewDB(testdbname, &db).ok());
// const int thread_num = 100;
// std::vector<std::thread> threads;
// auto test_start = std::chrono::high_resolution_clock::now();
// for (int i = 0; i < thread_num; ++i) {
// threads.emplace_back([=]() { thread_task_put(db, i); });
// threads.emplace_back([=]() { thread_task_delete(db, i); });
// threads.emplace_back([=]() { thread_task_createindex(db, i); });
// threads.emplace_back([=]() { thread_task_deleteindex(db, i); });
// }
// for (auto& th : threads) {
// if (th.joinable()) {
// th.join();
// }
// }
// auto test_end = std::chrono::high_resolution_clock::now();
// long long total_duration = std::chrono::duration_cast<std::chrono::milliseconds>(test_end - test_start).count();
// // 计算吞吐量
// double throughput = static_cast<double>(total_operations) / (total_duration / 1000.0);
// // 计算平均延迟
// long long sum_latency = 0;
// for (auto& lat : latencies) {
// sum_latency += lat;
// }
// double avg_latency = static_cast<double>(sum_latency) / latencies.size();
// std::cout << "Total Operations: " << total_operations.load() << std::endl;
// std::cout << "Throughput (ops/sec): " << throughput << std::endl;
// std::cout << "Average Latency (us): " << avg_latency << std::endl;
// std::cout << "PUT operations: " << put_count.load() << std::endl;
// std::cout << "DELETE operations: " << delete_count.load() << std::endl;
// std::cout << "CREATE INDEX operations: " << create_index_count.load() << std::endl;
// std::cout << "DELETE INDEX operations: " << delete_index_count.load() << std::endl;
// delete db;
// }
//并发测试
TEST(NewDBTest, ConcurrencyTest) {
NewDB* db;
ASSERT_TRUE(OpenNewDB(testdbname, &db).ok());
const int thread_num = 100;
const int operations_per_thread = 10;
std::vector<std::thread> threads;
auto start = std::chrono::high_resolution_clock::now();
// 启动线程
for (int i = 0; i < thread_num; ++i) {
threads.emplace_back(thread_task_put, db, i, operations_per_thread);
}
// 等待线程完成
for (auto& th : threads) {
th.join();
}
auto end = std::chrono::high_resolution_clock::now();
long long total_duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
// 计算吞吐量
double throughput = static_cast<double>(total_operations.load()) / (total_duration / 1000.0);
// 计算平均延迟
long long sum_latency = 0;
for (auto& lat : latencies) {
sum_latency += lat;
}
double avg_latency = static_cast<double>(sum_latency) / latencies.size();
//double avg_latency = static_cast<double>(total_duration) * 1000.0 / total_operations.load();
// 转换为毫秒
double total_latency_ms = static_cast<double>(sum_latency) / 1000.0;
std::cout << "Total Operations: " << total_operations.load() << std::endl;
std::cout << "Total Duration: " << total_duration << std::endl;
std::cout << "Throughput (ops/sec): " << throughput << std::endl;
std::cout << "Average Latency (us): " << avg_latency << std::endl;
std::cout << "Total Latency (ms): " << total_latency_ms << std::endl;
delete db;
}
int main(int argc, char** argv) {
// 设置全局随机种子
SetGlobalSeed(static_cast<unsigned>(time(nullptr)));
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

+ 236
- 0
test/index_test.cc 查看文件

@ -0,0 +1,236 @@
#include "gtest/gtest.h"
#include "db/NewDB.h" // NewDB 的头文件
#include "leveldb/env.h"
#include "leveldb/db.h"
#include <iostream>
using namespace leveldb;
// 打开数据库的辅助函数
Status OpenDB(std::string dbName, NewDB** db) {
Options options = Options();
options.create_if_missing = true;
return NewDB::Open(options, dbName, db);
}
std::string testdbname = "testdb1";
// 测试 NewDB::Put_fields 方法和二级索引的创建
TEST(TestNewDB, PutFieldsWithIndexTest) {
// 创建 NewDB 实例
NewDB* db;
if (OpenDB(testdbname, &db).ok() == false) {
std::cerr << "open db failed" << std::endl;
abort();
}
// 创建索引字段
db->CreateIndexOnField("address");
// 定义插入的字段数据
std::string key1 = "k_1";
FieldArray fields1 = {
{"name", "Customer#000000001"},
{"address", "IVhzIApeRb"},
{"phone", "25-989-741-2988"}
};
// 定义字段数组，用于获取数据
FieldArray retrieved_fields;
// 使用 NewDB::Put_fields 插入数据
db->Put_fields(WriteOptions(), key1, fields1);
// // 检查索引是否成功创建
// EXPECT_TRUE(db->IsFieldIndexed("address")); // 检查字段是否已索引
// 获取并验证字段数据
db->Get_fields(ReadOptions(), key1, &retrieved_fields);
EXPECT_EQ(retrieved_fields.size(), fields1.size());
// 验证插入的每个字段
for (size_t i = 0; i < fields1.size(); ++i) {
EXPECT_EQ(fields1[i].first, retrieved_fields[i].first); // 字段名
EXPECT_EQ(fields1[i].second, retrieved_fields[i].second); // 字段值
}
// 插入第二条数据
std::string key2 = "k_2";
FieldArray fields2 = {
{"name", "Customer#000000002"},
{"address", "TXkjZEdIrZ"},
{"phone", "25-123-456-7890"}
};
db->Put_fields(WriteOptions(), key2, fields2);
// 获取并验证第二条数据
FieldArray retrieved_fields2;
db->Get_fields(ReadOptions(), key2, &retrieved_fields2);
EXPECT_EQ(retrieved_fields2.size(), fields2.size());
for (size_t i = 0; i < fields2.size(); ++i) {
EXPECT_EQ(fields2[i].first, retrieved_fields2[i].first); // 字段名
EXPECT_EQ(fields2[i].second, retrieved_fields2[i].second); // 字段值
}
// 清理数据库
delete db;
}
TEST(TestNewDB, DeleteDataTest) {
// 创建 NewDB 实例
NewDB* db;
ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// 插入测试数据
std::string key = "k_3";
FieldArray fields = {
{"name", "Customer#000000003"},
{"address", "AddressToDelete"},
{"phone", "25-789-123-4567"}
};
ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok());
// 删除数据
ASSERT_TRUE(db->Delete(WriteOptions(), key));
// 验证数据已被删除
FieldArray retrieved_fields;
Status s = db->Get_fields(ReadOptions(), key, &retrieved_fields);
ASSERT_FALSE(s.ok()); // 数据应该不存在
// 清理数据库
delete db;
}
TEST(TestNewDB, QueryByIndexTest) {
// 创建 NewDB 实例
NewDB* db;
ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// 创建索引字段
db->CreateIndexOnField("address");
// 插入测试数据
std::string key1 = "k_4";
FieldArray fields1 = {
{"name", "Customer#000000004"},
{"address", "AddressToFind"},
{"phone", "25-987-654-3210"}
};
ASSERT_TRUE(db->Put_fields(WriteOptions(), key1, fields1).ok());
// 查询索引
Field field_to_query{"address", "AddressToFind"};
std::vector<std::string> matching_keys = db->QueryByIndex(field_to_query);
// 验证查询结果
ASSERT_EQ(matching_keys.size(), 1);
EXPECT_EQ(matching_keys[0], key1);
// 清理数据库
delete db;
}
TEST(TestNewDB, DeleteIndexTest) {
// 创建 NewDB 实例
NewDB* db;
ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// 创建索引字段
db->CreateIndexOnField("address");
// 插入测试数据
std::string key1 = "k_5";
FieldArray fields1 = {
{"name", "Customer#000000005"},
{"address", "AddressToDeleteIndex"},
{"phone", "25-123-456-7890"}
};
ASSERT_TRUE(db->Put_fields(WriteOptions(), key1, fields1).ok());
// 删除索引
ASSERT_TRUE(db->DeleteIndex("address:AddressToDeleteIndex_k_5"));
// 尝试查询已删除的索引
Field field_to_query{"address", "AddressToDeleteIndex"};
std::vector<std::string> matching_keys = db->QueryByIndex(field_to_query);
// 验证查询结果为空
EXPECT_TRUE(matching_keys.empty());
// 清理数据库
delete db;
}
TEST(TestNewDB, UpdateFieldsAndQueryIndexTest) {
// 创建 NewDB 实例
NewDB* db;
ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// 创建索引字段
db->CreateIndexOnField("address");
// 插入初始数据
std::string key = "k_6";
FieldArray initial_fields = {
{"name", "Customer#000000006"},
{"address", "InitialAddress"},
{"phone", "25-987-654-3210"}
};
ASSERT_TRUE(db->Put_fields(WriteOptions(), key, initial_fields).ok());
// 验证初始数据的索引查询
Field query_field1{"address", "InitialAddress"};
std::vector<std::string> matching_keys = db->QueryByIndex(query_field1);
ASSERT_EQ(matching_keys.size(), 1);
EXPECT_EQ(matching_keys[0], key);
// 更新数据
FieldArray updated_fields = {
{"name", "Customer#000000006"},
{"address", "UpdatedAddress"},
{"phone", "25-987-654-3210"}
};
ASSERT_TRUE(db->Put_fields(WriteOptions(), key, updated_fields).ok());
// 获取并验证字段数据
FieldArray retrieved_fields;
db->Get_fields(ReadOptions(), key, &retrieved_fields);
EXPECT_EQ(retrieved_fields.size(), updated_fields.size());
// 验证插入的每个字段
for (size_t i = 0; i < updated_fields.size(); ++i) {
EXPECT_EQ(updated_fields[i].first, retrieved_fields[i].first); // 字段名
EXPECT_EQ(updated_fields[i].second, retrieved_fields[i].second); // 字段值
}
// 验证新的 address 字段值存在于索引中
Field query_field2{"address", "UpdatedAddress"};
matching_keys = db->QueryByIndex(query_field2);
ASSERT_EQ(matching_keys.size(), 1);
EXPECT_EQ(matching_keys[0], key);
// 验证旧的 address 字段值不再存在于索引中
matching_keys = db->QueryByIndex(query_field1);
ASSERT_EQ(matching_keys.size(), 0);
EXPECT_TRUE(matching_keys.empty());
// 清理数据库
// db->Delete(WriteOptions(), "k_1");
// db->Delete(WriteOptions(), "k_2");
// db->Delete(WriteOptions(), "k_3");
// db->Delete(WriteOptions(), "k_4");
// db->Delete(WriteOptions(), "k_5");
// db->Delete(WriteOptions(), "k_6");
delete db;
}
// 主函数运行所有测试
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

+ 436
- 0
test/performance_test.cc 查看文件

@ -0,0 +1,436 @@
#include "gtest/gtest.h"
#include "db/NewDB.h" // NewDB 的头文件
#include "leveldb/env.h"
#include "leveldb/db.h"
#include <iostream>
#include <random>
#include <ctime>
#include <chrono> // 用于计时
#include <vector> // 用于存储延迟
using namespace std;
using namespace leveldb;
Status OpenDB(std::string dbName, NewDB** db) {
Options options = Options();
options.create_if_missing = true;
return NewDB::Open(options, dbName, db);
}
// 全局的随机数引擎
std::default_random_engine rng;
// 设置随机种子
void SetGlobalSeed(unsigned seed) {
rng.seed(seed);
}
// 生成随机字符串
std::string GenerateRandomString(size_t length) {
static const char alphanum[] =
"0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
std::uniform_int_distribution<int> dist(0, sizeof(alphanum) - 2);
std::string str(length, 0);
for (size_t i = 0; i < length; ++i) {
str[i] = alphanum[dist(rng)];
}
return str;
}
std::string testdbname = "testdb2";
// TEST(TestNewDB, PersistentIndexTest) {
// // 创建 NewDB 实例
// NewDB* db;
// ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// // 批量插入数据
// std::vector<std::pair<std::string, FieldArray>> bulk_data;
// const int num_records = 1000; // 适当减少记录数，以便测试
// for (int i = 0; i < num_records; ++i) {
// std::string key = "k_" + GenerateRandomString(10);
// FieldArray fields = {
// {"name", GenerateRandomString(10)},
// {"address", GenerateRandomString(25)},
// {"phone", GenerateRandomString(11)}
// };
// bulk_data.emplace_back(key, fields);
// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok());
// }
// // 创建索引字段
// db->CreateIndexOnField("address");
// // 关闭数据库
// delete db;
// // 重新打开数据库
// ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// // 验证索引字段是否仍然存在
// for (const auto& [key, fields] : bulk_data) {
// // 获取并验证字段数据
// FieldArray retrieved_fields;
// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok());
// EXPECT_EQ(retrieved_fields.size(), fields.size());
// // 验证索引是否能正确找到对应的键
// Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询
// std::vector<std::string> matching_keys = db->QueryByIndex(field_to_query);
// EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键
// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end());
// }
// // 关闭数据库
// delete db;
// }
// TEST(TestNewDB, BulkCreateIndexAndQueryTest) {
// // 创建 NewDB 实例
// NewDB* db;
// ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// // 批量插入数据
// std::vector<std::pair<std::string, FieldArray>> bulk_data;
// const int num_records = 10000;
// for (int i = 0; i < num_records; ++i) {
// std::string key = "k_" + GenerateRandomString(10);
// FieldArray fields = {
// {"name", GenerateRandomString(10)},
// {"address", GenerateRandomString(25)},
// {"phone", GenerateRandomString(11)}
// };
// bulk_data.emplace_back(key, fields);
// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok());
// }
// // 创建索引字段
// db->CreateIndexOnField("address");
// // 验证插入的数据和索引
// for (const auto& [key, fields] : bulk_data) {
// // 获取并验证字段数据
// FieldArray retrieved_fields;
// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok());
// EXPECT_EQ(retrieved_fields.size(), fields.size());
// // 验证插入的每个字段
// for (size_t i = 0; i < fields.size(); ++i) {
// // EXPECT_EQ(fields[i].first, retrieved_fields[i].first); // 字段名
// // EXPECT_EQ(fields[i].second, retrieved_fields[i].second); // 字段值
// if (fields[i].second != retrieved_fields[i].second) {
// std::cerr << "Mismatch at index: " << i
// << ", Key: " << key
// << ", Field Name: " << fields[i].first
// << ", Expected: " << fields[i].second
// << ", Retrieved: " << retrieved_fields[i].second
// << std::endl;
// }
// }
// // 验证索引是否能正确找到对应的键
// Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询
// std::vector<std::string> matching_keys = db->QueryByIndex(field_to_query);
// EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键
// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end());
// }
// delete db;
// }
TEST(TestNewDB, QueryByIndexPerformance) {
// 创建 NewDB 实例
NewDB* db;
ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// 批量插入数据
std::vector<std::pair<std::string, FieldArray>> bulk_data;
const int num_records = 10000;
for (int i = 0; i < num_records; ++i) {
std::string key = "k_" + GenerateRandomString(10);
FieldArray fields = {
{"name", GenerateRandomString(10)},
{"address", GenerateRandomString(25)},
{"phone", GenerateRandomString(11)}
};
bulk_data.emplace_back(key, fields);
ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok());
}
// 创建索引字段
db->CreateIndexOnField("address");
// 测试查询延迟
auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间
// 验证插入的数据和索引
for (const auto& [key, fields] : bulk_data) {
// 获取并验证字段数据
FieldArray retrieved_fields;
ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok());
EXPECT_EQ(retrieved_fields.size(), fields.size());
// 验证插入的每个字段
// for (size_t i = 0; i < fields.size(); ++i) {
// // EXPECT_EQ(fields[i].first, retrieved_fields[i].first); // 字段名
// // EXPECT_EQ(fields[i].second, retrieved_fields[i].second); // 字段值
// if (fields[i].second != retrieved_fields[i].second) {
// std::cerr << "Mismatch at index: " << i
// << ", Key: " << key
// << ", Field Name: " << fields[i].first
// << ", Expected: " << fields[i].second
// << ", Retrieved: " << retrieved_fields[i].second
// << std::endl;
// }
// }
// 验证索引是否能正确找到对应的键
Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询
std::vector<std::string> matching_keys = db->QueryByIndex(field_to_query);
EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键
EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end());
}
auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
cout << "Throughput: " << num_records * 1000 / duration << " OPS" << endl;
cout << "Total Latency " << duration << " ms" << endl;
delete db;
}
// TEST(TestNewDB, FindKeysByFieldPerformance) {
// // 创建 NewDB 实例
// NewDB* db;
// ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// // 批量插入数据
// std::vector<std::pair<std::string, FieldArray>> bulk_data;
// const int num_records = 100000;
// for (int i = 0; i < num_records; ++i) {
// std::string key = "k_" + GenerateRandomString(10);
// FieldArray fields = {
// {"name", GenerateRandomString(10)},
// {"address", GenerateRandomString(25)},
// {"phone", GenerateRandomString(11)}
// };
// bulk_data.emplace_back(key, fields);
// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok());
// }
// // 创建索引字段
// db->CreateIndexOnField("address");
// // 测试查询延迟
// auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间
// // 验证插入的数据和索引
// for (const auto& [key, fields] : bulk_data) {
// // 获取并验证字段数据
// FieldArray retrieved_fields;
// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok());
// EXPECT_EQ(retrieved_fields.size(), fields.size());
// // 验证插入的每个字段
// // for (size_t i = 0; i < fields.size(); ++i) {
// // // EXPECT_EQ(fields[i].first, retrieved_fields[i].first); // 字段名
// // // EXPECT_EQ(fields[i].second, retrieved_fields[i].second); // 字段值
// // if (fields[i].second != retrieved_fields[i].second) {
// // std::cerr << "Mismatch at index: " << i
// // << ", Key: " << key
// // << ", Field Name: " << fields[i].first
// // << ", Expected: " << fields[i].second
// // << ", Retrieved: " << retrieved_fields[i].second
// // << std::endl;
// // }
// // }
// // 验证索引是否能正确找到对应的键
// Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询
// std::vector<std::string> matching_keys = db->FindKeysByField(field_to_query);
// EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键
// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end());
// }
// auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间
// auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
// cout << "Throughput: " << num_records * 1000 / duration << " OPS" << endl;
// cout << "Total Latency " << duration << " ms" << endl;
// delete db;
// }
//测试创建二级索引后批量插入、以及删除索引后的查询
// TEST(TestNewDB, BulkPutFieldsAndDeleteIndexPerformance) {
// // 创建 NewDB 实例
// NewDB* db;
// ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// db->CreateIndexOnField("address");
// // 批量插入数据
// std::vector<std::pair<std::string, FieldArray>> bulk_data;
// const int num_records = 150000;
// for (int i = 0; i < num_records; ++i) {
// std::string key = "k_" + GenerateRandomString(10);
// FieldArray fields = {
// {"name", GenerateRandomString(10)},
// {"address", GenerateRandomString(25)},
// {"phone", GenerateRandomString(11)}
// };
// bulk_data.emplace_back(key, fields);
// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok());
// }
// // 验证插入的数据和索引
// for (const auto& [key, fields] : bulk_data) {
// // 获取并验证字段数据
// FieldArray retrieved_fields;
// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok());
// EXPECT_EQ(retrieved_fields.size(), fields.size());
// // 验证插入的每个字段
// for (size_t i = 0; i < fields.size(); ++i) {
// EXPECT_EQ(fields[i].first, retrieved_fields[i].first); // 字段名
// EXPECT_EQ(fields[i].second, retrieved_fields[i].second); // 字段值
// }
// // 验证索引是否能正确找到对应的键
// Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询
// std::vector<std::string> matching_keys = db->QueryByIndex(field_to_query);
// EXPECT_FALSE(matching_keys.empty()); // 应该能找到对应的键
// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end());
// }
// // 测试插入吞吐量
// auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间
// // 删除部分数据
// const int num_updates = 100000;
// for (int i = 0; i < num_updates; ++i) {
// auto& [key, fields] = bulk_data[i];
// ASSERT_TRUE(db->Delete(WriteOptions(), key));
// }
// // for (int i = 0; i < num_updates; ++i) {
// // Field query_field{"address", bulk_data[i].second[1].second};
// // std::vector<std::string> matching_keys = db->QueryByIndex(query_field);
// // EXPECT_TRUE(matching_keys.empty());
// // }
// // 删除索引
// ASSERT_TRUE(db->DeleteIndex("address"));
// auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间
// auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
// // 计算吞吐量并输出
// cout << "Throughput: " << num_updates * 1000 / duration << " OPS" << endl;
// cout << "Total Latency " << duration << " ms" << endl;
// // // 验证删除索引后无法再通过索引查询到数据
// // for (const auto& [key, fields] : bulk_data) {
// // Field field_to_query{"address", fields[1].second}; // 使用 address 字段进行查询
// // std::vector<std::string> matching_keys = db->QueryByIndex(field_to_query);
// // EXPECT_TRUE(matching_keys.empty()); // 索引已被删除，应该找不到任何匹配的键
// // }
// delete db;
// }
// TEST(TestNewDB, UpdateFieldsAndQueryIndexPerformance) {
// // 创建 NewDB 实例
// NewDB* db;
// ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// // 创建索引字段
// ASSERT_TRUE(db->CreateIndexOnField("address"));
// // 批量插入数据
// const int num_records = 200000;
// std::vector<std::pair<std::string, FieldArray>> initial_bulk_data;
// for (int i = 0; i < num_records; ++i) {
// std::string key = "k_" + GenerateRandomString(10);
// FieldArray fields = {
// {"name", GenerateRandomString(10)},
// {"address", GenerateRandomString(25)},
// {"phone", GenerateRandomString(11)}
// };
// initial_bulk_data.emplace_back(key, fields);
// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, fields).ok());
// }
// //验证初始数据的索引查询
// for (const auto& [key, fields] : initial_bulk_data) {
// Field query_field{"address", fields[1].second}; // 使用 address 字段进行查询
// std::vector<std::string> matching_keys = db->QueryByIndex(query_field);
// ASSERT_FALSE(matching_keys.empty()); // 应该能找到对应的键
// EXPECT_NE(std::find(matching_keys.begin(), matching_keys.end(), key), matching_keys.end());
// }
// // 更新部分数据
// const int num_updates = 100000;
// std::vector<std::pair<std::string, FieldArray>> updated_bulk_data;
// // 测试插入吞吐量
// auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间
// for (int i = 0; i < num_updates; ++i) {
// auto& [key, fields] = initial_bulk_data[i];
// FieldArray updated_fields = {
// {"name", fields[0].second},
// {"address", GenerateRandomString(15)}, // 更新 address
// {"phone", fields[2].second}
// };
// updated_bulk_data.emplace_back(key, updated_fields);
// ASSERT_TRUE(db->Put_fields(WriteOptions(), key, updated_fields).ok());
// }
// auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间
// auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
// // 计算吞吐量并输出
// cout << "Throughput: " << num_updates * 1000 / duration << " OPS" << endl;
// cout << "Total Latency " << duration << " ms" << endl;
// // 验证更新后的数据和索引
// for (const auto& [key, updated_fields] : updated_bulk_data) {
// // 获取并验证字段数据
// FieldArray retrieved_fields;
// ASSERT_TRUE(db->Get_fields(ReadOptions(), Slice(key), &retrieved_fields).ok());
// EXPECT_EQ(retrieved_fields.size(), updated_fields.size());
// // 验证插入的每个字段
// for (size_t i = 0; i < updated_fields.size(); ++i) {
// EXPECT_EQ(updated_fields[i].first, retrieved_fields[i].first); // 字段名
// EXPECT_EQ(updated_fields[i].second, retrieved_fields[i].second); // 字段值
// }
// // 验证新的 address 字段值存在于索引中
// Field query_field_new{"address", updated_fields[1].second};
// std::vector<std::string> matching_keys_new = db->QueryByIndex(query_field_new);
// ASSERT_FALSE(matching_keys_new.empty()); // 新地址应该能找到对应的键
// EXPECT_NE(std::find(matching_keys_new.begin(), matching_keys_new.end(), key), matching_keys_new.end());
// }
// for (int i = 0; i < num_updates; ++i) {
// // 验证旧的 address 字段值不再存在于索引中
// Field query_field_old{"address", initial_bulk_data[i].second[1].second};
// std::vector<std::string> matching_keys_old = db->QueryByIndex(query_field_old);
// EXPECT_TRUE(matching_keys_old.empty()); // 旧地址不应该再找到对应的键
// }
// delete db;
// }
int main(int argc, char** argv) {
// 设置全局随机种子
SetGlobalSeed(static_cast<unsigned>(time(nullptr)));
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

+ 107
- 0
test/throughput_test.cc 查看文件

@ -0,0 +1,107 @@
#include "gtest/gtest.h"
#include "db/NewDB.h" // NewDB 的头文件
#include "leveldb/env.h"
#include "leveldb/db.h"
#include <chrono> // 用于计时
#include <random> // 用于生成随机数
#include <vector> // 用于存储延迟
using namespace std;
using namespace leveldb;
// 随机生成字符串的辅助函数
std::string GenerateRandomString(size_t length) {
const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
const size_t max_index = sizeof(charset) - 1;
std::random_device rd;
std::mt19937 generator(rd());
std::uniform_int_distribution<size_t> distribution(0, max_index);
std::string random_str;
for (size_t i = 0; i < length; ++i) {
random_str += charset[distribution(generator)];
}
return random_str;
}
// 打开数据库的辅助函数
Status OpenDB(std::string dbName, NewDB** db) {
Options options;
options.create_if_missing = true;
Status s = NewDB::Open(options, dbName, db);
if (!s.ok()) {
cerr << "Error opening database: " << s.ToString() << endl;
}
return s;
}
std::string testdbname = "testdb_for_performance";
// 延迟测试
void TestLatency(int num_operations, std::vector<int64_t>& lat_res) {
int64_t latency = 0;
auto end_time = std::chrono::steady_clock::now();
auto last_time = end_time;
for (int i = 0; i < num_operations; ++i) {
// 记录操作的延迟
end_time = std::chrono::steady_clock::now();
latency = std::chrono::duration_cast<std::chrono::milliseconds>(
end_time - last_time).count();
last_time = end_time;
lat_res.emplace_back(latency);
}
}
// 吞吐量测试
TEST(TestNewDB, PutFieldsThroughputTest) {
// 创建 NewDB 实例
NewDB* db;
ASSERT_TRUE(OpenDB(testdbname, &db).ok());
// 插入索引字段
db->CreateIndexOnField("address");
// 延迟测试
vector<int64_t> latencies; // 存储每次插入的延迟
int num_inserts = 1000; // 插入的条数
// 测试插入吞吐量
auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间
for (int i = 0; i < num_inserts; ++i) {
std::string key = "k_" + std::to_string(i);
// 随机生成字段内容
FieldArray fields = {
{"name", "Customer#" + std::to_string(i)},
{"address", GenerateRandomString(10)}, // 随机生成地址
{"phone", GenerateRandomString(12)} // 随机生成电话号码
};
db->Put_fields(WriteOptions(), key, fields);
}
// 延迟测试
TestLatency(num_inserts, latencies);
auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();
// 计算吞吐量并输出
cout << "Throughput: " << num_inserts * 1000 / duration << " OPS" << endl;
// 输出延迟
// for (size_t i = 0; i < latencies.size(); ++i) {
// cout << "Latency for operation " << i + 1 << ": " << latencies[i] << " ms" << endl;
// }
cout << "Latency for operation " << latencies.size()-1 << ": " << latencies[latencies.size()-1] << " ms" << endl;
// 清理数据库
delete db;
}
// 主函数运行所有测试
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

撰写 预览
正在加载...
取消
保存