From 8f88688b490209ee91b28e55f0d06cf6b5ea5862 Mon Sep 17 00:00:00 2001 From: alexfisher <1823748191@qq.com> Date: Thu, 5 Dec 2024 02:18:30 +0800 Subject: [PATCH] update version_1 --- db/db_impl.cc | 2 +- test/benchmark_4leveldb.cpp | 287 ++++++++++++++++++++++++++------------------ 2 files changed, 174 insertions(+), 115 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 56d51ec..9f51f7b 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1234,7 +1234,7 @@ Status DBImpl::Get(const ReadOptions& options, const Slice& key, mem->Unref(); if (imm != nullptr) imm->Unref(); current->Unref(); - + if(!s.ok())return s; if(value->c_str()[0]==0x00){ *value=value->substr(1); return s; diff --git a/test/benchmark_4leveldb.cpp b/test/benchmark_4leveldb.cpp index 7573a04..26a62e0 100644 --- a/test/benchmark_4leveldb.cpp +++ b/test/benchmark_4leveldb.cpp @@ -1,141 +1,200 @@ -#include -#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include "leveldb/db.h" - -// 配置 -const int TEST_EXPONENT = 5; -const int TEST_FREQUENCY = static_cast(std::pow(10, TEST_EXPONENT)); -const int MIN_STR_LEN = 255; -const int MAX_STR_LEN = 1024; -const std::string DB_PATH = "db_benchmark"; -const std::string CHARSET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - -// 多语言 -const std::string BASE_VALUE = "こんにちは世界！Hello World! Привет, мир! ¡Hola Mundo! 你好，世界！Bonjour le monde! Hallo Welt!"; -// 莎士比亚 -// const std::string BASE_VALUE = "To be, or not to be, that is the question: Whether 'tis nobler in the mind to suffer the slings and arrows of outrageous fortune, or to take arms against a sea of troubles and by opposing end them."; -// 超长字符 -// const std::string BASE_VALUE = []() { -// std::string base = "壹贰叁肆伍陆柒捌玖拾"; -// std::string long_text; -// for (int i = 0; i < 100; ++i) { // 重复 100 次 -// long_text += base; -// } -// return long_text; -// }(); - - -// 随机字符串生成 -std::string randomStr() { - int len = rand() % (MAX_STR_LEN - MIN_STR_LEN + 1) + MIN_STR_LEN; - std::string str(len, '\0'); - for (int i = 0; i < len; ++i) { - str[i] = CHARSET[rand() % CHARSET.size()]; +#include "leveldb/write_batch.h" +#include "leveldb/iterator.h" +#include // For stat to get file size on Unix-like systems +#include // For directory reading on Unix-like systems + +#define THREAD_COUNT 16 // 线程数量 +#define PUT_THREAD_COUNT (THREAD_COUNT / 3) // Put线程数量 +#define DELETE_THREAD_COUNT (THREAD_COUNT / 3) // Delete线程数量 +#define ITERATE_THREAD_COUNT (THREAD_COUNT - PUT_THREAD_COUNT - DELETE_THREAD_COUNT) // Iterate线程数量 +#define VALUE_SIZE 1000 // Value的默认大小 +#define DATABASE_PATH "db_benchmark" // 数据库路径 + +std::mutex put_mutex; +std::mutex delete_mutex; +std::mutex iterate_mutex; + +std::pair put_time_count={0,0}; +std::pair delete_time_count={0,0}; +std::pair iterate_time_count={0,0}; + +// Helper function to generate a random string of a given length +std::string GenerateRandomString(size_t length) { + const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + std::default_random_engine rng(std::random_device{}()); + std::uniform_int_distribution dist(0, sizeof(charset) - 2); + + std::string result; + result.reserve(length); + for (size_t i = 0; i < length; ++i) { + result += charset[dist(rng)]; } - return str; + return result; } -// 计算并输出耗时的模板函数 -template -void measureTime(const std::string& operation, Func func) { - auto start = std::chrono::system_clock::now(); - func(); - auto end = std::chrono::system_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - - double seconds = double(duration.count()) * std::chrono::microseconds::period::num / std::chrono::microseconds::period::den; - - // 输出格式化信息 - std::cout << "Operation: " << operation << "\n"; - std::cout << "Number of operations: " << TEST_FREQUENCY << "\n"; - std::cout << "Total time: " - << std::fixed << std::setprecision(6) << seconds << " seconds\n"; - std::cout << "Average time per operation: " - << std::fixed << std::setprecision(6) - << (seconds / TEST_FREQUENCY) * 1e6 << " microseconds\n"; - std::cout << "========================================\n"; +void PutData(leveldb::DB* db, int thread_id, int num_entries, size_t value_size) { + leveldb::WriteOptions write_options; + write_options.sync = false; + + auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 + + for (int i = 0; i < num_entries; ++i) { + std::string key = "key_" + std::to_string(thread_id) + "_" + std::to_string(i); + std::string value = GenerateRandomString(value_size); + + leveldb::WriteBatch batch; + batch.Put(key, value); + db->Write(write_options, &batch); + } + + auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 + auto duration = std::chrono::duration_cast(end_time - start_time).count(); + put_mutex.lock(); + put_time_count.first+=duration; + put_time_count.second+=num_entries; + put_mutex.unlock(); } -int main() { - leveldb::DB* db; - leveldb::Options options; - options.create_if_missing = true; +void DeleteData(leveldb::DB* db, int thread_id, int num_entries) { + leveldb::WriteOptions write_options; + write_options.sync = false; - // 打开数据库 - leveldb::Status status = leveldb::DB::Open(options, DB_PATH, &db); - assert(status.ok()); - std::cout << "db open: " << DB_PATH << std::endl; + auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 + + for (int i = 0; i < num_entries; ++i) { + std::string key = "key_" + std::to_string(thread_id) + "_" + std::to_string(i); + + leveldb::WriteBatch batch; + batch.Delete(key); + db->Write(write_options, &batch); + } + auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 + auto duration = std::chrono::duration_cast(end_time - start_time).count(); + delete_mutex.lock(); + delete_time_count.first+=duration; + delete_time_count.second+=num_entries; + delete_mutex.unlock(); +} - srand(2017); +void IterateData(leveldb::DB* db, leveldb::ReadOptions& read_options) { + std::unique_ptr it(db->NewIterator(read_options)); - // 生成测试数据 - std::string keys[TEST_FREQUENCY]; - for (int i = 0; i < TEST_FREQUENCY; ++i) { - keys[i] = randomStr(); + auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 + + for (it->SeekToFirst(); it->Valid(); it->Next()) { + // 这里可以选择是否打印键值对，或者仅遍历不做任何操作 + std::cout << "Key: " << it->key().ToString() << ", Value: " << it->value().ToString() << "\n"; } - std::string value = BASE_VALUE; - for (int i = 0; i < 4; ++i) { - value += value; // 扩展 base value + + if (!it->status().ok()) { + std::cerr << "Error during iteration: " << it->status().ToString() << "\n"; } - // 测试添加 - measureTime("ADD", [&]() { - for (int i = 0; i < TEST_FREQUENCY; ++i) { - status = db->Put(leveldb::WriteOptions(), keys[i], value); - assert(status.ok()); - } - }); - - // 测试获取 - measureTime("GET", [&]() { - std::string retrievedValues[TEST_FREQUENCY]; - for (int i = 0; i < TEST_FREQUENCY; ++i) { - status = db->Get(leveldb::ReadOptions(), keys[i], &retrievedValues[i]); - assert(status.ok()); - assert(retrievedValues[i] == value); // 验证获取结果 - } - }); - - // 测试修改 - measureTime("UPDATE", [&]() { - std::string newValue = value + value; - for (int i = 0; i < TEST_FREQUENCY; ++i) { - status = db->Put(leveldb::WriteOptions(), keys[i], newValue); - assert(status.ok()); - } - }); + auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 + auto duration = std::chrono::duration_cast(end_time - start_time).count(); + iterate_mutex.lock(); + iterate_time_count.first+=duration; + iterate_time_count.second++; + iterate_mutex.unlock(); +} - // 测试删除 - measureTime("DELETE", [&]() { - for (int i = 0; i < TEST_FREQUENCY; ++i) { - status = db->Delete(leveldb::WriteOptions(), keys[i]); - assert(status.ok()); - } - }); +// Function to calculate the total size of all files in the database directory +uint64_t CalculateDatabaseSize(const std::string& db_path) { + uint64_t total_size = 0; + DIR* dir = opendir(db_path.c_str()); + if (dir == nullptr) { + std::cerr << "Failed to open directory: " << db_path << "\n"; + return total_size; + } - if (db) { - delete db; - db = nullptr; + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (entry->d_type == DT_REG) { // Only consider regular files + std::string full_path = db_path + "/" + entry->d_name; + struct stat file_stat; + if (stat(full_path.c_str(), &file_stat) == 0) { + total_size += file_stat.st_size; + } + } } - std::cout << "Test completed, database has been closed." << std::endl; - // Delete database directory + closedir(dir); + return total_size; +} + +void CleanupDatabase(const std::string& db_path) { + /// Delete database directory #ifdef _WIN32 - std::string command = "rd /s /q \"" + DB_PATH + "\""; // Windows delete directory + std::string command = "rd /s /q \"" + db_path + "\""; // Windows delete directory #else - std::string command = "rm -rf \"" + DB_PATH + "\""; // Linux/macOS delete directory + std::string command = "rm -rf \"" + db_path + "\""; // Linux/macOS delete directory #endif - if (std::system(command.c_str()) == 0) { std::cout << "Database directory has been successfully deleted" << std::endl; } else { std::cerr << "Warning: Failed to delete the database directory. Please check manually!" << std::endl; } +} + +int main() { + leveldb::DB* db; + leveldb::Options options; + options.create_if_missing = true; + leveldb::Status status = leveldb::DB::Open(options, DATABASE_PATH, &db); + if (!status.ok()) { + std::cerr << "Unable to open/create database: " << status.ToString() << "\n"; + return 1; + } + + const int entries_per_thread = 1000000; // 每个线程执行的操作次数 + std::vector threads; + + // Create snapshot for iterate threads + leveldb::ReadOptions read_options; + read_options.snapshot = db->GetSnapshot(); + + // Start threads for Put operations + for (int i = 0; i < PUT_THREAD_COUNT; ++i) { + threads.emplace_back(PutData, db, i, entries_per_thread, VALUE_SIZE); + } + + // Start threads for Delete operations + for (int i = 0; i < DELETE_THREAD_COUNT; ++i) { + threads.emplace_back(DeleteData, db, i, entries_per_thread); + } + std::this_thread::sleep_for(std::chrono::seconds(10)); + // Start threads for Iterate operations + for (int i = 0; i < ITERATE_THREAD_COUNT; ++i) { + threads.emplace_back(IterateData, db, std::ref(read_options)); + } + + // Wait for all threads to finish + for (auto& th : threads) { + if (th.joinable()) th.join(); + } + threads.clear(); + + // Release the snapshot after all threads have finished + db->ReleaseSnapshot(read_options.snapshot); + + // Close the database + delete db; + std::cout<<"Put average time(per second):"<