xxy
9 months ago
10 changed files with 308 additions and 174 deletions
Split View
Diff Options
-
+1 -1benchmarks/db_bench.cc
-
+59 -53db/db_impl.cc
-
+4 -2db/db_impl.h
-
+3 -0db/dbformat.h
-
+1 -1include/leveldb/db.h
-
+1 -1include/leveldb/options.h
-
+173 -114test/benchmark_4leveldb.cpp
-
+2 -2test/test.cpp
-
+55 -0util/coding.cc
-
+9 -0util/coding.h
+ 1
- 1
benchmarks/db_bench.cc
View File
+ 59
- 53
db/db_impl.cc
View File
+ 4
- 2
db/db_impl.h
View File
+ 3
- 0
db/dbformat.h
View File
+ 1
- 1
include/leveldb/db.h
View File
+ 1
- 1
include/leveldb/options.h
View File
+ 173
- 114
test/benchmark_4leveldb.cpp
View File
| @ -1,141 +1,200 @@ | |||
| #include <cassert> | |||
| #include <string> | |||
| #include <thread> | |||
| #include <vector> | |||
| #include <iostream> | |||
| #include <iomanip> | |||
| #include <cmath> | |||
| #include <chrono> | |||
| #include <random> | |||
| #include <string> | |||
| #include <mutex> | |||
| #include "leveldb/db.h" | |||
| // 配置 | |||
| const int TEST_EXPONENT = 5; | |||
| const int TEST_FREQUENCY = static_cast<int>(std::pow(10, TEST_EXPONENT)); | |||
| const int MIN_STR_LEN = 255; | |||
| const int MAX_STR_LEN = 1024; | |||
| const std::string DB_PATH = "db_benchmark"; | |||
| const std::string CHARSET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; | |||
| // 多语言 | |||
| const std::string BASE_VALUE = "こんにちは世界！Hello World! Привет, мир! ¡Hola Mundo! 你好，世界！Bonjour le monde! Hallo Welt!"; | |||
| // 莎士比亚 | |||
| // const std::string BASE_VALUE = "To be, or not to be, that is the question: Whether 'tis nobler in the mind to suffer the slings and arrows of outrageous fortune, or to take arms against a sea of troubles and by opposing end them."; | |||
| // 超长字符 | |||
| // const std::string BASE_VALUE = []() { | |||
| // std::string base = "壹贰叁肆伍陆柒捌玖拾"; | |||
| // std::string long_text; | |||
| // for (int i = 0; i < 100; ++i) { // 重复 100 次 | |||
| // long_text += base; | |||
| // } | |||
| // return long_text; | |||
| // }(); | |||
| // 随机字符串生成 | |||
| int len = rand() % (MAX_STR_LEN - MIN_STR_LEN + 1) + MIN_STR_LEN; | |||
| std::string str(len, '\0'); | |||
| for (int i = 0; i < len; ++i) { | |||
| str[i] = CHARSET[rand() % CHARSET.size()]; | |||
| #include "leveldb/write_batch.h" | |||
| #include "leveldb/iterator.h" | |||
| #include <sys/stat.h> // For stat to get file size on Unix-like systems | |||
| #include <dirent.h> // For directory reading on Unix-like systems | |||
| #define THREAD_COUNT 16 // 线程数量 | |||
| #define PUT_THREAD_COUNT (THREAD_COUNT / 3) // Put线程数量 | |||
| #define DELETE_THREAD_COUNT (THREAD_COUNT / 3) // Delete线程数量 | |||
| #define ITERATE_THREAD_COUNT (THREAD_COUNT - PUT_THREAD_COUNT - DELETE_THREAD_COUNT) // Iterate线程数量 | |||
| #define VALUE_SIZE 1000 // Value的默认大小 | |||
| #define DATABASE_PATH "db_benchmark" // 数据库路径 | |||
| std::mutex put_mutex; | |||
| std::mutex delete_mutex; | |||
| std::mutex iterate_mutex; | |||
| std::pair<int,int> put_time_count={0,0}; | |||
| std::pair<int,int> delete_time_count={0,0}; | |||
| std::pair<int,int> iterate_time_count={0,0}; | |||
| // Helper function to generate a random string of a given length | |||
| std::string GenerateRandomString(size_t length) { | |||
| const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; | |||
| std::default_random_engine rng(std::random_device{}()); | |||
| std::uniform_int_distribution<int> dist(0, sizeof(charset) - 2); | |||
| std::string result; | |||
| result.reserve(length); | |||
| for (size_t i = 0; i < length; ++i) { | |||
| result += charset[dist(rng)]; | |||
| } | |||
| return str; | |||
| return result; | |||
| } | |||
| // 计算并输出耗时的模板函数 | |||
| template<typename Func> | |||
| void measureTime(const std::string& operation, Func func) { | |||
| auto start = std::chrono::system_clock::now(); | |||
| func(); | |||
| auto end = std::chrono::system_clock::now(); | |||
| auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start); | |||
| double seconds = double(duration.count()) * std::chrono::microseconds::period::num / std::chrono::microseconds::period::den; | |||
| // 输出格式化信息 | |||
| std::cout << "Operation: " << operation << "\n"; | |||
| std::cout << "Number of operations: " << TEST_FREQUENCY << "\n"; | |||
| std::cout << "Total time: " | |||
| << std::fixed << std::setprecision(6) << seconds << " seconds\n"; | |||
| std::cout << "Average time per operation: " | |||
| << std::fixed << std::setprecision(6) | |||
| << (seconds / TEST_FREQUENCY) * 1e6 << " microseconds\n"; | |||
| std::cout << "========================================\n"; | |||
| void PutData(leveldb::DB* db, int thread_id, int num_entries, size_t value_size) { | |||
| leveldb::WriteOptions write_options; | |||
| write_options.sync = false; | |||
| auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 | |||
| for (int i = 0; i < num_entries; ++i) { | |||
| std::string key = "key_" + std::to_string(thread_id) + "_" + std::to_string(i); | |||
| std::string value = GenerateRandomString(value_size); | |||
| leveldb::WriteBatch batch; | |||
| batch.Put(key, value); | |||
| db->Write(write_options, &batch); | |||
| } | |||
| auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 | |||
| auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count(); | |||
| put_mutex.lock(); | |||
| put_time_count.first+=duration; | |||
| put_time_count.second+=num_entries; | |||
| put_mutex.unlock(); | |||
| } | |||
| int main() { | |||
| leveldb::DB* db; | |||
| leveldb::Options options; | |||
| options.create_if_missing = true; | |||
| void DeleteData(leveldb::DB* db, int thread_id, int num_entries) { | |||
| leveldb::WriteOptions write_options; | |||
| write_options.sync = false; | |||
| // 打开数据库 | |||
| leveldb::Status status = leveldb::DB::Open(options, DB_PATH, &db); | |||
| assert(status.ok()); | |||
| std::cout << "db open: " << DB_PATH << std::endl; | |||
| auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 | |||
| for (int i = 0; i < num_entries; ++i) { | |||
| std::string key = "key_" + std::to_string(thread_id) + "_" + std::to_string(i); | |||
| leveldb::WriteBatch batch; | |||
| batch.Delete(key); | |||
| db->Write(write_options, &batch); | |||
| } | |||
| auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 | |||
| auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count(); | |||
| delete_mutex.lock(); | |||
| delete_time_count.first+=duration; | |||
| delete_time_count.second+=num_entries; | |||
| delete_mutex.unlock(); | |||
| } | |||
| srand(2017); | |||
| void IterateData(leveldb::DB* db, leveldb::ReadOptions& read_options) { | |||
| std::unique_ptr<leveldb::Iterator> it(db->NewIterator(read_options)); | |||
| // 生成测试数据 | |||
| std::string keys[TEST_FREQUENCY]; | |||
| for (int i = 0; i < TEST_FREQUENCY; ++i) { | |||
| keys[i] = randomStr(); | |||
| auto start_time = std::chrono::high_resolution_clock::now(); // 记录开始时间 | |||
| for (it->SeekToFirst(); it->Valid(); it->Next()) { | |||
| // 这里可以选择是否打印键值对，或者仅遍历不做任何操作 | |||
| std::cout << "Key: " << it->key().ToString() << ", Value: " << it->value().ToString() << "\n"; | |||
| } | |||
| std::string value = BASE_VALUE; | |||
| for (int i = 0; i < 4; ++i) { | |||
| value += value; // 扩展 base value | |||
| if (!it->status().ok()) { | |||
| std::cerr << "Error during iteration: " << it->status().ToString() << "\n"; | |||
| } | |||
| // 测试添加 | |||
| measureTime("ADD", [&]() { | |||
| for (int i = 0; i < TEST_FREQUENCY; ++i) { | |||
| status = db->Put(leveldb::WriteOptions(), keys[i], value); | |||
| assert(status.ok()); | |||
| } | |||
| }); | |||
| // 测试获取 | |||
| measureTime("GET", [&]() { | |||
| std::string retrievedValues[TEST_FREQUENCY]; | |||
| for (int i = 0; i < TEST_FREQUENCY; ++i) { | |||
| status = db->Get(leveldb::ReadOptions(), keys[i], &retrievedValues[i]); | |||
| assert(status.ok()); | |||
| assert(retrievedValues[i] == value); // 验证获取结果 | |||
| } | |||
| }); | |||
| // 测试修改 | |||
| measureTime("UPDATE", [&]() { | |||
| std::string newValue = value + value; | |||
| for (int i = 0; i < TEST_FREQUENCY; ++i) { | |||
| status = db->Put(leveldb::WriteOptions(), keys[i], newValue); | |||
| assert(status.ok()); | |||
| } | |||
| }); | |||
| auto end_time = std::chrono::high_resolution_clock::now(); // 记录结束时间 | |||
| auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count(); | |||
| iterate_mutex.lock(); | |||
| iterate_time_count.first+=duration; | |||
| iterate_time_count.second++; | |||
| iterate_mutex.unlock(); | |||
| } | |||
| // 测试删除 | |||
| measureTime("DELETE", [&]() { | |||
| for (int i = 0; i < TEST_FREQUENCY; ++i) { | |||
| status = db->Delete(leveldb::WriteOptions(), keys[i]); | |||
| assert(status.ok()); | |||
| } | |||
| }); | |||
| // Function to calculate the total size of all files in the database directory | |||
| uint64_t CalculateDatabaseSize(const std::string& db_path) { | |||
| uint64_t total_size = 0; | |||
| DIR* dir = opendir(db_path.c_str()); | |||
| if (dir == nullptr) { | |||
| std::cerr << "Failed to open directory: " << db_path << "\n"; | |||
| return total_size; | |||
| } | |||
| if (db) { | |||
| delete db; | |||
| db = nullptr; | |||
| struct dirent* entry; | |||
| while ((entry = readdir(dir)) != nullptr) { | |||
| if (entry->d_type == DT_REG) { // Only consider regular files | |||
| std::string full_path = db_path + "/" + entry->d_name; | |||
| struct stat file_stat; | |||
| if (stat(full_path.c_str(), &file_stat) == 0) { | |||
| total_size += file_stat.st_size; | |||
| } | |||
| } | |||
| } | |||
| std::cout << "Test completed, database has been closed." << std::endl; | |||
| // Delete database directory | |||
| closedir(dir); | |||
| return total_size; | |||
| } | |||
| void CleanupDatabase(const std::string& db_path) { | |||
| /// Delete database directory | |||
| #ifdef _WIN32 | |||
| std::string command = "rd /s /q \"" + DB_PATH + "\""; // Windows delete directory | |||
| std::string command = "rd /s /q \"" + db_path + "\""; // Windows delete directory | |||
| #else | |||
| std::string command = "rm -rf \"" + DB_PATH + "\""; // Linux/macOS delete directory | |||
| std::string command = "rm -rf \"" + db_path + "\""; // Linux/macOS delete directory | |||
| #endif | |||
| if (std::system(command.c_str()) == 0) { | |||
| std::cout << "Database directory has been successfully deleted" << std::endl; | |||
| } else { | |||
| std::cerr << "Warning: Failed to delete the database directory. Please check manually!" << std::endl; | |||
| } | |||
| } | |||
| int main() { | |||
| leveldb::DB* db; | |||
| leveldb::Options options; | |||
| options.create_if_missing = true; | |||
| leveldb::Status status = leveldb::DB::Open(options, DATABASE_PATH, &db); | |||
| if (!status.ok()) { | |||
| std::cerr << "Unable to open/create database: " << status.ToString() << "\n"; | |||
| return 1; | |||
| } | |||
| const int entries_per_thread = 1000000; // 每个线程执行的操作次数 | |||
| std::vector<std::thread> threads; | |||
| // Create snapshot for iterate threads | |||
| leveldb::ReadOptions read_options; | |||
| read_options.snapshot = db->GetSnapshot(); | |||
| // Start threads for Put operations | |||
| for (int i = 0; i < PUT_THREAD_COUNT; ++i) { | |||
| threads.emplace_back(PutData, db, i, entries_per_thread, VALUE_SIZE); | |||
| } | |||
| // Start threads for Delete operations | |||
| for (int i = 0; i < DELETE_THREAD_COUNT; ++i) { | |||
| threads.emplace_back(DeleteData, db, i, entries_per_thread); | |||
| } | |||
| std::this_thread::sleep_for(std::chrono::seconds(10)); | |||
| // Start threads for Iterate operations | |||
| for (int i = 0; i < ITERATE_THREAD_COUNT; ++i) { | |||
| threads.emplace_back(IterateData, db, std::ref(read_options)); | |||
| } | |||
| // Wait for all threads to finish | |||
| for (auto& th : threads) { | |||
| if (th.joinable()) th.join(); | |||
| } | |||
| threads.clear(); | |||
| // Release the snapshot after all threads have finished | |||
| db->ReleaseSnapshot(read_options.snapshot); | |||
| // Close the database | |||
| delete db; | |||
| std::cout<<"Put average time(per second):"<<put_time_count.first<<" "<<put_time_count.second<<std::endl; | |||
| std::cout<<"Delete average time(per second):"<<delete_time_count.first<<" "<<delete_time_count.second<<std::endl; | |||
| std::cout<<"Iterate average time(per second):"<<iterate_time_count.first<<" "<<iterate_time_count.second<<std::endl; | |||
| // Calculate and print the total size of the database files | |||
| uint64_t db_size = CalculateDatabaseSize(DATABASE_PATH); | |||
| std::cout << "Total size of database files: " << db_size << " bytes\n"; | |||
| // Cleanup the database | |||
| CleanupDatabase(DATABASE_PATH); | |||
| return 0; | |||
| } | |||
| } | |||