From 154154357fe9c0a2c92cb10ec7e3767c47e938aa Mon Sep 17 00:00:00 2001 From: wangxuefei <10225501435@stu.ecnu.edu.cn> Date: Sat, 4 Jan 2025 22:28:04 +0800 Subject: [PATCH] fix small BUG and modify report.md --- report.md | 107 ++++++++++++++++++++++++++++++------------------------- test/db_test3.cc | 29 +++++++-------- test/db_test4.cc | 4 ++- test/db_test5.cc | 1 + 4 files changed, 75 insertions(+), 66 deletions(-) diff --git a/report.md b/report.md index 85c25df..1d139e7 100644 --- a/report.md +++ b/report.md @@ -327,34 +327,41 @@ return DB::Delete(options, key); 但是这又带来了一个问题，我们该如何管理slot_page这个文件？当插入新的kv时，我们需要在这个slot_page中分配新的slot，在GC删除某个kv时，我们需要将对应的slot进行释放。这里我们选择在内存中维护一个可线性扩展的bitmap。这个bitmap中每一个bit标识了当前slot_page文件中对应slot是否被使用，是为1，不是为0。这样一来，在插入新kv时，我们可以用bitmap来分配一个新的slot（将bitmap中第一个为0的bit设置为1），将内容进行写入；在GC删除某个kv时，我们将这个slot对应的bitmap中的bit重置为0即可。 -### 5. 功能测试 -### 5.1 在 LevelDB 的 value 中实现字段功能 -#### 5.1.1 功能测试 -1. 能否以字段形式插入并读取数据 -2. 能否以通过字段值查询对应的 key +### 3. 功能测试 +#### 3.1 在 LevelDB 的 value 中实现字段功能 +1. 以字段形式插入,读取数据 +2. 根据 key 删除数据 +3. 通过字段值查询对应的 key + +**测试流程：** + +1. 写入4条数据； +2. 读取这四条数据； +3. 检查读取的数据跟写入的是否一致； +4. 通过字段值查询对应的 key； +5. 删除查找到的 keys 中的第一个key; +6. 通过4中的字段值查询对应的 key，查找到的数目比4中少一个。 + +**测试代码：** ```` -Status OpenDB(std::string dbName, DB **db) { - Options options; - options.create_if_missing = true; - return DB::Open(options, dbName, db); -} - TEST(TestSchema, Basic) { - DB *db; + DB* db; WriteOptions writeOptions; ReadOptions readOptions; - if(OpenDB("testdb", &db).ok() == false) { + if (!OpenDB("testdb_function", &db).ok()) { std::cerr << "open db failed" << std::endl; abort(); } + std::string key0 = "k_0"; std::string key1 = "k_1"; std::string key2 = "k_2"; + std::string key3 = "k_3"; + FieldArray fields0 = {{"name", "myc&wxf"}}; FieldArray fields1 = { {"name", "Customer1"}, {"address", "IVhzIApeRb"}, {"phone", "25-989-741-2988"} }; - FieldArray fields2 = { {"name", "Customer1"}, {"address", "ecnu"}, @@ -363,59 +370,61 @@ TEST(TestSchema, Basic) { FieldArray fields3 = { {"name", "Customer2"}, {"address", "ecnu"}, - {"phone", "111111111"} + {"phone", "11111"} }; - // 序列化并插入 - std::string value1 = SerializeValue(fields1); - std::string value2 = SerializeValue(fields2); - std::string value3 = SerializeValue(fields3); - db->Put(leveldb::WriteOptions(), key1, value1); - db->Put(leveldb::WriteOptions(), key2, value2); - db->Put(leveldb::WriteOptions(), key2, value3); - - // 读取并反序列化 - std::string value_ret; - db->Get(leveldb::ReadOptions(), key1, &value_ret); - auto fields_ret = ParseValue(value_ret); - + db->Put_Fields(leveldb::WriteOptions(), key0, fields0); + db->Put_Fields(leveldb::WriteOptions(), key1, fields1); + db->Put_Fields(leveldb::WriteOptions(), key2, fields2); + db->Put_Fields(leveldb::WriteOptions(), key3, fields3); + FieldArray fields_ret_0; + FieldArray fields_ret_1; + FieldArray fields_ret_2; + FieldArray fields_ret_3; + db->Get_Fields(leveldb::ReadOptions(), key0, fields_ret_0); + db->Get_Fields(leveldb::ReadOptions(), key1, fields_ret_1); + db->Get_Fields(leveldb::ReadOptions(), key2, fields_ret_2); + db->Get_Fields(leveldb::ReadOptions(), key3, fields_ret_3); // 检查反序列化结果 - ASSERT_EQ(fields_ret.size(), fields1.size()); - for (size_t i = 0; i < fields_ret.size(); ++i) { - ASSERT_EQ(fields_ret[i].first, fields1[i].first); - ASSERT_EQ(fields_ret[i].second, fields1[i].second); + ASSERT_EQ(fields_ret_0.size(), fields0.size()); + for (size_t i = 0; i < fields_ret_0.size(); ++i) { + ASSERT_EQ(fields_ret_0[i].name, fields0[i].name); + ASSERT_EQ(fields_ret_0[i].value, fields0[i].value); + } + ASSERT_EQ(fields_ret_1.size(), fields1.size()); + for (size_t i = 0; i < fields_ret_1.size(); ++i) { + ASSERT_EQ(fields_ret_1[i].name, fields1[i].name); +// ASSERT_EQ(fields_ret_1[i].value, fields1[i].value); } - // 测试查找功能 - Field query_field = {"name", "Customer2"}; + Field query_field = {"name", "Customer1"}; std::vector found_keys = FindKeysByField(db, query_field); - std::cout << "找到的key有：" << found_keys.size() << "个" << std::endl; - + // 删除查找到的第一个 key + const std::string& key = found_keys[0]; + db->Delete(leveldb::WriteOptions(), key); + // 再次查找 + std::vector found_deleted_keys = FindKeysByField(db, query_field); // 关闭数据库 delete db; } - -int main(int argc, char **argv) { +int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } ```` -#### 5.1.2 测试结果 -插入三条数据，name 字段分别为： Customer1, Customer1, Customer2 +**测试结果：** -先根据 "name":"customer1"查找，结果为： -![图片](./pic/test_field_1.png) -在根据"name":"customer2"查找，结果为： -![图片](./pic/test_field_2.png) -### 5.2 +#### 3.2 测试并发插入和读取数据 +#### 3.3 测试 GC +#### 3.4 测试 单元测试: 1. 测试插入超过初始slot_page等slot数量之后，是否还能正常插入，检查slot_page文件等线性可扩展性 2. 测试插入后，进行删除，等待GC完成后再读取value和vlog的大小，看看GC过程是否正常进行。 -性能测试: -1. 测试插入的吞吐 -2. 测试在只有删除的情况下，GC的效率 -3. 测试在插入和删除不同比重的负载下，系统的吞吐情况 +### 4. 性能测试: +#### 4.1 测试吞吐量 +#### 4.2 测试延迟 +#### 4.3 测试写放大 吞吐率下降很多 写放大下降很多 diff --git a/test/db_test3.cc b/test/db_test3.cc index 1cdafe4..6de0644 100644 --- a/test/db_test3.cc +++ b/test/db_test3.cc @@ -30,6 +30,9 @@ std::vector FindKeysByField(leveldb::DB* db, const Field& field) { } } std::cout << "Found " << keys.size() << " keys" << std::endl; + for (const auto& key : keys) { + std::cout << "Found key: " << key << std::endl; + } delete it; return keys; } @@ -44,23 +47,20 @@ TEST(TestSchema, Basic) { DB* db; WriteOptions writeOptions; ReadOptions readOptions; - if (!OpenDB("testdb", &db).ok()) { + if (!OpenDB("testdb_function", &db).ok()) { std::cerr << "open db failed" << std::endl; abort(); } - // std::string key = "key"; std::string key0 = "k_0"; std::string key1 = "k_1"; std::string key2 = "k_2"; std::string key3 = "k_3"; - // std::string value = "value"; FieldArray fields0 = {{"name", "myc&wxf"}}; FieldArray fields1 = { {"name", "Customer1"}, {"address", "IVhzIApeRb"}, {"phone", "25-989-741-2988"} }; - FieldArray fields2 = { {"name", "Customer1"}, {"address", "ecnu"}, @@ -71,43 +71,40 @@ TEST(TestSchema, Basic) { {"address", "ecnu"}, {"phone", "11111"} }; - // db->Put(writeOptions, key, value); - // std::cout << "put_value: " << value << std::endl; db->Put_Fields(leveldb::WriteOptions(), key0, fields0); db->Put_Fields(leveldb::WriteOptions(), key1, fields1); db->Put_Fields(leveldb::WriteOptions(), key2, fields2); db->Put_Fields(leveldb::WriteOptions(), key3, fields3); - // std::string value_ret; - // db->Get(readOptions, key, &value_ret); - // std::cout << "get_value: " << value_ret << std::endl; - // 读取并反序列化 FieldArray fields_ret_0; FieldArray fields_ret_1; + FieldArray fields_ret_2; + FieldArray fields_ret_3; db->Get_Fields(leveldb::ReadOptions(), key0, fields_ret_0); db->Get_Fields(leveldb::ReadOptions(), key1, fields_ret_1); - - + db->Get_Fields(leveldb::ReadOptions(), key2, fields_ret_2); + db->Get_Fields(leveldb::ReadOptions(), key3, fields_ret_3); // 检查反序列化结果 ASSERT_EQ(fields_ret_0.size(), fields0.size()); for (size_t i = 0; i < fields_ret_0.size(); ++i) { ASSERT_EQ(fields_ret_0[i].name, fields0[i].name); ASSERT_EQ(fields_ret_0[i].value, fields0[i].value); } - ASSERT_EQ(fields_ret_1.size(), fields1.size()); for (size_t i = 0; i < fields_ret_1.size(); ++i) { ASSERT_EQ(fields_ret_1[i].name, fields1[i].name); // ASSERT_EQ(fields_ret_1[i].value, fields1[i].value); } - // 测试查找功能 Field query_field = {"name", "Customer1"}; std::vector found_keys = FindKeysByField(db, query_field); - std::cout << "找到的key有：" << found_keys.size() << "个" << std::endl; + // 删除查找到的第一个 key + const std::string& key = found_keys[0]; + db->Delete(leveldb::WriteOptions(), key); + // 再次查找 + std::vector found_deleted_keys = FindKeysByField(db, query_field); // 关闭数据库 delete db; } - int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/test/db_test4.cc b/test/db_test4.cc index fca9c46..b08ecc9 100644 --- a/test/db_test4.cc +++ b/test/db_test4.cc @@ -2,9 +2,11 @@ // Created by 马也驰 on 2024/12/20. // -#include "../db/slotpage.h" +#include #include +#include "../db/slotpage.h" + #define SLOTNUM 8192 void printVector(const std::vector& vec) { diff --git a/test/db_test5.cc b/test/db_test5.cc index 9d7154f..c5a962a 100644 --- a/test/db_test5.cc +++ b/test/db_test5.cc @@ -2,6 +2,7 @@ // Created by 马也驰 on 2024/12/22. // +#include #include #include #include