augurier
9 months ago
8 changed files with 496 additions and 181 deletions
Split View
Diff Options
-
+11 -2CMakeLists.txt
-
+32 -46fielddb/field_db.cpp
-
+13 -5fielddb/field_db.h
-
+80 -24fielddb/request.cpp
-
+4 -2fielddb/request.h
-
+11 -102test/basic_function_test.cc
-
+182 -0test/helper.cc
-
+163 -0test/parallel_test.cc
+ 11
- 2
CMakeLists.txt
View File
+ 32
- 46
fielddb/field_db.cpp
View File
+ 13
- 5
fielddb/field_db.h
View File
+ 80
- 24
fielddb/request.cpp
View File
+ 4
- 2
fielddb/request.h
View File
+ 11
- 102
test/basic_function_test.cc
View File
+ 182
- 0
test/helper.cc
View File
| @ -0,0 +1,182 @@ | |||
| #include "gtest/gtest.h" | |||
| // #include "leveldb/env.h" | |||
| // #include "leveldb/db.h" | |||
| #include "fielddb/field_db.h" | |||
| #include <random> | |||
| using namespace fielddb; | |||
| constexpr int value_size = 2048; | |||
| constexpr int data_size = 128 << 20; | |||
| #define AGE_RANGE 100 | |||
| std::vector<std::string> cities = { | |||
| "Beijing", "Shanghai", "Guangzhou", "Shenzhen", "Hangzhou", | |||
| "Chengdu", "Chongqing", "Wuhan", "Suzhou", "Tianjin" | |||
| }; | |||
| //检查insert和queryByIndex的数据是否对应 | |||
| std::set<std::string> shanghaiKeys; | |||
| std::set<std::string> age20Keys; | |||
| //复杂的测试要注意这两个全局变量，目前只有InsertFieldData和InsertOneField会往里加，并且没有清理 | |||
| Status OpenDB(std::string dbName, FieldDB **db) { | |||
| Options options; | |||
| options.create_if_missing = true; | |||
| return FieldDB::OpenFieldDB(options, dbName, db); | |||
| } | |||
| // void ClearDB(FieldDB *db){ | |||
| // //destroy和恢复没做前先用这个清理数据库，否则跑不同的数据多做几次测试会污染 | |||
| // WriteOptions writeOptions; | |||
| // int key_num = data_size / value_size; | |||
| // for (int i = 0; i < key_num; i++) { | |||
| // int key_ = i+1; | |||
| // std::string key = std::to_string(key_); | |||
| // Status s = db->Delete(WriteOptions(), key); | |||
| // ASSERT_TRUE(s.ok()); | |||
| // } | |||
| // } | |||
| //只插一条特定数据的测试 | |||
| void InsertOneField(FieldDB *db, std::string key = "0") { | |||
| WriteOptions writeOptions; | |||
| FieldArray fields = { | |||
| {"name", "special#" + key}, | |||
| {"address", "Shanghai"}, | |||
| {"age", "20"} | |||
| }; | |||
| Status s = db->PutFields(WriteOptions(), key, fields); | |||
| ASSERT_TRUE(s.ok()); | |||
| shanghaiKeys.insert(key); | |||
| age20Keys.insert(key); | |||
| } | |||
| //与上面对应 | |||
| void GetOneField(FieldDB *db, std::string key = "0") { | |||
| ReadOptions readOptions; | |||
| FieldArray fields_ret; | |||
| Status s = db->GetFields(readOptions, key, &fields_ret); | |||
| ASSERT_TRUE(s.ok()); | |||
| for (const Field& pairs : fields_ret) { | |||
| if (pairs.first == "name"){ | |||
| ASSERT_EQ(pairs.second, "special#" + key); | |||
| } else if (pairs.first == "address"){ | |||
| ASSERT_EQ(pairs.second, "Shanghai"); | |||
| } else if (pairs.first == "age"){ | |||
| ASSERT_EQ(pairs.second, "20"); | |||
| } else assert(false); | |||
| } | |||
| } | |||
| void InsertFieldData(FieldDB *db, int seed = 0/*随机种子*/) { | |||
| std::cout << "-------inserting-------" << std::endl; | |||
| WriteOptions writeOptions; | |||
| int key_num = data_size / value_size; | |||
| // srand线程不安全，这种可以保证多线程时随机序列也一致 | |||
| std::mt19937 rng(seed); | |||
| for (int i = 0; i < key_num; i++) { | |||
| int randThisTime = rng(); //确保读写一个循环只rand一次，否则随机序列会不一致 | |||
| //让批量写入的key>0, 单独写入的key<=0,方便测试观察 | |||
| int key_ = std::abs(randThisTime) % key_num + 1; | |||
| std::string key = std::to_string(key_); | |||
| std::string name = "customer#" + std::to_string(key_); | |||
| std::string address = cities[randThisTime % cities.size()]; | |||
| std::string age = std::to_string(std::abs(randThisTime) % AGE_RANGE); | |||
| FieldArray fields = { | |||
| {"name", name}, | |||
| {"address", address}, | |||
| {"age", age} | |||
| }; | |||
| if (address == "Shanghai") { | |||
| shanghaiKeys.insert(key); | |||
| } | |||
| if (age == "20") { | |||
| age20Keys.insert(key); | |||
| } | |||
| Status s = db->PutFields(WriteOptions(), key, fields); | |||
| ASSERT_TRUE(s.ok()); | |||
| } | |||
| } | |||
| //并发时不一定能读到，加个参数控制 | |||
| void GetFieldData(FieldDB *db, bool allowNotFound, int seed = 0) { | |||
| std::cout << "-------getting-------" << std::endl; | |||
| ReadOptions readOptions; | |||
| int key_num = data_size / value_size; | |||
| // 点查 | |||
| std::mt19937 rng(seed); | |||
| for (int i = 0; i < 100; i++) { | |||
| int randThisTime = rng(); | |||
| int key_ = std::abs(randThisTime) % key_num + 1; | |||
| std::string key = std::to_string(key_); | |||
| FieldArray fields_ret; | |||
| Status s = db->GetFields(readOptions, key, &fields_ret); | |||
| if (!allowNotFound){ //必须读到 | |||
| // if (!s.ok()){ | |||
| // std::cout << key << std::endl; | |||
| // } | |||
| ASSERT_TRUE(s.ok()); | |||
| } else { //不必须读到，但只要读到address必须正确 | |||
| if(s.IsNotFound()) continue; | |||
| } | |||
| for (const Field& pairs : fields_ret) { | |||
| if (pairs.first == "name"){ | |||
| } else if (pairs.first == "address"){ | |||
| std::string city = pairs.second; | |||
| ASSERT_NE(std::find(cities.begin(), cities.end(), city), cities.end()); | |||
| } else if (pairs.first == "age"){ | |||
| int age = std::stoi(pairs.second); | |||
| ASSERT_TRUE(age >= 0 && age < AGE_RANGE); | |||
| } else assert(false); | |||
| } | |||
| } | |||
| } | |||
| void findKeysByCity(FieldDB *db) { | |||
| std::cout << "-------getting field address-------" << std::endl; | |||
| Field field = {"address", "Shanghai"}; | |||
| std::vector<std::string> resKeys = db->FindKeysByField(field); | |||
| //打印比较，因为shanghaikey可能被后写入的、其他address的key覆盖，打印出的后一个数应该小于前一个数 | |||
| //如果随机种子相同，每次打印出的两个数也应该相同 | |||
| std::cout << "address: " << shanghaiKeys.size() << " " << resKeys.size() << std::endl; | |||
| for (const std::string &key : resKeys){ | |||
| ASSERT_NE(std::find(shanghaiKeys.begin(), shanghaiKeys.end(), key), shanghaiKeys.end()); | |||
| } | |||
| } | |||
| // haveIndex表明数据库有没有该索引（address） | |||
| void findKeysByCityIndex(FieldDB *db, bool haveIndex) { | |||
| std::cout << "-------getting field address by index-------" << std::endl; | |||
| Field field = {"address", "Shanghai"}; | |||
| Status s; | |||
| std::vector<std::string> resKeys = db->QueryByIndex(field, &s); | |||
| if (haveIndex) ASSERT_TRUE(s.ok()); | |||
| else { | |||
| ASSERT_TRUE(s.IsNotFound()); | |||
| return; | |||
| } | |||
| std::cout << "address: " << shanghaiKeys.size() << " " << resKeys.size() << std::endl;//打印比较 | |||
| for (const std::string &key : resKeys){ | |||
| ASSERT_NE(std::find(shanghaiKeys.begin(), shanghaiKeys.end(), key), shanghaiKeys.end()); | |||
| } | |||
| } | |||
| void findKeysByAgeIndex(FieldDB *db, bool haveIndex) { | |||
| std::cout << "-------getting field age by index-------" << std::endl; | |||
| Field field = {"age", "20"}; | |||
| Status s; | |||
| std::vector<std::string> resKeys = db->QueryByIndex(field, &s); | |||
| if (haveIndex) ASSERT_TRUE(s.ok()); | |||
| else { | |||
| ASSERT_TRUE(s.IsNotFound()); | |||
| return; | |||
| } | |||
| std::cout << "age: " << age20Keys.size() << " " << resKeys.size() << std::endl; | |||
| for (const std::string &key : resKeys){ | |||
| ASSERT_NE(std::find(age20Keys.begin(), age20Keys.end(), key), age20Keys.end()); | |||
| } | |||
| } | |||
+ 163
- 0
test/parallel_test.cc
View File
| @ -0,0 +1,163 @@ | |||
| #include "gtest/gtest.h" | |||
| #include <thread> | |||
| // #include "leveldb/env.h" | |||
| // #include "leveldb/db.h" | |||
| #include "fielddb/field_db.h" | |||
| #include "test/helper.cc" | |||
| using namespace fielddb; | |||
| // 测试中read/write都表示带索引的读写 | |||
| //读写有索引数据的并发 | |||
| TEST(TestReadWrite, Parallel) { | |||
| fielddb::DestroyDB("testdb2.1",Options()); | |||
| FieldDB *db = new FieldDB(); | |||
| if(OpenDB("testdb2.1", &db).ok() == false) { | |||
| std::cerr << "open db failed" << std::endl; | |||
| abort(); | |||
| } | |||
| // ClearDB(db); | |||
| int thread_num_ = 5; | |||
| std::vector<std::thread> threads(thread_num_); | |||
| //二写三读 | |||
| for (size_t i = 0; i < thread_num_; i++) | |||
| { | |||
| if (i == 0) {//写随机序列0 | |||
| threads[i] = std::thread(InsertFieldData, db, 0); | |||
| } else if (i == 1) | |||
| {//写随机序列1 | |||
| threads[i] = std::thread(InsertFieldData, db, 1); | |||
| } else {//读 | |||
| bool allowNotFound = true; | |||
| threads[i] = std::thread(GetFieldData, db, allowNotFound, 0); | |||
| } | |||
| } | |||
| for (auto& t : threads) { | |||
| if (t.joinable()) { | |||
| t.join(); | |||
| } | |||
| } | |||
| // 此时写已完成，一定能读到两次写 | |||
| bool allowNotFound = false; | |||
| GetFieldData(db, allowNotFound); | |||
| GetFieldData(db, allowNotFound, 1); | |||
| findKeysByCity(db); | |||
| delete db; | |||
| } | |||
| //创建索引与写有该索引数据的并发 | |||
| TEST(TestWriteCreatei, Parallel) { | |||
| fielddb::DestroyDB("testdb2.2",Options()); | |||
| FieldDB *db = new FieldDB(); | |||
| if(OpenDB("testdb2.2", &db).ok() == false) { | |||
| std::cerr << "open db failed" << std::endl; | |||
| abort(); | |||
| } | |||
| // ClearDB(db); | |||
| shanghaiKeys.clear(); | |||
| InsertFieldData(db); | |||
| int thread_num_ = 2; | |||
| std::vector<std::thread> threads(thread_num_); | |||
| for (size_t i = 0; i < thread_num_; i++) | |||
| { | |||
| if (i == 0) {//创建索引 | |||
| threads[i] = std::thread([db](){ | |||
| db->CreateIndexOnField("address"); | |||
| std::cout << "finish create index\n"; | |||
| }); | |||
| } else {//写 | |||
| threads[i] = std::thread([db](){ | |||
| while (db->GetIndexStatus("address") == NotExist){ | |||
| continue; //开始创建了再并发的写 | |||
| } | |||
| InsertOneField(db); //先插一条 | |||
| }); | |||
| } | |||
| } | |||
| for (auto& t : threads) { | |||
| if (t.joinable()) { | |||
| t.join(); | |||
| } | |||
| } | |||
| //检查索引是否创建成功 | |||
| bool haveIndex = true; | |||
| findKeysByCityIndex(db, haveIndex); | |||
| //检查写入是否成功 | |||
| GetOneField(db); | |||
| delete db; | |||
| } | |||
| //创建删除不同索引的并发 | |||
| TEST(TestCreateiCreatei, Parallel) { | |||
| fielddb::DestroyDB("testdb2.3",Options()); | |||
| FieldDB *db = new FieldDB(); | |||
| if(OpenDB("testdb2.3", &db).ok() == false) { | |||
| std::cerr << "open db failed" << std::endl; | |||
| abort(); | |||
| } | |||
| // ClearDB(db); | |||
| shanghaiKeys.clear(); | |||
| age20Keys.clear(); | |||
| InsertFieldData(db); | |||
| int thread_num_ = 3; | |||
| std::vector<std::thread> threads(thread_num_); | |||
| for (size_t i = 0; i < thread_num_; i++) | |||
| { | |||
| //3线程并发创建索引address | |||
| threads[i] = std::thread([db](){ | |||
| db->CreateIndexOnField("address"); | |||
| std::cout << "finish create index address\n"; | |||
| }); | |||
| } | |||
| for (auto& t : threads) { | |||
| if (t.joinable()) { | |||
| t.join(); | |||
| } | |||
| } | |||
| //检查索引是否创建成功 | |||
| bool haveIndex = true; | |||
| findKeysByCityIndex(db, haveIndex); | |||
| findKeysByAgeIndex(db, false); | |||
| for (size_t i = 0; i < thread_num_; i++) | |||
| { | |||
| if (i == 0 || i == 1) {//2线程删除索引address | |||
| threads[i] = std::thread([db](){ | |||
| db->DeleteIndex("address"); | |||
| std::cout << "finish delete index address\n"; | |||
| }); | |||
| } else {//1线程创建索引age | |||
| threads[i] = std::thread([db](){ | |||
| db->CreateIndexOnField("age"); | |||
| std::cout << "finish create index age\n"; | |||
| }); | |||
| } | |||
| } | |||
| for (auto& t : threads) { | |||
| if (t.joinable()) { | |||
| t.join(); | |||
| } | |||
| } | |||
| //检查 | |||
| findKeysByCityIndex(db, false); | |||
| findKeysByAgeIndex(db, true); | |||
| delete db; | |||
| } | |||
| int main(int argc, char** argv) { | |||
| // All tests currently run with the same read-only file limits. | |||
| testing::InitGoogleTest(&argc, argv); | |||
| return RUN_ALL_TESTS(); | |||
| } | |||