From d6995373eadc459a536b8aaea0d7ca6f0e371c32 Mon Sep 17 00:00:00 2001 From: augurier <14434658+augurier@user.noreply.gitee.com> Date: Thu, 5 Dec 2024 14:19:56 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=AE=8Cbug=EF=BC=8C=E8=A1=A5?= =?UTF-8?q?=E5=85=85lab1=E6=B5=8B=E8=AF=95=EF=BC=8C=E8=B7=91=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 2 + db/db_impl.cc | 15 +++++-- fielddb/field_db.cpp | 89 +++++++++++++++++++------------------ fielddb/field_db.h | 39 +++++++++------- test/lab1_test.cc | 116 ++++++++++++++++++++++++++++++++++++------------ util/serialize_value.cc | 4 +- util/serialize_value.h | 49 ++++++++++---------- 7 files changed, 197 insertions(+), 117 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 981cda2..31536ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -192,6 +192,8 @@ target_sources(leveldb "util/status.cc" "util/serialize_value.h" "util/serialize_value.cc" + "fielddb/field_db.cpp" + "fielddb/field_db.h" # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install". $<$:PUBLIC> diff --git a/db/db_impl.cc b/db/db_impl.cc index aaff6fb..4a7d25c 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1173,8 +1173,17 @@ Status DBImpl::GetFields(const ReadOptions& options, const Slice& key, return s; } -std::vector DBImpl::FindKeysByField(Field &field){//todo - return std::vector(); +std::vector DBImpl::FindKeysByField(Field &field){ + std::vector result; + auto iter = NewIterator(ReadOptions()); + for(iter->SeekToFirst();iter->Valid();iter->Next()) { + std::string k = iter->key().ToString(); + InternalFieldArray fields(iter->value()); + if(fields.HasField(field)) { + result.push_back(iter->key().ToString()); + } + } + return result; } Iterator* DBImpl::NewIterator(const ReadOptions& options) { @@ -1510,7 +1519,7 @@ Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) { Status DB::PutFields(const WriteOptions& opt, const Slice& key, const FieldArray& fields) { std::string value = SerializeValue(fields); - DB::Put(opt, key, value); + return DB::Put(opt, key, value); } Status DB::Delete(const WriteOptions& opt, const Slice& key) { diff --git a/fielddb/field_db.cpp b/fielddb/field_db.cpp index 5d28813..d867411 100644 --- a/fielddb/field_db.cpp +++ b/fielddb/field_db.cpp @@ -11,40 +11,53 @@ namespace fielddb { using namespace leveldb; //TODO:打开fieldDB -Status FieldDB::OpenFieldDB(const Options& options,const std::string& name,DB** dbptr) { +Status FieldDB::OpenFieldDB(const Options& options, + const std::string& name, FieldDB** dbptr) { // options.env->CreateDir("./abc") - *dbptr = new FieldDB(options,name); - return Status::OK(); -} - -Status FieldDB::Recover() { - -} + if(*dbptr == nullptr){ + return Status::NotSupported(name, "new a fieldDb first\n"); + } -FieldDB::FieldDB(const Options& options,const std::string& name) { Status status; - status = Open(options, name+"_indexDB", &indexDB); - if(!status.ok()) return; - status = Open(options, name+"_kvDB", &kvDB); - if(!status.ok()) return; - status = Open(options, name+"_metaDB", &metaDB); - if(!status.ok()) return; - - Recover(); + DB *indexdb, *kvdb, *metadb; + status = Open(options, name+"_indexDB", &indexdb); + if(!status.ok()) return status; + + status = Open(options, name+"_kvDB", &kvdb); + if(!status.ok()) return status; + status = Open(options, name+"_metaDB", &metadb); + if(!status.ok()) return status; + + (*dbptr)->indexDB_ = indexdb; + (*dbptr)->kvDB_ = kvdb; + (*dbptr)->metaDB_ = metadb; + (*dbptr)->dbname_ = name; + + status = (*dbptr)->Recover(); + return status; } +// todo +Status FieldDB::Recover() { + // + return Status::OK(); +} Status FieldDB::Put(const WriteOptions &options, const Slice &key, const Slice &value) { - return kvDB->Put(options, key, value); + return kvDB_->Put(options, key, value); } // TODO：需要对是否进行index更新做处理 -Status FieldDB::PutFields(const WriteOptions &, const Slice &key, const FieldArray &fields) { - return Status::OK(); +Status FieldDB::PutFields(const WriteOptions &Options, + const Slice &key, const FieldArray &fields) { + // + return kvDB_->PutFields(Options, key, fields); } +// todo: 删除有索引的key时indexdb也要同步 Status FieldDB::Delete(const WriteOptions &options, const Slice &key) { - return kvDB->Delete(options, key); + // + return kvDB_->Delete(options, key); } // TODO：根据updates里面的东西，要对是否需要更新index进行分别处理 Status FieldDB::Write(const WriteOptions &options, WriteBatch *updates) { @@ -52,56 +65,44 @@ Status FieldDB::Write(const WriteOptions &options, WriteBatch *updates) { } Status FieldDB::Get(const ReadOptions &options, const Slice &key, std::string *value) { - return kvDB->Get(options, key, value); + return kvDB_->Get(options, key, value); } Status FieldDB::GetFields(const ReadOptions &options, const Slice &key, FieldArray *fields) { - std::string value; - Status status; - status = kvDB->Get(options, key, &value); - if(status.ok() == false) return status; - fields = ParseValue(value); - return status; + return kvDB_->GetFields(options, key, fields); + } std::vector FieldDB::FindKeysByField(Field &field) { - std::vector result; - auto iter = kvDB->NewIterator(ReadOptions()); - for(iter->SeekToFirst();iter->Valid();iter->Next()) { - InternalFieldArray fields(iter->value()); - if(fields.HasField(field)) { - result.push_back(iter->key().ToString()); - } - } - return result; + return kvDB_->FindKeysByField(field); } Iterator * FieldDB::NewIterator(const ReadOptions &options) { - return kvDB->NewIterator(options); + return kvDB_->NewIterator(options); } // TODO：使用统一seq进行snapshot管理 const Snapshot * FieldDB::GetSnapshot() { - return kvDB->GetSnapshot(); + return kvDB_->GetSnapshot(); } // TODO：同上 void FieldDB::ReleaseSnapshot(const Snapshot *snapshot) { - kvDB->ReleaseSnapshot(snapshot); + kvDB_->ReleaseSnapshot(snapshot); } bool FieldDB::GetProperty(const Slice &property, std::string *value) { - return kvDB->GetProperty(property, value) | indexDB->GetProperty(property, value); + return kvDB_->GetProperty(property, value) | indexDB_->GetProperty(property, value); } void FieldDB::GetApproximateSizes(const Range *range, int n, uint64_t *sizes) { uint64_t temp = 0; - kvDB->GetApproximateSizes(range, n, sizes); - indexDB->GetApproximateSizes(range, n, &temp); + kvDB_->GetApproximateSizes(range, n, sizes); + indexDB_->GetApproximateSizes(range, n, &temp); *sizes += temp; } void FieldDB::CompactRange(const Slice *begin, const Slice *end) { - kvDB->CompactRange(begin, end); + kvDB_->CompactRange(begin, end); } } // end of namespace \ No newline at end of file diff --git a/fielddb/field_db.h b/fielddb/field_db.h index 411f564..6a3c845 100644 --- a/fielddb/field_db.h +++ b/fielddb/field_db.h @@ -1,3 +1,7 @@ +# ifndef FIELD_DB_H +# define FIELD_DB_H + +#include "port/port_stdcxx.h" #include "db/db_impl.h" #include #include @@ -7,16 +11,16 @@ #include "leveldb/options.h" #include "leveldb/slice.h" #include "leveldb/status.h" -#include "port/port_stdcxx.h" + #include "fielddb/request.h" namespace fielddb { using namespace leveldb; -class FieldDB : leveldb::DB { +class FieldDB : DB { public: - FieldDB() = default; - FieldDB(const Options& options,const std::string& name); -/*lab1的要求*/ + //用的时候必须FieldDB *db = new FieldDB()再open，不能像之前一样DB *db + FieldDB() : indexDB_(nullptr), kvDB_(nullptr), metaDB_(nullptr) {}; +/*lab1的要求，作为db派生类要实现的虚函数*/ Status Put(const WriteOptions &options, const Slice &key, const Slice &value) override; Status PutFields(const WriteOptions &, const Slice &key, const FieldArray &fields) override; Status Delete(const WriteOptions &options, const Slice &key) override; @@ -31,29 +35,32 @@ public: void GetApproximateSizes(const Range *range, int n, uint64_t *sizes) override; void CompactRange(const Slice *begin, const Slice *end) override; /*与索引相关*/ - bool CreateIndexOnField(const std::string& field_name); - bool DeleteIndex(std::string &field_name); - std::vector QueryByIndex(Field &field); + Status CreateIndexOnField(const std::string& field_name); + Status DeleteIndex(std::string &field_name); + std::vector QueryByIndex(Field &field, Status *s); - static Status OpenFieldDB(const Options& options,const std::string& name,DB** dbptr); + static Status OpenFieldDB(const Options& options,const std::string& name,FieldDB** dbptr); private: //根据metaDB的内容进行恢复 Status Recover(); private: - leveldb::DB *metaDB; - leveldb::DB *indexDB; - leveldb::DB *kvDB; + std::string dbname_; + + leveldb::DB *metaDB_; + leveldb::DB *indexDB_; + leveldb::DB *kvDB_; enum IndexStatus{ Creating, Deleting, Exist }; - std::map index; - port::Mutex _mutex; // mutex for taskqueue - std::deque taskqueue; + std::map index_; + leveldb::port::Mutex mutex_; // mutex for taskqueue + std::deque taskqueue_; }; -} // end of namespace \ No newline at end of file +} // end of namespace +# endif \ No newline at end of file diff --git a/test/lab1_test.cc b/test/lab1_test.cc index 94b0ee4..7c9a50d 100644 --- a/test/lab1_test.cc +++ b/test/lab1_test.cc @@ -1,46 +1,104 @@ #include "gtest/gtest.h" -#include "leveldb/env.h" -#include "leveldb/db.h" -using namespace leveldb; -using Field = std::pair; // field_name:field_value -using FieldArray = std::vector>; +// #include "leveldb/env.h" +// #include "leveldb/db.h" +#include "fielddb/field_db.h" +using namespace fielddb; -Status OpenDB(std::string dbName, DB **db) { +constexpr int value_size = 2048; +constexpr int data_size = 128 << 20; +std::vector cities = { + "Beijing", "Shanghai", "Guangzhou", "Shenzhen", "Hangzhou", + "Chengdu", "Chongqing", "Wuhan", "Suzhou", "Tianjin" + }; +std::vector shanghaiKeys; + +Status OpenDB(std::string dbName, FieldDB **db) { Options options; options.create_if_missing = true; - return DB::Open(options, dbName, db); + return FieldDB::OpenFieldDB(options, dbName, db); } -TEST(TestLab1, Basic) { - DestroyDB("testdb",Options()); - DB *db; +void ClearDB(FieldDB *db){ + //destroy和恢复没做前先用这个清理数据库，否则跑不同的数据多做几次测试会污染 + WriteOptions writeOptions; + int key_num = data_size / value_size; + for (int i = 0; i < key_num; i++) { + int key_ = i+1; + std::string key = std::to_string(key_); + Status s = db->Delete(WriteOptions(), key); + ASSERT_TRUE(s.ok()); + } +} - if(OpenDB("testdb", &db).ok() == false) { - std::cerr << "open db failed" << std::endl; - abort(); - } - std::string key = "k_1"; - +void InsertFieldData(FieldDB *db) { + WriteOptions writeOptions; + int key_num = data_size / value_size; + srand(0); + + for (int i = 0; i < key_num; i++) { + int randThisTime = rand(); //确保读写一个循环只rand一次，否则随机序列会不一致 + int key_ = randThisTime % key_num+1; + std::string key = std::to_string(key_); + + std::string name = "customer#" + std::to_string(key_); + std::string address = cities[randThisTime % cities.size()]; FieldArray fields = { - {"name", "Customer#000000001"}, - {"address", "IVhzIApeRb"}, - {"phone", "25-989-741-2988"} + {"name", name}, + {"address", address} }; + if (address == "Shanghai") { + shanghaiKeys.push_back(key); + } + Status s = db->PutFields(WriteOptions(), key, fields); + ASSERT_TRUE(s.ok()); + } +} - // 序列化并插入 - db->PutFields(WriteOptions(), key, fields); - - // 读取并反序列化 +void GetFieldData(FieldDB *db) { + ReadOptions readOptions; + int key_num = data_size / value_size; + + // 点查 + srand(0); + for (int i = 0; i < 100; i++) { + int randThisTime = rand(); + int key_ = randThisTime % key_num+1; + std::string key = std::to_string(key_); FieldArray fields_ret; - db->GetFields(ReadOptions(), key, &fields_ret); - // ASSERT_EQ(fields, fields_ret); 顺序不一样 + Status s = db->GetFields(readOptions, key, &fields_ret); + ASSERT_TRUE(s.ok()); for (const Field& pairs : fields_ret) { - ASSERT_NE(std::find(fields.begin(), fields.end(), pairs), fields.end()); + if (pairs.first == "name"){ + + } else if (pairs.first == "address"){ + std::string city = pairs.second; + ASSERT_NE(std::find(cities.begin(), cities.end(), city), cities.end()); + } else assert(false); } + } +} - //todo - Field field = {"name", "Customer#000000001"}; - std::vector resKeys = db->FindKeysByField(field); +void findKeysByCity(FieldDB *db) { + Field field = {"address", "Shanghai"}; + std::vector resKeys = db->FindKeysByField(field); + // std::cout << shanghaiKeys.size() << " " << resKeys.size() << std::endl; + for (const std::string &key : resKeys){ + ASSERT_NE(std::find(shanghaiKeys.begin(), shanghaiKeys.end(), key), shanghaiKeys.end()); + } +} + +TEST(TestLab1, Basic) { + // DestroyDB("testdb",Options()); + FieldDB *db = new FieldDB(); + + if(OpenDB("testdb", &db).ok() == false) { + std::cerr << "open db failed" << std::endl; + abort(); + } + // ClearDB(db); + InsertFieldData(db); + GetFieldData(db); + findKeysByCity(db); } diff --git a/util/serialize_value.cc b/util/serialize_value.cc index a1bdb08..b93d8b7 100644 --- a/util/serialize_value.cc +++ b/util/serialize_value.cc @@ -36,8 +36,8 @@ FieldArray *ParseValue(const std::string& value_str){ } else { std::cout << "name and val not match!" << std::endl; } - nameSlice = Slice(); - valSlice = Slice(); + nameSlice.clear(); + valSlice.clear(); } return res; } diff --git a/util/serialize_value.h b/util/serialize_value.h index ff6e68c..a1ca30a 100644 --- a/util/serialize_value.h +++ b/util/serialize_value.h @@ -16,41 +16,44 @@ FieldArray *ParseValue(const std::string& value_str); class InternalFieldArray { public: -using FieldMap = std::map; + using FieldMap = std::map; -InternalFieldArray(const FieldArray &fields, bool to_map = false):fields(fields),isMapped(false) { - if(to_map) Map(); -} + InternalFieldArray(const FieldArray &fields, bool to_map = false): + fields(fields),isMapped(false) { + if(to_map) Map(); + } -InternalFieldArray(const std::string& value_str) { - Slice valueSlice(value_str); - Slice nameSlice,valSlice; - while(GetLengthPrefixedSlice(&valueSlice, &nameSlice)) { - if(GetLengthPrefixedSlice(&valueSlice, &valueSlice)) { - map[nameSlice.ToString()] = valueSlice.ToString(); - } else { - std::cout << "name and val not match!" << std::endl; + InternalFieldArray(const Slice value_slice) { + Slice valueSlice = value_slice; + Slice nameSlice, valSlice; + while(GetLengthPrefixedSlice(&valueSlice, &nameSlice)) { + if(GetLengthPrefixedSlice(&valueSlice, &valSlice)) { + map[nameSlice.ToString()] = valSlice.ToString(); + } else { + std::cout << "name and val not match!" << std::endl; + } + nameSlice.clear(); + valSlice.clear(); } - nameSlice.clear(); - valSlice.clear(); + isMapped = true; } -} -InternalFieldArray(const Slice& slice):leveldb::InternalFieldArray(slice.ToString()) {} + InternalFieldArray(const std::string& value_str) + :leveldb::InternalFieldArray(Slice(value_str)) {} -//将vector变为用map存 -void Map(); + //将vector变为用map存 + void Map(); -std::string Serialize(); + std::string Serialize(); -bool HasField(const Field& field); + bool HasField(const Field& field); private: -bool isMapped; -const FieldArray fields; -FieldMap map; + bool isMapped; + const FieldArray fields; + FieldMap map; }; } #endif \ No newline at end of file