From d9ff6d8a3d1b85e5df308c29165bd5f43437bd29 Mon Sep 17 00:00:00 2001 From: cyq <1056374449@qq.com> Date: Fri, 3 Jan 2025 21:25:34 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E9=83=A8=E5=88=86?= =?UTF-8?q?=E6=B3=A8=E9=87=8A=EF=BC=8C=E6=8E=A8=E8=BF=9B=E4=BA=86=E9=83=A8?= =?UTF-8?q?=E5=88=86=E5=AE=9E=E9=AA=8C=E6=8A=A5=E5=91=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 139 ++++++++++++++++++++++++++++++++++++++++- benchmarks/db_bench_FieldDB.cc | 68 ++++++++++---------- fielddb/field_db.h | 38 +++++------ 3 files changed, 190 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index a2a0b7f..95e0c60 100644 --- a/README.md +++ b/README.md @@ -642,12 +642,147 @@ TestNormalRecover：创索引、批量写、此时之前测试都检测过能被 TestParalRecover**该测试比较特别，需要运行两次**：创索引 -> 并发：线程0批量写，线程1write，线程2delete，线程3 在单条插入后，deletedb。线程3导致了其他线程错误，测试会终止（模拟数据库崩溃），这会导致各线程在各种奇怪的时间点崩溃。此时注释掉上半部分代码，运行下半部分：单条写入能被读到，并检测一致性。 这里我们运行了几十次，前半部分的崩溃报错有多种，但后半部分的运行都是成功的。同时也追踪了恢复的运行过程，确实有数据从metadb中被正确解析。 -### 3.2 性能测试 -测试、分析、优化 +### 3.2 性能测试、分析、优化 + +#### 3.2.1 性能测量的实现 +我们主要采用了外部测量和内部测量相互结合的方式，来评估数据库系统的性能表现和定位性能瓶颈。外部测量的方式主要借助于benchmark完成。内部测量的主要采用插桩来测量数据库的各个部分的性能损耗情况。 + +相较于原版的leveldb，FieldDB增加了Field和二级索引功能，因此我们针对新的常见使用场景，增加了benchmark的测试点。新增部分有： +```c++ + "CreateIndex," //创建索引 + "FindKeysByField," //得到包含所有Field的KV对（不使用索引） + "QueryByIndex," //通过索引得到对应的主键 + "DeleteIndex," //删除索引 + "WriteSeqWhileCreating," //创建索引的同时顺序写 + "WriteSeqWhileDeleting," //删除索引的同时顺序写 + "WriteRandomWhileCreating," //创建索引的同时随机写 + "WriteRandomWhileDeleting," //删除索引的同时随机写 + "ReadSeqWhileCreating," //创建索引的同时顺序读 + "ReadSeqWhileDeleting," //删除索引的同时顺序读 + "ReadRandomWhileCreating," //创建索引的同时随机读 + "ReadRandomWhileDeleting," //删除索引的同时随机读 + "WriteRandomWithIndex," //随机写带索引的键值 + "WriteSeqWithIndex," //顺序写带索引的键值 + "WriteSeqWhileIndependentCCD," //在不断创建删除索引的情况下，顺序写与创删索引无关的数据 + "WriteSeqWhileCCD," //在不断创建删除索引的情况下，顺序写与创删索引有关的数据 +``` +通过上述新增加的benchmark，可以更加全面的了解增加了新功能后的，各个常见使用场景下的FieldDB的性能指标。各个benchmark的具体实现可以在`/becnmarks/db_becnh_FieldDB.cc`中找到。 + +为了能够进一步的定位性能瓶颈，我们对于操作的关键路径进行了层次化的插桩分析，实现更加精准的性能测量。根据外部测量得到的数据，相较于leveldb，对于读性能，FieldDB几乎没有影响，但是对于写性能，FieldDB性能有所下降，因此我们着重使用插桩分析了写入的关键路径。由于所收集的数据如下： +```c++ +int count = 0;//总计完成请求数量 +int count_Batch = 0;//总计完成的Batch数量 +int count_Batch_Sub = 0;//总计完成的Batch_sub数量 +uint64_t elapsed = 0;//总计时间消耗 + +uint64_t construct_elapsed = 0;//构建写入内容消耗 +uint64_t construct_BatchReq_init_elapsed = 0;//请求初始化消耗 +uint64_t construct_BatchReq_elapsed = 0;//构建batch的消耗 +uint64_t construct_BatchReq_Sub_elapsed = 0;//构建batch_sub消耗 +uint64_t construct_BatchReq_perSub_elapsed = 0;//每个Batch_sub消耗 +uint64_t construct_FieldsReq_Read_elapsed = 0;//构建时读取的消耗 + +uint64_t write_elapsed = 0;//写入的总耗时 +uint64_t write_meta_elapsed = 0;//写入meta的耗时 +uint64_t write_index_elapsed = 0;//写入index的耗时 +uint64_t write_kv_elapsed = 0;//写入kv的耗时 +uint64_t write_clean_elapsed = 0;//清除meta的耗时 + +uint64_t write_bytes = 0; +uint64_t write_step = 500 * 1024 * 1024; +uint64_t write_bytes_lim = write_step; + +uint64_t temp_elapsed = 0; + +uint64_t waiting_elasped = 0; //等待耗时 + +inline void dumpStatistics() { + if(count && count % 500000 == 0 || write_bytes && write_bytes > write_bytes_lim) { + std::cout << "=====================================================\n"; + std::cout << "Total Count : " << count; + std::cout << "\tTotal Write Bytes(MB) : " << write_bytes / 1048576.0 << std::endl; + std::cout << "Average Time(ms) : " << elapsed * 1.0 / count; + std::cout << "\tAverage Write rates(MB/s) : " << write_bytes / 1048576.0 / elapsed * 1000000 << std::endl; + std::cout << "Construct Time(ms) : " << construct_elapsed * 1.0 / count << std::endl; + std::cout << "\tConstruct BatchReq Init Time(ms) : " << construct_BatchReq_init_elapsed * 1.0 / count << std::endl; + std::cout << "\tConstruct BatchReq Time(ms) : " << construct_BatchReq_elapsed * 1.0 / count << std::endl; + std::cout << "\tConstruct BatchReq Sub Time(ms) : " << construct_BatchReq_Sub_elapsed * 1.0 / count << std::endl; + std::cout << "\tConstruct BatchReq perSub Time(ms) : " << construct_BatchReq_perSub_elapsed * 1.0 / count_Batch_Sub << std::endl; + std::cout << "\tConstruct FieldsReq Read Time(ms) : " << construct_FieldsReq_Read_elapsed * 1.0 / count << std::endl; + std::cout << "Write Time(ms) : " << write_elapsed * 1.0 / count << std::endl; + std::cout << "\tWrite Meta Time(ms) : " << write_meta_elapsed * 1.0 / count << std::endl; + std::cout << "\tWrite Index Time(ms) : " << write_index_elapsed * 1.0 / count << std::endl; + std::cout << "\tWrite KV Time(ms) : " << write_kv_elapsed * 1.0 / count << std::endl; + std::cout << "\tWrite Clean Time(ms) : " << write_clean_elapsed * 1.0 / count << std::endl; + std::cout << "TaskQueue Size : " << taskqueue_.size() << std::endl; + std::cout << "temp_elased : " << temp_elapsed * 1.0 / count << std::endl; + std::cout << "waiting elapsed : " << waiting_elasped * 1.0 / count << std::endl; + std::cout << "=====================================================\n"; + write_bytes_lim = write_bytes + write_step; + std::fflush(stdout); + } +} +``` +数据收集的方式如下所示（仅展示部分）： +```c++ +Status FieldDB::HandleRequest(Request &req, const WriteOptions &op) { + uint64_t start_ = env_->NowMicros(); + MutexLock L(&mutex_); + taskqueue_.push_back(&req); + while(true){ + uint64_t start_waiting = env_->NowMicros(); + while(req.isPending() || !req.done && &req != taskqueue_.front()) { + req.cond_.Wait(); + } + waiting_elasped += env_->NowMicros() - start_waiting; + if(req.done) { + elapsed += env_->NowMicros() - start_; + count ++; + dumpStatistics(); + return req.s; //在返回时自动释放锁L + } + Request *tail = GetHandleInterval(); + WriteBatch KVBatch,IndexBatch,MetaBatch; + SliceHashSet batchKeySet; + Status status; +/************************************************************************/ +``` + +#### 3.2.2 性能分析与优化（需要加上相关的性能分析结果） + +通过外部测量和内部测量，我们定位到了许多的值得优化的点，并进行迭代，下面将对于两个比较显著的点进行阐述： + +1. 消除冗余的写入请求 + +通过插桩分析，我们发现对于一个普通的写入，理论上只有对于kvDB写入的流量，但是在实际写入的时候，对于kvDB、metaDB、indexDB都会产生时间消耗。通过审阅代码，我们发现原因在于产生了对于空WriteBatch的写入，因此在写入之前对于WriteBatch的大小进行判断，如果是一个空的WriteBatch，则直接跳过实际写入流程。 + +2. 使用Slice代替std::string + +进一步分析了写入流程的各个部分后，我们认为实际写入数据库部分的写入消耗已经到了理论上限，在现有的结构上并没有优化的办法。同时，在写入流程中的读数据库也无法很好地优化。我们将目光聚焦于构建请求部分的实现上的优化。经过了代码分析，我们发现了在一开始实现的时候为了实现的便捷，我们大量的使用了stl库的`std::string`，但是有每次`string`的拷贝构造问题，会导致数据的多次在内存中的拷贝操作，在数据量较大的时候，我们认为对于性能会产生影响。 + +基于上述的考量，我们通过几轮的commit，将`request`内部的数据结构、相关辅助数据结构以及实现方式全部尽可能的使用`Slice`替换`std::string`。经过测试，我们发现性能确实有所提高。 + + +#### 3.2.3 最终版本的性能分析（草稿） + +1. 对于leveldb本身的一些分析（着重于多线程性能方面） + +2. 对于FieldDB的分析 + +1） 所有涉及读取性能的：和原版leveldb相比，几乎没有任何的损耗，还是非常好的 +2） 常规的写入性能：有所下降，但是由于是因为需要增加读操作，无法避免 +3） 对于创删索引：总体态度是虽然没有比较对象，但是总体可以接受 +4） 对于创删索引和写并发：如果是无关的，那么还是保持了高吞吐；如果是相关的，那么不得不受限于创删索引 ## 4. 问题与解决 +1. 包装一层性能减半？ + +2. + ## 5. 潜在优化点 1. 使用一些高性能并发设计模式，如reactor来优化多线程写入时由于锁竞争导致的性能问题 2. 采用一些高性能的库，如dpdk等 +3. 使用一些基于polling的请求处理手段等 +4. 对于各个log进行合并，减少写放大 ## 6. 分工 diff --git a/benchmarks/db_bench_FieldDB.cc b/benchmarks/db_bench_FieldDB.cc index a061fcc..8dfe049 100644 --- a/benchmarks/db_bench_FieldDB.cc +++ b/benchmarks/db_bench_FieldDB.cc @@ -50,41 +50,41 @@ using namespace fielddb; // sstables -- Print sstable info // heapprofile -- Dump a heap profile (if supported by this port) static const char* FLAGS_benchmarks = - // "fillseq," - // "fillsync," - // "fillrandom," - // "overwrite," - // "readrandom," - // "readrandom," // Extra run to allow previous compactions to quiesce - // "readseq," - // "readreverse," - // "compact," - // "readrandom," - // "readseq," - // "readreverse," - // "fill100K," - // "crc32c," - // "CreateIndex," - // "FindKeysByField," - // "QueryByIndex," - // "DeleteIndex," - // "compact," - // "WriteSeqWhileCreating," - // "WriteSeqWhileDeleting," - // "compact," - // "WriteRandomWhileCreating," - // "WriteRandomWhileDeleting," - // "compact," - // "ReadSeqWhileCreating," - // "ReadSeqWhileDeleting," - // "ReadRandomWhileCreating," - // "ReadRandomWhileDeleting," - // "WriteRandomWithIndex," - // "WriteSeqWithIndex," "fillseq," - "WriteSeqWhileIndependentCCD," - "fillseq," - "WriteSeqWhileCCD," + "fillsync," + "fillrandom," + "overwrite," + "readrandom," + "readrandom," // Extra run to allow previous compactions to quiesce + "readseq," + "readreverse," + "compact," + "readrandom," + "readseq," + "readreverse," + "fill100K," + "crc32c," + "CreateIndex," //创建索引 + "FindKeysByField," //得到包含所有Field的KV对（不使用索引） + "QueryByIndex," //通过索引得到对应的主键 + "DeleteIndex," //删除索引 + "compact," + "WriteSeqWhileCreating," //创建索引的同时顺序写 + "WriteSeqWhileDeleting," //删除索引的同时顺序写 + "compact," + "WriteRandomWhileCreating," //创建索引的同时随机写 + "WriteRandomWhileDeleting," //删除索引的同时随机写 + "compact," + "ReadSeqWhileCreating," //创建索引的同时顺序读 + "ReadSeqWhileDeleting," //删除索引的同时顺序读 + "ReadRandomWhileCreating," //创建索引的同时随机读 + "ReadRandomWhileDeleting," //删除索引的同时随机读 + "WriteRandomWithIndex," //随机写带索引的键值 + "WriteSeqWithIndex," //顺序写带索引的键值 + "fillseq," + "WriteSeqWhileIndependentCCD," //在不断创建删除索引的情况下，顺序写与创删索引无关的数据 + "fillseq," + "WriteSeqWhileCCD," //在不断创建删除索引的情况下，顺序写与创删索引有关的数据 ; // Number of key/values to place in database diff --git a/fielddb/field_db.h b/fielddb/field_db.h index ed9b87f..ef981e9 100644 --- a/fielddb/field_db.h +++ b/fielddb/field_db.h @@ -92,23 +92,23 @@ private: Request *GetHandleInterval(); //获得任务队列中的待处理区间，区间划分规则和原因见文档 // private: -// int count = 0; -// int count_Batch = 0; -// int count_Batch_Sub = 0; -// uint64_t elapsed = 0; - -// uint64_t construct_elapsed = 0; -// uint64_t construct_BatchReq_init_elapsed = 0; -// uint64_t construct_BatchReq_elapsed = 0; -// uint64_t construct_BatchReq_Sub_elapsed = 0; -// uint64_t construct_BatchReq_perSub_elapsed = 0; -// uint64_t construct_FieldsReq_Read_elapsed = 0; - -// uint64_t write_elapsed = 0; -// uint64_t write_meta_elapsed = 0; -// uint64_t write_index_elapsed = 0; -// uint64_t write_kv_elapsed = 0; -// uint64_t write_clean_elapsed = 0; +// int count = 0;//总计完成请求数量 +// int count_Batch = 0;//总计完成的Batch数量 +// int count_Batch_Sub = 0;//总计完成的Batch_sub数量 +// uint64_t elapsed = 0;//总计时间消耗 + +// uint64_t construct_elapsed = 0;//构建写入内容消耗 +// uint64_t construct_BatchReq_init_elapsed = 0;//请求初始化消耗 +// uint64_t construct_BatchReq_elapsed = 0;//构建batch的消耗 +// uint64_t construct_BatchReq_Sub_elapsed = 0;//构建batch_sub消耗 +// uint64_t construct_BatchReq_perSub_elapsed = 0;//每个Batch_sub消耗 +// uint64_t construct_FieldsReq_Read_elapsed = 0;//构建时读取的消耗 + +// uint64_t write_elapsed = 0;//写入的总耗时 +// uint64_t write_meta_elapsed = 0;//写入meta的耗时 +// uint64_t write_index_elapsed = 0;//写入index的耗时 +// uint64_t write_kv_elapsed = 0;//写入kv的耗时 +// uint64_t write_clean_elapsed = 0;//清除meta的耗时 // uint64_t write_bytes = 0; // uint64_t write_step = 500 * 1024 * 1024; @@ -116,7 +116,7 @@ private: // uint64_t temp_elapsed = 0; -// uint64_t waiting_elasped = 0; +// uint64_t waiting_elasped = 0;//等待耗时 // inline void dumpStatistics() { // if(count && count % 500000 == 0 || write_bytes && write_bytes > write_bytes_lim) { @@ -145,7 +145,7 @@ private: // std::fflush(stdout); // } // } -}; +// }; Status DestroyDB(const std::string& name, const Options& options);