From 8657ec37484d53a58dd64534d9af070c5adf29d3 Mon Sep 17 00:00:00 2001 From: cyq <1056374449@qq.com> Date: Sat, 21 Dec 2024 18:09:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E4=BA=86recover=EF=BC=8C?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BA=86=E6=9C=89=E5=85=B3ParseValuede?= =?UTF-8?q?=E7=9A=84=E5=86=85=E5=AD=98=E6=B3=84=E6=BC=8F=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E5=92=8C=E6=9C=89=E5=85=B3Transput=E5=92=8CTransDelete?= =?UTF-8?q?=E7=9A=84=E6=BD=9C=E5=9C=A8=E5=86=85=E5=AD=98=E8=AE=BF=E9=97=AE?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/db_impl.cc | 2 +- fielddb/field_db.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ fielddb/meta.cpp | 16 +++++++++------- fielddb/meta.h | 4 ++-- fielddb/request.cpp | 17 ++++++++++++----- util/serialize_value.cc | 5 +++-- util/serialize_value.h | 2 +- 7 files changed, 71 insertions(+), 18 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 49db131..6879b82 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1169,7 +1169,7 @@ Status DBImpl::GetFields(const ReadOptions& options, const Slice& key, FieldArray* fields) { std::string value; Status s = DBImpl::Get(options, key, &value); - *fields = *ParseValue(value); + ParseValue(value,fields); return s; } diff --git a/fielddb/field_db.cpp b/fielddb/field_db.cpp index c5c5ad4..0eea72e 100644 --- a/fielddb/field_db.cpp +++ b/fielddb/field_db.cpp @@ -6,10 +6,13 @@ #include #include "leveldb/db.h" #include "leveldb/env.h" +#include "leveldb/iterator.h" #include "leveldb/options.h" +#include "leveldb/slice.h" #include "leveldb/status.h" #include "leveldb/write_batch.h" #include "db/write_batch_internal.h" +#include "util/coding.h" #include "util/mutexlock.h" #include "util/serialize_value.h" #include "fielddb/encode_index.h" @@ -53,7 +56,47 @@ Status FieldDB::OpenFieldDB(const Options& options, Status FieldDB::Recover() { //TODO: //1. 遍历所有Index类型的meta，重建内存中的index_状态表 + Iterator *Iter = indexDB_->NewIterator(ReadOptions()); + std::string IndexKey; + Iter->SeekToFirst(); + while(Iter->Valid()) { + IndexKey = Iter->value().ToString(); + ParsedInternalIndexKey ParsedIndex; + ParseInternalIndexKey(Slice(IndexKey),&ParsedIndex); + index_[ParsedIndex.name_.ToString()] = {Exist,nullptr}; + std::cout << "Existed Index : " << ParsedIndex.name_.ToString() << std::endl; + + //构建下一个搜索的对象，在原来的fieldname的基础上加一个最大的ascii字符（不可见字符） + //TODO：不知道这个做法有没有道理 + std::string Seek; + PutLengthPrefixedSlice(&Seek, ParsedIndex.name_); + Seek.push_back(0xff); + Iter->Seek(Slice(Seek)); + } + delete Iter; //2. 寻找所有KV类型的meta，再次提交一遍请求 + Iter = metaDB_->NewIterator(ReadOptions()); + Slice MetaValue; + Iter->SeekToFirst(); + while (Iter->Valid()) { + MetaValue = Iter->key(); + MetaType type = MetaType(DecodeFixed32(MetaValue.data())); + MetaValue.remove_prefix(4);//移除头上的metaType的部分 + if(type == KV_Creating) { + FieldArray fields; + ParseValue(Iter->value().ToString(), &fields); + PutFields(WriteOptions(), MetaValue, fields); + } else if(type == KV_Deleting) { + Delete(WriteOptions(), MetaValue); + } else { + assert(0 && "Invalid MetaType"); + } + } + delete Iter; + //在所有的请求完成后，会自动把metaDB的内容清空。 + Iter = metaDB_->NewIterator(ReadOptions()); + Iter->SeekToFirst(); + std::cout << "Iter Valid : " << Iter->Valid() << std::endl; //3. 等待所有请求完成 return Status::OK(); } diff --git a/fielddb/meta.cpp b/fielddb/meta.cpp index a02f585..13ee09d 100644 --- a/fielddb/meta.cpp +++ b/fielddb/meta.cpp @@ -22,22 +22,24 @@ using namespace leveldb; //对于含有index field的put/delete的meta编码为 (KV|Key,Value) -void MetaKV::TransPut(Slice &MetaKey,Slice &MetaValue) { +void MetaKV::TransPut(std::string &MetaKey,std::string &MetaValue) { MetaKey.clear(); MetaValue.clear(); - std::string buf; + //这里的改动是为了防止潜在的段错误。原来的写法中，slice(buf)对应的buf是局部的，在函数返回后，buf被销毁 + //但是slice中的指针指向的是析构的string对象的部分内存 + std::string &buf = MetaKey; PutFixed32(&buf, KV_Creating); PutLengthPrefixedSlice(&buf, Slice(*name)); - MetaKey = Slice(buf); - MetaValue = Slice(*value); + // MetaKey = Slice(buf); + // MetaValue = Slice(*value); } -void MetaKV::TransDelete(Slice &MetaKey) { +void MetaKV::TransDelete(std::string &MetaKey) { MetaKey.clear(); - std::string buf; + std::string &buf = MetaKey; PutFixed32(&buf, KV_Deleting); PutLengthPrefixedSlice(&buf, Slice(*name)); - MetaKey = Slice(buf); + // MetaKey = Slice(buf); } class CleanerHandler : public WriteBatch::Handler { diff --git a/fielddb/meta.h b/fielddb/meta.h index 3dba911..2766e88 100644 --- a/fielddb/meta.h +++ b/fielddb/meta.h @@ -37,8 +37,8 @@ class MetaKV { public: MetaKV(std::string *field_name,std::string *field_value = nullptr): name(field_name),value(field_value) { } - void TransPut(Slice &MetaKey,Slice &MetaValue); - void TransDelete(Slice &MetaKey); + void TransPut(std::string &MetaKey,std::string &MetaValue); + void TransDelete(std::string &MetaKey); private: std::string *name; std::string *value; diff --git a/fielddb/request.cpp b/fielddb/request.cpp index ce2f18e..d2c2d8b 100644 --- a/fielddb/request.cpp +++ b/fielddb/request.cpp @@ -1,5 +1,6 @@ #include "fielddb/request.h" #include +#include #include "leveldb/slice.h" #include "leveldb/status.h" #include "leveldb/write_batch.h" @@ -54,7 +55,8 @@ void FieldsReq::ConstructBatch(WriteBatch &KVBatch,WriteBatch &IndexBatch, if (s.IsNotFound()){ oldFields = nullptr; } else if (s.ok()) { //得到数据库之前key的fields, 判断需不需要删除其中潜在的索引 - oldFields = ParseValue(val_str); + oldFields = new FieldArray; + oldFields = ParseValue(val_str,oldFields); } else { assert(0); } @@ -99,11 +101,11 @@ void FieldsReq::ConstructBatch(WriteBatch &KVBatch,WriteBatch &IndexBatch, //2.对于没有冲突但含有索引操作的put，构建metaKV，这里直接将KV对简单编码后写入metaDB if(HasIndex || HasOldIndex) { - Slice MetaKey,MetaValue; + std::string MetaKey,MetaValue; std::string serialized = SerializeValue(*Fields); MetaKV MKV = MetaKV(Key,&serialized); MKV.TransPut(MetaKey, MetaValue); - MetaBatch.Put(MetaKey, MetaValue); + MetaBatch.Put(MetaKey, serialized); //3.1对于含有索引的oldfield删除索引 @@ -135,6 +137,8 @@ void FieldsReq::ConstructBatch(WriteBatch &KVBatch,WriteBatch &IndexBatch, } //优化：对于3.1，3.2中都有的索引只写一次 } + + if(oldFields) delete oldFields; } @@ -154,7 +158,9 @@ void DeleteReq::ConstructBatch(WriteBatch &KVBatch,WriteBatch &IndexBatch, //2.3 如果存在field的索引状态是Creating或者Deleting，那么在那个队列上面进行等待 std::string val_str; DB->kvDB_->Get(ReadOptions(), *Key, &val_str); - FieldArray *Fields = ParseValue(val_str); + FieldArray *Fields = new FieldArray; + ParseValue(val_str,Fields); + KVBatch.Delete(Slice(*Key)); bool HasIndex = false; { @@ -176,7 +182,7 @@ void DeleteReq::ConstructBatch(WriteBatch &KVBatch,WriteBatch &IndexBatch, } //2.对于没有冲突但含有索引操作的delete，构建metaKV，这里直接将KV对简单编码后写入metaDB if(HasIndex) { - Slice MetaKey; + std::string MetaKey; MetaKV MKV = MetaKV(Key); MKV.TransDelete(MetaKey); //meta中写入一个delete不需要value MetaBatch.Put(MetaKey, Slice()); @@ -192,6 +198,7 @@ void DeleteReq::ConstructBatch(WriteBatch &KVBatch,WriteBatch &IndexBatch, } } } + delete Fields; } /*******iCreateReq*******/ diff --git a/util/serialize_value.cc b/util/serialize_value.cc index 562360b..88aa844 100644 --- a/util/serialize_value.cc +++ b/util/serialize_value.cc @@ -20,9 +20,10 @@ std::string SerializeValue(const FieldArray& fields){ return result; } -FieldArray *ParseValue(const std::string& value_str){ +FieldArray *ParseValue(const std::string& value_str,FieldArray *fields){ Slice valueSlice(value_str); - FieldArray *res = new FieldArray; + // FieldArray *res = new FieldArray; + FieldArray *res = fields; Slice nameSlice = Slice(); Slice valSlice = Slice(); std::string nameStr; diff --git a/util/serialize_value.h b/util/serialize_value.h index b769fb8..a337bc6 100644 --- a/util/serialize_value.h +++ b/util/serialize_value.h @@ -12,7 +12,7 @@ using Field = std::pair; // field_name:field_value using FieldArray = std::vector>; std::string SerializeValue(const FieldArray& fields); -FieldArray *ParseValue(const std::string& value_str); +FieldArray *ParseValue(const std::string& value_str, FieldArray *fields); class InternalFieldArray { public: