diff --git a/CMakeLists.txt b/CMakeLists.txt index 7307c47..8fd8ce7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -522,6 +522,10 @@ add_executable(db_test1 "${PROJECT_SOURCE_DIR}/test/test.cpp" ) target_link_libraries(db_test1 PRIVATE leveldb gtest) +add_executable(db_test2 +"${PROJECT_SOURCE_DIR}/test/test2.cpp" + ) +target_link_libraries(db_test2 PRIVATE leveldb gtest) add_executable(db_test_bench "${PROJECT_SOURCE_DIR}/test/benchmark_4leveldb.cpp" ) diff --git a/db/db_impl.cc b/db/db_impl.cc index 1924794..1c8ea59 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1829,24 +1829,23 @@ void DBImpl::GarbageCollect() { // 更新当前偏移 current_offset += sizeof(uint64_t); - // Now seek to the actual data position and read the value - cur_valuelog.seekg(current_offset); - char* value_buf = new char[val_len]; - cur_valuelog.read(value_buf, val_len); - if (!cur_valuelog.good()) { - delete[] key_buf; - delete[] key_buf_len; - delete[] value_buf_len; - delete[] value_buf; - cur_valuelog.close(); - std::cerr << "Failed to read file: " << valuelog_name << std::endl; - break; - } + // // Now seek to the actual data position and read the value + // cur_valuelog.seekg(current_offset); + // char* value_buf = new char[val_len]; + // cur_valuelog.read(value_buf, val_len); + // if (!cur_valuelog.good()) { + // delete[] key_buf; + // delete[] key_buf_len; + // delete[] value_buf_len; + // delete[] value_buf; + // cur_valuelog.close(); + // std::cerr << "Failed to read file: " << valuelog_name << std::endl; + // break; + // } current_offset += val_len; - // Assign the read value data to the Slice - value = Slice(value_buf, val_len); - // std::cout< a, std::vector b) { - if (a.size() != b.size()){ - return false; - } - for (size_t i = 0; i < a.size(); ++i) { - if (a[i] != b[i]){ - return false; - } - } - return true; -} - -std::string SerializeValue(const FieldArray& fields){ - std::string res_=""; - PutVarint64(&res_,(uint64_t)fields.size()); - for(auto pr:fields){ - PutLengthPrefixedSlice(&res_, pr.first); - PutLengthPrefixedSlice(&res_, pr.second); - } - return res_; -} - - // 鍙嶅簭鍒楀寲涓哄瓧娈垫暟缁? -void DeserializeValue(const std::string& value_str,FieldArray* res){ - Slice slice=Slice(value_str.c_str()); - uint64_t siz; - bool tmpres=GetVarint64(&slice,&siz); - assert(tmpres); - res->clear(); - for(int i=0;iemplace_back(value_name,value); - } -} - -Status Get_keys_by_field(DB *db,const ReadOptions& options, const Field field,std::vector *keys){ - auto it=db->NewIterator(options); - it->SeekToFirst(); - keys->clear(); - while(it->Valid()){ - auto val=it->value(); - FieldArray arr; - auto str_val=std::string(val.data(),val.size()); - DeserializeValue(str_val,&arr); - for(auto pr:arr){ - if(pr.first==field.first&&pr.second==field.second){ - Slice key=it->key(); - keys->push_back(std::string(key.data(),key.size())); - break; - } - } - it->Next(); - } - delete it; - return Status::OK(); -} TEST(Test, CheckGetFields) { DB *db; @@ -158,37 +89,37 @@ TEST(Test, CheckSearchKey) { delete db; } -// TEST(Test, LARGE_DATA_COMPACT_TEST) { -// DB *db; -// WriteOptions writeOptions; -// ReadOptions readOptions; -// if(OpenDB("testdb_for_XOY_large", &db).ok() == false) { -// std::cerr << "open db failed" << std::endl; -// abort(); -// } -// std::vector values; -// for(int i=0;i<500000;i++){ -// std::string key=std::to_string(i); -// std::string value; -// for(int j=0;j<1000;j++){ -// value+=std::to_string(i); -// } -// values.push_back(value); -// db->Put(writeOptions,key,value); -// } -// for(int i=0;i<500000;i++){ -// std::string key=std::to_string(i); -// std::string value; -// Status s=db->Get(readOptions,key,&value); -// assert(s.ok()); -// if(values[i]!=value){ -// std::cout< values; + for(int i=0;i<500000;i++){ + std::string key=std::to_string(i); + std::string value; + for(int j=0;j<1000;j++){ + value+=std::to_string(i); + } + values.push_back(value); + db->Put(writeOptions,key,value); + } + // for(int i=0;i<500000;i++){ + // std::string key=std::to_string(i); + // std::string value; + // Status s=db->Get(readOptions,key,&value); + // assert(s.ok()); + // if(values[i]!=value){ + // std::cout< +#include + +using namespace std::chrono; +using namespace leveldb; + +using Field=std::pair; +using FieldArray=std::vector>; + +int data_number=100000; + +Status OpenDB(std::string dbName, DB **db) { + Options options; + options.max_file_size=16*1024; + options.write_buffer_size=32*1024; + options.create_if_missing = true; + return DB::Open(options, dbName, db); +} + + +TEST(Test, Garbage_Collect_TEST) { + DB *db; + WriteOptions writeOptions; + ReadOptions readOptions; + if(OpenDB("testdb_for_XOY_large", &db).ok() == false) { + std::cerr << "open db failed" << std::endl; + abort(); + } + std::vector values; + for(int i=0;iPut(writeOptions,key,value); + } + // for(int i=0;iPut(writeOptions,key,value); + // } + // Measure GC time + auto start_time = high_resolution_clock::now(); + db->TEST_GarbageCollect(); + auto end_time = high_resolution_clock::now(); + + auto duration = duration_cast(end_time - start_time); + std::cout << "GC finished. Time taken: " << duration.count() << " ms" << std::endl; + + + for(int i=0;iGet(readOptions,key,&value); + assert(s.ok()); + if(values[i]!=value){ + std::cout<& files, int num_workers, std::vector>* chunks); +bool CompareFieldArray(const FieldArray &a, const FieldArray &b); +bool CompareKey(const std::vector a, std::vector b); +std::string SerializeValue(const FieldArray& fields); +void DeserializeValue(const std::string& value_str,FieldArray* res); +Status Get_keys_by_field(DB *db,const ReadOptions& options, const Field field,std::vector *keys); + + } // namespace leveldb #endif // STORAGE_LEVELDB_UTIL_CODING_H_