diff --git a/CMakeLists.txt b/CMakeLists.txt index 54b14a1..0fa80be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,4 +528,9 @@ target_link_libraries(db_test2 PRIVATE leveldb) add_executable(ttl_test "${PROJECT_SOURCE_DIR}/test/ttl_test.cc" ) -target_link_libraries(ttl_test PRIVATE leveldb gtest) \ No newline at end of file +target_link_libraries(ttl_test PRIVATE leveldb gtest) + +add_executable(ttl_garbage_collection + "${PROJECT_SOURCE_DIR}/test/ttl_garbage_collection.cc" +) +target_link_libraries(ttl_garbage_collection PRIVATE leveldb gtest) \ No newline at end of file diff --git a/db/db_impl.cc b/db/db_impl.cc index a42e75c..22ab80d 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1202,6 +1202,10 @@ Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val) { return DB::Put(o, key, val); } +Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val,uint64_t ttl) { + return DB::Put(o, key, val,ttl); +} + Status DBImpl::Delete(const WriteOptions& options, const Slice& key) { return DB::Delete(options, key); } @@ -1491,7 +1495,27 @@ void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) { // can call if they wish Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) { WriteBatch batch; - batch.Put(key, value); + int len=value.size()+sizeof(uint64_t); + char* new_data=new char[len]; + time_t now = time(nullptr);//ttl单位为秒 + uint64_t ttl=INT64_MAX; + memcpy(new_data,value.data(),value.size()); + memcpy(new_data+len-sizeof(uint64_t),(char*)(&ttl),sizeof(uint64_t)); + Slice newValue=Slice(new_data,len); + batch.Put(key, newValue); + return Write(opt, &batch); +} + +Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value,uint64_t ttl) { + WriteBatch batch; + int len=value.size()+sizeof(uint64_t); + char* new_data=new char[len]; + time_t now = time(nullptr);//ttl单位为秒 + ttl+=static_cast(now); + memcpy(new_data,value.data(),value.size()); + memcpy(new_data+len-sizeof(uint64_t),(char*)(&ttl),sizeof(uint64_t)); + Slice newValue=Slice(new_data,len); + batch.Put(key, newValue); return Write(opt, &batch); } diff --git a/db/db_impl.h b/db/db_impl.h index c7b0172..f9ae1ab 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -38,6 +38,8 @@ class DBImpl : public DB { // Implementations of the DB interface Status Put(const WriteOptions&, const Slice& key, const Slice& value) override; + Status Put(const WriteOptions&, const Slice& key, + const Slice& value,uint64_t ttl) override; Status Delete(const WriteOptions&, const Slice& key) override; Status Write(const WriteOptions& options, WriteBatch* updates) override; Status Get(const ReadOptions& options, const Slice& key, diff --git a/db/db_test.cc b/db/db_test.cc index a4a84cd..4a05522 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2117,6 +2117,9 @@ class ModelDB : public DB { Status Put(const WriteOptions& o, const Slice& k, const Slice& v) override { return DB::Put(o, k, v); } + Status Put(const WriteOptions& o, const Slice& k, const Slice& v,uint64_t ttl) override { + return DB::Put(o, k, v,ttl); + } Status Delete(const WriteOptions& o, const Slice& key) override { return DB::Delete(o, key); } diff --git a/db/memtable.cc b/db/memtable.cc index 4f09340..1cf4b33 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -103,7 +103,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) { Slice memkey = key.memtable_key(); Table::Iterator iter(&table_); iter.Seek(memkey.data()); - if (iter.Valid()) { + while (iter.Valid()) { // entry format is: // klength varint32 // userkey char[klength] @@ -123,7 +123,14 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) { switch (static_cast(tag & 0xff)) { case kTypeValue: { Slice v = GetLengthPrefixedSlice(key_ptr + key_length); - value->assign(v.data(), v.size()); + uint64_t ttl=*(uint64_t*)(v.data()+v.size()-sizeof(uint64_t)); + time_t now = time(nullptr); + if(ttl < static_cast(now)){ + iter.Next(); + continue; + } + value->assign(v.data(), v.size()-sizeof(uint64_t)); + return true; } case kTypeDeletion: @@ -131,6 +138,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) { return true; } } + else break; } return false; } diff --git a/db/version_set.cc b/db/version_set.cc index a7d2cce..109e5c9 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -271,6 +271,11 @@ static void SaveValue(void* arg, const Slice& ikey, const Slice& v) { uint64_t ttl=*(uint64_t*)(v.data()+v.size()-sizeof(uint64_t)); if(ttl < static_cast(now))return; } + if(parsed_key.type == kTypeValue){ + time_t now = time(nullptr); + uint64_t ttl=*(uint64_t*)(v.data()+v.size()-8); + if(ttl < static_cast(now))return; + } s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted; if (s->state == kFound) { s->value->assign(v.data(), v.size()-sizeof(uint64_t)); diff --git a/include/leveldb/db.h b/include/leveldb/db.h index bf4eec5..04470bb 100644 --- a/include/leveldb/db.h +++ b/include/leveldb/db.h @@ -149,7 +149,7 @@ class LEVELDB_EXPORT DB { // ----------------------------For TTL----------------------------- // 为当前key设置ttl，过期后自动失效 virtual Status Put(const WriteOptions& options, const Slice& key, - const Slice& value, uint64_t ttl) = 0; + const Slice& value, uint64_t ttl)=0; }; // Destroy the contents of the specified database. diff --git a/test/ttl_garbage_collection.cc b/test/ttl_garbage_collection.cc new file mode 100644 index 0000000..c680278 --- /dev/null +++ b/test/ttl_garbage_collection.cc @@ -0,0 +1,73 @@ +#include "gtest/gtest.h" + +#include "leveldb/env.h" +#include "leveldb/db.h" + + +using namespace leveldb; + +constexpr int value_size = 2048; +constexpr int data_size = 128 << 20; +constexpr int key_num = 50000; + +Status OpenDB(std::string dbName, DB **db) { + Options options; + options.create_if_missing = true; + return DB::Open(options, dbName, db); +} + +void InsertData(DB *db, uint64_t ttl/* second */) { + WriteOptions writeOptions; + + for (int i = 0; i < key_num; i++) { + int key_ = i; + std::string key = std::to_string(key_); + std::string value(value_size, 'a'); + db->Put(writeOptions, key, value, ttl); + } +} + +TEST(TestTTL, ReadTTL) { + DB *db; + if(OpenDB("test_garbage_db", &db).ok() == false) { + std::cerr << "open db failed" << std::endl; + abort(); + } + + uint64_t ttl = 10; + + InsertData(db, ttl); + + ReadOptions readOptions; + Status status; + srand(static_cast(time(0))); + for (int i = 0; i < key_num; i++) { + int key_ = i; + std::string key = std::to_string(key_); + std::string value; + status = db->Get(readOptions, key, &value); + ASSERT_TRUE(status.ok()); + } + + Env::Default()->SleepForMicroseconds((ttl+5) * 1000000); + + InsertData(db, ttl); + + for (int i = 0; i < key_num; i++) { + int key_ = i; + std::string key = std::to_string(key_); + std::string value; + status = db->Get(readOptions, key, &value); + if(status.ok()){ + std::cout<<"!!"; + } + ASSERT_TRUE(status.ok()); + } +} + + +int main(int argc, char** argv) { + // All tests currently run with the same read-only file limits. + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/ttl_test.cc b/test/ttl_test.cc index 06f4cda..beafb94 100644 --- a/test/ttl_test.cc +++ b/test/ttl_test.cc @@ -10,6 +10,7 @@ using namespace leveldb; constexpr int value_size = 2048; constexpr int data_size = 128 << 20; +constexpr int key_num = 10000; Status OpenDB(std::string dbName, DB **db) { Options options; @@ -19,31 +20,15 @@ Status OpenDB(std::string dbName, DB **db) { void InsertData(DB *db, uint64_t ttl/* second */) { WriteOptions writeOptions; - int key_num = data_size / value_size; - srand(static_cast(time(0))); for (int i = 0; i < key_num; i++) { - int key_ = rand() % key_num+1; + int key_ = i; std::string key = std::to_string(key_); std::string value(value_size, 'a'); db->Put(writeOptions, key, value, ttl); } } -void GetData(DB *db, int size = (1 << 30)) { - ReadOptions readOptions; - int key_num = data_size / value_size; - - // 点查 - srand(static_cast(time(0))); - for (int i = 0; i < 100; i++) { - int key_ = rand() % key_num+1; - std::string key = std::to_string(key_); - std::string value; - db->Get(readOptions, key, &value); - } -} - TEST(TestTTL, ReadTTL) { DB *db; if(OpenDB("testdb", &db).ok() == false) { @@ -51,60 +36,62 @@ TEST(TestTTL, ReadTTL) { abort(); } - uint64_t ttl = 20; + uint64_t ttl = 10; InsertData(db, ttl); ReadOptions readOptions; Status status; - int key_num = data_size / value_size; srand(static_cast(time(0))); - for (int i = 0; i < 100; i++) { - int key_ = rand() % key_num+1; + for (int i = 0; i < key_num; i++) { + int key_ = i; std::string key = std::to_string(key_); std::string value; status = db->Get(readOptions, key, &value); ASSERT_TRUE(status.ok()); } - Env::Default()->SleepForMicroseconds(ttl * 1000000); + Env::Default()->SleepForMicroseconds((ttl+5) * 1000000); - for (int i = 0; i < 100; i++) { - int key_ = rand() % key_num+1; + for (int i = 0; i < key_num; i++) { + int key_ = i; std::string key = std::to_string(key_); std::string value; status = db->Get(readOptions, key, &value); + if(status.ok()){ + std::cout<<"!!"; + } ASSERT_FALSE(status.ok()); } } -TEST(TestTTL, CompactionTTL) { - DB *db; +// TEST(TestTTL, CompactionTTL) { +// DB *db; - if(OpenDB("testdb", &db).ok() == false) { - std::cerr << "open db failed" << std::endl; - abort(); - } +// if(OpenDB("testdb", &db).ok() == false) { +// std::cerr << "open db failed" << std::endl; +// abort(); +// } - uint64_t ttl = 20; - InsertData(db, ttl); +// uint64_t ttl = 20; +// InsertData(db, ttl); - leveldb::Range ranges[1]; - ranges[0] = leveldb::Range("-", "A"); - uint64_t sizes[1]; - db->GetApproximateSizes(ranges, 1, sizes); - ASSERT_GT(sizes[0], 0); +// leveldb::Range ranges[1]; +// ranges[0] = leveldb::Range("-", "A"); +// uint64_t sizes[1]; +// db->GetApproximateSizes(ranges, 1, sizes); +// ASSERT_GT(sizes[0], 0); - Env::Default()->SleepForMicroseconds(ttl * 1000000); +// Env::Default()->SleepForMicroseconds(ttl * 1000000); - db->CompactRange(nullptr, nullptr); +// db->CompactRange(nullptr, nullptr); - leveldb::Range ranges[1]; - ranges[0] = leveldb::Range("-", "A"); - uint64_t sizes[1]; - db->GetApproximateSizes(ranges, 1, sizes); - ASSERT_EQ(sizes[0], 0); -} +// leveldb::Range ranges[1]; +// ranges[0] = leveldb::Range("-", "A"); +// uint64_t sizes[1]; +// db->GetApproximateSizes(ranges, 1, sizes); +// ASSERT_EQ(sizes[0], 0); +// } int main(int argc, char** argv) {