building_data_management_systems.Xuanzhou.2024Fall.DaSE
/
LevelDB_KV_Sep
23
0
派生 0
代码 话题 0 合并请求 0 版本发布 0 百科 动态
浏览代码

testing

main
wesley 5 个月前
父节点
9a36e35ed6
当前提交
af7c4c5041
共有 4 个文件被更改，包括 87 次插入 和 47 次删除
合并视图
  1. +2
    -2
      include/leveldb/options.h
  2. +1
    -1
      table/vtable_manager.cc
  3. +44
    -19
      test/test_bench.cc
  4. +40
    -25
      test/test_bench_gc.cc

+ 2
- 2
include/leveldb/options.h 查看文件

@ -101,7 +101,7 @@ struct LEVELDB_EXPORT Options {
size_t block_size = 4 * 1024; size_t block_size = 4 * 1024;
// Threshold of value size that decide whether to separate the key and value // Threshold of value size that decide whether to separate the key and value
size_t kv_sep_size = 1; size_t kv_sep_size = 1000;
// Number of keys between restart points for delta encoding of keys. // Number of keys between restart points for delta encoding of keys.
// This parameter can be changed dynamically. Most clients should // This parameter can be changed dynamically. Most clients should
@ -118,7 +118,7 @@ struct LEVELDB_EXPORT Options {
// initially populating a large database. // initially populating a large database.
size_t max_file_size = 2 * 1024 * 1024; size_t max_file_size = 2 * 1024 * 1024;
size_t gc_size_threshold = 1024 * 1024 * 1024; size_t gc_size_threshold = 128 * 1024 * 1024;
// Compress blocks using the specified compression algorithm. This // Compress blocks using the specified compression algorithm. This
// parameter can be changed dynamically. // parameter can be changed dynamically.

+ 1
- 1
table/vtable_manager.cc 查看文件

@ -181,7 +181,7 @@ void VTableManager::BackgroudGC(void* gc_info) {
} }
auto end_time = std::chrono::steady_clock::now(); auto end_time = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "gc lasts: " << duration << "micros" << std::endl; std::cout << "gc lasts: " << duration << " micros" << std::endl;
} }
void VTableManager::RefVTable(uint64_t file_num) { void VTableManager::RefVTable(uint64_t file_num) {

+ 44
- 19
test/test_bench.cc 查看文件

@ -11,11 +11,11 @@ using namespace leveldb;
// Number of key/values to operate in database // Number of key/values to operate in database
constexpr int num_ = 100000; constexpr int num_ = 100000;
// Size of each value // Size of each value
constexpr int value_size_ = 1000; constexpr int value_size_ = 2000;
// Number of read operations // Number of read operations
constexpr int reads_ = 100000; constexpr int reads_ = 100000;
// Number of findkeysbyfield operations // Number of findkeysbyfield operations
constexpr int search_ = 20; constexpr int search_ = 10;
int64_t bytes_ = 0; int64_t bytes_ = 0;
@ -32,8 +32,8 @@ void InsertData(DB *db, std::vector &lats) {
bytes_ = 0; bytes_ = 0;
int64_t bytes = 0; int64_t bytes = 0;
srand(0); srand(0);
std::mt19937 value_seed(100); // std::mt19937 value_seed(100);
std::uniform_int_distribution<int> value_range(10, 2048); // std::uniform_int_distribution<int> value_range(500, 2048);
int64_t latency = 0; int64_t latency = 0;
auto end_time = std::chrono::steady_clock::now(); auto end_time = std::chrono::steady_clock::now();
auto last_time = end_time; auto last_time = end_time;
@ -41,8 +41,8 @@ void InsertData(DB *db, std::vector &lats) {
for (int i = 0; i < num_; i++) { for (int i = 0; i < num_; i++) {
int key_ = rand() % num_+1; int key_ = rand() % num_+1;
int value_ = std::rand() % (num_ + 1); int value_ = std::rand() % (num_ + 1);
int value_size = value_range(value_seed); // int value_size = value_range(value_seed);
std::string value(value_size, 'a'); std::string value(value_size_, 'a');
std::string key = std::to_string(key_); std::string key = std::to_string(key_);
FieldArray field_array = { FieldArray field_array = {
{"1", value}, {"1", value},
@ -174,29 +174,28 @@ TEST(TestBench, Throughput) {
InsertData(db, lats); InsertData(db, lats);
auto end_time = std::chrono::steady_clock::now(); auto end_time = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "Throughput of Put(): " << std::fixed << num_ * 1e6 / duration << " ops/s" << std::endl; // std::cout << "Throughput of Put(): " << std::fixed << num_ * 1e6 / duration << " ops/s" << std::endl;
std::cout << "Throughput of Put(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl; std::cout << "Throughput of Put(): " << std::fixed << std::setprecision(2) << std::endl << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl; // << " MB/s" << std::endl << std::endl;
// Get() // Get()
start_time = std::chrono::steady_clock::now(); start_time = std::chrono::steady_clock::now();
GetData(db, lats); GetData(db, lats);
end_time = std::chrono::steady_clock::now(); end_time = std::chrono::steady_clock::now();
duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "Throughput of Get(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl; // std::cout << "Throughput of Get(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
std::cout << "Throughput of Get(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl; std::cout << "Throughput of Get(): " << std::fixed << std::setprecision(2) << std::endl << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl; // << " MB/s" << std::endl << std::endl;
// Iterator() // Iterator()
start_time = std::chrono::steady_clock::now(); start_time = std::chrono::steady_clock::now();
ReadOrdered(db, lats); ReadOrdered(db, lats);
end_time = std::chrono::steady_clock::now(); end_time = std::chrono::steady_clock::now();
duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "Throughput of Iterator(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl; // std::cout << "Throughput of Iterator(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
std::cout << "Throughput of Iterator(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl; std::cout << "Throughput of Iterator(): " << std::fixed << std::setprecision(2) << std::endl << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl; // << " MB/s" << std::endl << std::endl;
// FindKeysbyField() // FindKeysbyField()
start_time = std::chrono::steady_clock::now(); start_time = std::chrono::steady_clock::now();
SearchField(db, lats); SearchField(db, lats);
end_time = std::chrono::steady_clock::now(); end_time = std::chrono::steady_clock::now();
duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "Throughput of FindKeysbyField(): " << std::setprecision(3) << search_ * 1e6 / duration << " ops/s" << std::endl << std::endl; std::cout << "Throughput of FindKeysbyField(): " << std::fixed << std::setprecision(2) << std::endl << search_ * 1e6 / duration << std::endl; // << " ops/s" << std::endl << std::endl;
delete db; delete db;
} }
@ -230,33 +229,59 @@ TEST(TestBench, Latency) {
double put_avg = std::get<0>(put_latency); double put_avg = std::get<0>(put_latency);
double put_p75 = std::get<1>(put_latency); double put_p75 = std::get<1>(put_latency);
double put_p99 = std::get<2>(put_latency); double put_p99 = std::get<2>(put_latency);
std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << put_avg * 1e-3 << " micros/op, " << put_p75 * 1e-3 << " micros/op, " << put_p99 * 1e-3 << " micros/op" << std::endl << std::endl; std::cout << "Put Latency (avg, P75, P99): " << std::fixed << std::endl << std::setprecision(2) << put_avg * 1e-3 // << " micros/op, "
<< std::endl << put_p75 * 1e-3 // << " micros/op, "
<< std::endl << put_p99 * 1e-3 //<< " micros/op"
<< std::endl << std::endl;
GetData(db, get_lats); GetData(db, get_lats);
std::tuple<double, double, double> get_latency = calc_lat(get_lats); std::tuple<double, double, double> get_latency = calc_lat(get_lats);
double get_avg = std::get<0>(get_latency); double get_avg = std::get<0>(get_latency);
double get_p75 = std::get<1>(get_latency); double get_p75 = std::get<1>(get_latency);
double get_p99 = std::get<2>(get_latency); double get_p99 = std::get<2>(get_latency);
std::cout << "Get Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << get_avg * 1e-3 << " micros/op, " << get_p75 * 1e-3 << " micros/op, " << get_p99 * 1e-3 << " micros/op" << std::endl << std::endl; std::cout << "Put Latency (avg, P75, P99): " << std::fixed << std::endl << std::setprecision(2) << get_avg * 1e-3 // << " micros/op, "
<< std::endl << get_p75 * 1e-3 // << " micros/op, "
<< std::endl << get_p99 * 1e-3 //<< " micros/op"
<< std::endl << std::endl;
ReadOrdered(db, iter_lats); ReadOrdered(db, iter_lats);
std::tuple<double, double, double> iter_latency = calc_lat(iter_lats); std::tuple<double, double, double> iter_latency = calc_lat(iter_lats);
double iter_avg = std::get<0>(iter_latency); double iter_avg = std::get<0>(iter_latency);
double iter_p75 = std::get<1>(iter_latency); double iter_p75 = std::get<1>(iter_latency);
double iter_p99 = std::get<2>(iter_latency); double iter_p99 = std::get<2>(iter_latency);
std::cout << "Iterator Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << iter_avg * 1e-3 << " micros/op, " << iter_p75 * 1e-3 << " micros/op, " << iter_p99 * 1e-3 << " micros/op" << std::endl << std::endl; std::cout << "Put Latency (avg, P75, P99): " << std::fixed << std::endl << std::setprecision(2) << iter_avg * 1e-3 // << " micros/op, "
<< std::endl << iter_p75 * 1e-3 // << " micros/op, "
<< std::endl << iter_p99 * 1e-3 //<< " micros/op"
<< std::endl << std::endl;
SearchField(db, search_lats); SearchField(db, search_lats);
std::tuple<double, double, double> search_latency = calc_lat(search_lats); std::tuple<double, double, double> search_latency = calc_lat(search_lats);
double search_avg = std::get<0>(search_latency); double search_avg = std::get<0>(search_latency);
double search_p75 = std::get<1>(search_latency); double search_p75 = std::get<1>(search_latency);
double search_p99 = std::get<2>(search_latency); double search_p99 = std::get<2>(search_latency);
std::cout << "FindKeysByField Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << search_avg * 1e-3 << " micros/op, " << search_p75 * 1e-3 << " micros/op, " << search_p99 * 1e-3 << " micros/op" << std::endl << std::endl; std::cout << "Put Latency (avg, P75, P99): " << std::fixed << std::endl << std::setprecision(2) << search_avg * 1e-3 // << " micros/op, "
<< std::endl << search_p75 * 1e-3 // << " micros/op, "
<< std::endl << search_p99 * 1e-3 //<< " micros/op"
<< std::endl << std::endl;
delete db; delete db;
} }
TEST(TestBench, Write) {
DB *db;
if(OpenDB("testdb", &db).ok() == false) {
std::cerr << "open db failed" << std::endl;
abort();
}
std::vector<int64_t> lats;
InsertData(db, lats);
std::cout << "Write Size: " << bytes_ << " bytes altogether";
delete db;
}
int main(int argc, char **argv) { int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv); testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();

+ 40
- 25
test/test_bench_gc.cc 查看文件

@ -33,7 +33,7 @@ void InsertData(DB *db, std::vector &lats) {
int64_t bytes = 0; int64_t bytes = 0;
srand(0); srand(0);
std::mt19937 value_seed(100); std::mt19937 value_seed(100);
std::uniform_int_distribution<int> value_range(10, 2048); std::uniform_int_distribution<int> value_range(500, 2048);
int64_t latency = 0; int64_t latency = 0;
auto end_time = std::chrono::steady_clock::now(); auto end_time = std::chrono::steady_clock::now();
auto last_time = end_time; auto last_time = end_time;
@ -148,17 +148,17 @@ void SearchField(DB *db, std::vector &lats) {
} }
} }
// // Insert many k/vs in order to start background GC // Insert many k/vs in order to start background GC
// void InsertMany(DB *db) { void InsertMany(DB *db) {
// std::vector<int64_t> lats; std::vector<int64_t> lats;
// for (int i = 0; i < 10; i++) { for (int i = 0; i < 2; i++) {
// InsertData(db, lats); InsertData(db, lats);
// GetData(db, lats); GetData(db, lats);
// db->CompactRange(nullptr, nullptr); db->CompactRange(nullptr, nullptr);
// std::cout << "put and get " << i << " of Many" << std::endl; std::cout << "put and get " << i << " of Many" << std::endl;
// } }
// } }
double CalculatePercentile(const std::vector<int64_t>& latencies, double percentile) { double CalculatePercentile(const std::vector<int64_t>& latencies, double percentile) {
if (latencies.empty()) return 0.0; if (latencies.empty()) return 0.0;
@ -197,61 +197,76 @@ TEST(TestBench, WithGC) {
return std::make_tuple(avg, p75, p99); return std::make_tuple(avg, p75, p99);
}; };
InsertMany(db);
// Put() // Put()
auto start_time = std::chrono::steady_clock::now(); auto start_time = std::chrono::steady_clock::now();
InsertData(db, put_lats); InsertData(db, put_lats);
auto end_time = std::chrono::steady_clock::now(); auto end_time = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "Throughput of Put(): " << std::fixed << num_ * 1e6 / duration << " ops/s" << std::endl; // std::cout << "Throughput of Put(): " << std::fixed << num_ * 1e6 / duration << " ops/s" << std::endl;
std::cout << "Throughput of Put(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl; std::cout << std::endl << "Write size: " << bytes_ << " bytes" << std::endl << std::endl;
std::cout << "Throughput of Put(): " << std::fixed << std::setprecision(2) << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl << std::endl; // << " MB/s" << std::endl << std::endl;
std::tuple<double, double, double> put_latency = calc_lat(put_lats); std::tuple<double, double, double> put_latency = calc_lat(put_lats);
double put_avg = std::get<0>(put_latency); double put_avg = std::get<0>(put_latency);
double put_p75 = std::get<1>(put_latency); double put_p75 = std::get<1>(put_latency);
double put_p99 = std::get<2>(put_latency); double put_p99 = std::get<2>(put_latency);
std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << put_avg * 1e-3 << " micros/op, " << put_p75 * 1e-3 << " micros/op, " << put_p99 * 1e-3 << " micros/op" << std::endl << std::endl; std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(2) << put_avg * 1e-3 // << " micros/op, "
<< std::endl << put_p75 * 1e-3 // << " micros/op, "
<< std::endl << put_p99 * 1e-3 // << " micros/op"
<< std::endl << std::endl;
// Get() // Get()
start_time = std::chrono::steady_clock::now(); start_time = std::chrono::steady_clock::now();
GetData(db, get_lats); GetData(db, get_lats);
end_time = std::chrono::steady_clock::now(); end_time = std::chrono::steady_clock::now();
duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "Throughput of Get(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl; // std::cout << "Throughput of Get(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
std::cout << "Throughput of Get(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl; std::cout << "Throughput of Get(): " << std::fixed << std::setprecision(2) << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl << std::endl; // << " MB/s" << std::endl << std::endl;
std::tuple<double, double, double> get_latency = calc_lat(get_lats); std::tuple<double, double, double> get_latency = calc_lat(get_lats);
double get_avg = std::get<0>(get_latency); double get_avg = std::get<0>(get_latency);
double get_p75 = std::get<1>(get_latency); double get_p75 = std::get<1>(get_latency);
double get_p99 = std::get<2>(get_latency); double get_p99 = std::get<2>(get_latency);
std::cout << "Get Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << get_avg * 1e-3 << " micros/op, " << get_p75 * 1e-3 << " micros/op, " << get_p99 * 1e-3 << " micros/op" << std::endl << std::endl; std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(2) << get_avg * 1e-3 // << " micros/op, "
<< std::endl << get_p75 * 1e-3 // << " micros/op, "
<< std::endl << get_p99 * 1e-3 // << " micros/op"
<< std::endl << std::endl;
// Iterator() // Iterator()
start_time = std::chrono::steady_clock::now(); start_time = std::chrono::steady_clock::now();
ReadOrdered(db, iter_lats); ReadOrdered(db, iter_lats);
end_time = std::chrono::steady_clock::now(); end_time = std::chrono::steady_clock::now();
duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "Throughput of Iterator(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl; // std::cout << "Throughput of Iterator(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
std::cout << "Throughput of Iterator(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl; std::cout << "Throughput of Iterator(): " << std::fixed << std::setprecision(2) << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl << std::endl; // << " MB/s" << std::endl << std::endl;
std::tuple<double, double, double> iter_latency = calc_lat(iter_lats); std::tuple<double, double, double> iter_latency = calc_lat(iter_lats);
double iter_avg = std::get<0>(iter_latency); double iter_avg = std::get<0>(iter_latency);
double iter_p75 = std::get<1>(iter_latency); double iter_p75 = std::get<1>(iter_latency);
double iter_p99 = std::get<2>(iter_latency); double iter_p99 = std::get<2>(iter_latency);
std::cout << "Iterator Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << iter_avg * 1e-3 << " micros/op, " << iter_p75 * 1e-3 << " micros/op, " << iter_p99 * 1e-3 << " micros/op" << std::endl << std::endl; std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(2) << iter_avg * 1e-3 // << " micros/op, "
<< std::endl << iter_p75 * 1e-3 // << " micros/op, "
<< std::endl << iter_p99 * 1e-3 // << " micros/op"
<< std::endl << std::endl;
// FindKeysByField() // FindKeysByField()
start_time = std::chrono::steady_clock::now(); start_time = std::chrono::steady_clock::now();
SearchField(db, search_lats); SearchField(db, search_lats);
end_time = std::chrono::steady_clock::now(); end_time = std::chrono::steady_clock::now();
duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count(); duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
std::cout << "Throughput of FindKeysbyField(): " << std::setprecision(3) << search_ * 1e6 / duration << " ops/s" << std::endl << std::endl; std::cout << "Throughput of FindKeysbyField(): " << std::setprecision(2) << search_ * 1e6 / duration << std::endl << std::endl; // << " ops/s" << std::endl << std::endl;
std::tuple<double, double, double> search_latency = calc_lat(search_lats); std::tuple<double, double, double> search_latency = calc_lat(search_lats);
double search_avg = std::get<0>(search_latency); double search_avg = std::get<0>(search_latency);
double search_p75 = std::get<1>(search_latency); double search_p75 = std::get<1>(search_latency);
double search_p99 = std::get<2>(search_latency); double search_p99 = std::get<2>(search_latency);
std::cout << "FindKeysByField Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << search_avg * 1e-3 << " micros/op, " << search_p75 * 1e-3 << " micros/op, " << search_p99 * 1e-3 << " micros/op" << std::endl << std::endl; std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(2) << search_avg * 1e-3 // << " micros/op, "
<< std::endl << search_p75 * 1e-3 // << " micros/op, "
<< std::endl << search_p99 * 1e-3 // << " micros/op"
<< std::endl << std::endl;
delete db; delete db;
} }

撰写 预览
||||||
​x
 
​
000:0
正在加载...
取消
保存