testing

include/leveldb/options.h

@@ -101,7 +101,7 @@ struct LEVELDB_EXPORT Options {
   size_t block_size = 4 * 1024;
 
   // Threshold of value size that decide whether to separate the key and value
-  size_t kv_sep_size = 1;
+  size_t kv_sep_size = 1000;
 
   // Number of keys between restart points for delta encoding of keys.
   // This parameter can be changed dynamically.  Most clients should
@@ -118,7 +118,7 @@ struct LEVELDB_EXPORT Options {
   // initially populating a large database.
   size_t max_file_size = 2 * 1024 * 1024;
 
-  size_t gc_size_threshold = 1024 * 1024 * 1024;
+  size_t gc_size_threshold = 128 * 1024 * 1024;
 
   // Compress blocks using the specified compression algorithm.  This
   // parameter can be changed dynamically.

table/vtable_manager.cc

@@ -181,7 +181,7 @@ void VTableManager::BackgroudGC(void* gc_info) {
   }
   auto end_time = std::chrono::steady_clock::now();
   auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "gc lasts: " << duration << "micros" << std::endl;
+  std::cout << "gc lasts: " << duration << " micros" << std::endl;
 }
 
 void VTableManager::RefVTable(uint64_t file_num) {

test/test_bench.cc

@@ -11,11 +11,11 @@ using namespace leveldb;
 // Number of key/values to operate in database
 constexpr int num_ = 100000;
 // Size of each value
-constexpr int value_size_ = 1000;
+constexpr int value_size_ = 2000;
 // Number of read operations
 constexpr int reads_ = 100000;
 // Number of findkeysbyfield operations
-constexpr int search_ = 20;
+constexpr int search_ = 10;
 
 int64_t bytes_ = 0;
 
@@ -32,8 +32,8 @@ void InsertData(DB *db, std::vector &lats) {
   bytes_ = 0;
   int64_t bytes = 0;
   srand(0);
-  std::mt19937 value_seed(100);
-  std::uniform_int_distribution<int> value_range(10, 2048);
+  // std::mt19937 value_seed(100);
+  // std::uniform_int_distribution<int> value_range(500, 2048);
   int64_t latency = 0;
   auto end_time = std::chrono::steady_clock::now();
   auto last_time = end_time;
@@ -41,8 +41,8 @@ void InsertData(DB *db, std::vector &lats) {
   for (int i = 0; i < num_; i++) {
     int key_ = rand() % num_+1;
     int value_ = std::rand() % (num_ + 1);
-    int value_size = value_range(value_seed);
-    std::string value(value_size, 'a');
+    // int value_size = value_range(value_seed);
+    std::string value(value_size_, 'a');
     std::string key = std::to_string(key_);
     FieldArray field_array = {
       {"1", value},
@@ -174,29 +174,28 @@ TEST(TestBench, Throughput) {
   InsertData(db, lats);
   auto end_time = std::chrono::steady_clock::now();
   auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "Throughput of Put(): " << std::fixed << num_ * 1e6 / duration << " ops/s" << std::endl;
-  std::cout << "Throughput of Put(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl;
-  
+  // std::cout << "Throughput of Put(): " << std::fixed << num_ * 1e6 / duration << " ops/s" << std::endl;
+  std::cout << "Throughput of Put(): " << std::fixed << std::setprecision(2) << std::endl << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl; // << " MB/s" << std::endl << std::endl;
   // Get()
   start_time = std::chrono::steady_clock::now();
   GetData(db, lats);
   end_time = std::chrono::steady_clock::now();
   duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "Throughput of Get(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
-  std::cout << "Throughput of Get(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl;
+  // std::cout << "Throughput of Get(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
+  std::cout << "Throughput of Get(): " << std::fixed << std::setprecision(2) << std::endl << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl; // << " MB/s" << std::endl << std::endl;
   // Iterator()
   start_time = std::chrono::steady_clock::now();
   ReadOrdered(db, lats);
   end_time = std::chrono::steady_clock::now();
   duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "Throughput of Iterator(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
-  std::cout << "Throughput of Iterator(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl;
+  // std::cout << "Throughput of Iterator(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
+  std::cout << "Throughput of Iterator(): " << std::fixed << std::setprecision(2) << std::endl << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl; // << " MB/s" << std::endl << std::endl;
   // FindKeysbyField()
   start_time = std::chrono::steady_clock::now();
   SearchField(db, lats);
   end_time = std::chrono::steady_clock::now();
   duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "Throughput of FindKeysbyField(): " << std::setprecision(3) << search_ * 1e6 / duration << " ops/s" << std::endl << std::endl;
+  std::cout << "Throughput of FindKeysbyField(): " << std::fixed << std::setprecision(2) << std::endl << search_ * 1e6 / duration << std::endl; // << " ops/s" << std::endl << std::endl;
   
   delete db;
 }
@@ -230,33 +229,59 @@ TEST(TestBench, Latency) {
   double put_avg = std::get<0>(put_latency);
   double put_p75 = std::get<1>(put_latency);
   double put_p99 = std::get<2>(put_latency);
-  std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << put_avg * 1e-3 << " micros/op, " << put_p75 * 1e-3 << " micros/op, " << put_p99 * 1e-3 << " micros/op" << std::endl << std::endl;
+  std::cout << "Put Latency (avg, P75, P99): " << std::fixed << std::endl << std::setprecision(2) << put_avg * 1e-3 // << " micros/op, " 
+   << std::endl << put_p75 * 1e-3 // << " micros/op, " 
+   << std::endl << put_p99 * 1e-3 //<< " micros/op" 
+   << std::endl << std::endl;
 
   GetData(db, get_lats);
   std::tuple<double, double, double> get_latency = calc_lat(get_lats);
   double get_avg = std::get<0>(get_latency);
   double get_p75 = std::get<1>(get_latency);
   double get_p99 = std::get<2>(get_latency);
-  std::cout << "Get Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << get_avg * 1e-3 << " micros/op, " << get_p75 * 1e-3 << " micros/op, " << get_p99 * 1e-3 << " micros/op" << std::endl << std::endl; 
+  std::cout << "Put Latency (avg, P75, P99): " << std::fixed << std::endl << std::setprecision(2) << get_avg * 1e-3 // << " micros/op, " 
+   << std::endl << get_p75 * 1e-3 // << " micros/op, " 
+   << std::endl << get_p99 * 1e-3 //<< " micros/op" 
+   << std::endl << std::endl;
 
   ReadOrdered(db, iter_lats);
   std::tuple<double, double, double> iter_latency = calc_lat(iter_lats);
   double iter_avg = std::get<0>(iter_latency);
   double iter_p75 = std::get<1>(iter_latency);
   double iter_p99 = std::get<2>(iter_latency);
-  std::cout << "Iterator Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << iter_avg * 1e-3 << " micros/op, " << iter_p75 * 1e-3 << " micros/op, " << iter_p99 * 1e-3 << " micros/op" << std::endl << std::endl;
+  std::cout << "Put Latency (avg, P75, P99): " << std::fixed << std::endl << std::setprecision(2) << iter_avg * 1e-3 // << " micros/op, " 
+   << std::endl << iter_p75 * 1e-3 // << " micros/op, " 
+   << std::endl << iter_p99 * 1e-3 //<< " micros/op" 
+   << std::endl << std::endl;
 
   SearchField(db, search_lats);
   std::tuple<double, double, double> search_latency = calc_lat(search_lats);
   double search_avg = std::get<0>(search_latency);
   double search_p75 = std::get<1>(search_latency);
   double search_p99 = std::get<2>(search_latency);
-  std::cout << "FindKeysByField Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << search_avg * 1e-3 << " micros/op, " << search_p75 * 1e-3 << " micros/op, " << search_p99 * 1e-3 << " micros/op" << std::endl << std::endl; 
-  
+  std::cout << "Put Latency (avg, P75, P99): " << std::fixed << std::endl << std::setprecision(2) << search_avg * 1e-3 // << " micros/op, " 
+   << std::endl << search_p75 * 1e-3 // << " micros/op, " 
+   << std::endl << search_p99 * 1e-3 //<< " micros/op" 
+   << std::endl << std::endl;
+
   delete db;
 
 }
 
+TEST(TestBench, Write) {
+  DB *db;
+  if(OpenDB("testdb", &db).ok() == false) {
+    std::cerr << "open db failed" << std::endl;
+    abort();
+  }
+
+  std::vector<int64_t> lats;
+
+  InsertData(db, lats);
+  std::cout << "Write Size: " << bytes_ << " bytes altogether";
+  
+  delete db;
+}
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();

test/test_bench_gc.cc

@@ -33,7 +33,7 @@ void InsertData(DB *db, std::vector &lats) {
   int64_t bytes = 0;
   srand(0);
   std::mt19937 value_seed(100);
-  std::uniform_int_distribution<int> value_range(10, 2048);
+  std::uniform_int_distribution<int> value_range(500, 2048);
   int64_t latency = 0;
   auto end_time = std::chrono::steady_clock::now();
   auto last_time = end_time;
@@ -148,17 +148,17 @@ void SearchField(DB *db, std::vector &lats) {
   }
 }
 
-// // Insert many k/vs in order to start background GC
-// void InsertMany(DB *db) {
-//   std::vector<int64_t> lats;
-//   for (int i = 0; i < 10; i++) {
-//     InsertData(db, lats);
+// Insert many k/vs in order to start background GC
+void InsertMany(DB *db) {
+  std::vector<int64_t> lats;
+  for (int i = 0; i < 2; i++) {
+    InsertData(db, lats);
     
-//     GetData(db, lats);
-//     db->CompactRange(nullptr, nullptr);
-//     std::cout << "put and get " << i << " of Many" << std::endl;
-//   }
-// }
+    GetData(db, lats);
+    db->CompactRange(nullptr, nullptr);
+    std::cout << "put and get " << i << " of Many" << std::endl;
+  }
+}
 
 double CalculatePercentile(const std::vector<int64_t>& latencies, double percentile) {
   if (latencies.empty()) return 0.0;
@@ -197,61 +197,76 @@ TEST(TestBench, WithGC) {
     return std::make_tuple(avg, p75, p99);
   };
 
+  InsertMany(db);
+
   // Put()
   auto start_time = std::chrono::steady_clock::now();
   InsertData(db, put_lats);
   auto end_time = std::chrono::steady_clock::now();
   auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "Throughput of Put(): " << std::fixed << num_ * 1e6 / duration << " ops/s" << std::endl;
-  std::cout << "Throughput of Put(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl;
+  // std::cout << "Throughput of Put(): " << std::fixed << num_ * 1e6 / duration << " ops/s" << std::endl;
+
+  std::cout << std::endl << "Write size: " << bytes_ << " bytes" << std::endl << std::endl;
+  std::cout << "Throughput of Put(): " << std::fixed << std::setprecision(2) << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl << std::endl; // << " MB/s" << std::endl << std::endl;
   
   std::tuple<double, double, double> put_latency = calc_lat(put_lats);
   double put_avg = std::get<0>(put_latency);
   double put_p75 = std::get<1>(put_latency);
   double put_p99 = std::get<2>(put_latency);
-  std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << put_avg * 1e-3 << " micros/op, " << put_p75 * 1e-3 << " micros/op, " << put_p99 * 1e-3 << " micros/op" << std::endl << std::endl;
+  std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(2) << put_avg * 1e-3 // << " micros/op, " 
+  << std::endl << put_p75 * 1e-3 // << " micros/op, " 
+  << std::endl << put_p99 * 1e-3 // << " micros/op" 
+  << std::endl << std::endl;
   
   // Get()
   start_time = std::chrono::steady_clock::now();
   GetData(db, get_lats);
   end_time = std::chrono::steady_clock::now();
   duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "Throughput of Get(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
-  std::cout << "Throughput of Get(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl;
+  // std::cout << "Throughput of Get(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
+  std::cout << "Throughput of Get(): " << std::fixed << std::setprecision(2) << (bytes_ / 1048576.0) / (duration * 1e-6)
 << std::endl << std::endl; // << " MB/s" << std::endl << std::endl;
 
   std::tuple<double, double, double> get_latency = calc_lat(get_lats);
   double get_avg = std::get<0>(get_latency);
   double get_p75 = std::get<1>(get_latency);
   double get_p99 = std::get<2>(get_latency);
-  std::cout << "Get Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << get_avg * 1e-3 << " micros/op, " << get_p75 * 1e-3 << " micros/op, " << get_p99 * 1e-3 << " micros/op" << std::endl << std::endl; 
+  std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(2) << get_avg * 1e-3 // << " micros/op, " 
+  << std::endl << get_p75 * 1e-3 // << " micros/op, " 
+  << std::endl << get_p99 * 1e-3 // << " micros/op" 
+  << std::endl << std::endl;
 
   // Iterator()
   start_time = std::chrono::steady_clock::now();
   ReadOrdered(db, iter_lats);
   end_time = std::chrono::steady_clock::now();
   duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "Throughput of Iterator(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
-  std::cout << "Throughput of Iterator(): " << std::setprecision(3) << (bytes_ / 1048576.0) / (duration * 1e-6) << " MB/s" << std::endl << std::endl;
-
+  // std::cout << "Throughput of Iterator(): " << std::fixed << reads_ * 1e6 / duration << " ops/s" << std::endl;
+  std::cout << "Throughput of Iterator(): " << std::fixed << std::setprecision(2) << (bytes_ / 1048576.0) / (duration * 1e-6) << std::endl << std::endl; // << " MB/s" << std::endl << std::endl;
   std::tuple<double, double, double> iter_latency = calc_lat(iter_lats);
   double iter_avg = std::get<0>(iter_latency);
   double iter_p75 = std::get<1>(iter_latency);
   double iter_p99 = std::get<2>(iter_latency);
-  std::cout << "Iterator Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << iter_avg * 1e-3 << " micros/op, " << iter_p75 * 1e-3 << " micros/op, " << iter_p99 * 1e-3 << " micros/op" << std::endl << std::endl;
-  
+  std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(2) << iter_avg * 1e-3 // << " micros/op, " 
+  << std::endl << iter_p75 * 1e-3 // << " micros/op, " 
+  << std::endl << iter_p99 * 1e-3 // << " micros/op" 
+  << std::endl << std::endl;
+
   // FindKeysByField()
   start_time = std::chrono::steady_clock::now();
   SearchField(db, search_lats);
   end_time = std::chrono::steady_clock::now();
   duration = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
-  std::cout << "Throughput of FindKeysbyField(): " << std::setprecision(3) << search_ * 1e6 / duration << " ops/s" << std::endl << std::endl;
+  std::cout << "Throughput of FindKeysbyField(): " << std::setprecision(2) << search_ * 1e6 / duration << std::endl << std::endl; // << " ops/s" << std::endl << std::endl;
 
   std::tuple<double, double, double> search_latency = calc_lat(search_lats);
   double search_avg = std::get<0>(search_latency);
   double search_p75 = std::get<1>(search_latency);
   double search_p99 = std::get<2>(search_latency);
-  std::cout << "FindKeysByField Latency (avg, P75, P99): " << std::endl << std::setprecision(3) << search_avg * 1e-3 << " micros/op, " << search_p75 * 1e-3 << " micros/op, " << search_p99 * 1e-3 << " micros/op" << std::endl << std::endl; 
-  
+  std::cout << "Put Latency (avg, P75, P99): " << std::endl << std::setprecision(2) << search_avg * 1e-3 // << " micros/op, " 
+  << std::endl << search_p75 * 1e-3 // << " micros/op, " 
+  << std::endl << search_p99 * 1e-3 // << " micros/op" 
+  << std::endl << std::endl;
+
   delete db;
 }