roughly completed

benchmarks/db_bench.cc

@@ -20,6 +20,7 @@
 #include "util/mutexlock.h"
 #include "util/random.h"
 #include "util/testutil.h"
+#include "db/fields.h"
 
 // Comma-separated list of operations to run in the specified order
 //   Actual benchmarks:
@@ -55,14 +56,15 @@ static const char* FLAGS_benchmarks =
     "readreverse,"
     "compact,"
     "readrandom,"
+    "findkeysbyfield,"
     "readseq,"
     "readreverse,"
-    "fill100K,"
-    "crc32c,"
-    "snappycomp,"
-    "snappyuncomp,"
-    "zstdcomp,"
-    "zstduncomp,";
+    "fill100K,";
+// "crc32c,"
+// "snappycomp,"
+// "snappyuncomp,"
+// "zstdcomp,"
+// "zstduncomp,";
 
 // Number of key/values to place in database
 static int FLAGS_num = 1000000;
@@ -70,6 +72,9 @@ static int FLAGS_num = 1000000;
 // Number of read operations to do.  If negative, do FLAGS_num reads.
 static int FLAGS_reads = -1;
 
+// Number of given fields used in FindKeysByField test. If negative, write in half of FLAGS_num targets with given field.
+static int FLAGS_num_fields = 80000;
+
 // Number of concurrent threads to run.
 static int FLAGS_threads = 1;
 
@@ -124,7 +129,7 @@ static bool FLAGS_reuse_logs = false;
 static bool FLAGS_compression = true;
 
 // Use the db with the following name.
-static const char* FLAGS_db = nullptr;
+static const char* FLAGS_db = "benchmark_db";
 
 // ZSTD compression level to try out
 static int FLAGS_zstd_compression_level = 1;
@@ -438,6 +443,7 @@ class Benchmark {
   int heap_counter_;
   CountComparator count_comparator_;
   int total_thread_count_;
+  int num_fields; // 插入的fields数量
 
   void PrintHeader() {
     const int kKeySize = 16 + FLAGS_key_prefix;
@@ -530,7 +536,8 @@ class Benchmark {
         reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
         heap_counter_(0),
         count_comparator_(BytewiseComparator()),
-        total_thread_count_(0) {
+        total_thread_count_(0),
+        num_fields(FLAGS_num_fields < 0 ? FLAGS_num / 2 : FLAGS_num_fields) {
     std::vector<std::string> files;
     g_env->GetChildren(FLAGS_db, &files);
     for (size_t i = 0; i < files.size(); i++) {
@@ -615,6 +622,8 @@ class Benchmark {
         method = &Benchmark::SeekRandom;
       } else if (name == Slice("seekordered")) {
         method = &Benchmark::SeekOrdered;
+      } else if (name == Slice("findkeysbyfield")) {
+        method = &Benchmark::FindKeysByField;
       } else if (name == Slice("readhot")) {
         method = &Benchmark::ReadHot;
       } else if (name == Slice("readrandomsmall")) {
@@ -852,8 +861,11 @@ class Benchmark {
       for (int j = 0; j < entries_per_batch_; j++) {
         const int k = seq ? i + j : thread->rand.Uniform(FLAGS_num);
         key.Set(k);
-        batch.Put(key.slice(), gen.Generate(value_size_));
-        bytes += value_size_ + key.slice().size();
+        FieldArray fields = {{"field1", "value1_" + std::to_string(i)}, {"field2", "value2_"}};
+        Fields ffields(fields);
+        db_->PutFields(WriteOptions(), key.slice(), ffields);
+        // batch.Put(key.slice(), gen.Generate(value_size_));
+        bytes += ffields.size() + key.slice().size();
         thread->stats.FinishedSingleOp();
       }
       s = db_->Write(write_options_, &batch);
@@ -935,6 +947,72 @@ class Benchmark {
     }
   }
 
+  void WriteTargetSeq(ThreadState* thread) { WriteGiven(thread, true); }
+
+  void WriteTargetRandom(ThreadState* thread) { WriteGiven(thread, false); }
+
+  void WriteGiven(ThreadState* thread, bool seq) {
+    if (num_ != FLAGS_num) {
+      char msg[100];
+      std::snprintf(msg, sizeof(msg), "(%d ops)", num_);
+      thread->stats.AddMessage(msg);
+    }
+
+    RandomGenerator gen;
+    WriteBatch batch;
+    Status s;
+    int64_t bytes = 0;
+    KeyBuffer key;
+    for (int i = 0; i < num_; i += entries_per_batch_) {
+      batch.Clear();
+      for (int j = 0; j < entries_per_batch_; j++) {
+        const int k = seq ? i + j : thread->rand.Uniform(FLAGS_num);
+        key.Set(k);
+
+        FieldArray fields;
+        auto value = gen.Generate(value_size_);
+        if (i < num_fields) {
+          fields = {
+              {"field1", value.ToString()},
+              {"field2", "value2_"},
+          };
+        } else {
+          fields = {
+              {"field1", value.ToString()},
+          };
+        }
+
+        Fields ffields(fields);
+        db_->PutFields(WriteOptions(), key.slice(), ffields);
+        bytes += ffields.size() + key.slice().size();
+
+        thread->stats.FinishedSingleOp();
+      }
+      s = db_->Write(write_options_, &batch);
+      if (!s.ok()) {
+        std::fprintf(stderr, "put error: %s\n", s.ToString().c_str());
+        std::exit(1);
+      }
+    }
+    thread->stats.AddBytes(bytes);
+  }
+
+  void FindKeysByField(ThreadState* thread){
+    int found = 0;
+    FieldArray fields_to_find = {{"field2", "value2_"}};
+
+    std::string dbname_ = "benchmark_db";
+    Options options;
+    options.create_if_missing = true;
+    DBImpl* impl = new DBImpl(options, dbname_);
+
+    std::vector<std::string> found_keys = Fields::FindKeysByFields(db_, fields_to_find, impl);
+    found = found_keys.size();
+    char msg[100];
+    snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_fields);
+    thread->stats.AddMessage(msg);
+  }
+
   void SeekRandom(ThreadState* thread) {
     ReadOptions options;
     int found = 0;
@@ -1097,6 +1175,8 @@ int main(int argc, char** argv) {
       FLAGS_num = n;
     } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
       FLAGS_reads = n;
+    } else if (sscanf(argv[i], "--num_fields=%d%c", &n, &junk) == 1) {
+      FLAGS_num_fields = n;
     } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) {
       FLAGS_threads = n;
     } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {

test/bench_test.cc

@@ -42,7 +42,7 @@ void InsertFields(DB *db, std::vector &lats) {
   for (int i = 0; i < num_; ++i) {
     int key_ = rand() % num_ + 1;
     std::string key = std::to_string(key_);
-    FieldArray fields = {{"field" + std::to_string(key_), "old_value_" + std::to_string(key)}};
+    FieldArray fields = {{"field" + std::to_string(key_), "old_value_" + std::to_string(key_)}};
     Fields f(fields);
     auto start_time = std::chrono::steady_clock::now();
     db->PutFields(writeOptions, Slice(key), f);
@@ -84,7 +84,14 @@ void FindKeys(DB *db, std::vector &lats) {
     int key_ = rand() % num_ + 1;
     FieldArray fields_to_find = {{"field" + std::to_string(key_), "old_value_" + std::to_string(key_)}};
     auto start_time = std::chrono::steady_clock::now();
-    Fields::FindKeysByFields(db, fields_to_find);
+
+    std::string dbname_ = "benchmark_db";
+    Options options;
+    options.create_if_missing = true;
+    DBImpl* impl = new DBImpl(options, dbname_);
+
+
+    Fields::FindKeysByFields(db, fields_to_find, impl);
     auto end_time = std::chrono::steady_clock::now();
     lats.emplace_back(std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count());
   }
@@ -102,14 +109,29 @@ double CalculatePercentile(const std::vector& latencies, double percent
   return sorted_latencies[index];
 }
 
+void SetupData(DB *db) {
+  std::vector<int64_t> lats;
+  InsertData(db, lats);
+}
+
+void SetupFields(DB *db) {
+  std::vector<int64_t> lats;
+  InsertFields(db, lats);
+}
+
 template<typename Func>
-void RunBenchmark(const char* name, Func func) {
+void RunBenchmark(const char* name, Func func, bool setup_data = true, bool setup_fields = false) {
   DB *db;
+  std::string rm_command = "rm -rf testdb_bench";
+  system(rm_command.c_str());
   if (!OpenDB("testdb_bench", &db).ok()) {
     std::cerr << "open db failed" << std::endl;
     abort();
   }
 
+  if (setup_data) SetupData(db);
+  if (setup_fields) SetupFields(db);
+
   std::vector<int64_t> lats;
   auto start_time = std::chrono::steady_clock::now();
   func(db, lats);
@@ -131,14 +153,15 @@ void RunBenchmark(const char* name, Func func) {
   delete db;
 }
 
-class BenchTest : public ::testing::TestWithParam<double> {};
+// TEST(BenchTest, PutLatency) { RunBenchmark("Put", InsertData, false, false); }
+// TEST(BenchTest, PutFieldsLatency) { RunBenchmark("PutFields", InsertFields, false, false); }
 
-TEST_P(BenchTest, PutLatency) { RunBenchmark("Put", InsertData); }
-TEST_P(BenchTest, PutLatency) { RunBenchmark("PutFields", InsertFields); }
-TEST_P(BenchTest, GetLatency) { RunBenchmark("Get", GetData); }
-TEST_P(BenchTest, IteratorLatency) { RunBenchmark("Iterator", ReadOrdered); }
-TEST_P(BenchTest, FindKeysByFieldLatency) { RunBenchmark("FindKeysByFields", FindKeys); } 
+// TEST(BenchTest, GetLatency) { RunBenchmark("Get", GetData, true, false); }
+// TEST(BenchTest, IteratorLatency) { RunBenchmark("Iterator", ReadOrdered, true, false); }
 
+TEST(BenchTest, FindKeysByFieldLatency) {
+  RunBenchmark("FindKeysByFields", FindKeys, false, true);
+}
 
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);