Merge commit '75426ffb064de60dc9ecf06ca882334abae8738e' into cyq

benchmarks/db_bench_FieldDB.cc

@@ -63,6 +63,22 @@ static const char* FLAGS_benchmarks =
     "readreverse,"
     "fill100K,"
     "crc32c,"
+    "readwhilewriting,"
+    "CreateIndex,"
+    "FindKeysByField,"
+    "QueryByIndex,"
+    "DeleteIndex,"
+    "compact,"
+    "WriteSeqWhileCreating,"
+    "WriteSeqWhileDeleting,"
+    "compact,"
+    "WriteRandomWhileCreating,"
+    "WriteRandomWhileDeleting,"
+    "compact,"
+    "ReadSeqWhileCreating,"
+    "ReadSeqWhileDeleting,"
+    "ReadRandomWhileCreating,"
+    "ReadRandomWhileDeleting,"
     "snappycomp,"
     "snappyuncomp,"
     "zstdcomp,"
@@ -543,7 +559,7 @@ class Benchmark {
       }
     }
     if (!FLAGS_use_existing_db) {
-      DestroyDB(FLAGS_db, Options());
+      fielddb::DestroyDB(FLAGS_db, Options());
     }
   }
 
@@ -635,6 +651,38 @@ class Benchmark {
         method = &Benchmark::Compact;
       } else if (name == Slice("crc32c")) {
         method = &Benchmark::Crc32c;
+      } else if (name == Slice("CreateIndex")) {
+        method = &Benchmark::CreateIndex;
+      } else if (name == Slice("FindKeysByField")) {
+        method = &Benchmark::FindKeysByField;
+      } else if (name == Slice("QueryByIndex")) {
+        method = &Benchmark::QueryByIndex;
+      } else if (name == Slice("DeleteIndex")) {
+        method = &Benchmark::DeleteIndex;
+      } else if (name == Slice("WriteSeqWhileCreating")) {
+        num_threads++;
+        method = &Benchmark::WriteSeqWhileCreating;
+      } else if (name == Slice("WriteSeqWhileDeleting")) {
+        num_threads++;
+        method = &Benchmark::WriteSeqWhileDeleting;
+      } else if (name == Slice("WriteRandomWhileCreating")) {
+        num_threads++;
+        method = &Benchmark::WriteRandomWhileCreating;
+      } else if (name == Slice("WriteRandomWhileDeleting")) {
+        num_threads++;
+        method = &Benchmark::WriteRandomWhileDeleting;
+      } else if (name == Slice("ReadSeqWhileCreating")) {
+        num_threads++;
+        method = &Benchmark::ReadSeqWhileCreating;
+      } else if (name == Slice("ReadSeqWhileDeleting")) {
+        num_threads++;
+        method = &Benchmark::ReadSeqWhileDeleting;
+      } else if (name == Slice("ReadRandomWhileCreating")) {
+        num_threads++;
+        method = &Benchmark::ReadRandomWhileCreating;
+      } else if (name == Slice("ReadRandomWhileDeleting")) {
+        num_threads++;
+        method = &Benchmark::ReadRandomWhileDeleting;
       } else if (name == Slice("snappycomp")) {
         method = &Benchmark::SnappyCompress;
       } else if (name == Slice("snappyuncomp")) {
@@ -664,7 +712,7 @@ class Benchmark {
         } else {
           delete db_;
           db_ = nullptr;
-          //DestroyDB(FLAGS_db, Options());
+          fielddb::DestroyDB(FLAGS_db, Options());
           Open();
         }
       }
@@ -821,7 +869,6 @@ class Benchmark {
     options.compression =
         FLAGS_compression ? kSnappyCompression : kNoCompression;
     // Status s = DB::Open(options, FLAGS_db, &db_);
-    fielddb::DestroyDB(FLAGS_db, options);
     db_ = new FieldDB();
     Status s = FieldDB::OpenFieldDB(options, FLAGS_db, &db_);
     if (!s.ok()) {
@@ -1078,6 +1125,210 @@ class Benchmark {
       g_env->RemoveFile(fname);
     }
   }
+
+  void CreateIndex(ThreadState* thread) {
+    db_->CreateIndexOnField("age", write_options_);
+  }
+
+  void FindKeysByField(ThreadState* thread) {
+    Field f = {"age", "20"};
+    std::vector<std::string> res;
+    res = db_->FindKeysByField(f);
+  }
+
+  void QueryByIndex(ThreadState* thread) {
+    Field f = {"age", "20"};
+    Status s;
+    db_->QueryByIndex(f, &s);    
+  }
+
+  void DeleteIndex(ThreadState* thread) {
+    db_->DeleteIndex("age", write_options_);
+  }
+  
+  void WriteSeqWhileCreating(ThreadState* thread) {
+    if (thread->tid > 0) {
+      WriteSeq(thread);
+    } else {
+      // Special thread that keeps creating index until other threads are done.
+      if (db_->GetIndexStatus("age") != IndexStatus::NotExist) {
+          std::fprintf(stderr, "index status error in WriteWhileCreating\n");
+          std::exit(1);
+      }      
+
+      while (true) {
+        if (db_->GetIndexStatus("age") == IndexStatus::Exist) {
+          break;
+        }
+
+        db_->CreateIndexOnField("age", write_options_);
+      }
+
+      // Do not count any of the preceding work/delay in stats.
+      thread->stats.Start();
+    }
+  }
+
+  void WriteSeqWhileDeleting(ThreadState* thread) {
+    if (thread->tid > 0) {
+      WriteSeq(thread);
+    } else {
+      // Special thread that keeps creating index until other threads are done.
+      if (db_->GetIndexStatus("age") != IndexStatus::Exist) {
+          std::fprintf(stderr, "index status error in WriteWhileDeleting\n");
+          std::exit(1);
+      }      
+
+      while (true) {
+        if (db_->GetIndexStatus("age") == IndexStatus::NotExist) {
+          break;
+        }
+
+        db_->DeleteIndex("age", write_options_);
+      }
+
+      // Do not count any of the preceding work/delay in stats.
+      thread->stats.Start();
+    }
+  }
+
+  void WriteRandomWhileCreating(ThreadState* thread) {
+    if (thread->tid > 0) {
+      WriteRandom(thread);
+    } else {
+      // Special thread that keeps creating index until other threads are done.
+      if (db_->GetIndexStatus("age") != IndexStatus::NotExist) {
+          std::fprintf(stderr, "index status error in WriteWhileCreating\n");
+          std::exit(1);
+      }      
+
+      while (true) {
+        if (db_->GetIndexStatus("age") == IndexStatus::Exist) {
+          break;
+        }
+
+        db_->CreateIndexOnField("age", write_options_);
+      }
+
+      // Do not count any of the preceding work/delay in stats.
+      thread->stats.Start();
+    }
+  }
+
+  void WriteRandomWhileDeleting(ThreadState* thread) {
+    if (thread->tid > 0) {
+      WriteRandom(thread);
+    } else {
+      // Special thread that keeps creating index until other threads are done.
+      if (db_->GetIndexStatus("age") != IndexStatus::Exist) {
+          std::fprintf(stderr, "index status error in WriteWhileDeleting\n");
+          std::exit(1);
+      }      
+
+      while (true) {
+        if (db_->GetIndexStatus("age") == IndexStatus::NotExist) {
+          break;
+        }
+
+        db_->DeleteIndex("age", write_options_);
+      }
+
+      // Do not count any of the preceding work/delay in stats.
+      thread->stats.Start();
+    }
+  }
+
+  void ReadSeqWhileCreating(ThreadState* thread) {
+    if (thread->tid > 0) {
+      ReadSequential(thread);
+    } else {
+      // Special thread that keeps creating index until other threads are done.
+      if (db_->GetIndexStatus("age") != IndexStatus::NotExist) {
+          std::fprintf(stderr, "index status error in WriteWhileCreating\n");
+          std::exit(1);
+      }      
+
+      while (true) {
+        if (db_->GetIndexStatus("age") == IndexStatus::Exist) {
+          break;
+        }
+
+        db_->CreateIndexOnField("age", write_options_);
+      }
+
+      // Do not count any of the preceding work/delay in stats.
+      thread->stats.Start();
+    }
+  }
+
+  void ReadSeqWhileDeleting(ThreadState* thread) {
+    if (thread->tid > 0) {
+      ReadSequential(thread);
+    } else {
+      // Special thread that keeps creating index until other threads are done.
+      if (db_->GetIndexStatus("age") != IndexStatus::Exist) {
+          std::fprintf(stderr, "index status error in WriteWhileDeleting\n");
+          std::exit(1);
+      }      
+
+      while (true) {
+        if (db_->GetIndexStatus("age") == IndexStatus::NotExist) {
+          break;
+        }
+
+        db_->DeleteIndex("age", write_options_);
+      }
+
+      // Do not count any of the preceding work/delay in stats.
+      thread->stats.Start();
+    }
+  }
+
+  void ReadRandomWhileCreating(ThreadState* thread) {
+    if (thread->tid > 0) {
+      ReadRandom(thread);
+    } else {
+      // Special thread that keeps creating index until other threads are done.
+      if (db_->GetIndexStatus("age") != IndexStatus::NotExist) {
+          std::fprintf(stderr, "index status error in WriteWhileCreating\n");
+          std::exit(1);
+      }      
+
+      while (true) {
+        if (db_->GetIndexStatus("age") == IndexStatus::Exist) {
+          break;
+        }
+
+        db_->CreateIndexOnField("age", write_options_);
+      }
+
+      // Do not count any of the preceding work/delay in stats.
+      thread->stats.Start();
+    }
+  }
+
+  void ReadRandomWhileDeleting(ThreadState* thread) {
+    if (thread->tid > 0) {
+      ReadRandom(thread);
+    } else {
+      // Special thread that keeps creating index until other threads are done.
+      if (db_->GetIndexStatus("age") != IndexStatus::Exist) {
+          std::fprintf(stderr, "index status error in WriteWhileDeleting\n");
+          std::exit(1);
+      }      
+
+      while (true) {
+        if (db_->GetIndexStatus("age") == IndexStatus::NotExist) {
+          break;
+        }
+
+        db_->DeleteIndex("age", write_options_);
+      }
+
+      // Do not count any of the preceding work/delay in stats.
+      thread->stats.Start();
+    }
+  }
 };
 
 }  // namespace leveldb

benchmarks/db_bench_testDB.cc

@@ -544,7 +544,7 @@ class Benchmark {
       }
     }
     if (!FLAGS_use_existing_db) {
-      DestroyDB(FLAGS_db, Options());
+      testdb::DestroyDB(FLAGS_db, Options());
     }
   }
 
@@ -665,7 +665,7 @@ class Benchmark {
         } else {
           delete db_;
           db_ = nullptr;
-          DestroyDB(FLAGS_db, Options());
+          testdb::DestroyDB(FLAGS_db, Options());
           Open();
         }
       }

testdb/testdb.cc

@@ -2,6 +2,7 @@
 #include "db/db_impl.h"
 #include <memory>
 #include "leveldb/status.h"
+#include "testdb.h"
 using namespace testdb;
 
 Status testDB::OpentestDB(Options& options,
@@ -93,7 +94,7 @@ void testDB::CompactRange(const Slice *begin, const Slice *end) {
     kvDB_->CompactRange(begin, end);
 }
 
-Status 
DestroyDB(const std::string& name, const Options& options) {
+Status testdb::DestroyDB(const std::string& name, const Options& options) {
   Status s;
   s = leveldb::DestroyDB(name+"_kvDB", options);
   assert(s.ok());
@@ -108,4 +109,4 @@ testDB::~testDB() {
     delete kvDB_;
     // delete indexDB_;
     // delete metaDB_;
-}
+}

util/serialize_value.cc

@@ -52,6 +52,7 @@ FieldArray *ParseValue(const std::string& value_str,FieldArray *fields){
             res->emplace_back(nameStr, valStr);
         } else {
             std::cout << "name and val not match! From ParseValue" << std::endl;
+            assert(0);
         }
         nameSlice.clear();
         valSlice.clear();