Sfoglia il codice sorgente

roughly completed

main
VirgilZhu 8 mesi fa
parent
commit
accd7e616c
2 ha cambiato i file con 122 aggiunte e 19 eliminazioni
  1. +90
    -10
      benchmarks/db_bench.cc
  2. +32
    -9
      test/bench_test.cc

+ 90
- 10
benchmarks/db_bench.cc Vedi File

@ -20,6 +20,7 @@
#include "util/mutexlock.h"
#include "util/random.h"
#include "util/testutil.h"
#include "db/fields.h"
// Comma-separated list of operations to run in the specified order
// Actual benchmarks:
@ -55,14 +56,15 @@ static const char* FLAGS_benchmarks =
"readreverse,"
"compact,"
"readrandom,"
"findkeysbyfield,"
"readseq,"
"readreverse,"
"fill100K,"
"crc32c,"
"snappycomp,"
"snappyuncomp,"
"zstdcomp,"
"zstduncomp,";
"fill100K,";
// "crc32c,"
// "snappycomp,"
// "snappyuncomp,"
// "zstdcomp,"
// "zstduncomp,";
// Number of key/values to place in database
static int FLAGS_num = 1000000;
@ -70,6 +72,9 @@ static int FLAGS_num = 1000000;
// Number of read operations to do. If negative, do FLAGS_num reads.
static int FLAGS_reads = -1;
// Number of given fields used in FindKeysByField test. If negative, write in half of FLAGS_num targets with given field.
static int FLAGS_num_fields = 80000;
// Number of concurrent threads to run.
static int FLAGS_threads = 1;
@ -124,7 +129,7 @@ static bool FLAGS_reuse_logs = false;
static bool FLAGS_compression = true;
// Use the db with the following name.
static const char* FLAGS_db = nullptr;
static const char* FLAGS_db = "benchmark_db";
// ZSTD compression level to try out
static int FLAGS_zstd_compression_level = 1;
@ -438,6 +443,7 @@ class Benchmark {
int heap_counter_;
CountComparator count_comparator_;
int total_thread_count_;
int num_fields; // 插入的fields数量
void PrintHeader() {
const int kKeySize = 16 + FLAGS_key_prefix;
@ -530,7 +536,8 @@ class Benchmark {
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
heap_counter_(0),
count_comparator_(BytewiseComparator()),
total_thread_count_(0) {
total_thread_count_(0),
num_fields(FLAGS_num_fields < 0 ? FLAGS_num / 2 : FLAGS_num_fields) {
std::vector<std::string> files;
g_env->GetChildren(FLAGS_db, &files);
for (size_t i = 0; i < files.size(); i++) {
@ -615,6 +622,8 @@ class Benchmark {
method = &Benchmark::SeekRandom;
} else if (name == Slice("seekordered")) {
method = &Benchmark::SeekOrdered;
} else if (name == Slice("findkeysbyfield")) {
method = &Benchmark::FindKeysByField;
} else if (name == Slice("readhot")) {
method = &Benchmark::ReadHot;
} else if (name == Slice("readrandomsmall")) {
@ -852,8 +861,11 @@ class Benchmark {
for (int j = 0; j < entries_per_batch_; j++) {
const int k = seq ? i + j : thread->rand.Uniform(FLAGS_num);
key.Set(k);
batch.Put(key.slice(), gen.Generate(value_size_));
bytes += value_size_ + key.slice().size();
FieldArray fields = {{"field1", "value1_" + std::to_string(i)}, {"field2", "value2_"}};
Fields ffields(fields);
db_->PutFields(WriteOptions(), key.slice(), ffields);
// batch.Put(key.slice(), gen.Generate(value_size_));
bytes += ffields.size() + key.slice().size();
thread->stats.FinishedSingleOp();
}
s = db_->Write(write_options_, &batch);
@ -935,6 +947,72 @@ class Benchmark {
}
}
void WriteTargetSeq(ThreadState* thread) { WriteGiven(thread, true); }
void WriteTargetRandom(ThreadState* thread) { WriteGiven(thread, false); }
void WriteGiven(ThreadState* thread, bool seq) {
if (num_ != FLAGS_num) {
char msg[100];
std::snprintf(msg, sizeof(msg), "(%d ops)", num_);
thread->stats.AddMessage(msg);
}
RandomGenerator gen;
WriteBatch batch;
Status s;
int64_t bytes = 0;
KeyBuffer key;
for (int i = 0; i < num_; i += entries_per_batch_) {
batch.Clear();
for (int j = 0; j < entries_per_batch_; j++) {
const int k = seq ? i + j : thread->rand.Uniform(FLAGS_num);
key.Set(k);
FieldArray fields;
auto value = gen.Generate(value_size_);
if (i < num_fields) {
fields = {
{"field1", value.ToString()},
{"field2", "value2_"},
};
} else {
fields = {
{"field1", value.ToString()},
};
}
Fields ffields(fields);
db_->PutFields(WriteOptions(), key.slice(), ffields);
bytes += ffields.size() + key.slice().size();
thread->stats.FinishedSingleOp();
}
s = db_->Write(write_options_, &batch);
if (!s.ok()) {
std::fprintf(stderr, "put error: %s\n", s.ToString().c_str());
std::exit(1);
}
}
thread->stats.AddBytes(bytes);
}
void FindKeysByField(ThreadState* thread){
int found = 0;
FieldArray fields_to_find = {{"field2", "value2_"}};
std::string dbname_ = "benchmark_db";
Options options;
options.create_if_missing = true;
DBImpl* impl = new DBImpl(options, dbname_);
std::vector<std::string> found_keys = Fields::FindKeysByFields(db_, fields_to_find, impl);
found = found_keys.size();
char msg[100];
snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_fields);
thread->stats.AddMessage(msg);
}
void SeekRandom(ThreadState* thread) {
ReadOptions options;
int found = 0;
@ -1097,6 +1175,8 @@ int main(int argc, char** argv) {
FLAGS_num = n;
} else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
FLAGS_reads = n;
} else if (sscanf(argv[i], "--num_fields=%d%c", &n, &junk) == 1) {
FLAGS_num_fields = n;
} else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) {
FLAGS_threads = n;
} else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {

+ 32
- 9
test/bench_test.cc Vedi File

@ -42,7 +42,7 @@ void InsertFields(DB *db, std::vector &lats) {
for (int i = 0; i < num_; ++i) {
int key_ = rand() % num_ + 1;
std::string key = std::to_string(key_);
FieldArray fields = {{"field" + std::to_string(key_), "old_value_" + std::to_string(key)}};
FieldArray fields = {{"field" + std::to_string(key_), "old_value_" + std::to_string(key_)}};
Fields f(fields);
auto start_time = std::chrono::steady_clock::now();
db->PutFields(writeOptions, Slice(key), f);
@ -84,7 +84,14 @@ void FindKeys(DB *db, std::vector &lats) {
int key_ = rand() % num_ + 1;
FieldArray fields_to_find = {{"field" + std::to_string(key_), "old_value_" + std::to_string(key_)}};
auto start_time = std::chrono::steady_clock::now();
Fields::FindKeysByFields(db, fields_to_find);
std::string dbname_ = "benchmark_db";
Options options;
options.create_if_missing = true;
DBImpl* impl = new DBImpl(options, dbname_);
Fields::FindKeysByFields(db, fields_to_find, impl);
auto end_time = std::chrono::steady_clock::now();
lats.emplace_back(std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count());
}
@ -102,14 +109,29 @@ double CalculatePercentile(const std::vector& latencies, double percent
return sorted_latencies[index];
}
void SetupData(DB *db) {
std::vector<int64_t> lats;
InsertData(db, lats);
}
void SetupFields(DB *db) {
std::vector<int64_t> lats;
InsertFields(db, lats);
}
template<typename Func>
void RunBenchmark(const char* name, Func func) {
void RunBenchmark(const char* name, Func func, bool setup_data = true, bool setup_fields = false) {
DB *db;
std::string rm_command = "rm -rf testdb_bench";
system(rm_command.c_str());
if (!OpenDB("testdb_bench", &db).ok()) {
std::cerr << "open db failed" << std::endl;
abort();
}
if (setup_data) SetupData(db);
if (setup_fields) SetupFields(db);
std::vector<int64_t> lats;
auto start_time = std::chrono::steady_clock::now();
func(db, lats);
@ -131,14 +153,15 @@ void RunBenchmark(const char* name, Func func) {
delete db;
}
class BenchTest : public ::testing::TestWithParam<double> {};
// TEST(BenchTest, PutLatency) { RunBenchmark("Put", InsertData, false, false); }
// TEST(BenchTest, PutFieldsLatency) { RunBenchmark("PutFields", InsertFields, false, false); }
TEST_P(BenchTest, PutLatency) { RunBenchmark("Put", InsertData); }
TEST_P(BenchTest, PutLatency) { RunBenchmark("PutFields", InsertFields); }
TEST_P(BenchTest, GetLatency) { RunBenchmark("Get", GetData); }
TEST_P(BenchTest, IteratorLatency) { RunBenchmark("Iterator", ReadOrdered); }
TEST_P(BenchTest, FindKeysByFieldLatency) { RunBenchmark("FindKeysByFields", FindKeys); }
// TEST(BenchTest, GetLatency) { RunBenchmark("Get", GetData, true, false); }
// TEST(BenchTest, IteratorLatency) { RunBenchmark("Iterator", ReadOrdered, true, false); }
TEST(BenchTest, FindKeysByFieldLatency) {
RunBenchmark("FindKeysByFields", FindKeys, false, true);
}
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);

Caricamento…
Annulla
Salva