979 рядки
28 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include <sys/types.h>
  5. #include <stdio.h>
  6. #include <stdlib.h>
  7. #include "db/db_impl.h"
  8. #include "db/version_set.h"
  9. #include "leveldb/cache.h"
  10. #include "leveldb/db.h"
  11. #include "leveldb/env.h"
  12. #include "leveldb/write_batch.h"
  13. #include "port/port.h"
  14. #include "util/crc32c.h"
  15. #include "util/histogram.h"
  16. #include "util/mutexlock.h"
  17. #include "util/random.h"
  18. #include "util/testutil.h"
  19. // Comma-separated list of operations to run in the specified order
  20. // Actual benchmarks:
  21. // fillseq -- write N values in sequential key order in async mode
  22. // fillrandom -- write N values in random key order in async mode
  23. // overwrite -- overwrite N values in random key order in async mode
  24. // fillsync -- write N/100 values in random key order in sync mode
  25. // fill100K -- write N/1000 100K values in random order in async mode
  26. // deleteseq -- delete N keys in sequential order
  27. // deleterandom -- delete N keys in random order
  28. // readseq -- read N times sequentially
  29. // readreverse -- read N times in reverse order
  30. // readrandom -- read N times in random order
  31. // readmissing -- read N missing keys in random order
  32. // readhot -- read N times in random order from 1% section of DB
  33. // seekrandom -- N random seeks
  34. // crc32c -- repeated crc32c of 4K of data
  35. // acquireload -- load N*1000 times
  36. // Meta operations:
  37. // compact -- Compact the entire DB
  38. // stats -- Print DB stats
  39. // sstables -- Print sstable info
  40. // heapprofile -- Dump a heap profile (if supported by this port)
  41. static const char* FLAGS_benchmarks =
  42. "fillseq,"
  43. "fillsync,"
  44. "fillrandom,"
  45. "overwrite,"
  46. "readrandom,"
  47. "readrandom," // Extra run to allow previous compactions to quiesce
  48. "readseq,"
  49. "readreverse,"
  50. "compact,"
  51. "readrandom,"
  52. "readseq,"
  53. "readreverse,"
  54. "fill100K,"
  55. "crc32c,"
  56. "snappycomp,"
  57. "snappyuncomp,"
  58. "acquireload,"
  59. ;
  60. // Number of key/values to place in database
  61. static int FLAGS_num = 1000000;
  62. // Number of read operations to do. If negative, do FLAGS_num reads.
  63. static int FLAGS_reads = -1;
  64. // Number of concurrent threads to run.
  65. static int FLAGS_threads = 1;
  66. // Size of each value
  67. static int FLAGS_value_size = 100;
  68. // Arrange to generate values that shrink to this fraction of
  69. // their original size after compression
  70. static double FLAGS_compression_ratio = 0.5;
  71. // Print histogram of operation timings
  72. static bool FLAGS_histogram = false;
  73. // Number of bytes to buffer in memtable before compacting
  74. // (initialized to default value by "main")
  75. static int FLAGS_write_buffer_size = 0;
  76. // Number of bytes to use as a cache of uncompressed data.
  77. // Negative means use default settings.
  78. static int FLAGS_cache_size = -1;
  79. // Maximum number of files to keep open at the same time (use default if == 0)
  80. static int FLAGS_open_files = 0;
  81. // Bloom filter bits per key.
  82. // Negative means use default settings.
  83. static int FLAGS_bloom_bits = -1;
  84. // If true, do not destroy the existing database. If you set this
  85. // flag and also specify a benchmark that wants a fresh database, that
  86. // benchmark will fail.
  87. static bool FLAGS_use_existing_db = false;
  88. // Use the db with the following name.
  89. static const char* FLAGS_db = NULL;
  90. namespace leveldb {
  91. namespace {
  92. // Helper for quickly generating random data.
  93. class RandomGenerator {
  94. private:
  95. std::string data_;
  96. int pos_;
  97. public:
  98. RandomGenerator() {
  99. // We use a limited amount of data over and over again and ensure
  100. // that it is larger than the compression window (32KB), and also
  101. // large enough to serve all typical value sizes we want to write.
  102. Random rnd(301);
  103. std::string piece;
  104. while (data_.size() < 1048576) {
  105. // Add a short fragment that is as compressible as specified
  106. // by FLAGS_compression_ratio.
  107. test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
  108. data_.append(piece);
  109. }
  110. pos_ = 0;
  111. }
  112. Slice Generate(int len) {
  113. if (pos_ + len > data_.size()) {
  114. pos_ = 0;
  115. assert(len < data_.size());
  116. }
  117. pos_ += len;
  118. return Slice(data_.data() + pos_ - len, len);
  119. }
  120. };
  121. static Slice TrimSpace(Slice s) {
  122. int start = 0;
  123. while (start < s.size() && isspace(s[start])) {
  124. start++;
  125. }
  126. int limit = s.size();
  127. while (limit > start && isspace(s[limit-1])) {
  128. limit--;
  129. }
  130. return Slice(s.data() + start, limit - start);
  131. }
  132. static void AppendWithSpace(std::string* str, Slice msg) {
  133. if (msg.empty()) return;
  134. if (!str->empty()) {
  135. str->push_back(' ');
  136. }
  137. str->append(msg.data(), msg.size());
  138. }
  139. class Stats {
  140. private:
  141. double start_;
  142. double finish_;
  143. double seconds_;
  144. int done_;
  145. int next_report_;
  146. int64_t bytes_;
  147. double last_op_finish_;
  148. Histogram hist_;
  149. std::string message_;
  150. public:
  151. Stats() { Start(); }
  152. void Start() {
  153. next_report_ = 100;
  154. last_op_finish_ = start_;
  155. hist_.Clear();
  156. done_ = 0;
  157. bytes_ = 0;
  158. seconds_ = 0;
  159. start_ = Env::Default()->NowMicros();
  160. finish_ = start_;
  161. message_.clear();
  162. }
  163. void Merge(const Stats& other) {
  164. hist_.Merge(other.hist_);
  165. done_ += other.done_;
  166. bytes_ += other.bytes_;
  167. seconds_ += other.seconds_;
  168. if (other.start_ < start_) start_ = other.start_;
  169. if (other.finish_ > finish_) finish_ = other.finish_;
  170. // Just keep the messages from one thread
  171. if (message_.empty()) message_ = other.message_;
  172. }
  173. void Stop() {
  174. finish_ = Env::Default()->NowMicros();
  175. seconds_ = (finish_ - start_) * 1e-6;
  176. }
  177. void AddMessage(Slice msg) {
  178. AppendWithSpace(&message_, msg);
  179. }
  180. void FinishedSingleOp() {
  181. if (FLAGS_histogram) {
  182. double now = Env::Default()->NowMicros();
  183. double micros = now - last_op_finish_;
  184. hist_.Add(micros);
  185. if (micros > 20000) {
  186. fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
  187. fflush(stderr);
  188. }
  189. last_op_finish_ = now;
  190. }
  191. done_++;
  192. if (done_ >= next_report_) {
  193. if (next_report_ < 1000) next_report_ += 100;
  194. else if (next_report_ < 5000) next_report_ += 500;
  195. else if (next_report_ < 10000) next_report_ += 1000;
  196. else if (next_report_ < 50000) next_report_ += 5000;
  197. else if (next_report_ < 100000) next_report_ += 10000;
  198. else if (next_report_ < 500000) next_report_ += 50000;
  199. else next_report_ += 100000;
  200. fprintf(stderr, "... finished %d ops%30s\r", done_, "");
  201. fflush(stderr);
  202. }
  203. }
  204. void AddBytes(int64_t n) {
  205. bytes_ += n;
  206. }
  207. void Report(const Slice& name) {
  208. // Pretend at least one op was done in case we are running a benchmark
  209. // that does not call FinishedSingleOp().
  210. if (done_ < 1) done_ = 1;
  211. std::string extra;
  212. if (bytes_ > 0) {
  213. // Rate is computed on actual elapsed time, not the sum of per-thread
  214. // elapsed times.
  215. double elapsed = (finish_ - start_) * 1e-6;
  216. char rate[100];
  217. snprintf(rate, sizeof(rate), "%6.1f MB/s",
  218. (bytes_ / 1048576.0) / elapsed);
  219. extra = rate;
  220. }
  221. AppendWithSpace(&extra, message_);
  222. fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
  223. name.ToString().c_str(),
  224. seconds_ * 1e6 / done_,
  225. (extra.empty() ? "" : " "),
  226. extra.c_str());
  227. if (FLAGS_histogram) {
  228. fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
  229. }
  230. fflush(stdout);
  231. }
  232. };
  233. // State shared by all concurrent executions of the same benchmark.
  234. struct SharedState {
  235. port::Mutex mu;
  236. port::CondVar cv;
  237. int total;
  238. // Each thread goes through the following states:
  239. // (1) initializing
  240. // (2) waiting for others to be initialized
  241. // (3) running
  242. // (4) done
  243. int num_initialized;
  244. int num_done;
  245. bool start;
  246. SharedState() : cv(&mu) { }
  247. };
  248. // Per-thread state for concurrent executions of the same benchmark.
  249. struct ThreadState {
  250. int tid; // 0..n-1 when running in n threads
  251. Random rand; // Has different seeds for different threads
  252. Stats stats;
  253. SharedState* shared;
  254. ThreadState(int index)
  255. : tid(index),
  256. rand(1000 + index) {
  257. }
  258. };
  259. } // namespace
  260. class Benchmark {
  261. private:
  262. Cache* cache_;
  263. const FilterPolicy* filter_policy_;
  264. DB* db_;
  265. int num_;
  266. int value_size_;
  267. int entries_per_batch_;
  268. WriteOptions write_options_;
  269. int reads_;
  270. int heap_counter_;
  271. void PrintHeader() {
  272. const int kKeySize = 16;
  273. PrintEnvironment();
  274. fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
  275. fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
  276. FLAGS_value_size,
  277. static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
  278. fprintf(stdout, "Entries: %d\n", num_);
  279. fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
  280. ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
  281. / 1048576.0));
  282. fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
  283. (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_)
  284. / 1048576.0));
  285. PrintWarnings();
  286. fprintf(stdout, "------------------------------------------------\n");
  287. }
  288. void PrintWarnings() {
  289. #if defined(__GNUC__) && !defined(__OPTIMIZE__)
  290. fprintf(stdout,
  291. "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
  292. );
  293. #endif
  294. #ifndef NDEBUG
  295. fprintf(stdout,
  296. "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
  297. #endif
  298. // See if snappy is working by attempting to compress a compressible string
  299. const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy";
  300. std::string compressed;
  301. if (!port::Snappy_Compress(text, sizeof(text), &compressed)) {
  302. fprintf(stdout, "WARNING: Snappy compression is not enabled\n");
  303. } else if (compressed.size() >= sizeof(text)) {
  304. fprintf(stdout, "WARNING: Snappy compression is not effective\n");
  305. }
  306. }
  307. void PrintEnvironment() {
  308. fprintf(stderr, "LevelDB: version %d.%d\n",
  309. kMajorVersion, kMinorVersion);
  310. #if defined(__linux)
  311. time_t now = time(NULL);
  312. fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
  313. FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
  314. if (cpuinfo != NULL) {
  315. char line[1000];
  316. int num_cpus = 0;
  317. std::string cpu_type;
  318. std::string cache_size;
  319. while (fgets(line, sizeof(line), cpuinfo) != NULL) {
  320. const char* sep = strchr(line, ':');
  321. if (sep == NULL) {
  322. continue;
  323. }
  324. Slice key = TrimSpace(Slice(line, sep - 1 - line));
  325. Slice val = TrimSpace(Slice(sep + 1));
  326. if (key == "model name") {
  327. ++num_cpus;
  328. cpu_type = val.ToString();
  329. } else if (key == "cache size") {
  330. cache_size = val.ToString();
  331. }
  332. }
  333. fclose(cpuinfo);
  334. fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
  335. fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
  336. }
  337. #endif
  338. }
  339. public:
  340. Benchmark()
  341. : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL),
  342. filter_policy_(FLAGS_bloom_bits >= 0
  343. ? NewBloomFilterPolicy(FLAGS_bloom_bits)
  344. : NULL),
  345. db_(NULL),
  346. num_(FLAGS_num),
  347. value_size_(FLAGS_value_size),
  348. entries_per_batch_(1),
  349. reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
  350. heap_counter_(0) {
  351. std::vector<std::string> files;
  352. Env::Default()->GetChildren(FLAGS_db, &files);
  353. for (int i = 0; i < files.size(); i++) {
  354. if (Slice(files[i]).starts_with("heap-")) {
  355. Env::Default()->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
  356. }
  357. }
  358. if (!FLAGS_use_existing_db) {
  359. DestroyDB(FLAGS_db, Options());
  360. }
  361. }
  362. ~Benchmark() {
  363. delete db_;
  364. delete cache_;
  365. delete filter_policy_;
  366. }
  367. void Run() {
  368. PrintHeader();
  369. Open();
  370. const char* benchmarks = FLAGS_benchmarks;
  371. while (benchmarks != NULL) {
  372. const char* sep = strchr(benchmarks, ',');
  373. Slice name;
  374. if (sep == NULL) {
  375. name = benchmarks;
  376. benchmarks = NULL;
  377. } else {
  378. name = Slice(benchmarks, sep - benchmarks);
  379. benchmarks = sep + 1;
  380. }
  381. // Reset parameters that may be overriddden bwlow
  382. num_ = FLAGS_num;
  383. reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
  384. value_size_ = FLAGS_value_size;
  385. entries_per_batch_ = 1;
  386. write_options_ = WriteOptions();
  387. void (Benchmark::*method)(ThreadState*) = NULL;
  388. bool fresh_db = false;
  389. int num_threads = FLAGS_threads;
  390. if (name == Slice("fillseq")) {
  391. fresh_db = true;
  392. method = &Benchmark::WriteSeq;
  393. } else if (name == Slice("fillbatch")) {
  394. fresh_db = true;
  395. entries_per_batch_ = 1000;
  396. method = &Benchmark::WriteSeq;
  397. } else if (name == Slice("fillrandom")) {
  398. fresh_db = true;
  399. method = &Benchmark::WriteRandom;
  400. } else if (name == Slice("overwrite")) {
  401. fresh_db = false;
  402. method = &Benchmark::WriteRandom;
  403. } else if (name == Slice("fillsync")) {
  404. fresh_db = true;
  405. num_ /= 1000;
  406. write_options_.sync = true;
  407. method = &Benchmark::WriteRandom;
  408. } else if (name == Slice("fill100K")) {
  409. fresh_db = true;
  410. num_ /= 1000;
  411. value_size_ = 100 * 1000;
  412. method = &Benchmark::WriteRandom;
  413. } else if (name == Slice("readseq")) {
  414. method = &Benchmark::ReadSequential;
  415. } else if (name == Slice("readreverse")) {
  416. method = &Benchmark::ReadReverse;
  417. } else if (name == Slice("readrandom")) {
  418. method = &Benchmark::ReadRandom;
  419. } else if (name == Slice("readmissing")) {
  420. method = &Benchmark::ReadMissing;
  421. } else if (name == Slice("seekrandom")) {
  422. method = &Benchmark::SeekRandom;
  423. } else if (name == Slice("readhot")) {
  424. method = &Benchmark::ReadHot;
  425. } else if (name == Slice("readrandomsmall")) {
  426. reads_ /= 1000;
  427. method = &Benchmark::ReadRandom;
  428. } else if (name == Slice("deleteseq")) {
  429. method = &Benchmark::DeleteSeq;
  430. } else if (name == Slice("deleterandom")) {
  431. method = &Benchmark::DeleteRandom;
  432. } else if (name == Slice("readwhilewriting")) {
  433. num_threads++; // Add extra thread for writing
  434. method = &Benchmark::ReadWhileWriting;
  435. } else if (name == Slice("compact")) {
  436. method = &Benchmark::Compact;
  437. } else if (name == Slice("crc32c")) {
  438. method = &Benchmark::Crc32c;
  439. } else if (name == Slice("acquireload")) {
  440. method = &Benchmark::AcquireLoad;
  441. } else if (name == Slice("snappycomp")) {
  442. method = &Benchmark::SnappyCompress;
  443. } else if (name == Slice("snappyuncomp")) {
  444. method = &Benchmark::SnappyUncompress;
  445. } else if (name == Slice("heapprofile")) {
  446. HeapProfile();
  447. } else if (name == Slice("stats")) {
  448. PrintStats("leveldb.stats");
  449. } else if (name == Slice("sstables")) {
  450. PrintStats("leveldb.sstables");
  451. } else {
  452. if (name != Slice()) { // No error message for empty name
  453. fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
  454. }
  455. }
  456. if (fresh_db) {
  457. if (FLAGS_use_existing_db) {
  458. fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n",
  459. name.ToString().c_str());
  460. method = NULL;
  461. } else {
  462. delete db_;
  463. db_ = NULL;
  464. DestroyDB(FLAGS_db, Options());
  465. Open();
  466. }
  467. }
  468. if (method != NULL) {
  469. RunBenchmark(num_threads, name, method);
  470. }
  471. }
  472. }
  473. private:
  474. struct ThreadArg {
  475. Benchmark* bm;
  476. SharedState* shared;
  477. ThreadState* thread;
  478. void (Benchmark::*method)(ThreadState*);
  479. };
  480. static void ThreadBody(void* v) {
  481. ThreadArg* arg = reinterpret_cast<ThreadArg*>(v);
  482. SharedState* shared = arg->shared;
  483. ThreadState* thread = arg->thread;
  484. {
  485. MutexLock l(&shared->mu);
  486. shared->num_initialized++;
  487. if (shared->num_initialized >= shared->total) {
  488. shared->cv.SignalAll();
  489. }
  490. while (!shared->start) {
  491. shared->cv.Wait();
  492. }
  493. }
  494. thread->stats.Start();
  495. (arg->bm->*(arg->method))(thread);
  496. thread->stats.Stop();
  497. {
  498. MutexLock l(&shared->mu);
  499. shared->num_done++;
  500. if (shared->num_done >= shared->total) {
  501. shared->cv.SignalAll();
  502. }
  503. }
  504. }
  505. void RunBenchmark(int n, Slice name,
  506. void (Benchmark::*method)(ThreadState*)) {
  507. SharedState shared;
  508. shared.total = n;
  509. shared.num_initialized = 0;
  510. shared.num_done = 0;
  511. shared.start = false;
  512. ThreadArg* arg = new ThreadArg[n];
  513. for (int i = 0; i < n; i++) {
  514. arg[i].bm = this;
  515. arg[i].method = method;
  516. arg[i].shared = &shared;
  517. arg[i].thread = new ThreadState(i);
  518. arg[i].thread->shared = &shared;
  519. Env::Default()->StartThread(ThreadBody, &arg[i]);
  520. }
  521. shared.mu.Lock();
  522. while (shared.num_initialized < n) {
  523. shared.cv.Wait();
  524. }
  525. shared.start = true;
  526. shared.cv.SignalAll();
  527. while (shared.num_done < n) {
  528. shared.cv.Wait();
  529. }
  530. shared.mu.Unlock();
  531. for (int i = 1; i < n; i++) {
  532. arg[0].thread->stats.Merge(arg[i].thread->stats);
  533. }
  534. arg[0].thread->stats.Report(name);
  535. for (int i = 0; i < n; i++) {
  536. delete arg[i].thread;
  537. }
  538. delete[] arg;
  539. }
  540. void Crc32c(ThreadState* thread) {
  541. // Checksum about 500MB of data total
  542. const int size = 4096;
  543. const char* label = "(4K per op)";
  544. std::string data(size, 'x');
  545. int64_t bytes = 0;
  546. uint32_t crc = 0;
  547. while (bytes < 500 * 1048576) {
  548. crc = crc32c::Value(data.data(), size);
  549. thread->stats.FinishedSingleOp();
  550. bytes += size;
  551. }
  552. // Print so result is not dead
  553. fprintf(stderr, "... crc=0x%x\r", static_cast<unsigned int>(crc));
  554. thread->stats.AddBytes(bytes);
  555. thread->stats.AddMessage(label);
  556. }
  557. void AcquireLoad(ThreadState* thread) {
  558. int dummy;
  559. port::AtomicPointer ap(&dummy);
  560. int count = 0;
  561. void *ptr = NULL;
  562. thread->stats.AddMessage("(each op is 1000 loads)");
  563. while (count < 100000) {
  564. for (int i = 0; i < 1000; i++) {
  565. ptr = ap.Acquire_Load();
  566. }
  567. count++;
  568. thread->stats.FinishedSingleOp();
  569. }
  570. if (ptr == NULL) exit(1); // Disable unused variable warning.
  571. }
  572. void SnappyCompress(ThreadState* thread) {
  573. RandomGenerator gen;
  574. Slice input = gen.Generate(Options().block_size);
  575. int64_t bytes = 0;
  576. int64_t produced = 0;
  577. bool ok = true;
  578. std::string compressed;
  579. while (ok && bytes < 1024 * 1048576) { // Compress 1G
  580. ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
  581. produced += compressed.size();
  582. bytes += input.size();
  583. thread->stats.FinishedSingleOp();
  584. }
  585. if (!ok) {
  586. thread->stats.AddMessage("(snappy failure)");
  587. } else {
  588. char buf[100];
  589. snprintf(buf, sizeof(buf), "(output: %.1f%%)",
  590. (produced * 100.0) / bytes);
  591. thread->stats.AddMessage(buf);
  592. thread->stats.AddBytes(bytes);
  593. }
  594. }
  595. void SnappyUncompress(ThreadState* thread) {
  596. RandomGenerator gen;
  597. Slice input = gen.Generate(Options().block_size);
  598. std::string compressed;
  599. bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
  600. int64_t bytes = 0;
  601. char* uncompressed = new char[input.size()];
  602. while (ok && bytes < 1024 * 1048576) { // Compress 1G
  603. ok = port::Snappy_Uncompress(compressed.data(), compressed.size(),
  604. uncompressed);
  605. bytes += input.size();
  606. thread->stats.FinishedSingleOp();
  607. }
  608. delete[] uncompressed;
  609. if (!ok) {
  610. thread->stats.AddMessage("(snappy failure)");
  611. } else {
  612. thread->stats.AddBytes(bytes);
  613. }
  614. }
  615. void Open() {
  616. assert(db_ == NULL);
  617. Options options;
  618. options.create_if_missing = !FLAGS_use_existing_db;
  619. options.block_cache = cache_;
  620. options.write_buffer_size = FLAGS_write_buffer_size;
  621. options.max_open_files = FLAGS_open_files;
  622. options.filter_policy = filter_policy_;
  623. Status s = DB::Open(options, FLAGS_db, &db_);
  624. if (!s.ok()) {
  625. fprintf(stderr, "open error: %s\n", s.ToString().c_str());
  626. exit(1);
  627. }
  628. }
  629. void WriteSeq(ThreadState* thread) {
  630. DoWrite(thread, true);
  631. }
  632. void WriteRandom(ThreadState* thread) {
  633. DoWrite(thread, false);
  634. }
  635. void DoWrite(ThreadState* thread, bool seq) {
  636. if (num_ != FLAGS_num) {
  637. char msg[100];
  638. snprintf(msg, sizeof(msg), "(%d ops)", num_);
  639. thread->stats.AddMessage(msg);
  640. }
  641. RandomGenerator gen;
  642. WriteBatch batch;
  643. Status s;
  644. int64_t bytes = 0;
  645. for (int i = 0; i < num_; i += entries_per_batch_) {
  646. batch.Clear();
  647. for (int j = 0; j < entries_per_batch_; j++) {
  648. const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
  649. char key[100];
  650. snprintf(key, sizeof(key), "%016d", k);
  651. batch.Put(key, gen.Generate(value_size_));
  652. bytes += value_size_ + strlen(key);
  653. thread->stats.FinishedSingleOp();
  654. }
  655. s = db_->Write(write_options_, &batch);
  656. if (!s.ok()) {
  657. fprintf(stderr, "put error: %s\n", s.ToString().c_str());
  658. exit(1);
  659. }
  660. }
  661. thread->stats.AddBytes(bytes);
  662. }
  663. void ReadSequential(ThreadState* thread) {
  664. Iterator* iter = db_->NewIterator(ReadOptions());
  665. int i = 0;
  666. int64_t bytes = 0;
  667. for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
  668. bytes += iter->key().size() + iter->value().size();
  669. thread->stats.FinishedSingleOp();
  670. ++i;
  671. }
  672. delete iter;
  673. thread->stats.AddBytes(bytes);
  674. }
  675. void ReadReverse(ThreadState* thread) {
  676. Iterator* iter = db_->NewIterator(ReadOptions());
  677. int i = 0;
  678. int64_t bytes = 0;
  679. for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) {
  680. bytes += iter->key().size() + iter->value().size();
  681. thread->stats.FinishedSingleOp();
  682. ++i;
  683. }
  684. delete iter;
  685. thread->stats.AddBytes(bytes);
  686. }
  687. void ReadRandom(ThreadState* thread) {
  688. ReadOptions options;
  689. std::string value;
  690. int found = 0;
  691. for (int i = 0; i < reads_; i++) {
  692. char key[100];
  693. const int k = thread->rand.Next() % FLAGS_num;
  694. snprintf(key, sizeof(key), "%016d", k);
  695. if (db_->Get(options, key, &value).ok()) {
  696. found++;
  697. }
  698. thread->stats.FinishedSingleOp();
  699. }
  700. char msg[100];
  701. snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
  702. thread->stats.AddMessage(msg);
  703. }
  704. void ReadMissing(ThreadState* thread) {
  705. ReadOptions options;
  706. std::string value;
  707. for (int i = 0; i < reads_; i++) {
  708. char key[100];
  709. const int k = thread->rand.Next() % FLAGS_num;
  710. snprintf(key, sizeof(key), "%016d.", k);
  711. db_->Get(options, key, &value);
  712. thread->stats.FinishedSingleOp();
  713. }
  714. }
  715. void ReadHot(ThreadState* thread) {
  716. ReadOptions options;
  717. std::string value;
  718. const int range = (FLAGS_num + 99) / 100;
  719. for (int i = 0; i < reads_; i++) {
  720. char key[100];
  721. const int k = thread->rand.Next() % range;
  722. snprintf(key, sizeof(key), "%016d", k);
  723. db_->Get(options, key, &value);
  724. thread->stats.FinishedSingleOp();
  725. }
  726. }
  727. void SeekRandom(ThreadState* thread) {
  728. ReadOptions options;
  729. std::string value;
  730. int found = 0;
  731. for (int i = 0; i < reads_; i++) {
  732. Iterator* iter = db_->NewIterator(options);
  733. char key[100];
  734. const int k = thread->rand.Next() % FLAGS_num;
  735. snprintf(key, sizeof(key), "%016d", k);
  736. iter->Seek(key);
  737. if (iter->Valid() && iter->key() == key) found++;
  738. delete iter;
  739. thread->stats.FinishedSingleOp();
  740. }
  741. char msg[100];
  742. snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
  743. thread->stats.AddMessage(msg);
  744. }
  745. void DoDelete(ThreadState* thread, bool seq) {
  746. RandomGenerator gen;
  747. WriteBatch batch;
  748. Status s;
  749. for (int i = 0; i < num_; i += entries_per_batch_) {
  750. batch.Clear();
  751. for (int j = 0; j < entries_per_batch_; j++) {
  752. const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
  753. char key[100];
  754. snprintf(key, sizeof(key), "%016d", k);
  755. batch.Delete(key);
  756. thread->stats.FinishedSingleOp();
  757. }
  758. s = db_->Write(write_options_, &batch);
  759. if (!s.ok()) {
  760. fprintf(stderr, "del error: %s\n", s.ToString().c_str());
  761. exit(1);
  762. }
  763. }
  764. }
  765. void DeleteSeq(ThreadState* thread) {
  766. DoDelete(thread, true);
  767. }
  768. void DeleteRandom(ThreadState* thread) {
  769. DoDelete(thread, false);
  770. }
  771. void ReadWhileWriting(ThreadState* thread) {
  772. if (thread->tid > 0) {
  773. ReadRandom(thread);
  774. } else {
  775. // Special thread that keeps writing until other threads are done.
  776. RandomGenerator gen;
  777. while (true) {
  778. {
  779. MutexLock l(&thread->shared->mu);
  780. if (thread->shared->num_done + 1 >= thread->shared->num_initialized) {
  781. // Other threads have finished
  782. break;
  783. }
  784. }
  785. const int k = thread->rand.Next() % FLAGS_num;
  786. char key[100];
  787. snprintf(key, sizeof(key), "%016d", k);
  788. Status s = db_->Put(write_options_, key, gen.Generate(value_size_));
  789. if (!s.ok()) {
  790. fprintf(stderr, "put error: %s\n", s.ToString().c_str());
  791. exit(1);
  792. }
  793. }
  794. // Do not count any of the preceding work/delay in stats.
  795. thread->stats.Start();
  796. }
  797. }
  798. void Compact(ThreadState* thread) {
  799. db_->CompactRange(NULL, NULL);
  800. }
  801. void PrintStats(const char* key) {
  802. std::string stats;
  803. if (!db_->GetProperty(key, &stats)) {
  804. stats = "(failed)";
  805. }
  806. fprintf(stdout, "\n%s\n", stats.c_str());
  807. }
  808. static void WriteToFile(void* arg, const char* buf, int n) {
  809. reinterpret_cast<WritableFile*>(arg)->Append(Slice(buf, n));
  810. }
  811. void HeapProfile() {
  812. char fname[100];
  813. snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
  814. WritableFile* file;
  815. Status s = Env::Default()->NewWritableFile(fname, &file);
  816. if (!s.ok()) {
  817. fprintf(stderr, "%s\n", s.ToString().c_str());
  818. return;
  819. }
  820. bool ok = port::GetHeapProfile(WriteToFile, file);
  821. delete file;
  822. if (!ok) {
  823. fprintf(stderr, "heap profiling not supported\n");
  824. Env::Default()->DeleteFile(fname);
  825. }
  826. }
  827. };
  828. } // namespace leveldb
  829. int main(int argc, char** argv) {
  830. FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
  831. FLAGS_open_files = leveldb::Options().max_open_files;
  832. std::string default_db_path;
  833. for (int i = 1; i < argc; i++) {
  834. double d;
  835. int n;
  836. char junk;
  837. if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
  838. FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
  839. } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
  840. FLAGS_compression_ratio = d;
  841. } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
  842. (n == 0 || n == 1)) {
  843. FLAGS_histogram = n;
  844. } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
  845. (n == 0 || n == 1)) {
  846. FLAGS_use_existing_db = n;
  847. } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
  848. FLAGS_num = n;
  849. } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
  850. FLAGS_reads = n;
  851. } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) {
  852. FLAGS_threads = n;
  853. } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
  854. FLAGS_value_size = n;
  855. } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
  856. FLAGS_write_buffer_size = n;
  857. } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
  858. FLAGS_cache_size = n;
  859. } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) {
  860. FLAGS_bloom_bits = n;
  861. } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
  862. FLAGS_open_files = n;
  863. } else if (strncmp(argv[i], "--db=", 5) == 0) {
  864. FLAGS_db = argv[i] + 5;
  865. } else {
  866. fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
  867. exit(1);
  868. }
  869. }
  870. // Choose a location for the test database if none given with --db=<path>
  871. if (FLAGS_db == NULL) {
  872. leveldb::Env::Default()->GetTestDirectory(&default_db_path);
  873. default_db_path += "/dbbench";
  874. FLAGS_db = default_db_path.c_str();
  875. }
  876. leveldb::Benchmark benchmark;
  877. benchmark.Run();
  878. return 0;
  879. }