小组成员:10215300402-朱维清 & 10222140408 谷杰

981 рядки
29 KiB

Release 1.18 Changes are: * Update version number to 1.18 * Replace the basic fprintf call with a call to fwrite in order to work around the apparent compiler optimization/rewrite failure that we are seeing with the new toolchain/iOS SDKs provided with Xcode6 and iOS8. * Fix ALL the header guards. * Createed a README.md with the LevelDB project description. * A new CONTRIBUTING file. * Don't implicitly convert uint64_t to size_t or int. Either preserve it as uint64_t, or explicitly cast. This fixes MSVC warnings about possible value truncation when compiling this code in Chromium. * Added a DumpFile() library function that encapsulates the guts of the "leveldbutil dump" command. This will allow clients to dump data to their log files instead of stdout. It will also allow clients to supply their own environment. * leveldb: Remove unused function 'ConsumeChar'. * leveldbutil: Remove unused member variables from WriteBatchItemPrinter. * OpenBSD, NetBSD and DragonflyBSD have _LITTLE_ENDIAN, so define PLATFORM_IS_LITTLE_ENDIAN like on FreeBSD. This fixes: * issue #143 * issue #198 * issue #249 * Switch from <cstdatomic> to <atomic>. The former never made it into the standard and doesn't exist in modern gcc versions at all. The later contains everything that leveldb was using from the former. This problem was noticed when porting to Portable Native Client where no memory barrier is defined. The fact that <cstdatomic> is missing normally goes unnoticed since memory barriers are defined for most architectures. * Make Hash() treat its input as unsigned. Before this change LevelDB files from platforms with different signedness of char were not compatible. This change fixes: issue #243 * Verify checksums of index/meta/filter blocks when paranoid_checks set. * Invoke all tools for iOS with xcrun. (This was causing problems with the new XCode 5.1.1 image on pulse.) * include <sys/stat.h> only once, and fix the following linter warning: "Found C system header after C++ system header" * When encountering a corrupted table file, return Status::Corruption instead of Status::InvalidArgument. * Support cygwin as build platform, patch is from https://code.google.com/p/leveldb/issues/detail?id=188 * Fix typo, merge patch from https://code.google.com/p/leveldb/issues/detail?id=159 * Fix typos and comments, and address the following two issues: * issue #166 * issue #241 * Add missing db synchronize after "fillseq" in the benchmark. * Removed unused variable in SeekRandom: value (issue #201)
10 роки тому
  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include <stdio.h>
  5. #include <stdlib.h>
  6. #include <sys/types.h>
  7. #include "leveldb/cache.h"
  8. #include "leveldb/db.h"
  9. #include "leveldb/env.h"
  10. #include "leveldb/filter_policy.h"
  11. #include "leveldb/write_batch.h"
  12. #include "port/port.h"
  13. #include "util/crc32c.h"
  14. #include "util/histogram.h"
  15. #include "util/mutexlock.h"
  16. #include "util/random.h"
  17. #include "util/testutil.h"
  18. // Comma-separated list of operations to run in the specified order
  19. // Actual benchmarks:
  20. // fillseq -- write N values in sequential key order in async mode
  21. // fillrandom -- write N values in random key order in async mode
  22. // overwrite -- overwrite N values in random key order in async mode
  23. // fillsync -- write N/100 values in random key order in sync mode
  24. // fill100K -- write N/1000 100K values in random order in async mode
  25. // deleteseq -- delete N keys in sequential order
  26. // deleterandom -- delete N keys in random order
  27. // readseq -- read N times sequentially
  28. // readreverse -- read N times in reverse order
  29. // readrandom -- read N times in random order
  30. // readmissing -- read N missing keys in random order
  31. // readhot -- read N times in random order from 1% section of DB
  32. // seekrandom -- N random seeks
  33. // open -- cost of opening a DB
  34. // crc32c -- repeated crc32c of 4K of data
  35. // Meta operations:
  36. // compact -- Compact the entire DB
  37. // stats -- Print DB stats
  38. // sstables -- Print sstable info
  39. // heapprofile -- Dump a heap profile (if supported by this port)
  40. static const char* FLAGS_benchmarks =
  41. "fillseq,"
  42. "fillsync,"
  43. "fillrandom,"
  44. "overwrite,"
  45. "readrandom,"
  46. "readrandom," // Extra run to allow previous compactions to quiesce
  47. "readseq,"
  48. "readreverse,"
  49. "compact,"
  50. "readrandom,"
  51. "readseq,"
  52. "readreverse,"
  53. "fill100K,"
  54. "crc32c,"
  55. "snappycomp,"
  56. "snappyuncomp,";
  57. // Number of key/values to place in database
  58. static int FLAGS_num = 1000000;
  59. // Number of read operations to do. If negative, do FLAGS_num reads.
  60. static int FLAGS_reads = -1;
  61. // Number of concurrent threads to run.
  62. static int FLAGS_threads = 1;
  63. // Size of each value
  64. static int FLAGS_value_size = 100;
  65. // Arrange to generate values that shrink to this fraction of
  66. // their original size after compression
  67. static double FLAGS_compression_ratio = 0.5;
  68. // Print histogram of operation timings
  69. static bool FLAGS_histogram = false;
  70. // Number of bytes to buffer in memtable before compacting
  71. // (initialized to default value by "main")
  72. static int FLAGS_write_buffer_size = 0;
  73. // Number of bytes written to each file.
  74. // (initialized to default value by "main")
  75. static int FLAGS_max_file_size = 0;
  76. // Approximate size of user data packed per block (before compression.
  77. // (initialized to default value by "main")
  78. static int FLAGS_block_size = 0;
  79. // Number of bytes to use as a cache of uncompressed data.
  80. // Negative means use default settings.
  81. static int FLAGS_cache_size = -1;
  82. // Maximum number of files to keep open at the same time (use default if == 0)
  83. static int FLAGS_open_files = 0;
  84. // Bloom filter bits per key.
  85. // Negative means use default settings.
  86. static int FLAGS_bloom_bits = -1;
  87. // If true, do not destroy the existing database. If you set this
  88. // flag and also specify a benchmark that wants a fresh database, that
  89. // benchmark will fail.
  90. static bool FLAGS_use_existing_db = false;
  91. // If true, reuse existing log/MANIFEST files when re-opening a database.
  92. static bool FLAGS_reuse_logs = false;
  93. // Use the db with the following name.
  94. static const char* FLAGS_db = nullptr;
  95. namespace leveldb {
  96. namespace {
  97. leveldb::Env* g_env = nullptr;
  98. // Helper for quickly generating random data.
  99. class RandomGenerator {
  100. private:
  101. std::string data_;
  102. int pos_;
  103. public:
  104. RandomGenerator() {
  105. // We use a limited amount of data over and over again and ensure
  106. // that it is larger than the compression window (32KB), and also
  107. // large enough to serve all typical value sizes we want to write.
  108. Random rnd(301);
  109. std::string piece;
  110. while (data_.size() < 1048576) {
  111. // Add a short fragment that is as compressible as specified
  112. // by FLAGS_compression_ratio.
  113. test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
  114. data_.append(piece);
  115. }
  116. pos_ = 0;
  117. }
  118. Slice Generate(size_t len) {
  119. if (pos_ + len > data_.size()) {
  120. pos_ = 0;
  121. assert(len < data_.size());
  122. }
  123. pos_ += len;
  124. return Slice(data_.data() + pos_ - len, len);
  125. }
  126. };
  127. #if defined(__linux)
  128. static Slice TrimSpace(Slice s) {
  129. size_t start = 0;
  130. while (start < s.size() && isspace(s[start])) {
  131. start++;
  132. }
  133. size_t limit = s.size();
  134. while (limit > start && isspace(s[limit - 1])) {
  135. limit--;
  136. }
  137. return Slice(s.data() + start, limit - start);
  138. }
  139. #endif
  140. static void AppendWithSpace(std::string* str, Slice msg) {
  141. if (msg.empty()) return;
  142. if (!str->empty()) {
  143. str->push_back(' ');
  144. }
  145. str->append(msg.data(), msg.size());
  146. }
  147. class Stats {
  148. private:
  149. double start_;
  150. double finish_;
  151. double seconds_;
  152. int done_;
  153. int next_report_;
  154. int64_t bytes_;
  155. double last_op_finish_;
  156. Histogram hist_;
  157. std::string message_;
  158. public:
  159. Stats() { Start(); }
  160. void Start() {
  161. next_report_ = 100;
  162. hist_.Clear();
  163. done_ = 0;
  164. bytes_ = 0;
  165. seconds_ = 0;
  166. message_.clear();
  167. start_ = finish_ = last_op_finish_ = g_env->NowMicros();
  168. }
  169. void Merge(const Stats& other) {
  170. hist_.Merge(other.hist_);
  171. done_ += other.done_;
  172. bytes_ += other.bytes_;
  173. seconds_ += other.seconds_;
  174. if (other.start_ < start_) start_ = other.start_;
  175. if (other.finish_ > finish_) finish_ = other.finish_;
  176. // Just keep the messages from one thread
  177. if (message_.empty()) message_ = other.message_;
  178. }
  179. void Stop() {
  180. finish_ = g_env->NowMicros();
  181. seconds_ = (finish_ - start_) * 1e-6;
  182. }
  183. void AddMessage(Slice msg) { AppendWithSpace(&message_, msg); }
  184. void FinishedSingleOp() {
  185. if (FLAGS_histogram) {
  186. double now = g_env->NowMicros();
  187. double micros = now - last_op_finish_;
  188. hist_.Add(micros);
  189. if (micros > 20000) {
  190. fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
  191. fflush(stderr);
  192. }
  193. last_op_finish_ = now;
  194. }
  195. done_++;
  196. if (done_ >= next_report_) {
  197. if (next_report_ < 1000)
  198. next_report_ += 100;
  199. else if (next_report_ < 5000)
  200. next_report_ += 500;
  201. else if (next_report_ < 10000)
  202. next_report_ += 1000;
  203. else if (next_report_ < 50000)
  204. next_report_ += 5000;
  205. else if (next_report_ < 100000)
  206. next_report_ += 10000;
  207. else if (next_report_ < 500000)
  208. next_report_ += 50000;
  209. else
  210. next_report_ += 100000;
  211. fprintf(stderr, "... finished %d ops%30s\r", done_, "");
  212. fflush(stderr);
  213. }
  214. }
  215. void AddBytes(int64_t n) { bytes_ += n; }
  216. void Report(const Slice& name) {
  217. // Pretend at least one op was done in case we are running a benchmark
  218. // that does not call FinishedSingleOp().
  219. if (done_ < 1) done_ = 1;
  220. std::string extra;
  221. if (bytes_ > 0) {
  222. // Rate is computed on actual elapsed time, not the sum of per-thread
  223. // elapsed times.
  224. double elapsed = (finish_ - start_) * 1e-6;
  225. char rate[100];
  226. snprintf(rate, sizeof(rate), "%6.1f MB/s",
  227. (bytes_ / 1048576.0) / elapsed);
  228. extra = rate;
  229. }
  230. AppendWithSpace(&extra, message_);
  231. fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", name.ToString().c_str(),
  232. seconds_ * 1e6 / done_, (extra.empty() ? "" : " "), extra.c_str());
  233. if (FLAGS_histogram) {
  234. fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
  235. }
  236. fflush(stdout);
  237. }
  238. };
  239. // State shared by all concurrent executions of the same benchmark.
  240. struct SharedState {
  241. port::Mutex mu;
  242. port::CondVar cv GUARDED_BY(mu);
  243. int total GUARDED_BY(mu);
  244. // Each thread goes through the following states:
  245. // (1) initializing
  246. // (2) waiting for others to be initialized
  247. // (3) running
  248. // (4) done
  249. int num_initialized GUARDED_BY(mu);
  250. int num_done GUARDED_BY(mu);
  251. bool start GUARDED_BY(mu);
  252. SharedState(int total)
  253. : cv(&mu), total(total), num_initialized(0), num_done(0), start(false) {}
  254. };
  255. // Per-thread state for concurrent executions of the same benchmark.
  256. struct ThreadState {
  257. int tid; // 0..n-1 when running in n threads
  258. Random rand; // Has different seeds for different threads
  259. Stats stats;
  260. SharedState* shared;
  261. ThreadState(int index) : tid(index), rand(1000 + index), shared(nullptr) {}
  262. };
  263. } // namespace
  264. class Benchmark {
  265. private:
  266. Cache* cache_;
  267. const FilterPolicy* filter_policy_;
  268. DB* db_;
  269. int num_;
  270. int value_size_;
  271. int entries_per_batch_;
  272. WriteOptions write_options_;
  273. int reads_;
  274. int heap_counter_;
  275. void PrintHeader() {
  276. const int kKeySize = 16;
  277. PrintEnvironment();
  278. fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
  279. fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
  280. FLAGS_value_size,
  281. static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
  282. fprintf(stdout, "Entries: %d\n", num_);
  283. fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
  284. ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) /
  285. 1048576.0));
  286. fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
  287. (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) /
  288. 1048576.0));
  289. PrintWarnings();
  290. fprintf(stdout, "------------------------------------------------\n");
  291. }
  292. void PrintWarnings() {
  293. #if defined(__GNUC__) && !defined(__OPTIMIZE__)
  294. fprintf(
  295. stdout,
  296. "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
  297. #endif
  298. #ifndef NDEBUG
  299. fprintf(stdout,
  300. "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
  301. #endif
  302. // See if snappy is working by attempting to compress a compressible string
  303. const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy";
  304. std::string compressed;
  305. if (!port::Snappy_Compress(text, sizeof(text), &compressed)) {
  306. fprintf(stdout, "WARNING: Snappy compression is not enabled\n");
  307. } else if (compressed.size() >= sizeof(text)) {
  308. fprintf(stdout, "WARNING: Snappy compression is not effective\n");
  309. }
  310. }
  311. void PrintEnvironment() {
  312. fprintf(stderr, "LevelDB: version %d.%d\n", kMajorVersion,
  313. kMinorVersion);
  314. #if defined(__linux)
  315. time_t now = time(nullptr);
  316. fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
  317. FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
  318. if (cpuinfo != nullptr) {
  319. char line[1000];
  320. int num_cpus = 0;
  321. std::string cpu_type;
  322. std::string cache_size;
  323. while (fgets(line, sizeof(line), cpuinfo) != nullptr) {
  324. const char* sep = strchr(line, ':');
  325. if (sep == nullptr) {
  326. continue;
  327. }
  328. Slice key = TrimSpace(Slice(line, sep - 1 - line));
  329. Slice val = TrimSpace(Slice(sep + 1));
  330. if (key == "model name") {
  331. ++num_cpus;
  332. cpu_type = val.ToString();
  333. } else if (key == "cache size") {
  334. cache_size = val.ToString();
  335. }
  336. }
  337. fclose(cpuinfo);
  338. fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
  339. fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
  340. }
  341. #endif
  342. }
  343. public:
  344. Benchmark()
  345. : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : nullptr),
  346. filter_policy_(FLAGS_bloom_bits >= 0
  347. ? NewBloomFilterPolicy(FLAGS_bloom_bits)
  348. : nullptr),
  349. db_(nullptr),
  350. num_(FLAGS_num),
  351. value_size_(FLAGS_value_size),
  352. entries_per_batch_(1),
  353. reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
  354. heap_counter_(0) {
  355. std::vector<std::string> files;
  356. g_env->GetChildren(FLAGS_db, &files);
  357. for (size_t i = 0; i < files.size(); i++) {
  358. if (Slice(files[i]).starts_with("heap-")) {
  359. g_env->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
  360. }
  361. }
  362. if (!FLAGS_use_existing_db) {
  363. DestroyDB(FLAGS_db, Options());
  364. }
  365. }
  366. ~Benchmark() {
  367. delete db_;
  368. delete cache_;
  369. delete filter_policy_;
  370. }
  371. void Run() {
  372. PrintHeader();
  373. Open();
  374. const char* benchmarks = FLAGS_benchmarks;
  375. while (benchmarks != nullptr) {
  376. const char* sep = strchr(benchmarks, ',');
  377. Slice name;
  378. if (sep == nullptr) {
  379. name = benchmarks;
  380. benchmarks = nullptr;
  381. } else {
  382. name = Slice(benchmarks, sep - benchmarks);
  383. benchmarks = sep + 1;
  384. }
  385. // Reset parameters that may be overridden below
  386. num_ = FLAGS_num;
  387. reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
  388. value_size_ = FLAGS_value_size;
  389. entries_per_batch_ = 1;
  390. write_options_ = WriteOptions();
  391. void (Benchmark::*method)(ThreadState*) = nullptr;
  392. bool fresh_db = false;
  393. int num_threads = FLAGS_threads;
  394. if (name == Slice("open")) {
  395. method = &Benchmark::OpenBench;
  396. num_ /= 10000;
  397. if (num_ < 1) num_ = 1;
  398. } else if (name == Slice("fillseq")) {
  399. fresh_db = true;
  400. method = &Benchmark::WriteSeq;
  401. } else if (name == Slice("fillbatch")) {
  402. fresh_db = true;
  403. entries_per_batch_ = 1000;
  404. method = &Benchmark::WriteSeq;
  405. } else if (name == Slice("fillrandom")) {
  406. fresh_db = true;
  407. method = &Benchmark::WriteRandom;
  408. } else if (name == Slice("overwrite")) {
  409. fresh_db = false;
  410. method = &Benchmark::WriteRandom;
  411. } else if (name == Slice("fillsync")) {
  412. fresh_db = true;
  413. num_ /= 1000;
  414. write_options_.sync = true;
  415. method = &Benchmark::WriteRandom;
  416. } else if (name == Slice("fill100K")) {
  417. fresh_db = true;
  418. num_ /= 1000;
  419. value_size_ = 100 * 1000;
  420. method = &Benchmark::WriteRandom;
  421. } else if (name == Slice("readseq")) {
  422. method = &Benchmark::ReadSequential;
  423. } else if (name == Slice("readreverse")) {
  424. method = &Benchmark::ReadReverse;
  425. } else if (name == Slice("readrandom")) {
  426. method = &Benchmark::ReadRandom;
  427. } else if (name == Slice("readmissing")) {
  428. method = &Benchmark::ReadMissing;
  429. } else if (name == Slice("seekrandom")) {
  430. method = &Benchmark::SeekRandom;
  431. } else if (name == Slice("readhot")) {
  432. method = &Benchmark::ReadHot;
  433. } else if (name == Slice("readrandomsmall")) {
  434. reads_ /= 1000;
  435. method = &Benchmark::ReadRandom;
  436. } else if (name == Slice("deleteseq")) {
  437. method = &Benchmark::DeleteSeq;
  438. } else if (name == Slice("deleterandom")) {
  439. method = &Benchmark::DeleteRandom;
  440. } else if (name == Slice("readwhilewriting")) {
  441. num_threads++; // Add extra thread for writing
  442. method = &Benchmark::ReadWhileWriting;
  443. } else if (name == Slice("compact")) {
  444. method = &Benchmark::Compact;
  445. } else if (name == Slice("crc32c")) {
  446. method = &Benchmark::Crc32c;
  447. } else if (name == Slice("snappycomp")) {
  448. method = &Benchmark::SnappyCompress;
  449. } else if (name == Slice("snappyuncomp")) {
  450. method = &Benchmark::SnappyUncompress;
  451. } else if (name == Slice("heapprofile")) {
  452. HeapProfile();
  453. } else if (name == Slice("stats")) {
  454. PrintStats("leveldb.stats");
  455. } else if (name == Slice("sstables")) {
  456. PrintStats("leveldb.sstables");
  457. } else {
  458. if (!name.empty()) { // No error message for empty name
  459. fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
  460. }
  461. }
  462. if (fresh_db) {
  463. if (FLAGS_use_existing_db) {
  464. fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n",
  465. name.ToString().c_str());
  466. method = nullptr;
  467. } else {
  468. delete db_;
  469. db_ = nullptr;
  470. DestroyDB(FLAGS_db, Options());
  471. Open();
  472. }
  473. }
  474. if (method != nullptr) {
  475. RunBenchmark(num_threads, name, method);
  476. }
  477. }
  478. }
  479. private:
  480. struct ThreadArg {
  481. Benchmark* bm;
  482. SharedState* shared;
  483. ThreadState* thread;
  484. void (Benchmark::*method)(ThreadState*);
  485. };
  486. static void ThreadBody(void* v) {
  487. ThreadArg* arg = reinterpret_cast<ThreadArg*>(v);
  488. SharedState* shared = arg->shared;
  489. ThreadState* thread = arg->thread;
  490. {
  491. MutexLock l(&shared->mu);
  492. shared->num_initialized++;
  493. if (shared->num_initialized >= shared->total) {
  494. shared->cv.SignalAll();
  495. }
  496. while (!shared->start) {
  497. shared->cv.Wait();
  498. }
  499. }
  500. thread->stats.Start();
  501. (arg->bm->*(arg->method))(thread);
  502. thread->stats.Stop();
  503. {
  504. MutexLock l(&shared->mu);
  505. shared->num_done++;
  506. if (shared->num_done >= shared->total) {
  507. shared->cv.SignalAll();
  508. }
  509. }
  510. }
  511. void RunBenchmark(int n, Slice name,
  512. void (Benchmark::*method)(ThreadState*)) {
  513. SharedState shared(n);
  514. ThreadArg* arg = new ThreadArg[n];
  515. for (int i = 0; i < n; i++) {
  516. arg[i].bm = this;
  517. arg[i].method = method;
  518. arg[i].shared = &shared;
  519. arg[i].thread = new ThreadState(i);
  520. arg[i].thread->shared = &shared;
  521. g_env->StartThread(ThreadBody, &arg[i]);
  522. }
  523. shared.mu.Lock();
  524. while (shared.num_initialized < n) {
  525. shared.cv.Wait();
  526. }
  527. shared.start = true;
  528. shared.cv.SignalAll();
  529. while (shared.num_done < n) {
  530. shared.cv.Wait();
  531. }
  532. shared.mu.Unlock();
  533. for (int i = 1; i < n; i++) {
  534. arg[0].thread->stats.Merge(arg[i].thread->stats);
  535. }
  536. arg[0].thread->stats.Report(name);
  537. for (int i = 0; i < n; i++) {
  538. delete arg[i].thread;
  539. }
  540. delete[] arg;
  541. }
  542. void Crc32c(ThreadState* thread) {
  543. // Checksum about 500MB of data total
  544. const int size = 4096;
  545. const char* label = "(4K per op)";
  546. std::string data(size, 'x');
  547. int64_t bytes = 0;
  548. uint32_t crc = 0;
  549. while (bytes < 500 * 1048576) {
  550. crc = crc32c::Value(data.data(), size);
  551. thread->stats.FinishedSingleOp();
  552. bytes += size;
  553. }
  554. // Print so result is not dead
  555. fprintf(stderr, "... crc=0x%x\r", static_cast<unsigned int>(crc));
  556. thread->stats.AddBytes(bytes);
  557. thread->stats.AddMessage(label);
  558. }
  559. void SnappyCompress(ThreadState* thread) {
  560. RandomGenerator gen;
  561. Slice input = gen.Generate(Options().block_size);
  562. int64_t bytes = 0;
  563. int64_t produced = 0;
  564. bool ok = true;
  565. std::string compressed;
  566. while (ok && bytes < 1024 * 1048576) { // Compress 1G
  567. ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
  568. produced += compressed.size();
  569. bytes += input.size();
  570. thread->stats.FinishedSingleOp();
  571. }
  572. if (!ok) {
  573. thread->stats.AddMessage("(snappy failure)");
  574. } else {
  575. char buf[100];
  576. snprintf(buf, sizeof(buf), "(output: %.1f%%)",
  577. (produced * 100.0) / bytes);
  578. thread->stats.AddMessage(buf);
  579. thread->stats.AddBytes(bytes);
  580. }
  581. }
  582. void SnappyUncompress(ThreadState* thread) {
  583. RandomGenerator gen;
  584. Slice input = gen.Generate(Options().block_size);
  585. std::string compressed;
  586. bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
  587. int64_t bytes = 0;
  588. char* uncompressed = new char[input.size()];
  589. while (ok && bytes < 1024 * 1048576) { // Compress 1G
  590. ok = port::Snappy_Uncompress(compressed.data(), compressed.size(),
  591. uncompressed);
  592. bytes += input.size();
  593. thread->stats.FinishedSingleOp();
  594. }
  595. delete[] uncompressed;
  596. if (!ok) {
  597. thread->stats.AddMessage("(snappy failure)");
  598. } else {
  599. thread->stats.AddBytes(bytes);
  600. }
  601. }
  602. void Open() {
  603. assert(db_ == nullptr);
  604. Options options;
  605. options.env = g_env;
  606. options.create_if_missing = !FLAGS_use_existing_db;
  607. options.block_cache = cache_;
  608. options.write_buffer_size = FLAGS_write_buffer_size;
  609. options.max_file_size = FLAGS_max_file_size;
  610. options.block_size = FLAGS_block_size;
  611. options.max_open_files = FLAGS_open_files;
  612. options.filter_policy = filter_policy_;
  613. options.reuse_logs = FLAGS_reuse_logs;
  614. Status s = DB::Open(options, FLAGS_db, &db_);
  615. if (!s.ok()) {
  616. fprintf(stderr, "open error: %s\n", s.ToString().c_str());
  617. exit(1);
  618. }
  619. }
  620. void OpenBench(ThreadState* thread) {
  621. for (int i = 0; i < num_; i++) {
  622. delete db_;
  623. Open();
  624. thread->stats.FinishedSingleOp();
  625. }
  626. }
  627. void WriteSeq(ThreadState* thread) { DoWrite(thread, true); }
  628. void WriteRandom(ThreadState* thread) { DoWrite(thread, false); }
  629. void DoWrite(ThreadState* thread, bool seq) {
  630. if (num_ != FLAGS_num) {
  631. char msg[100];
  632. snprintf(msg, sizeof(msg), "(%d ops)", num_);
  633. thread->stats.AddMessage(msg);
  634. }
  635. RandomGenerator gen;
  636. WriteBatch batch;
  637. Status s;
  638. int64_t bytes = 0;
  639. for (int i = 0; i < num_; i += entries_per_batch_) {
  640. batch.Clear();
  641. for (int j = 0; j < entries_per_batch_; j++) {
  642. const int k = seq ? i + j : (thread->rand.Next() % FLAGS_num);
  643. char key[100];
  644. snprintf(key, sizeof(key), "%016d", k);
  645. batch.Put(key, gen.Generate(value_size_));
  646. bytes += value_size_ + strlen(key);
  647. thread->stats.FinishedSingleOp();
  648. }
  649. s = db_->Write(write_options_, &batch);
  650. if (!s.ok()) {
  651. fprintf(stderr, "put error: %s\n", s.ToString().c_str());
  652. exit(1);
  653. }
  654. }
  655. thread->stats.AddBytes(bytes);
  656. }
  657. void ReadSequential(ThreadState* thread) {
  658. Iterator* iter = db_->NewIterator(ReadOptions());
  659. int i = 0;
  660. int64_t bytes = 0;
  661. for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
  662. bytes += iter->key().size() + iter->value().size();
  663. thread->stats.FinishedSingleOp();
  664. ++i;
  665. }
  666. delete iter;
  667. thread->stats.AddBytes(bytes);
  668. }
  669. void ReadReverse(ThreadState* thread) {
  670. Iterator* iter = db_->NewIterator(ReadOptions());
  671. int i = 0;
  672. int64_t bytes = 0;
  673. for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) {
  674. bytes += iter->key().size() + iter->value().size();
  675. thread->stats.FinishedSingleOp();
  676. ++i;
  677. }
  678. delete iter;
  679. thread->stats.AddBytes(bytes);
  680. }
  681. void ReadRandom(ThreadState* thread) {
  682. ReadOptions options;
  683. std::string value;
  684. int found = 0;
  685. for (int i = 0; i < reads_; i++) {
  686. char key[100];
  687. const int k = thread->rand.Next() % FLAGS_num;
  688. snprintf(key, sizeof(key), "%016d", k);
  689. if (db_->Get(options, key, &value).ok()) {
  690. found++;
  691. }
  692. thread->stats.FinishedSingleOp();
  693. }
  694. char msg[100];
  695. snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
  696. thread->stats.AddMessage(msg);
  697. }
  698. void ReadMissing(ThreadState* thread) {
  699. ReadOptions options;
  700. std::string value;
  701. for (int i = 0; i < reads_; i++) {
  702. char key[100];
  703. const int k = thread->rand.Next() % FLAGS_num;
  704. snprintf(key, sizeof(key), "%016d.", k);
  705. db_->Get(options, key, &value);
  706. thread->stats.FinishedSingleOp();
  707. }
  708. }
  709. void ReadHot(ThreadState* thread) {
  710. ReadOptions options;
  711. std::string value;
  712. const int range = (FLAGS_num + 99) / 100;
  713. for (int i = 0; i < reads_; i++) {
  714. char key[100];
  715. const int k = thread->rand.Next() % range;
  716. snprintf(key, sizeof(key), "%016d", k);
  717. db_->Get(options, key, &value);
  718. thread->stats.FinishedSingleOp();
  719. }
  720. }
  721. void SeekRandom(ThreadState* thread) {
  722. ReadOptions options;
  723. int found = 0;
  724. for (int i = 0; i < reads_; i++) {
  725. Iterator* iter = db_->NewIterator(options);
  726. char key[100];
  727. const int k = thread->rand.Next() % FLAGS_num;
  728. snprintf(key, sizeof(key), "%016d", k);
  729. iter->Seek(key);
  730. if (iter->Valid() && iter->key() == key) found++;
  731. delete iter;
  732. thread->stats.FinishedSingleOp();
  733. }
  734. char msg[100];
  735. snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
  736. thread->stats.AddMessage(msg);
  737. }
  738. void DoDelete(ThreadState* thread, bool seq) {
  739. RandomGenerator gen;
  740. WriteBatch batch;
  741. Status s;
  742. for (int i = 0; i < num_; i += entries_per_batch_) {
  743. batch.Clear();
  744. for (int j = 0; j < entries_per_batch_; j++) {
  745. const int k = seq ? i + j : (thread->rand.Next() % FLAGS_num);
  746. char key[100];
  747. snprintf(key, sizeof(key), "%016d", k);
  748. batch.Delete(key);
  749. thread->stats.FinishedSingleOp();
  750. }
  751. s = db_->Write(write_options_, &batch);
  752. if (!s.ok()) {
  753. fprintf(stderr, "del error: %s\n", s.ToString().c_str());
  754. exit(1);
  755. }
  756. }
  757. }
  758. void DeleteSeq(ThreadState* thread) { DoDelete(thread, true); }
  759. void DeleteRandom(ThreadState* thread) { DoDelete(thread, false); }
  760. void ReadWhileWriting(ThreadState* thread) {
  761. if (thread->tid > 0) {
  762. ReadRandom(thread);
  763. } else {
  764. // Special thread that keeps writing until other threads are done.
  765. RandomGenerator gen;
  766. while (true) {
  767. {
  768. MutexLock l(&thread->shared->mu);
  769. if (thread->shared->num_done + 1 >= thread->shared->num_initialized) {
  770. // Other threads have finished
  771. break;
  772. }
  773. }
  774. const int k = thread->rand.Next() % FLAGS_num;
  775. char key[100];
  776. snprintf(key, sizeof(key), "%016d", k);
  777. Status s = db_->Put(write_options_, key, gen.Generate(value_size_));
  778. if (!s.ok()) {
  779. fprintf(stderr, "put error: %s\n", s.ToString().c_str());
  780. exit(1);
  781. }
  782. }
  783. // Do not count any of the preceding work/delay in stats.
  784. thread->stats.Start();
  785. }
  786. }
  787. void Compact(ThreadState* thread) { db_->CompactRange(nullptr, nullptr); }
  788. void PrintStats(const char* key) {
  789. std::string stats;
  790. if (!db_->GetProperty(key, &stats)) {
  791. stats = "(failed)";
  792. }
  793. fprintf(stdout, "\n%s\n", stats.c_str());
  794. }
  795. static void WriteToFile(void* arg, const char* buf, int n) {
  796. reinterpret_cast<WritableFile*>(arg)->Append(Slice(buf, n));
  797. }
  798. void HeapProfile() {
  799. char fname[100];
  800. snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
  801. WritableFile* file;
  802. Status s = g_env->NewWritableFile(fname, &file);
  803. if (!s.ok()) {
  804. fprintf(stderr, "%s\n", s.ToString().c_str());
  805. return;
  806. }
  807. bool ok = port::GetHeapProfile(WriteToFile, file);
  808. delete file;
  809. if (!ok) {
  810. fprintf(stderr, "heap profiling not supported\n");
  811. g_env->DeleteFile(fname);
  812. }
  813. }
  814. };
  815. } // namespace leveldb
  816. int main(int argc, char** argv) {
  817. FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
  818. FLAGS_max_file_size = leveldb::Options().max_file_size;
  819. FLAGS_block_size = leveldb::Options().block_size;
  820. FLAGS_open_files = leveldb::Options().max_open_files;
  821. std::string default_db_path;
  822. for (int i = 1; i < argc; i++) {
  823. double d;
  824. int n;
  825. char junk;
  826. if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
  827. FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
  828. } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
  829. FLAGS_compression_ratio = d;
  830. } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
  831. (n == 0 || n == 1)) {
  832. FLAGS_histogram = n;
  833. } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
  834. (n == 0 || n == 1)) {
  835. FLAGS_use_existing_db = n;
  836. } else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 &&
  837. (n == 0 || n == 1)) {
  838. FLAGS_reuse_logs = n;
  839. } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
  840. FLAGS_num = n;
  841. } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
  842. FLAGS_reads = n;
  843. } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) {
  844. FLAGS_threads = n;
  845. } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
  846. FLAGS_value_size = n;
  847. } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
  848. FLAGS_write_buffer_size = n;
  849. } else if (sscanf(argv[i], "--max_file_size=%d%c", &n, &junk) == 1) {
  850. FLAGS_max_file_size = n;
  851. } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {
  852. FLAGS_block_size = n;
  853. } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
  854. FLAGS_cache_size = n;
  855. } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) {
  856. FLAGS_bloom_bits = n;
  857. } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
  858. FLAGS_open_files = n;
  859. } else if (strncmp(argv[i], "--db=", 5) == 0) {
  860. FLAGS_db = argv[i] + 5;
  861. } else {
  862. fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
  863. exit(1);
  864. }
  865. }
  866. leveldb::g_env = leveldb::Env::Default();
  867. // Choose a location for the test database if none given with --db=<path>
  868. if (FLAGS_db == nullptr) {
  869. leveldb::g_env->GetTestDirectory(&default_db_path);
  870. default_db_path += "/dbbench";
  871. FLAGS_db = default_db_path.c_str();
  872. }
  873. leveldb::Benchmark benchmark;
  874. benchmark.Run();
  875. return 0;
  876. }