小组成员:10215300402-朱维清 & 10222140408 谷杰
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

721 lines
22 KiB

Add WITHOUT ROWID to SQLite benchmark. The SQLite-specific schema feature is documented at https://www.sqlite.org/withoutrowid.html and https://www.sqlite.org/rowidtable.html. By default, SQLite stores each table in a B-tree keyed by an integer, called the ROWID. Any index, including the PRIMARY KEY index, is a separate B-tree mapping index keys to ROWIDs. Tables without ROWIDs are stored in a B-tree keyed by the primary key. Additional indexes (the PRIMARY KEY index is implicitly built into the table) are stored as B-trees mapping index keys to row primary keys. This CL introduces a boolean --use-rowids flag to db_bench_sqlite. When the flag is false (default), the schema of the test table includes WITHOUT ROWID. The test table uses a primary key, so adding WITHOUT ROWID to the schema reduces the number of B-trees used by the benchmark from 2 to 1. This brings SQLite's disk usage closer to LevelDB. When WITHOUT ROWID is used, SQLite fares better (than today) on benchmarks with small (16-byte) keys, and worse on benchmarks with large (100kb) keys. Baseline results: fillseq : 21.310 micros/op; 5.2 MB/s fillseqsync : 146.377 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.065 micros/op; 53.6 MB/s fillrandom : 34.767 micros/op; 3.2 MB/s fillrandsync : 159.943 micros/op; 0.7 MB/s (10000 ops) fillrandbatch : 15.055 micros/op; 7.3 MB/s overwrite : 43.660 micros/op; 2.5 MB/s overwritebatch : 27.691 micros/op; 4.0 MB/s readrandom : 12.725 micros/op; readseq : 2.602 micros/op; 36.7 MB/s fillrand100K : 606.333 micros/op; 157.3 MB/s (1000 ops) fillseq100K : 657.457 micros/op; 145.1 MB/s (1000 ops) readseq : 46.523 micros/op; 2049.9 MB/s readrand100K : 54.943 micros/op; Results after this CL: fillseq : 16.231 micros/op; 6.8 MB/s fillseqsync : 147.460 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.294 micros/op; 48.2 MB/s fillrandom : 27.871 micros/op; 4.0 MB/s fillrandsync : 141.979 micros/op; 0.8 MB/s (10000 ops) fillrandbatch : 16.087 micros/op; 6.9 MB/s overwrite : 26.829 micros/op; 4.1 MB/s overwritebatch : 19.014 micros/op; 5.8 MB/s readrandom : 11.657 micros/op; readseq : 0.155 micros/op; 615.0 MB/s fillrand100K : 816.812 micros/op; 116.8 MB/s (1000 ops) fillseq100K : 754.689 micros/op; 126.4 MB/s (1000 ops) readseq : 47.112 micros/op; 2024.3 MB/s readrand100K : 287.679 micros/op; Results after this CL, with --use-rowids=1 fillseq : 20.655 micros/op; 5.4 MB/s fillseqsync : 146.408 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.045 micros/op; 54.1 MB/s fillrandom : 34.080 micros/op; 3.2 MB/s fillrandsync : 154.582 micros/op; 0.7 MB/s (10000 ops) fillrandbatch : 14.404 micros/op; 7.7 MB/s overwrite : 42.928 micros/op; 2.6 MB/s overwritebatch : 27.829 micros/op; 4.0 MB/s readrandom : 12.835 micros/op; readseq : 2.483 micros/op; 38.4 MB/s fillrand100K : 603.265 micros/op; 158.1 MB/s (1000 ops) fillseq100K : 662.473 micros/op; 144.0 MB/s (1000 ops) readseq : 45.478 micros/op; 2097.0 MB/s readrand100K : 54.439 micros/op; PiperOrigin-RevId: 283407101
5 years ago
Add WITHOUT ROWID to SQLite benchmark. The SQLite-specific schema feature is documented at https://www.sqlite.org/withoutrowid.html and https://www.sqlite.org/rowidtable.html. By default, SQLite stores each table in a B-tree keyed by an integer, called the ROWID. Any index, including the PRIMARY KEY index, is a separate B-tree mapping index keys to ROWIDs. Tables without ROWIDs are stored in a B-tree keyed by the primary key. Additional indexes (the PRIMARY KEY index is implicitly built into the table) are stored as B-trees mapping index keys to row primary keys. This CL introduces a boolean --use-rowids flag to db_bench_sqlite. When the flag is false (default), the schema of the test table includes WITHOUT ROWID. The test table uses a primary key, so adding WITHOUT ROWID to the schema reduces the number of B-trees used by the benchmark from 2 to 1. This brings SQLite's disk usage closer to LevelDB. When WITHOUT ROWID is used, SQLite fares better (than today) on benchmarks with small (16-byte) keys, and worse on benchmarks with large (100kb) keys. Baseline results: fillseq : 21.310 micros/op; 5.2 MB/s fillseqsync : 146.377 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.065 micros/op; 53.6 MB/s fillrandom : 34.767 micros/op; 3.2 MB/s fillrandsync : 159.943 micros/op; 0.7 MB/s (10000 ops) fillrandbatch : 15.055 micros/op; 7.3 MB/s overwrite : 43.660 micros/op; 2.5 MB/s overwritebatch : 27.691 micros/op; 4.0 MB/s readrandom : 12.725 micros/op; readseq : 2.602 micros/op; 36.7 MB/s fillrand100K : 606.333 micros/op; 157.3 MB/s (1000 ops) fillseq100K : 657.457 micros/op; 145.1 MB/s (1000 ops) readseq : 46.523 micros/op; 2049.9 MB/s readrand100K : 54.943 micros/op; Results after this CL: fillseq : 16.231 micros/op; 6.8 MB/s fillseqsync : 147.460 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.294 micros/op; 48.2 MB/s fillrandom : 27.871 micros/op; 4.0 MB/s fillrandsync : 141.979 micros/op; 0.8 MB/s (10000 ops) fillrandbatch : 16.087 micros/op; 6.9 MB/s overwrite : 26.829 micros/op; 4.1 MB/s overwritebatch : 19.014 micros/op; 5.8 MB/s readrandom : 11.657 micros/op; readseq : 0.155 micros/op; 615.0 MB/s fillrand100K : 816.812 micros/op; 116.8 MB/s (1000 ops) fillseq100K : 754.689 micros/op; 126.4 MB/s (1000 ops) readseq : 47.112 micros/op; 2024.3 MB/s readrand100K : 287.679 micros/op; Results after this CL, with --use-rowids=1 fillseq : 20.655 micros/op; 5.4 MB/s fillseqsync : 146.408 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.045 micros/op; 54.1 MB/s fillrandom : 34.080 micros/op; 3.2 MB/s fillrandsync : 154.582 micros/op; 0.7 MB/s (10000 ops) fillrandbatch : 14.404 micros/op; 7.7 MB/s overwrite : 42.928 micros/op; 2.6 MB/s overwritebatch : 27.829 micros/op; 4.0 MB/s readrandom : 12.835 micros/op; readseq : 2.483 micros/op; 38.4 MB/s fillrand100K : 603.265 micros/op; 158.1 MB/s (1000 ops) fillseq100K : 662.473 micros/op; 144.0 MB/s (1000 ops) readseq : 45.478 micros/op; 2097.0 MB/s readrand100K : 54.439 micros/op; PiperOrigin-RevId: 283407101
5 years ago
Add WITHOUT ROWID to SQLite benchmark. The SQLite-specific schema feature is documented at https://www.sqlite.org/withoutrowid.html and https://www.sqlite.org/rowidtable.html. By default, SQLite stores each table in a B-tree keyed by an integer, called the ROWID. Any index, including the PRIMARY KEY index, is a separate B-tree mapping index keys to ROWIDs. Tables without ROWIDs are stored in a B-tree keyed by the primary key. Additional indexes (the PRIMARY KEY index is implicitly built into the table) are stored as B-trees mapping index keys to row primary keys. This CL introduces a boolean --use-rowids flag to db_bench_sqlite. When the flag is false (default), the schema of the test table includes WITHOUT ROWID. The test table uses a primary key, so adding WITHOUT ROWID to the schema reduces the number of B-trees used by the benchmark from 2 to 1. This brings SQLite's disk usage closer to LevelDB. When WITHOUT ROWID is used, SQLite fares better (than today) on benchmarks with small (16-byte) keys, and worse on benchmarks with large (100kb) keys. Baseline results: fillseq : 21.310 micros/op; 5.2 MB/s fillseqsync : 146.377 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.065 micros/op; 53.6 MB/s fillrandom : 34.767 micros/op; 3.2 MB/s fillrandsync : 159.943 micros/op; 0.7 MB/s (10000 ops) fillrandbatch : 15.055 micros/op; 7.3 MB/s overwrite : 43.660 micros/op; 2.5 MB/s overwritebatch : 27.691 micros/op; 4.0 MB/s readrandom : 12.725 micros/op; readseq : 2.602 micros/op; 36.7 MB/s fillrand100K : 606.333 micros/op; 157.3 MB/s (1000 ops) fillseq100K : 657.457 micros/op; 145.1 MB/s (1000 ops) readseq : 46.523 micros/op; 2049.9 MB/s readrand100K : 54.943 micros/op; Results after this CL: fillseq : 16.231 micros/op; 6.8 MB/s fillseqsync : 147.460 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.294 micros/op; 48.2 MB/s fillrandom : 27.871 micros/op; 4.0 MB/s fillrandsync : 141.979 micros/op; 0.8 MB/s (10000 ops) fillrandbatch : 16.087 micros/op; 6.9 MB/s overwrite : 26.829 micros/op; 4.1 MB/s overwritebatch : 19.014 micros/op; 5.8 MB/s readrandom : 11.657 micros/op; readseq : 0.155 micros/op; 615.0 MB/s fillrand100K : 816.812 micros/op; 116.8 MB/s (1000 ops) fillseq100K : 754.689 micros/op; 126.4 MB/s (1000 ops) readseq : 47.112 micros/op; 2024.3 MB/s readrand100K : 287.679 micros/op; Results after this CL, with --use-rowids=1 fillseq : 20.655 micros/op; 5.4 MB/s fillseqsync : 146.408 micros/op; 0.8 MB/s (10000 ops) fillseqbatch : 2.045 micros/op; 54.1 MB/s fillrandom : 34.080 micros/op; 3.2 MB/s fillrandsync : 154.582 micros/op; 0.7 MB/s (10000 ops) fillrandbatch : 14.404 micros/op; 7.7 MB/s overwrite : 42.928 micros/op; 2.6 MB/s overwritebatch : 27.829 micros/op; 4.0 MB/s readrandom : 12.835 micros/op; readseq : 2.483 micros/op; 38.4 MB/s fillrand100K : 603.265 micros/op; 158.1 MB/s (1000 ops) fillseq100K : 662.473 micros/op; 144.0 MB/s (1000 ops) readseq : 45.478 micros/op; 2097.0 MB/s readrand100K : 54.439 micros/op; PiperOrigin-RevId: 283407101
5 years ago
  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include <sqlite3.h>
  5. #include <stdio.h>
  6. #include <stdlib.h>
  7. #include "util/histogram.h"
  8. #include "util/random.h"
  9. #include "util/testutil.h"
  10. // Comma-separated list of operations to run in the specified order
  11. // Actual benchmarks:
  12. //
  13. // fillseq -- write N values in sequential key order in async mode
  14. // fillseqsync -- write N/100 values in sequential key order in sync mode
  15. // fillseqbatch -- batch write N values in sequential key order in async mode
  16. // fillrandom -- write N values in random key order in async mode
  17. // fillrandsync -- write N/100 values in random key order in sync mode
  18. // fillrandbatch -- batch write N values in sequential key order in async mode
  19. // overwrite -- overwrite N values in random key order in async mode
  20. // fillrand100K -- write N/1000 100K values in random order in async mode
  21. // fillseq100K -- write N/1000 100K values in sequential order in async mode
  22. // readseq -- read N times sequentially
  23. // readrandom -- read N times in random order
  24. // readrand100K -- read N/1000 100K values in sequential order in async mode
  25. static const char* FLAGS_benchmarks =
  26. "fillseq,"
  27. "fillseqsync,"
  28. "fillseqbatch,"
  29. "fillrandom,"
  30. "fillrandsync,"
  31. "fillrandbatch,"
  32. "overwrite,"
  33. "overwritebatch,"
  34. "readrandom,"
  35. "readseq,"
  36. "fillrand100K,"
  37. "fillseq100K,"
  38. "readseq,"
  39. "readrand100K,";
  40. // Number of key/values to place in database
  41. static int FLAGS_num = 1000000;
  42. // Number of read operations to do. If negative, do FLAGS_num reads.
  43. static int FLAGS_reads = -1;
  44. // Size of each value
  45. static int FLAGS_value_size = 100;
  46. // Print histogram of operation timings
  47. static bool FLAGS_histogram = false;
  48. // Arrange to generate values that shrink to this fraction of
  49. // their original size after compression
  50. static double FLAGS_compression_ratio = 0.5;
  51. // Page size. Default 1 KB.
  52. static int FLAGS_page_size = 1024;
  53. // Number of pages.
  54. // Default cache size = FLAGS_page_size * FLAGS_num_pages = 4 MB.
  55. static int FLAGS_num_pages = 4096;
  56. // If true, do not destroy the existing database. If you set this
  57. // flag and also specify a benchmark that wants a fresh database, that
  58. // benchmark will fail.
  59. static bool FLAGS_use_existing_db = false;
  60. // If true, the SQLite table has ROWIDs.
  61. static bool FLAGS_use_rowids = false;
  62. // If true, we allow batch writes to occur
  63. static bool FLAGS_transaction = true;
  64. // If true, we enable Write-Ahead Logging
  65. static bool FLAGS_WAL_enabled = true;
  66. // Use the db with the following name.
  67. static const char* FLAGS_db = nullptr;
  68. inline static void ExecErrorCheck(int status, char* err_msg) {
  69. if (status != SQLITE_OK) {
  70. fprintf(stderr, "SQL error: %s\n", err_msg);
  71. sqlite3_free(err_msg);
  72. exit(1);
  73. }
  74. }
  75. inline static void StepErrorCheck(int status) {
  76. if (status != SQLITE_DONE) {
  77. fprintf(stderr, "SQL step error: status = %d\n", status);
  78. exit(1);
  79. }
  80. }
  81. inline static void ErrorCheck(int status) {
  82. if (status != SQLITE_OK) {
  83. fprintf(stderr, "sqlite3 error: status = %d\n", status);
  84. exit(1);
  85. }
  86. }
  87. inline static void WalCheckpoint(sqlite3* db_) {
  88. // Flush all writes to disk
  89. if (FLAGS_WAL_enabled) {
  90. sqlite3_wal_checkpoint_v2(db_, nullptr, SQLITE_CHECKPOINT_FULL, nullptr,
  91. nullptr);
  92. }
  93. }
  94. namespace leveldb {
  95. // Helper for quickly generating random data.
  96. namespace {
  97. class RandomGenerator {
  98. private:
  99. std::string data_;
  100. int pos_;
  101. public:
  102. RandomGenerator() {
  103. // We use a limited amount of data over and over again and ensure
  104. // that it is larger than the compression window (32KB), and also
  105. // large enough to serve all typical value sizes we want to write.
  106. Random rnd(301);
  107. std::string piece;
  108. while (data_.size() < 1048576) {
  109. // Add a short fragment that is as compressible as specified
  110. // by FLAGS_compression_ratio.
  111. test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
  112. data_.append(piece);
  113. }
  114. pos_ = 0;
  115. }
  116. Slice Generate(int len) {
  117. if (pos_ + len > data_.size()) {
  118. pos_ = 0;
  119. assert(len < data_.size());
  120. }
  121. pos_ += len;
  122. return Slice(data_.data() + pos_ - len, len);
  123. }
  124. };
  125. static Slice TrimSpace(Slice s) {
  126. int start = 0;
  127. while (start < s.size() && isspace(s[start])) {
  128. start++;
  129. }
  130. int limit = s.size();
  131. while (limit > start && isspace(s[limit - 1])) {
  132. limit--;
  133. }
  134. return Slice(s.data() + start, limit - start);
  135. }
  136. } // namespace
  137. class Benchmark {
  138. private:
  139. sqlite3* db_;
  140. int db_num_;
  141. int num_;
  142. int reads_;
  143. double start_;
  144. double last_op_finish_;
  145. int64_t bytes_;
  146. std::string message_;
  147. Histogram hist_;
  148. RandomGenerator gen_;
  149. Random rand_;
  150. // State kept for progress messages
  151. int done_;
  152. int next_report_; // When to report next
  153. void PrintHeader() {
  154. const int kKeySize = 16;
  155. PrintEnvironment();
  156. fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
  157. fprintf(stdout, "Values: %d bytes each\n", FLAGS_value_size);
  158. fprintf(stdout, "Entries: %d\n", num_);
  159. fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
  160. ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) /
  161. 1048576.0));
  162. PrintWarnings();
  163. fprintf(stdout, "------------------------------------------------\n");
  164. }
  165. void PrintWarnings() {
  166. #if defined(__GNUC__) && !defined(__OPTIMIZE__)
  167. fprintf(
  168. stdout,
  169. "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
  170. #endif
  171. #ifndef NDEBUG
  172. fprintf(stdout,
  173. "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
  174. #endif
  175. }
  176. void PrintEnvironment() {
  177. fprintf(stderr, "SQLite: version %s\n", SQLITE_VERSION);
  178. #if defined(__linux)
  179. time_t now = time(nullptr);
  180. fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
  181. FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
  182. if (cpuinfo != nullptr) {
  183. char line[1000];
  184. int num_cpus = 0;
  185. std::string cpu_type;
  186. std::string cache_size;
  187. while (fgets(line, sizeof(line), cpuinfo) != nullptr) {
  188. const char* sep = strchr(line, ':');
  189. if (sep == nullptr) {
  190. continue;
  191. }
  192. Slice key = TrimSpace(Slice(line, sep - 1 - line));
  193. Slice val = TrimSpace(Slice(sep + 1));
  194. if (key == "model name") {
  195. ++num_cpus;
  196. cpu_type = val.ToString();
  197. } else if (key == "cache size") {
  198. cache_size = val.ToString();
  199. }
  200. }
  201. fclose(cpuinfo);
  202. fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
  203. fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
  204. }
  205. #endif
  206. }
  207. void Start() {
  208. start_ = Env::Default()->NowMicros() * 1e-6;
  209. bytes_ = 0;
  210. message_.clear();
  211. last_op_finish_ = start_;
  212. hist_.Clear();
  213. done_ = 0;
  214. next_report_ = 100;
  215. }
  216. void FinishedSingleOp() {
  217. if (FLAGS_histogram) {
  218. double now = Env::Default()->NowMicros() * 1e-6;
  219. double micros = (now - last_op_finish_) * 1e6;
  220. hist_.Add(micros);
  221. if (micros > 20000) {
  222. fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
  223. fflush(stderr);
  224. }
  225. last_op_finish_ = now;
  226. }
  227. done_++;
  228. if (done_ >= next_report_) {
  229. if (next_report_ < 1000)
  230. next_report_ += 100;
  231. else if (next_report_ < 5000)
  232. next_report_ += 500;
  233. else if (next_report_ < 10000)
  234. next_report_ += 1000;
  235. else if (next_report_ < 50000)
  236. next_report_ += 5000;
  237. else if (next_report_ < 100000)
  238. next_report_ += 10000;
  239. else if (next_report_ < 500000)
  240. next_report_ += 50000;
  241. else
  242. next_report_ += 100000;
  243. fprintf(stderr, "... finished %d ops%30s\r", done_, "");
  244. fflush(stderr);
  245. }
  246. }
  247. void Stop(const Slice& name) {
  248. double finish = Env::Default()->NowMicros() * 1e-6;
  249. // Pretend at least one op was done in case we are running a benchmark
  250. // that does not call FinishedSingleOp().
  251. if (done_ < 1) done_ = 1;
  252. if (bytes_ > 0) {
  253. char rate[100];
  254. snprintf(rate, sizeof(rate), "%6.1f MB/s",
  255. (bytes_ / 1048576.0) / (finish - start_));
  256. if (!message_.empty()) {
  257. message_ = std::string(rate) + " " + message_;
  258. } else {
  259. message_ = rate;
  260. }
  261. }
  262. fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", name.ToString().c_str(),
  263. (finish - start_) * 1e6 / done_, (message_.empty() ? "" : " "),
  264. message_.c_str());
  265. if (FLAGS_histogram) {
  266. fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
  267. }
  268. fflush(stdout);
  269. }
  270. public:
  271. enum Order { SEQUENTIAL, RANDOM };
  272. enum DBState { FRESH, EXISTING };
  273. Benchmark()
  274. : db_(nullptr),
  275. db_num_(0),
  276. num_(FLAGS_num),
  277. reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
  278. bytes_(0),
  279. rand_(301) {
  280. std::vector<std::string> files;
  281. std::string test_dir;
  282. Env::Default()->GetTestDirectory(&test_dir);
  283. Env::Default()->GetChildren(test_dir, &files);
  284. if (!FLAGS_use_existing_db) {
  285. for (int i = 0; i < files.size(); i++) {
  286. if (Slice(files[i]).starts_with("dbbench_sqlite3")) {
  287. std::string file_name(test_dir);
  288. file_name += "/";
  289. file_name += files[i];
  290. Env::Default()->DeleteFile(file_name.c_str());
  291. }
  292. }
  293. }
  294. }
  295. ~Benchmark() {
  296. int status = sqlite3_close(db_);
  297. ErrorCheck(status);
  298. }
  299. void Run() {
  300. PrintHeader();
  301. Open();
  302. const char* benchmarks = FLAGS_benchmarks;
  303. while (benchmarks != nullptr) {
  304. const char* sep = strchr(benchmarks, ',');
  305. Slice name;
  306. if (sep == nullptr) {
  307. name = benchmarks;
  308. benchmarks = nullptr;
  309. } else {
  310. name = Slice(benchmarks, sep - benchmarks);
  311. benchmarks = sep + 1;
  312. }
  313. bytes_ = 0;
  314. Start();
  315. bool known = true;
  316. bool write_sync = false;
  317. if (name == Slice("fillseq")) {
  318. Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1);
  319. WalCheckpoint(db_);
  320. } else if (name == Slice("fillseqbatch")) {
  321. Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1000);
  322. WalCheckpoint(db_);
  323. } else if (name == Slice("fillrandom")) {
  324. Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1);
  325. WalCheckpoint(db_);
  326. } else if (name == Slice("fillrandbatch")) {
  327. Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1000);
  328. WalCheckpoint(db_);
  329. } else if (name == Slice("overwrite")) {
  330. Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1);
  331. WalCheckpoint(db_);
  332. } else if (name == Slice("overwritebatch")) {
  333. Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1000);
  334. WalCheckpoint(db_);
  335. } else if (name == Slice("fillrandsync")) {
  336. write_sync = true;
  337. Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1);
  338. WalCheckpoint(db_);
  339. } else if (name == Slice("fillseqsync")) {
  340. write_sync = true;
  341. Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1);
  342. WalCheckpoint(db_);
  343. } else if (name == Slice("fillrand100K")) {
  344. Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1);
  345. WalCheckpoint(db_);
  346. } else if (name == Slice("fillseq100K")) {
  347. Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1);
  348. WalCheckpoint(db_);
  349. } else if (name == Slice("readseq")) {
  350. ReadSequential();
  351. } else if (name == Slice("readrandom")) {
  352. Read(RANDOM, 1);
  353. } else if (name == Slice("readrand100K")) {
  354. int n = reads_;
  355. reads_ /= 1000;
  356. Read(RANDOM, 1);
  357. reads_ = n;
  358. } else {
  359. known = false;
  360. if (name != Slice()) { // No error message for empty name
  361. fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
  362. }
  363. }
  364. if (known) {
  365. Stop(name);
  366. }
  367. }
  368. }
  369. void Open() {
  370. assert(db_ == nullptr);
  371. int status;
  372. char file_name[100];
  373. char* err_msg = nullptr;
  374. db_num_++;
  375. // Open database
  376. std::string tmp_dir;
  377. Env::Default()->GetTestDirectory(&tmp_dir);
  378. snprintf(file_name, sizeof(file_name), "%s/dbbench_sqlite3-%d.db",
  379. tmp_dir.c_str(), db_num_);
  380. status = sqlite3_open(file_name, &db_);
  381. if (status) {
  382. fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_));
  383. exit(1);
  384. }
  385. // Change SQLite cache size
  386. char cache_size[100];
  387. snprintf(cache_size, sizeof(cache_size), "PRAGMA cache_size = %d",
  388. FLAGS_num_pages);
  389. status = sqlite3_exec(db_, cache_size, nullptr, nullptr, &err_msg);
  390. ExecErrorCheck(status, err_msg);
  391. // FLAGS_page_size is defaulted to 1024
  392. if (FLAGS_page_size != 1024) {
  393. char page_size[100];
  394. snprintf(page_size, sizeof(page_size), "PRAGMA page_size = %d",
  395. FLAGS_page_size);
  396. status = sqlite3_exec(db_, page_size, nullptr, nullptr, &err_msg);
  397. ExecErrorCheck(status, err_msg);
  398. }
  399. // Change journal mode to WAL if WAL enabled flag is on
  400. if (FLAGS_WAL_enabled) {
  401. std::string WAL_stmt = "PRAGMA journal_mode = WAL";
  402. // LevelDB's default cache size is a combined 4 MB
  403. std::string WAL_checkpoint = "PRAGMA wal_autocheckpoint = 4096";
  404. status = sqlite3_exec(db_, WAL_stmt.c_str(), nullptr, nullptr, &err_msg);
  405. ExecErrorCheck(status, err_msg);
  406. status =
  407. sqlite3_exec(db_, WAL_checkpoint.c_str(), nullptr, nullptr, &err_msg);
  408. ExecErrorCheck(status, err_msg);
  409. }
  410. // Change locking mode to exclusive and create tables/index for database
  411. std::string locking_stmt = "PRAGMA locking_mode = EXCLUSIVE";
  412. std::string create_stmt =
  413. "CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))";
  414. if (!FLAGS_use_rowids) create_stmt += " WITHOUT ROWID";
  415. std::string stmt_array[] = {locking_stmt, create_stmt};
  416. int stmt_array_length = sizeof(stmt_array) / sizeof(std::string);
  417. for (int i = 0; i < stmt_array_length; i++) {
  418. status =
  419. sqlite3_exec(db_, stmt_array[i].c_str(), nullptr, nullptr, &err_msg);
  420. ExecErrorCheck(status, err_msg);
  421. }
  422. }
  423. void Write(bool write_sync, Order order, DBState state, int num_entries,
  424. int value_size, int entries_per_batch) {
  425. // Create new database if state == FRESH
  426. if (state == FRESH) {
  427. if (FLAGS_use_existing_db) {
  428. message_ = "skipping (--use_existing_db is true)";
  429. return;
  430. }
  431. sqlite3_close(db_);
  432. db_ = nullptr;
  433. Open();
  434. Start();
  435. }
  436. if (num_entries != num_) {
  437. char msg[100];
  438. snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
  439. message_ = msg;
  440. }
  441. char* err_msg = nullptr;
  442. int status;
  443. sqlite3_stmt *replace_stmt, *begin_trans_stmt, *end_trans_stmt;
  444. std::string replace_str = "REPLACE INTO test (key, value) VALUES (?, ?)";
  445. std::string begin_trans_str = "BEGIN TRANSACTION;";
  446. std::string end_trans_str = "END TRANSACTION;";
  447. // Check for synchronous flag in options
  448. std::string sync_stmt =
  449. (write_sync) ? "PRAGMA synchronous = FULL" : "PRAGMA synchronous = OFF";
  450. status = sqlite3_exec(db_, sync_stmt.c_str(), nullptr, nullptr, &err_msg);
  451. ExecErrorCheck(status, err_msg);
  452. // Preparing sqlite3 statements
  453. status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1, &replace_stmt,
  454. nullptr);
  455. ErrorCheck(status);
  456. status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
  457. &begin_trans_stmt, nullptr);
  458. ErrorCheck(status);
  459. status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, &end_trans_stmt,
  460. nullptr);
  461. ErrorCheck(status);
  462. bool transaction = (entries_per_batch > 1);
  463. for (int i = 0; i < num_entries; i += entries_per_batch) {
  464. // Begin write transaction
  465. if (FLAGS_transaction && transaction) {
  466. status = sqlite3_step(begin_trans_stmt);
  467. StepErrorCheck(status);
  468. status = sqlite3_reset(begin_trans_stmt);
  469. ErrorCheck(status);
  470. }
  471. // Create and execute SQL statements
  472. for (int j = 0; j < entries_per_batch; j++) {
  473. const char* value = gen_.Generate(value_size).data();
  474. // Create values for key-value pair
  475. const int k =
  476. (order == SEQUENTIAL) ? i + j : (rand_.Next() % num_entries);
  477. char key[100];
  478. snprintf(key, sizeof(key), "%016d", k);
  479. // Bind KV values into replace_stmt
  480. status = sqlite3_bind_blob(replace_stmt, 1, key, 16, SQLITE_STATIC);
  481. ErrorCheck(status);
  482. status = sqlite3_bind_blob(replace_stmt, 2, value, value_size,
  483. SQLITE_STATIC);
  484. ErrorCheck(status);
  485. // Execute replace_stmt
  486. bytes_ += value_size + strlen(key);
  487. status = sqlite3_step(replace_stmt);
  488. StepErrorCheck(status);
  489. // Reset SQLite statement for another use
  490. status = sqlite3_clear_bindings(replace_stmt);
  491. ErrorCheck(status);
  492. status = sqlite3_reset(replace_stmt);
  493. ErrorCheck(status);
  494. FinishedSingleOp();
  495. }
  496. // End write transaction
  497. if (FLAGS_transaction && transaction) {
  498. status = sqlite3_step(end_trans_stmt);
  499. StepErrorCheck(status);
  500. status = sqlite3_reset(end_trans_stmt);
  501. ErrorCheck(status);
  502. }
  503. }
  504. status = sqlite3_finalize(replace_stmt);
  505. ErrorCheck(status);
  506. status = sqlite3_finalize(begin_trans_stmt);
  507. ErrorCheck(status);
  508. status = sqlite3_finalize(end_trans_stmt);
  509. ErrorCheck(status);
  510. }
  511. void Read(Order order, int entries_per_batch) {
  512. int status;
  513. sqlite3_stmt *read_stmt, *begin_trans_stmt, *end_trans_stmt;
  514. std::string read_str = "SELECT * FROM test WHERE key = ?";
  515. std::string begin_trans_str = "BEGIN TRANSACTION;";
  516. std::string end_trans_str = "END TRANSACTION;";
  517. // Preparing sqlite3 statements
  518. status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
  519. &begin_trans_stmt, nullptr);
  520. ErrorCheck(status);
  521. status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, &end_trans_stmt,
  522. nullptr);
  523. ErrorCheck(status);
  524. status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &read_stmt, nullptr);
  525. ErrorCheck(status);
  526. bool transaction = (entries_per_batch > 1);
  527. for (int i = 0; i < reads_; i += entries_per_batch) {
  528. // Begin read transaction
  529. if (FLAGS_transaction && transaction) {
  530. status = sqlite3_step(begin_trans_stmt);
  531. StepErrorCheck(status);
  532. status = sqlite3_reset(begin_trans_stmt);
  533. ErrorCheck(status);
  534. }
  535. // Create and execute SQL statements
  536. for (int j = 0; j < entries_per_batch; j++) {
  537. // Create key value
  538. char key[100];
  539. int k = (order == SEQUENTIAL) ? i + j : (rand_.Next() % reads_);
  540. snprintf(key, sizeof(key), "%016d", k);
  541. // Bind key value into read_stmt
  542. status = sqlite3_bind_blob(read_stmt, 1, key, 16, SQLITE_STATIC);
  543. ErrorCheck(status);
  544. // Execute read statement
  545. while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {
  546. }
  547. StepErrorCheck(status);
  548. // Reset SQLite statement for another use
  549. status = sqlite3_clear_bindings(read_stmt);
  550. ErrorCheck(status);
  551. status = sqlite3_reset(read_stmt);
  552. ErrorCheck(status);
  553. FinishedSingleOp();
  554. }
  555. // End read transaction
  556. if (FLAGS_transaction && transaction) {
  557. status = sqlite3_step(end_trans_stmt);
  558. StepErrorCheck(status);
  559. status = sqlite3_reset(end_trans_stmt);
  560. ErrorCheck(status);
  561. }
  562. }
  563. status = sqlite3_finalize(read_stmt);
  564. ErrorCheck(status);
  565. status = sqlite3_finalize(begin_trans_stmt);
  566. ErrorCheck(status);
  567. status = sqlite3_finalize(end_trans_stmt);
  568. ErrorCheck(status);
  569. }
  570. void ReadSequential() {
  571. int status;
  572. sqlite3_stmt* pStmt;
  573. std::string read_str = "SELECT * FROM test ORDER BY key";
  574. status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &pStmt, nullptr);
  575. ErrorCheck(status);
  576. for (int i = 0; i < reads_ && SQLITE_ROW == sqlite3_step(pStmt); i++) {
  577. bytes_ += sqlite3_column_bytes(pStmt, 1) + sqlite3_column_bytes(pStmt, 2);
  578. FinishedSingleOp();
  579. }
  580. status = sqlite3_finalize(pStmt);
  581. ErrorCheck(status);
  582. }
  583. };
  584. } // namespace leveldb
  585. int main(int argc, char** argv) {
  586. std::string default_db_path;
  587. for (int i = 1; i < argc; i++) {
  588. double d;
  589. int n;
  590. char junk;
  591. if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
  592. FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
  593. } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
  594. (n == 0 || n == 1)) {
  595. FLAGS_histogram = n;
  596. } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
  597. FLAGS_compression_ratio = d;
  598. } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
  599. (n == 0 || n == 1)) {
  600. FLAGS_use_existing_db = n;
  601. } else if (sscanf(argv[i], "--use_rowids=%d%c", &n, &junk) == 1 &&
  602. (n == 0 || n == 1)) {
  603. FLAGS_use_rowids = n;
  604. } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
  605. FLAGS_num = n;
  606. } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
  607. FLAGS_reads = n;
  608. } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
  609. FLAGS_value_size = n;
  610. } else if (leveldb::Slice(argv[i]) == leveldb::Slice("--no_transaction")) {
  611. FLAGS_transaction = false;
  612. } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) {
  613. FLAGS_page_size = n;
  614. } else if (sscanf(argv[i], "--num_pages=%d%c", &n, &junk) == 1) {
  615. FLAGS_num_pages = n;
  616. } else if (sscanf(argv[i], "--WAL_enabled=%d%c", &n, &junk) == 1 &&
  617. (n == 0 || n == 1)) {
  618. FLAGS_WAL_enabled = n;
  619. } else if (strncmp(argv[i], "--db=", 5) == 0) {
  620. FLAGS_db = argv[i] + 5;
  621. } else {
  622. fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
  623. exit(1);
  624. }
  625. }
  626. // Choose a location for the test database if none given with --db=<path>
  627. if (FLAGS_db == nullptr) {
  628. leveldb::Env::Default()->GetTestDirectory(&default_db_path);
  629. default_db_path += "/dbbench";
  630. FLAGS_db = default_db_path.c_str();
  631. }
  632. leveldb::Benchmark benchmark;
  633. benchmark.Run();
  634. return 0;
  635. }