作者: 韩晨旭 10225101440 李畅 10225102463
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

376 lignes
11 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include <sys/types.h>
  5. #include <stdio.h>
  6. #include <stdlib.h>
  7. #include "db/db_impl.h"
  8. #include "db/version_set.h"
  9. #include "include/cache.h"
  10. #include "include/db.h"
  11. #include "include/env.h"
  12. #include "include/write_batch.h"
  13. #include "util/histogram.h"
  14. #include "util/random.h"
  15. #include "util/testutil.h"
  16. // Comma-separated list of operations to run in the specified order
  17. // Actual benchmarks:
  18. // writeseq -- write N values in sequential key order
  19. // writerandom -- write N values in random key order
  20. // writebig -- write N/1000 100K valuesin random order
  21. // readseq -- read N values sequentially
  22. // readrandom -- read N values in random order
  23. // Meta operations:
  24. // compact -- Compact the entire DB
  25. // heapprofile -- Dump a heap profile (if supported by this port)
  26. // sync -- switch to synchronous writes (not the default)
  27. // nosync -- switch to asynchronous writes (the default)
  28. // tenth -- divide N by 10 (i.e., following benchmarks are smaller)
  29. // normal -- reset N back to its normal value (1000000)
  30. static const char* FLAGS_benchmarks =
  31. "writeseq,"
  32. "writeseq,"
  33. "writerandom,"
  34. "sync,tenth,tenth,writerandom,nosync,normal,"
  35. "readseq,"
  36. "readrandom,"
  37. "compact,"
  38. "readseq,"
  39. "readrandom,"
  40. "writebig";
  41. // Number of key/values to place in database
  42. static int FLAGS_num = 1000000;
  43. // Size of each value
  44. static int FLAGS_value_size = 100;
  45. // Arrange to generate values that shrink to this fraction of
  46. // their original size after compression
  47. static double FLAGS_compression_ratio = 0.25;
  48. // Print histogram of operation timings
  49. static bool FLAGS_histogram = false;
  50. // Number of bytes to buffer in memtable before compacting
  51. static int FLAGS_write_buffer_size = 1 << 20;
  52. namespace leveldb {
  53. // Helper for quickly generating random data.
  54. namespace {
  55. class RandomGenerator {
  56. private:
  57. std::string data_;
  58. int pos_;
  59. public:
  60. RandomGenerator() {
  61. // We use a limited amount of data over and over again and ensure
  62. // that it is larger than the compression window (32KB), and also
  63. // large enough to serve all typical value sizes we want to write.
  64. Random rnd(301);
  65. std::string piece;
  66. while (data_.size() < 1048576) {
  67. // Add a short fragment that is as compressible as specified
  68. // by FLAGS_compression_ratio.
  69. test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
  70. data_.append(piece);
  71. }
  72. pos_ = 0;
  73. }
  74. Slice Generate(int len) {
  75. if (pos_ + len > data_.size()) {
  76. pos_ = 0;
  77. assert(len < data_.size());
  78. }
  79. pos_ += len;
  80. return Slice(data_.data() + pos_ - len, len);
  81. }
  82. };
  83. }
  84. class Benchmark {
  85. private:
  86. Cache* cache_;
  87. DB* db_;
  88. int num_;
  89. bool sync_;
  90. int heap_counter_;
  91. double start_;
  92. double last_op_finish_;
  93. int64_t bytes_;
  94. std::string message_;
  95. Histogram hist_;
  96. RandomGenerator gen_;
  97. Random rand_;
  98. // State kept for progress messages
  99. int done_;
  100. int next_report_; // When to report next
  101. void Start() {
  102. start_ = Env::Default()->NowMicros() * 1e-6;
  103. bytes_ = 0;
  104. message_.clear();
  105. last_op_finish_ = start_;
  106. hist_.Clear();
  107. done_ = 0;
  108. next_report_ = 100;
  109. }
  110. void FinishedSingleOp() {
  111. if (FLAGS_histogram) {
  112. double now = Env::Default()->NowMicros() * 1e-6;
  113. double micros = (now - last_op_finish_) * 1e6;
  114. hist_.Add(micros);
  115. if (micros > 20000) {
  116. fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
  117. fflush(stderr);
  118. }
  119. last_op_finish_ = now;
  120. }
  121. done_++;
  122. if (done_ >= next_report_) {
  123. if (next_report_ < 1000) {
  124. next_report_ += 100;
  125. } else if (next_report_ < 10000) {
  126. next_report_ += 1000;
  127. } else if (next_report_ < 100000) {
  128. next_report_ += 10000;
  129. } else {
  130. next_report_ += 100000;
  131. }
  132. fprintf(stderr, "... finished %d ops%30s\r", done_, "");
  133. fflush(stderr);
  134. }
  135. }
  136. void Stop(const Slice& name) {
  137. double finish = Env::Default()->NowMicros() * 1e-6;
  138. // Pretend at least one op was done in case we are running a benchmark
  139. // that does nto call FinishedSingleOp().
  140. if (done_ < 1) done_ = 1;
  141. if (bytes_ > 0) {
  142. char rate[100];
  143. snprintf(rate, sizeof(rate), "%5.1f MB/s",
  144. (bytes_ / 1048576.0) / (finish - start_));
  145. if (!message_.empty()) {
  146. message_.push_back(' ');
  147. }
  148. message_.append(rate);
  149. }
  150. fprintf(stdout, "%-12s : %10.3f micros/op;%s%s\n",
  151. name.ToString().c_str(),
  152. (finish - start_) * 1e6 / done_,
  153. (message_.empty() ? "" : " "),
  154. message_.c_str());
  155. if (FLAGS_histogram) {
  156. fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
  157. }
  158. fflush(stdout);
  159. }
  160. public:
  161. enum Order { SEQUENTIAL, RANDOM };
  162. Benchmark() : cache_(NewLRUCache(200<<20)),
  163. db_(NULL),
  164. num_(FLAGS_num),
  165. sync_(false),
  166. heap_counter_(0),
  167. bytes_(0),
  168. rand_(301) {
  169. std::vector<std::string> files;
  170. Env::Default()->GetChildren("/tmp/dbbench", &files);
  171. for (int i = 0; i < files.size(); i++) {
  172. if (Slice(files[i]).starts_with("heap-")) {
  173. Env::Default()->DeleteFile("/tmp/dbbench/" + files[i]);
  174. }
  175. }
  176. DestroyDB("/tmp/dbbench", Options());
  177. }
  178. ~Benchmark() {
  179. delete db_;
  180. delete cache_;
  181. }
  182. void Run() {
  183. Options options;
  184. options.create_if_missing = true;
  185. options.max_open_files = 10000;
  186. options.block_cache = cache_;
  187. options.write_buffer_size = FLAGS_write_buffer_size;
  188. Start();
  189. Status s = DB::Open(options, "/tmp/dbbench", &db_);
  190. Stop("open");
  191. if (!s.ok()) {
  192. fprintf(stderr, "open error: %s\n", s.ToString().c_str());
  193. exit(1);
  194. }
  195. const char* benchmarks = FLAGS_benchmarks;
  196. while (benchmarks != NULL) {
  197. const char* sep = strchr(benchmarks, ',');
  198. Slice name;
  199. if (sep == NULL) {
  200. name = benchmarks;
  201. benchmarks = NULL;
  202. } else {
  203. name = Slice(benchmarks, sep - benchmarks);
  204. benchmarks = sep + 1;
  205. }
  206. Start();
  207. if (name == Slice("writeseq")) {
  208. Write(SEQUENTIAL, num_, FLAGS_value_size);
  209. } else if (name == Slice("writerandom")) {
  210. Write(RANDOM, num_, FLAGS_value_size);
  211. } else if (name == Slice("writebig")) {
  212. Write(RANDOM, num_ / 1000, 100 * 1000);
  213. } else if (name == Slice("readseq")) {
  214. Read(SEQUENTIAL);
  215. } else if (name == Slice("readrandom")) {
  216. Read(RANDOM);
  217. } else if (name == Slice("compact")) {
  218. Compact();
  219. } else if (name == Slice("heapprofile")) {
  220. HeapProfile();
  221. } else if (name == Slice("sync")) {
  222. sync_ = true;
  223. } else if (name == Slice("nosync")) {
  224. sync_ = false;
  225. } else if (name == Slice("tenth")) {
  226. num_ = num_ / 10;
  227. } else if (name == Slice("normal")) {
  228. num_ = FLAGS_num;
  229. } else {
  230. fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
  231. }
  232. Stop(name);
  233. }
  234. }
  235. void Write(Order order, int num_entries, int value_size) {
  236. WriteBatch batch;
  237. Status s;
  238. std::string val;
  239. WriteOptions options;
  240. options.sync = sync_;
  241. for (int i = 0; i < num_entries; i++) {
  242. const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % FLAGS_num);
  243. char key[100];
  244. snprintf(key, sizeof(key), "%012d", k);
  245. batch.Clear();
  246. batch.Put(key, gen_.Generate(value_size));
  247. s = db_->Write(options, &batch);
  248. bytes_ += value_size + strlen(key);
  249. if (!s.ok()) {
  250. fprintf(stderr, "put error: %s\n", s.ToString().c_str());
  251. exit(1);
  252. }
  253. FinishedSingleOp();
  254. }
  255. }
  256. void Read(Order order) {
  257. ReadOptions options;
  258. if (order == SEQUENTIAL) {
  259. Iterator* iter = db_->NewIterator(options);
  260. int i = 0;
  261. for (iter->SeekToFirst(); i < num_ && iter->Valid(); iter->Next()) {
  262. bytes_ += iter->key().size() + iter->value().size();
  263. FinishedSingleOp();
  264. ++i;
  265. }
  266. delete iter;
  267. } else {
  268. std::string value;
  269. for (int i = 0; i < num_; i++) {
  270. char key[100];
  271. const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % FLAGS_num);
  272. snprintf(key, sizeof(key), "%012d", k);
  273. db_->Get(options, key, &value);
  274. FinishedSingleOp();
  275. }
  276. }
  277. }
  278. void Compact() {
  279. DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  280. dbi->TEST_CompactMemTable();
  281. int max_level_with_files = 1;
  282. for (int level = 1; level < config::kNumLevels; level++) {
  283. uint64_t v;
  284. char name[100];
  285. snprintf(name, sizeof(name), "leveldb.num-files-at-level%d", level);
  286. if (db_->GetProperty(name, &v) && v > 0) {
  287. max_level_with_files = level;
  288. }
  289. }
  290. for (int level = 0; level < max_level_with_files; level++) {
  291. dbi->TEST_CompactRange(level, "", "~");
  292. }
  293. }
  294. static void WriteToFile(void* arg, const char* buf, int n) {
  295. reinterpret_cast<WritableFile*>(arg)->Append(Slice(buf, n));
  296. }
  297. void HeapProfile() {
  298. char fname[100];
  299. snprintf(fname, sizeof(fname), "/tmp/dbbench/heap-%04d", ++heap_counter_);
  300. WritableFile* file;
  301. Status s = Env::Default()->NewWritableFile(fname, &file);
  302. if (!s.ok()) {
  303. message_ = s.ToString();
  304. return;
  305. }
  306. bool ok = port::GetHeapProfile(WriteToFile, file);
  307. delete file;
  308. if (!ok) {
  309. message_ = "not supported";
  310. Env::Default()->DeleteFile(fname);
  311. }
  312. }
  313. };
  314. }
  315. int main(int argc, char** argv) {
  316. for (int i = 1; i < argc; i++) {
  317. double d;
  318. int n;
  319. char junk;
  320. if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
  321. FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
  322. } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
  323. FLAGS_compression_ratio = d;
  324. } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
  325. (n == 0 || n == 1)) {
  326. FLAGS_histogram = n;
  327. } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
  328. FLAGS_num = n;
  329. } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
  330. FLAGS_value_size = n;
  331. } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
  332. FLAGS_write_buffer_size = n;
  333. } else {
  334. fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
  335. exit(1);
  336. }
  337. }
  338. leveldb::Benchmark benchmark;
  339. benchmark.Run();
  340. return 0;
  341. }