500 regels
13 KiB

  1. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file. See the AUTHORS file for names of contributors.
  4. #include "db/log_reader.h"
  5. #include "db/log_writer.h"
  6. #include "leveldb/env.h"
  7. #include "util/coding.h"
  8. #include "util/crc32c.h"
  9. #include "util/random.h"
  10. #include "util/testharness.h"
  11. namespace leveldb {
  12. namespace log {
  13. // Construct a string of the specified length made out of the supplied
  14. // partial string.
  15. static std::string BigString(const std::string& partial_string, size_t n) {
  16. std::string result;
  17. while (result.size() < n) {
  18. result.append(partial_string);
  19. }
  20. result.resize(n);
  21. return result;
  22. }
  23. // Construct a string from a number
  24. static std::string NumberString(int n) {
  25. char buf[50];
  26. snprintf(buf, sizeof(buf), "%d.", n);
  27. return std::string(buf);
  28. }
  29. // Return a skewed potentially long string
  30. static std::string RandomSkewedString(int i, Random* rnd) {
  31. return BigString(NumberString(i), rnd->Skewed(17));
  32. }
  33. class LogTest {
  34. private:
  35. class StringDest : public WritableFile {
  36. public:
  37. std::string contents_;
  38. virtual Status Close() { return Status::OK(); }
  39. virtual Status Flush() { return Status::OK(); }
  40. virtual Status Sync() { return Status::OK(); }
  41. virtual Status Append(const Slice& slice) {
  42. contents_.append(slice.data(), slice.size());
  43. return Status::OK();
  44. }
  45. };
  46. class StringSource : public SequentialFile {
  47. public:
  48. Slice contents_;
  49. bool force_error_;
  50. bool returned_partial_;
  51. StringSource() : force_error_(false), returned_partial_(false) { }
  52. virtual Status Read(size_t n, Slice* result, char* scratch) {
  53. ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error";
  54. if (force_error_) {
  55. force_error_ = false;
  56. returned_partial_ = true;
  57. return Status::Corruption("read error");
  58. }
  59. if (contents_.size() < n) {
  60. n = contents_.size();
  61. returned_partial_ = true;
  62. }
  63. *result = Slice(contents_.data(), n);
  64. contents_.remove_prefix(n);
  65. return Status::OK();
  66. }
  67. virtual Status Skip(uint64_t n) {
  68. if (n > contents_.size()) {
  69. contents_.clear();
  70. return Status::NotFound("in-memory file skipepd past end");
  71. }
  72. contents_.remove_prefix(n);
  73. return Status::OK();
  74. }
  75. };
  76. class ReportCollector : public Reader::Reporter {
  77. public:
  78. size_t dropped_bytes_;
  79. std::string message_;
  80. ReportCollector() : dropped_bytes_(0) { }
  81. virtual void Corruption(size_t bytes, const Status& status) {
  82. dropped_bytes_ += bytes;
  83. message_.append(status.ToString());
  84. }
  85. };
  86. StringDest dest_;
  87. StringSource source_;
  88. ReportCollector report_;
  89. bool reading_;
  90. Writer writer_;
  91. Reader reader_;
  92. // Record metadata for testing initial offset functionality
  93. static size_t initial_offset_record_sizes_[];
  94. static uint64_t initial_offset_last_record_offsets_[];
  95. public:
  96. LogTest() : reading_(false),
  97. writer_(&dest_),
  98. reader_(&source_, &report_, true/*checksum*/,
  99. 0/*initial_offset*/) {
  100. }
  101. void Write(const std::string& msg) {
  102. ASSERT_TRUE(!reading_) << "Write() after starting to read";
  103. writer_.AddRecord(Slice(msg));
  104. }
  105. size_t WrittenBytes() const {
  106. return dest_.contents_.size();
  107. }
  108. std::string Read() {
  109. if (!reading_) {
  110. reading_ = true;
  111. source_.contents_ = Slice(dest_.contents_);
  112. }
  113. std::string scratch;
  114. Slice record;
  115. if (reader_.ReadRecord(&record, &scratch)) {
  116. return record.ToString();
  117. } else {
  118. return "EOF";
  119. }
  120. }
  121. void IncrementByte(int offset, int delta) {
  122. dest_.contents_[offset] += delta;
  123. }
  124. void SetByte(int offset, char new_byte) {
  125. dest_.contents_[offset] = new_byte;
  126. }
  127. void ShrinkSize(int bytes) {
  128. dest_.contents_.resize(dest_.contents_.size() - bytes);
  129. }
  130. void FixChecksum(int header_offset, int len) {
  131. // Compute crc of type/len/data
  132. uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len);
  133. crc = crc32c::Mask(crc);
  134. EncodeFixed32(&dest_.contents_[header_offset], crc);
  135. }
  136. void ForceError() {
  137. source_.force_error_ = true;
  138. }
  139. size_t DroppedBytes() const {
  140. return report_.dropped_bytes_;
  141. }
  142. std::string ReportMessage() const {
  143. return report_.message_;
  144. }
  145. // Returns OK iff recorded error message contains "msg"
  146. std::string MatchError(const std::string& msg) const {
  147. if (report_.message_.find(msg) == std::string::npos) {
  148. return report_.message_;
  149. } else {
  150. return "OK";
  151. }
  152. }
  153. void WriteInitialOffsetLog() {
  154. for (int i = 0; i < 4; i++) {
  155. std::string record(initial_offset_record_sizes_[i],
  156. static_cast<char>('a' + i));
  157. Write(record);
  158. }
  159. }
  160. void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
  161. WriteInitialOffsetLog();
  162. reading_ = true;
  163. source_.contents_ = Slice(dest_.contents_);
  164. Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
  165. WrittenBytes() + offset_past_end);
  166. Slice record;
  167. std::string scratch;
  168. ASSERT_TRUE(!offset_reader->ReadRecord(&record, &scratch));
  169. delete offset_reader;
  170. }
  171. void CheckInitialOffsetRecord(uint64_t initial_offset,
  172. int expected_record_offset) {
  173. WriteInitialOffsetLog();
  174. reading_ = true;
  175. source_.contents_ = Slice(dest_.contents_);
  176. Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
  177. initial_offset);
  178. Slice record;
  179. std::string scratch;
  180. ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
  181. ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
  182. record.size());
  183. ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
  184. offset_reader->LastRecordOffset());
  185. ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
  186. delete offset_reader;
  187. }
  188. };
  189. size_t LogTest::initial_offset_record_sizes_[] =
  190. {10000, // Two sizable records in first block
  191. 10000,
  192. 2 * log::kBlockSize - 1000, // Span three blocks
  193. 1};
  194. uint64_t LogTest::initial_offset_last_record_offsets_[] =
  195. {0,
  196. kHeaderSize + 10000,
  197. 2 * (kHeaderSize + 10000),
  198. 2 * (kHeaderSize + 10000) +
  199. (2 * log::kBlockSize - 1000) + 3 * kHeaderSize};
  200. TEST(LogTest, Empty) {
  201. ASSERT_EQ("EOF", Read());
  202. }
  203. TEST(LogTest, ReadWrite) {
  204. Write("foo");
  205. Write("bar");
  206. Write("");
  207. Write("xxxx");
  208. ASSERT_EQ("foo", Read());
  209. ASSERT_EQ("bar", Read());
  210. ASSERT_EQ("", Read());
  211. ASSERT_EQ("xxxx", Read());
  212. ASSERT_EQ("EOF", Read());
  213. ASSERT_EQ("EOF", Read()); // Make sure reads at eof work
  214. }
  215. TEST(LogTest, ManyBlocks) {
  216. for (int i = 0; i < 100000; i++) {
  217. Write(NumberString(i));
  218. }
  219. for (int i = 0; i < 100000; i++) {
  220. ASSERT_EQ(NumberString(i), Read());
  221. }
  222. ASSERT_EQ("EOF", Read());
  223. }
  224. TEST(LogTest, Fragmentation) {
  225. Write("small");
  226. Write(BigString("medium", 50000));
  227. Write(BigString("large", 100000));
  228. ASSERT_EQ("small", Read());
  229. ASSERT_EQ(BigString("medium", 50000), Read());
  230. ASSERT_EQ(BigString("large", 100000), Read());
  231. ASSERT_EQ("EOF", Read());
  232. }
  233. TEST(LogTest, MarginalTrailer) {
  234. // Make a trailer that is exactly the same length as an empty record.
  235. const int n = kBlockSize - 2*kHeaderSize;
  236. Write(BigString("foo", n));
  237. ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
  238. Write("");
  239. Write("bar");
  240. ASSERT_EQ(BigString("foo", n), Read());
  241. ASSERT_EQ("", Read());
  242. ASSERT_EQ("bar", Read());
  243. ASSERT_EQ("EOF", Read());
  244. }
  245. TEST(LogTest, MarginalTrailer2) {
  246. // Make a trailer that is exactly the same length as an empty record.
  247. const int n = kBlockSize - 2*kHeaderSize;
  248. Write(BigString("foo", n));
  249. ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
  250. Write("bar");
  251. ASSERT_EQ(BigString("foo", n), Read());
  252. ASSERT_EQ("bar", Read());
  253. ASSERT_EQ("EOF", Read());
  254. ASSERT_EQ(0, DroppedBytes());
  255. ASSERT_EQ("", ReportMessage());
  256. }
  257. TEST(LogTest, ShortTrailer) {
  258. const int n = kBlockSize - 2*kHeaderSize + 4;
  259. Write(BigString("foo", n));
  260. ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
  261. Write("");
  262. Write("bar");
  263. ASSERT_EQ(BigString("foo", n), Read());
  264. ASSERT_EQ("", Read());
  265. ASSERT_EQ("bar", Read());
  266. ASSERT_EQ("EOF", Read());
  267. }
  268. TEST(LogTest, AlignedEof) {
  269. const int n = kBlockSize - 2*kHeaderSize + 4;
  270. Write(BigString("foo", n));
  271. ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
  272. ASSERT_EQ(BigString("foo", n), Read());
  273. ASSERT_EQ("EOF", Read());
  274. }
  275. TEST(LogTest, RandomRead) {
  276. const int N = 500;
  277. Random write_rnd(301);
  278. for (int i = 0; i < N; i++) {
  279. Write(RandomSkewedString(i, &write_rnd));
  280. }
  281. Random read_rnd(301);
  282. for (int i = 0; i < N; i++) {
  283. ASSERT_EQ(RandomSkewedString(i, &read_rnd), Read());
  284. }
  285. ASSERT_EQ("EOF", Read());
  286. }
  287. // Tests of all the error paths in log_reader.cc follow:
  288. TEST(LogTest, ReadError) {
  289. Write("foo");
  290. ForceError();
  291. ASSERT_EQ("EOF", Read());
  292. ASSERT_EQ(kBlockSize, DroppedBytes());
  293. ASSERT_EQ("OK", MatchError("read error"));
  294. }
  295. TEST(LogTest, BadRecordType) {
  296. Write("foo");
  297. // Type is stored in header[6]
  298. IncrementByte(6, 100);
  299. FixChecksum(0, 3);
  300. ASSERT_EQ("EOF", Read());
  301. ASSERT_EQ(3, DroppedBytes());
  302. ASSERT_EQ("OK", MatchError("unknown record type"));
  303. }
  304. TEST(LogTest, TruncatedTrailingRecord) {
  305. Write("foo");
  306. ShrinkSize(4); // Drop all payload as well as a header byte
  307. ASSERT_EQ("EOF", Read());
  308. ASSERT_EQ(kHeaderSize - 1, DroppedBytes());
  309. ASSERT_EQ("OK", MatchError("truncated record at end of file"));
  310. }
  311. TEST(LogTest, BadLength) {
  312. Write("foo");
  313. ShrinkSize(1);
  314. ASSERT_EQ("EOF", Read());
  315. ASSERT_EQ(kHeaderSize + 2, DroppedBytes());
  316. ASSERT_EQ("OK", MatchError("bad record length"));
  317. }
  318. TEST(LogTest, ChecksumMismatch) {
  319. Write("foo");
  320. IncrementByte(0, 10);
  321. ASSERT_EQ("EOF", Read());
  322. ASSERT_EQ(10, DroppedBytes());
  323. ASSERT_EQ("OK", MatchError("checksum mismatch"));
  324. }
  325. TEST(LogTest, UnexpectedMiddleType) {
  326. Write("foo");
  327. SetByte(6, kMiddleType);
  328. FixChecksum(0, 3);
  329. ASSERT_EQ("EOF", Read());
  330. ASSERT_EQ(3, DroppedBytes());
  331. ASSERT_EQ("OK", MatchError("missing start"));
  332. }
  333. TEST(LogTest, UnexpectedLastType) {
  334. Write("foo");
  335. SetByte(6, kLastType);
  336. FixChecksum(0, 3);
  337. ASSERT_EQ("EOF", Read());
  338. ASSERT_EQ(3, DroppedBytes());
  339. ASSERT_EQ("OK", MatchError("missing start"));
  340. }
  341. TEST(LogTest, UnexpectedFullType) {
  342. Write("foo");
  343. Write("bar");
  344. SetByte(6, kFirstType);
  345. FixChecksum(0, 3);
  346. ASSERT_EQ("bar", Read());
  347. ASSERT_EQ("EOF", Read());
  348. ASSERT_EQ(3, DroppedBytes());
  349. ASSERT_EQ("OK", MatchError("partial record without end"));
  350. }
  351. TEST(LogTest, UnexpectedFirstType) {
  352. Write("foo");
  353. Write(BigString("bar", 100000));
  354. SetByte(6, kFirstType);
  355. FixChecksum(0, 3);
  356. ASSERT_EQ(BigString("bar", 100000), Read());
  357. ASSERT_EQ("EOF", Read());
  358. ASSERT_EQ(3, DroppedBytes());
  359. ASSERT_EQ("OK", MatchError("partial record without end"));
  360. }
  361. TEST(LogTest, ErrorJoinsRecords) {
  362. // Consider two fragmented records:
  363. // first(R1) last(R1) first(R2) last(R2)
  364. // where the middle two fragments disappear. We do not want
  365. // first(R1),last(R2) to get joined and returned as a valid record.
  366. // Write records that span two blocks
  367. Write(BigString("foo", kBlockSize));
  368. Write(BigString("bar", kBlockSize));
  369. Write("correct");
  370. // Wipe the middle block
  371. for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
  372. SetByte(offset, 'x');
  373. }
  374. ASSERT_EQ("correct", Read());
  375. ASSERT_EQ("EOF", Read());
  376. const int dropped = DroppedBytes();
  377. ASSERT_LE(dropped, 2*kBlockSize + 100);
  378. ASSERT_GE(dropped, 2*kBlockSize);
  379. }
  380. TEST(LogTest, ReadStart) {
  381. CheckInitialOffsetRecord(0, 0);
  382. }
  383. TEST(LogTest, ReadSecondOneOff) {
  384. CheckInitialOffsetRecord(1, 1);
  385. }
  386. TEST(LogTest, ReadSecondTenThousand) {
  387. CheckInitialOffsetRecord(10000, 1);
  388. }
  389. TEST(LogTest, ReadSecondStart) {
  390. CheckInitialOffsetRecord(10007, 1);
  391. }
  392. TEST(LogTest, ReadThirdOneOff) {
  393. CheckInitialOffsetRecord(10008, 2);
  394. }
  395. TEST(LogTest, ReadThirdStart) {
  396. CheckInitialOffsetRecord(20014, 2);
  397. }
  398. TEST(LogTest, ReadFourthOneOff) {
  399. CheckInitialOffsetRecord(20015, 3);
  400. }
  401. TEST(LogTest, ReadFourthFirstBlockTrailer) {
  402. CheckInitialOffsetRecord(log::kBlockSize - 4, 3);
  403. }
  404. TEST(LogTest, ReadFourthMiddleBlock) {
  405. CheckInitialOffsetRecord(log::kBlockSize + 1, 3);
  406. }
  407. TEST(LogTest, ReadFourthLastBlock) {
  408. CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3);
  409. }
  410. TEST(LogTest, ReadFourthStart) {
  411. CheckInitialOffsetRecord(
  412. 2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
  413. 3);
  414. }
  415. TEST(LogTest, ReadEnd) {
  416. CheckOffsetPastEndReturnsNoRecords(0);
  417. }
  418. TEST(LogTest, ReadPastEnd) {
  419. CheckOffsetPastEndReturnsNoRecords(5);
  420. }
  421. } // namespace log
  422. } // namespace leveldb
  423. int main(int argc, char** argv) {
  424. return leveldb::test::RunAllTests();
  425. }