From ea2e9195fc241c8fe9f329679d7a424345c68c7b Mon Sep 17 00:00:00 2001 From: Sanjay Ghemawat Date: Thu, 27 Dec 2012 10:38:48 -0800 Subject: [PATCH] added utility to dump leveldb files --- Makefile | 7 +- build_detect_platform | 3 +- db/leveldb_main.cc | 238 ++++++++++++++++++++++++++++++++++++++++++++++++++ include/leveldb/db.h | 2 +- port/port_posix.h | 8 +- 5 files changed, 253 insertions(+), 5 deletions(-) create mode 100644 db/leveldb_main.cc diff --git a/Makefile b/Makefile index fdc66f9..fef085b 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ TESTS = \ version_set_test \ write_batch_test -PROGRAMS = db_bench $(TESTS) +PROGRAMS = db_bench leveldbutil $(TESTS) BENCHMARKS = db_bench_sqlite3 db_bench_tree_db LIBRARY = libleveldb.a @@ -69,7 +69,7 @@ SHARED = $(SHARED1) else # Update db.h if you change these. SHARED_MAJOR = 1 -SHARED_MINOR = 7 +SHARED_MINOR = 8 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) @@ -107,6 +107,9 @@ db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) +leveldbutil: db/leveldb_main.o $(LIBOBJECTS) + $(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS) + arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) diff --git a/build_detect_platform b/build_detect_platform index 7cdb793..5801d20 100755 --- a/build_detect_platform +++ b/build_detect_platform @@ -141,7 +141,8 @@ DIRS="$PREFIX/db $PREFIX/util $PREFIX/table" set -f # temporarily disable globbing so that our patterns aren't expanded PRUNE_TEST="-name *test*.cc -prune" PRUNE_BENCH="-name *_bench.cc -prune" -PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "` +PRUNE_TOOL="-name leveldb_main.cc -prune" +PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "` set +f # re-enable globbing diff --git a/db/leveldb_main.cc b/db/leveldb_main.cc new file mode 100644 index 0000000..995d761 --- /dev/null +++ b/db/leveldb_main.cc @@ -0,0 +1,238 @@ +// Copyright (c) 2012 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "db/dbformat.h" +#include "db/filename.h" +#include "db/log_reader.h" +#include "db/version_edit.h" +#include "db/write_batch_internal.h" +#include "leveldb/env.h" +#include "leveldb/iterator.h" +#include "leveldb/options.h" +#include "leveldb/status.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" +#include "util/logging.h" + +namespace leveldb { + +namespace { + +bool GuessType(const std::string& fname, FileType* type) { + size_t pos = fname.rfind('/'); + std::string basename; + if (pos == std::string::npos) { + basename = fname; + } else { + basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1); + } + uint64_t ignored; + return ParseFileName(basename, &ignored, type); +} + +// Notified when log reader encounters corruption. +class CorruptionReporter : public log::Reader::Reporter { + public: + virtual void Corruption(size_t bytes, const Status& status) { + printf("corruption: %d bytes; %s\n", + static_cast(bytes), + status.ToString().c_str()); + } +}; + +// Print contents of a log file. (*func)() is called on every record. +bool PrintLogContents(Env* env, const std::string& fname, + void (*func)(Slice)) { + SequentialFile* file; + Status s = env->NewSequentialFile(fname, &file); + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + return false; + } + CorruptionReporter reporter; + log::Reader reader(file, &reporter, true, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch)) { + printf("--- offset %llu; ", + static_cast(reader.LastRecordOffset())); + (*func)(record); + } + delete file; + return true; +} + +// Called on every item found in a WriteBatch. +class WriteBatchItemPrinter : public WriteBatch::Handler { + public: + uint64_t offset_; + uint64_t sequence_; + + virtual void Put(const Slice& key, const Slice& value) { + printf(" put '%s' '%s'\n", + EscapeString(key).c_str(), + EscapeString(value).c_str()); + } + virtual void Delete(const Slice& key) { + printf(" del '%s'\n", + EscapeString(key).c_str()); + } +}; + + +// Called on every log record (each one of which is a WriteBatch) +// found in a kLogFile. +static void WriteBatchPrinter(Slice record) { + if (record.size() < 12) { + printf("log record length %d is too small\n", + static_cast(record.size())); + return; + } + WriteBatch batch; + WriteBatchInternal::SetContents(&batch, record); + printf("sequence %llu\n", + static_cast(WriteBatchInternal::Sequence(&batch))); + WriteBatchItemPrinter batch_item_printer; + Status s = batch.Iterate(&batch_item_printer); + if (!s.ok()) { + printf(" error: %s\n", s.ToString().c_str()); + } +} + +bool DumpLog(Env* env, const std::string& fname) { + return PrintLogContents(env, fname, WriteBatchPrinter); +} + +// Called on every log record (each one of which is a WriteBatch) +// found in a kDescriptorFile. +static void VersionEditPrinter(Slice record) { + VersionEdit edit; + Status s = edit.DecodeFrom(record); + if (!s.ok()) { + printf("%s\n", s.ToString().c_str()); + return; + } + printf("%s", edit.DebugString().c_str()); +} + +bool DumpDescriptor(Env* env, const std::string& fname) { + return PrintLogContents(env, fname, VersionEditPrinter); +} + +bool DumpTable(Env* env, const std::string& fname) { + uint64_t file_size; + RandomAccessFile* file = NULL; + Table* table = NULL; + Status s = env->GetFileSize(fname, &file_size); + if (s.ok()) { + s = env->NewRandomAccessFile(fname, &file); + } + if (s.ok()) { + // We use the default comparator, which may or may not match the + // comparator used in this database. However this should not cause + // problems since we only use Table operations that do not require + // any comparisons. In particular, we do not call Seek or Prev. + s = Table::Open(Options(), file, file_size, &table); + } + if (!s.ok()) { + fprintf(stderr, "%s\n", s.ToString().c_str()); + delete table; + delete file; + return false; + } + + ReadOptions ro; + ro.fill_cache = false; + Iterator* iter = table->NewIterator(ro); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedInternalKey key; + if (!ParseInternalKey(iter->key(), &key)) { + printf("badkey '%s' => '%s'\n", + EscapeString(iter->key()).c_str(), + EscapeString(iter->value()).c_str()); + } else { + char kbuf[20]; + const char* type; + if (key.type == kTypeDeletion) { + type = "del"; + } else if (key.type == kTypeValue) { + type = "val"; + } else { + snprintf(kbuf, sizeof(kbuf), "%d", static_cast(key.type)); + type = kbuf; + } + printf("'%s' @ %8llu : %s => '%s'\n", + EscapeString(key.user_key).c_str(), + static_cast(key.sequence), + type, + EscapeString(iter->value()).c_str()); + } + } + s = iter->status(); + if (!s.ok()) { + printf("iterator error: %s\n", s.ToString().c_str()); + } + + delete iter; + delete table; + delete file; + return true; +} + +bool DumpFile(Env* env, const std::string& fname) { + FileType ftype; + if (!GuessType(fname, &ftype)) { + fprintf(stderr, "%s: unknown file type\n", fname.c_str()); + return false; + } + switch (ftype) { + case kLogFile: return DumpLog(env, fname); + case kDescriptorFile: return DumpDescriptor(env, fname); + case kTableFile: return DumpTable(env, fname); + + default: { + fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str()); + break; + } + } + return false; +} + +bool HandleDumpCommand(Env* env, char** files, int num) { + bool ok = true; + for (int i = 0; i < num; i++) { + ok &= DumpFile(env, files[i]); + } + return ok; +} + +} +} // namespace leveldb + +static void Usage() { + fprintf( + stderr, + "Usage: leveldbutil command...\n" + " dump files... -- dump contents of specified files\n" + ); +} + +int main(int argc, char** argv) { + leveldb::Env* env = leveldb::Env::Default(); + bool ok = true; + if (argc < 2) { + Usage(); + ok = false; + } else { + std::string command = argv[1]; + if (command == "dump") { + ok = leveldb::HandleDumpCommand(env, argv+2, argc-2); + } else { + Usage(); + ok = false; + } + } + return (ok ? 0 : 1); +} diff --git a/include/leveldb/db.h b/include/leveldb/db.h index 79142f5..f1e70a0 100644 --- a/include/leveldb/db.h +++ b/include/leveldb/db.h @@ -14,7 +14,7 @@ namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 7; +static const int kMinorVersion = 8; struct Options; struct ReadOptions; diff --git a/port/port_posix.h b/port/port_posix.h index 6ca352e..f2b89bf 100644 --- a/port/port_posix.h +++ b/port/port_posix.h @@ -26,11 +26,17 @@ #include #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\ - defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID) + defined(OS_DRAGONFLYBSD) #include #include #elif defined(OS_HPUX) #define PLATFORM_IS_LITTLE_ENDIAN false +#elif defined(OS_ANDROID) + // Due to a bug in the NDK x86 definition, + // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android. + // See http://code.google.com/p/android/issues/detail?id=39824 + #include + #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #else #include #endif