Sync with upstream @23860137.

Fix GCC -Wshadow warnings in LevelDB's public header files,
reported by Dustin.

Add in-memory Env implementation (helpers/memenv/*).
This enables users to create LevelDB databases in-memory.

Initialize ShardedLRUCache::last_id_ to zero.
This fixes a Valgrind warning.

(Also delete port/sha1_* which were removed upstream some time ago.)

Makefile

@@ -92,6 +92,7 @@ TESTS = \
 	env_test \
 	filename_test \
 	log_test \
+	memenv_test \
 	skiplist_test \
 	table_test \
 	version_edit_test \
@@ -102,6 +103,7 @@ PROGRAMS = db_bench $(TESTS)
 BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
 
 LIBRARY = libleveldb.a
+MEMENVLIBRARY = libmemenv.a
 
 all: $(LIBRARY)
 
@@ -109,7 +111,7 @@ check: $(PROGRAMS) $(TESTS)
 	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
 
 clean:
-	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o
+	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(MEMENVLIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o
 	-rm -rf ios-x86/* ios-arm/*
 	-rm build_config.mk
 
@@ -174,6 +176,13 @@ version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
 write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CC) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
 
+$(MEMENVLIBRARY) : helpers/memenv/memenv.o
+	rm -f $@
+	$(AR) -rs $@ helpers/memenv/memenv.o
+
+memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
+	$(CC) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@
+
 ifeq ($(PLATFORM), IOS)
 # For iOS, create universal object files to be used on both the simulator and
 # a device.
@@ -202,4 +211,3 @@ else
 .c.o:
 	$(CC) $(CFLAGS) $< -o $@
 endif
-

helpers/memenv/memenv.cc

@@ -0,0 +1,374 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "helpers/memenv/memenv.h"
+
+#include "leveldb/env.h"
+#include "leveldb/status.h"
+#include "port/port.h"
+#include "util/mutexlock.h"
+#include <map>
+#include <string.h>
+#include <string>
+#include <vector>
+
+namespace leveldb {
+
+namespace {
+
+class FileState {
+ public:
+  // FileStates are reference counted. The initial reference count is zero
+  // and the caller must call Ref() at least once.
+  FileState() : refs_(0), size_(0) {}
+
+  // Increase the reference count.
+  void Ref() {
+    MutexLock lock(&refs_mutex_);
+    ++refs_;
+  }
+
+  // Decrease the reference count. Delete if this is the last reference.
+  void Unref() {
+    bool do_delete = false;
+
+    {
+      MutexLock lock(&refs_mutex_);
+      --refs_;
+      assert(refs_ >= 0);
+      if (refs_ <= 0) {
+        do_delete = true;
+      }
+    }
+
+    if (do_delete) {
+      delete this;
+    }
+  }
+
+  uint64_t Size() const { return size_; }
+
+  Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
+    if (offset > size_) {
+      return Status::IOError("Offset greater than file size.");
+    }
+    const uint64_t available = size_ - offset;
+    if (n > available) {
+      n = available;
+    }
+    if (n == 0) {
+      *result = Slice();
+      return Status::OK();
+    }
+
+    size_t block = offset / kBlockSize;
+    size_t block_offset = offset % kBlockSize;
+
+    if (n <= kBlockSize - block_offset) {
+      // The requested bytes are all in the first block.
+      *result = Slice(blocks_[block] + block_offset, n);
+      return Status::OK();
+    }
+
+    size_t bytes_to_copy = n;
+    char* dst = scratch;
+
+    while (bytes_to_copy > 0) {
+      size_t avail = kBlockSize - block_offset;
+      if (avail > bytes_to_copy) {
+        avail = bytes_to_copy;
+      }
+      memcpy(dst, blocks_[block] + block_offset, avail);
+
+      bytes_to_copy -= avail;
+      dst += avail;
+      block++;
+      block_offset = 0;
+    }
+
+    *result = Slice(scratch, n);
+    return Status::OK();
+  }
+
+  Status Append(const Slice& data) {
+    const char* src = data.data();
+    size_t src_len = data.size();
+
+    while (src_len > 0) {
+      size_t avail;
+      size_t offset = size_ % kBlockSize;
+
+      if (offset != 0) {
+        // There is some room in the last block.
+        avail = kBlockSize - offset;
+      } else {
+        // No room in the last block; push new one.
+        blocks_.push_back(new char[kBlockSize]);
+        avail = kBlockSize;
+      }
+
+      if (avail > src_len) {
+        avail = src_len;
+      }
+      memcpy(blocks_.back() + offset, src, avail);
+      src_len -= avail;
+      src += avail;
+      size_ += avail;
+    }
+
+    return Status::OK();
+  }
+
+ private:
+  // Private since only Unref() should be used to delete it.
+  ~FileState() {
+    for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
+         ++i) {
+      delete [] *i;
+    }
+  }
+
+  // No copying allowed.
+  FileState(const FileState&);
+  void operator=(const FileState&);
+
+  port::Mutex refs_mutex_;
+  int refs_;  // Protected by refs_mutex_;
+
+  // The following fields are not protected by any mutex. They are only mutable
+  // while the file is being written, and concurrent access is not allowed
+  // to writable files.
+  std::vector<char*> blocks_;
+  uint64_t size_;
+
+  enum { kBlockSize = 8 * 1024 };
+};
+
+class SequentialFileImpl : public SequentialFile {
+ public:
+  explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
+    file_->Ref();
+  }
+
+  ~SequentialFileImpl() {
+    file_->Unref();
+  }
+
+  virtual Status Read(size_t n, Slice* result, char* scratch) {
+    Status s = file_->Read(pos_, n, result, scratch);
+    if (s.ok()) {
+      pos_ += result->size();
+    }
+    return s;
+  }
+
+  virtual Status Skip(uint64_t n) {
+    if (pos_ > file_->Size()) {
+      return Status::IOError("pos_ > file_->Size()");
+    }
+    const size_t available = file_->Size() - pos_;
+    if (n > available) {
+      n = available;
+    }
+    pos_ += n;
+    return Status::OK();
+  }
+
+ private:
+  FileState* file_;
+  size_t pos_;
+};
+
+class RandomAccessFileImpl : public RandomAccessFile {
+ public:
+  explicit RandomAccessFileImpl(FileState* file) : file_(file) {
+    file_->Ref();
+  }
+
+  ~RandomAccessFileImpl() {
+    file_->Unref();
+  }
+
+  virtual Status Read(uint64_t offset, size_t n, Slice* result,
+                      char* scratch) const {
+    return file_->Read(offset, n, result, scratch);
+  }
+
+ private:
+  FileState* file_;
+};
+
+class WritableFileImpl : public WritableFile {
+ public:
+  WritableFileImpl(FileState* file) : file_(file) {
+    file_->Ref();
+  }
+
+  ~WritableFileImpl() {
+    file_->Unref();
+  }
+
+  virtual Status Append(const Slice& data) {
+    return file_->Append(data);
+  }
+
+  virtual Status Close() { return Status::OK(); }
+  virtual Status Flush() { return Status::OK(); }
+  virtual Status Sync() { return Status::OK(); }
+
+ private:
+  FileState* file_;
+};
+
+class InMemoryEnv : public EnvWrapper {
+ public:
+  explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
+
+  virtual ~InMemoryEnv() {
+    for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
+      i->second->Unref();
+    }
+  }
+
+  // Partial implementation of the Env interface.
+  virtual Status NewSequentialFile(const std::string& fname,
+                                   SequentialFile** result) {
+    MutexLock lock(&mutex_);
+    if (file_map_.find(fname) == file_map_.end()) {
+      *result = NULL;
+      return Status::IOError(fname, "File not found");
+    }
+
+    *result = new SequentialFileImpl(file_map_[fname]);
+    return Status::OK();
+  }
+
+  virtual Status NewRandomAccessFile(const std::string& fname,
+                                     RandomAccessFile** result) {
+    MutexLock lock(&mutex_);
+    if (file_map_.find(fname) == file_map_.end()) {
+      *result = NULL;
+      return Status::IOError(fname, "File not found");
+    }
+
+    *result = new RandomAccessFileImpl(file_map_[fname]);
+    return Status::OK();
+  }
+
+  virtual Status NewWritableFile(const std::string& fname,
+                                 WritableFile** result) {
+    MutexLock lock(&mutex_);
+    if (file_map_.find(fname) != file_map_.end()) {
+      DeleteFileInternal(fname);
+    }
+
+    FileState* file = new FileState();
+    file->Ref();
+    file_map_[fname] = file;
+
+    *result = new WritableFileImpl(file);
+    return Status::OK();
+  }
+
+  virtual bool FileExists(const std::string& fname) {
+    MutexLock lock(&mutex_);
+    return file_map_.find(fname) != file_map_.end();
+  }
+
+  virtual Status GetChildren(const std::string& dir,
+                             std::vector<std::string>* result) {
+    MutexLock lock(&mutex_);
+    result->clear();
+
+    for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
+      const std::string& filename = i->first;
+
+      if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
+          Slice(filename).starts_with(Slice(dir))) {
+        result->push_back(filename.substr(dir.size() + 1));
+      }
+    }
+
+    return Status::OK();
+  }
+
+  void DeleteFileInternal(const std::string& fname) {
+    if (file_map_.find(fname) == file_map_.end()) {
+      return;
+    }
+
+    file_map_[fname]->Unref();
+    file_map_.erase(fname);
+  }
+
+  virtual Status DeleteFile(const std::string& fname) {
+    MutexLock lock(&mutex_);
+    if (file_map_.find(fname) == file_map_.end()) {
+      return Status::IOError(fname, "File not found");
+    }
+
+    DeleteFileInternal(fname);
+    return Status::OK();
+  }
+
+  virtual Status CreateDir(const std::string& dirname) {
+    return Status::OK();
+  }
+
+  virtual Status DeleteDir(const std::string& dirname) {
+    return Status::OK();
+  }
+
+  virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) {
+    MutexLock lock(&mutex_);
+    if (file_map_.find(fname) == file_map_.end()) {
+      return Status::IOError(fname, "File not found");
+    }
+
+    *file_size = file_map_[fname]->Size();
+    return Status::OK();
+  }
+
+  virtual Status RenameFile(const std::string& src,
+                            const std::string& target) {
+    MutexLock lock(&mutex_);
+    if (file_map_.find(src) == file_map_.end()) {
+      return Status::IOError(src, "File not found");
+    }
+
+    DeleteFileInternal(target);
+    file_map_[target] = file_map_[src];
+    file_map_.erase(src);
+    return Status::OK();
+  }
+
+  virtual Status LockFile(const std::string& fname, FileLock** lock) {
+    *lock = new FileLock;
+    return Status::OK();
+  }
+
+  virtual Status UnlockFile(FileLock* lock) {
+    delete lock;
+    return Status::OK();
+  }
+
+  virtual Status GetTestDirectory(std::string* path) {
+    *path = "/test";
+    return Status::OK();
+  }
+
+ private:
+  // Map from filenames to FileState objects, representing a simple file system.
+  typedef std::map<std::string, FileState*> FileSystem;
+  port::Mutex mutex_;
+  FileSystem file_map_;  // Protected by mutex_.
+};
+
+}
+
+Env* NewMemEnv(Env* base_env) {
+  return new InMemoryEnv(base_env);
+}
+
+}

helpers/memenv/memenv.h

@@ -0,0 +1,20 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
+#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
+
+namespace leveldb {
+
+class Env;
+
+// Returns a new environment that stores its data in memory and delegates
+// all non-file-storage tasks to base_env. The caller must delete the result
+// when it is no longer needed.
+// *base_env must remain live while the result is in use.
+Env* NewMemEnv(Env* base_env);
+
+}
+
+#endif  // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_

helpers/memenv/memenv_test.cc

@@ -0,0 +1,232 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "helpers/memenv/memenv.h"
+
+#include "db/db_impl.h"
+#include "leveldb/db.h"
+#include "leveldb/env.h"
+#include "util/testharness.h"
+#include <string>
+#include <vector>
+
+namespace leveldb {
+
+class MemEnvTest {
+ public:
+  Env* env_;
+
+  MemEnvTest()
+      : env_(NewMemEnv(Env::Default())) {
+  }
+  ~MemEnvTest() {
+    delete env_;
+  }
+};
+
+TEST(MemEnvTest, Basics) {
+  size_t file_size;
+  WritableFile* writable_file;
+  std::vector<std::string> children;
+
+  ASSERT_OK(env_->CreateDir("/dir"));
+
+  // Check that the directory is empty.
+  ASSERT_TRUE(!env_->FileExists("/dir/non_existent"));
+  ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
+  ASSERT_OK(env_->GetChildren("/dir", &children));
+  ASSERT_EQ(0, children.size());
+
+  // Create a file.
+  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+  delete writable_file;
+
+  // Check that the file exists.
+  ASSERT_TRUE(env_->FileExists("/dir/f"));
+  ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
+  ASSERT_EQ(0, file_size);
+  ASSERT_OK(env_->GetChildren("/dir", &children));
+  ASSERT_EQ(1, children.size());
+  ASSERT_EQ("f", children[0]);
+
+  // Write to the file.
+  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+  ASSERT_OK(writable_file->Append("abc"));
+  delete writable_file;
+
+  // Check for expected size.
+  ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
+  ASSERT_EQ(3, file_size);
+
+  // Check that renaming works.
+  ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
+  ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g"));
+  ASSERT_TRUE(!env_->FileExists("/dir/f"));
+  ASSERT_TRUE(env_->FileExists("/dir/g"));
+  ASSERT_OK(env_->GetFileSize("/dir/g", &file_size));
+  ASSERT_EQ(3, file_size);
+
+  // Check that opening non-existent file fails.
+  SequentialFile* seq_file;
+  RandomAccessFile* rand_file;
+  ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok());
+  ASSERT_TRUE(!seq_file);
+  ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok());
+  ASSERT_TRUE(!rand_file);
+
+  // Check that deleting works.
+  ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
+  ASSERT_OK(env_->DeleteFile("/dir/g"));
+  ASSERT_TRUE(!env_->FileExists("/dir/g"));
+  ASSERT_OK(env_->GetChildren("/dir", &children));
+  ASSERT_EQ(0, children.size());
+  ASSERT_OK(env_->DeleteDir("/dir"));
+}
+
+TEST(MemEnvTest, ReadWrite) {
+  WritableFile* writable_file;
+  SequentialFile* seq_file;
+  RandomAccessFile* rand_file;
+  Slice result;
+  char scratch[100];
+
+  ASSERT_OK(env_->CreateDir("/dir"));
+
+  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+  ASSERT_OK(writable_file->Append("hello "));
+  ASSERT_OK(writable_file->Append("world"));
+  delete writable_file;
+
+  // Read sequentially.
+  ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
+  ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
+  ASSERT_EQ(0, result.compare("hello"));
+  ASSERT_OK(seq_file->Skip(1));
+  ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
+  ASSERT_EQ(0, result.compare("world"));
+  ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
+  ASSERT_EQ(0, result.size());
+  ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
+  ASSERT_OK(seq_file->Read(1000, &result, scratch));
+  ASSERT_EQ(0, result.size());
+  delete seq_file;
+
+  // Random reads.
+  ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
+  ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
+  ASSERT_EQ(0, result.compare("world"));
+  ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
+  ASSERT_EQ(0, result.compare("hello"));
+  ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
+  ASSERT_EQ(0, result.compare("d"));
+
+  // Too high offset.
+  ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok());
+  delete rand_file;
+}
+
+TEST(MemEnvTest, Locks) {
+  FileLock* lock;
+
+  // These are no-ops, but we test they return success.
+  ASSERT_OK(env_->LockFile("some file", &lock));
+  ASSERT_OK(env_->UnlockFile(lock));
+}
+
+TEST(MemEnvTest, Misc) {
+  std::string test_dir;
+  ASSERT_OK(env_->GetTestDirectory(&test_dir));
+  ASSERT_TRUE(!test_dir.empty());
+
+  WritableFile* writable_file;
+  ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file));
+
+  // These are no-ops, but we test they return success.
+  ASSERT_OK(writable_file->Sync());
+  ASSERT_OK(writable_file->Flush());
+  ASSERT_OK(writable_file->Close());
+  delete writable_file;
+}
+
+TEST(MemEnvTest, LargeWrite) {
+  const size_t kWriteSize = 300 * 1024;
+  char* scratch = new char[kWriteSize * 2];
+
+  std::string write_data;
+  for (size_t i = 0; i < kWriteSize; ++i) {
+    write_data.append(1, static_cast<char>(i));
+  }
+
+  WritableFile* writable_file;
+  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+  ASSERT_OK(writable_file->Append("foo"));
+  ASSERT_OK(writable_file->Append(write_data));
+  delete writable_file;
+
+  SequentialFile* seq_file;
+  Slice result;
+  ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
+  ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
+  ASSERT_EQ(0, result.compare("foo"));
+
+  size_t read = 0;
+  std::string read_data;
+  while (read < kWriteSize) {
+    ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch));
+    read_data.append(result.data(), result.size());
+    read += result.size();
+  }
+  ASSERT_TRUE(write_data == read_data);
+  delete seq_file;
+  delete [] scratch;
+}
+
+TEST(MemEnvTest, DBTest) {
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_;
+  DB* db;
+
+  const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")};
+  const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")};
+
+  ASSERT_OK(DB::Open(options, "/dir/db", &db));
+  for (size_t i = 0; i < 3; ++i) {
+    ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i]));
+  }
+
+  for (size_t i = 0; i < 3; ++i) {
+    std::string res;
+    ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
+    ASSERT_TRUE(res == vals[i]);
+  }
+
+  Iterator* iterator = db->NewIterator(ReadOptions());
+  iterator->SeekToFirst();
+  for (size_t i = 0; i < 3; ++i) {
+    ASSERT_TRUE(iterator->Valid());
+    ASSERT_TRUE(keys[i] == iterator->key());
+    ASSERT_TRUE(vals[i] == iterator->value());
+    iterator->Next();
+  }
+  ASSERT_TRUE(!iterator->Valid());
+  delete iterator;
+
+  DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
+  ASSERT_OK(dbi->TEST_CompactMemTable());
+
+  for (size_t i = 0; i < 3; ++i) {
+    std::string res;
+    ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
+    ASSERT_TRUE(res == vals[i]);
+  }
+
+  delete db;
+}
+
+}
+
+int main(int argc, char** argv) {
+  return leveldb::test::RunAllTests();
+}

include/leveldb/env.h

@@ -258,8 +258,8 @@ extern Status ReadFileToString(Env* env, const std::string& fname,
 // functionality of another Env.
 class EnvWrapper : public Env {
  public:
-  // Initialize an EnvWrapper that delegates all calls to *target
-  explicit EnvWrapper(Env* target) : target_(target) { }
+  // Initialize an EnvWrapper that delegates all calls to *t
+  explicit EnvWrapper(Env* t) : target_(t) { }
   virtual ~EnvWrapper();
 
   // Return the target to which this Env forwards all calls

include/leveldb/slice.h

@@ -27,8 +27,8 @@ class Slice {
   // Create an empty slice.
   Slice() : data_(""), size_(0) { }
 
-  // Create a slice that refers to data[0,n-1].
-  Slice(const char* data, size_t n) : data_(data), size_(n) { }
+  // Create a slice that refers to d[0,n-1].
+  Slice(const char* d, size_t n) : data_(d), size_(n) { }
 
   // Create a slice that refers to the contents of "s"
   Slice(const std::string& s) : data_(s.data()), size_(s.size()) { }

port/sha1_portable.cc

@@ -1,298 +0,0 @@
-// Portions copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// This module provides a slow but portable implementation of
-// the SHA1 hash function.
-//
-// It is adapted from free code written by Paul E. Jones
-// <paulej@packetizer.com>.  See http://www.packetizer.com/security/sha1/
-//
-// The license for the original code is:
-/*
-  Copyright (C) 1998, 2009
-  Paul E. Jones <paulej@packetizer.com>
-
-  Freeware Public License (FPL)
-
-  This software is licensed as "freeware."  Permission to distribute
-  this software in source and binary forms, including incorporation
-  into other products, is hereby granted without a fee.  THIS SOFTWARE
-  IS PROVIDED 'AS IS' AND WITHOUT ANY EXPRESSED OR IMPLIED WARRANTIES,
-  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
-  AND FITNESS FOR A PARTICULAR PURPOSE.  THE AUTHOR SHALL NOT BE HELD
-  LIABLE FOR ANY DAMAGES RESULTING FROM THE USE OF THIS SOFTWARE, EITHER
-  DIRECTLY OR INDIRECTLY, INCLUDING, BUT NOT LIMITED TO, LOSS OF DATA
-  OR DATA BEING RENDERED INACCURATE.
-*/
-
-#include "port/sha1_portable.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-
-namespace leveldb {
-namespace port {
-
-/*
- *  Description:
- *      This class implements the Secure Hashing Standard as defined
- *      in FIPS PUB 180-1 published April 17, 1995.
- */
-
-/*
- *  This structure will hold context information for the hashing
- *  operation
- */
-typedef struct SHA1Context {
-  unsigned Message_Digest[5]; /* Message Digest (output)          */
-
-  unsigned Length_Low;        /* Message length in bits           */
-  unsigned Length_High;       /* Message length in bits           */
-
-  unsigned char Message_Block[64]; /* 512-bit message blocks      */
-  int Message_Block_Index;    /* Index into message block array   */
-
-  bool Computed;               /* Is the digest computed?          */
-  bool Corrupted;              /* Is the message digest corruped?  */
-} SHA1Context;
-
-/*
- *  Portability Issues:
- *      SHA-1 is defined in terms of 32-bit "words".  This code was
- *      written with the expectation that the processor has at least
- *      a 32-bit machine word size.  If the machine word size is larger,
- *      the code should still function properly.  One caveat to that
- *      is that the input functions taking characters and character
- *      arrays assume that only 8 bits of information are stored in each
- *      character.
- */
-
-/*
- *  Define the circular shift macro
- */
-#define SHA1CircularShift(bits,word) \
-                ((((word) << (bits)) & 0xFFFFFFFF) | \
-                ((word) >> (32-(bits))))
-
-/* Function prototypes */
-static void SHA1ProcessMessageBlock(SHA1Context *);
-static void SHA1PadMessage(SHA1Context *);
-
-// Initialize the SHA1Context in preparation for computing a new
-// message digest.
-static void SHA1Reset(SHA1Context* context) {
-  context->Length_Low             = 0;
-  context->Length_High            = 0;
-  context->Message_Block_Index    = 0;
-
-  context->Message_Digest[0]      = 0x67452301;
-  context->Message_Digest[1]      = 0xEFCDAB89;
-  context->Message_Digest[2]      = 0x98BADCFE;
-  context->Message_Digest[3]      = 0x10325476;
-  context->Message_Digest[4]      = 0xC3D2E1F0;
-
-  context->Computed   = false;
-  context->Corrupted  = false;
-}
-
-// This function will return the 160-bit message digest into the
-// Message_Digest array within the SHA1Context provided
-static bool SHA1Result(SHA1Context *context) {
-  if (context->Corrupted) {
-    return false;
-  }
-
-  if (!context->Computed) {
-    SHA1PadMessage(context);
-    context->Computed = true;
-  }
-  return true;
-}
-
-// This function accepts an array of bytes as the next portion of
-// the message.
-static void SHA1Input(SHA1Context         *context,
-                      const unsigned char *message_array,
-                      unsigned            length) {
-  if (!length) return;
-
-  if (context->Computed || context->Corrupted) {
-    context->Corrupted = true;
-    return;
-  }
-
-  while(length-- && !context->Corrupted) {
-    context->Message_Block[context->Message_Block_Index++] =
-        (*message_array & 0xFF);
-
-    context->Length_Low += 8;
-    /* Force it to 32 bits */
-    context->Length_Low &= 0xFFFFFFFF;
-    if (context->Length_Low == 0) {
-      context->Length_High++;
-      /* Force it to 32 bits */
-      context->Length_High &= 0xFFFFFFFF;
-      if (context->Length_High == 0)
-      {
-        /* Message is too long */
-        context->Corrupted = true;
-      }
-    }
-
-    if (context->Message_Block_Index == 64)
-    {
-      SHA1ProcessMessageBlock(context);
-    }
-
-    message_array++;
-  }
-}
-
-// This function will process the next 512 bits of the message stored
-// in the Message_Block array.
-static void SHA1ProcessMessageBlock(SHA1Context *context) {
-  const unsigned K[] =            // Constants defined in SHA-1
-      {
-        0x5A827999,
-        0x6ED9EBA1,
-        0x8F1BBCDC,
-        0xCA62C1D6
-      };
-  int         t;                  // Loop counter
-  unsigned    temp;               // Temporary word value
-  unsigned    W[80];            // Word sequence
-  unsigned    A, B, C, D, E;    // Word buffers
-
-  // Initialize the first 16 words in the array W
-  for(t = 0; t < 16; t++) {
-    W[t] = ((unsigned) context->Message_Block[t * 4]) << 24;
-    W[t] |= ((unsigned) context->Message_Block[t * 4 + 1]) << 16;
-    W[t] |= ((unsigned) context->Message_Block[t * 4 + 2]) << 8;
-    W[t] |= ((unsigned) context->Message_Block[t * 4 + 3]);
-  }
-
-  for(t = 16; t < 80; t++) {
-    W[t] = SHA1CircularShift(1,W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16]);
-  }
-
-  A = context->Message_Digest[0];
-  B = context->Message_Digest[1];
-  C = context->Message_Digest[2];
-  D = context->Message_Digest[3];
-  E = context->Message_Digest[4];
-
-  for(t = 0; t < 20; t++) {
-    temp =  SHA1CircularShift(5,A) +
-        ((B & C) | ((~B) & D)) + E + W[t] + K[0];
-    temp &= 0xFFFFFFFF;
-    E = D;
-    D = C;
-    C = SHA1CircularShift(30,B);
-    B = A;
-    A = temp;
-  }
-
-  for(t = 20; t < 40; t++) {
-    temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[1];
-    temp &= 0xFFFFFFFF;
-    E = D;
-    D = C;
-    C = SHA1CircularShift(30,B);
-    B = A;
-    A = temp;
-  }
-
-  for(t = 40; t < 60; t++) {
-    temp = SHA1CircularShift(5,A) +
-        ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2];
-    temp &= 0xFFFFFFFF;
-    E = D;
-    D = C;
-    C = SHA1CircularShift(30,B);
-    B = A;
-    A = temp;
-  }
-
-  for(t = 60; t < 80; t++) {
-    temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[3];
-    temp &= 0xFFFFFFFF;
-    E = D;
-    D = C;
-    C = SHA1CircularShift(30,B);
-    B = A;
-    A = temp;
-  }
-
-  context->Message_Digest[0] = (context->Message_Digest[0] + A) & 0xFFFFFFFF;
-  context->Message_Digest[1] = (context->Message_Digest[1] + B) & 0xFFFFFFFF;
-  context->Message_Digest[2] = (context->Message_Digest[2] + C) & 0xFFFFFFFF;
-  context->Message_Digest[3] = (context->Message_Digest[3] + D) & 0xFFFFFFFF;
-  context->Message_Digest[4] = (context->Message_Digest[4] + E) & 0xFFFFFFFF;
-
-  context->Message_Block_Index = 0;
-}
-
-// According to the standard, the message must be padded to an even
-// 512 bits.  The first padding bit must be a '1'.  The last 64 bits
-// represent the length of the original message.  All bits in between
-// should be 0.  This function will pad the message according to those
-// rules by filling the Message_Block array accordingly.  It will also
-// call SHA1ProcessMessageBlock() appropriately.  When it returns, it
-// can be assumed that the message digest has been computed.
-static void SHA1PadMessage(SHA1Context *context) {
-  // Check to see if the current message block is too small to hold
-  // the initial padding bits and length.  If so, we will pad the
-  // block, process it, and then continue padding into a second block.
-  if (context->Message_Block_Index > 55) {
-    context->Message_Block[context->Message_Block_Index++] = 0x80;
-    while(context->Message_Block_Index < 64) {
-      context->Message_Block[context->Message_Block_Index++] = 0;
-    }
-
-    SHA1ProcessMessageBlock(context);
-
-    while(context->Message_Block_Index < 56) {
-      context->Message_Block[context->Message_Block_Index++] = 0;
-    }
-  } else {
-    context->Message_Block[context->Message_Block_Index++] = 0x80;
-    while(context->Message_Block_Index < 56) {
-      context->Message_Block[context->Message_Block_Index++] = 0;
-    }
-  }
-
-  // Store the message length as the last 8 octets
-  context->Message_Block[56] = (context->Length_High >> 24) & 0xFF;
-  context->Message_Block[57] = (context->Length_High >> 16) & 0xFF;
-  context->Message_Block[58] = (context->Length_High >> 8) & 0xFF;
-  context->Message_Block[59] = (context->Length_High) & 0xFF;
-  context->Message_Block[60] = (context->Length_Low >> 24) & 0xFF;
-  context->Message_Block[61] = (context->Length_Low >> 16) & 0xFF;
-  context->Message_Block[62] = (context->Length_Low >> 8) & 0xFF;
-  context->Message_Block[63] = (context->Length_Low) & 0xFF;
-
-  SHA1ProcessMessageBlock(context);
-}
-
-
-void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array) {
-  SHA1Context context;
-  SHA1Reset(&context);
-  SHA1Input(&context, reinterpret_cast<const unsigned char*>(data), len);
-  bool ok = SHA1Result(&context);
-  if (!ok) {
-    fprintf(stderr, "Unexpected error in SHA1_Hash_Portable code\n");
-    exit(1);
-  }
-  for (int i = 0; i < 5; i++) {
-    uint32_t value = context.Message_Digest[i];
-    hash_array[i*4 + 0] = (value >> 24) & 0xff;
-    hash_array[i*4 + 1] = (value >> 16) & 0xff;
-    hash_array[i*4 + 2] = (value >> 8) & 0xff;
-    hash_array[i*4 + 3] = value & 0xff;
-  }
-}
-
-}
-}

port/sha1_portable.h

@@ -1,25 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_
-#define STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_
-
-#include <stddef.h>
-
-namespace leveldb {
-namespace port {
-
-// Compute the SHA1 hash value of "data[0..len-1]" and store it in
-// "hash_array[0..19]".  hash_array must have 20 bytes of space available.
-//
-// This function is portable but may not be as fast as a version
-// optimized for your platform.  It is provided as a default method
-// that can be used when porting leveldb to a new platform if no
-// better SHA1 hash implementation is available.
-void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array);
-
-}
-}
-
-#endif  // STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_

port/sha1_test.cc

@@ -1,39 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "port/port.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-namespace port {
-
-class SHA1 { };
-
-static std::string TestSHA1(const char* data, size_t len) {
-  char hash_val[20];
-  SHA1_Hash(data, len, hash_val);
-  char buf[41];
-  for (int i = 0; i < 20; i++) {
-    snprintf(buf + i * 2, 41 - i * 2,
-             "%02x",
-             static_cast<unsigned int>(static_cast<unsigned char>(
-                 hash_val[i])));
-  }
-  return std::string(buf, 40);
-}
-
-TEST(SHA1, Simple) {
-  ASSERT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", TestSHA1("", 0));
-  ASSERT_EQ("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", TestSHA1("hello", 5));
-  std::string x(10000, 'x');
-  ASSERT_EQ("f8c5cde791c5056cf515881e701c8a9ecb439a75",
-            TestSHA1(x.data(), x.size()));
-}
-
-}
-}
-
-int main(int argc, char** argv) {
-  return leveldb::test::RunAllTests();
-}

util/cache.cc

@@ -285,7 +285,8 @@ class ShardedLRUCache : public Cache {
   }
 
  public:
-  explicit ShardedLRUCache(size_t capacity) {
+  explicit ShardedLRUCache(size_t capacity)
+      : last_id_(0) {
     const size_t per_shard = (capacity + (kNumShards - 1)) / kNumShards;
     for (int s = 0; s < kNumShards; s++) {
       shard_[s].SetCapacity(per_shard);