Initial commit

src/kv/CMakeLists.txt

@@ -0,0 +1,6 @@
+add_library(leveldb_logger STATIC LevelDBLogger.cpp)
+target_link_libraries(leveldb_logger common leveldb)
+target_compile_options(leveldb_logger PRIVATE "-fno-rtti")
+
+target_add_lib(kv common leveldb leveldb_logger rocksdb)
+target_include_directories(kv PUBLIC ${CMAKE_SOURCE_DIR}/third_party/rocksdb/include)

src/kv/KVStore.cc

@@ -0,0 +1,27 @@
+#include "kv/KVStore.h"
+
+#include "common/utils/StringUtils.h"
+#include "kv/LevelDBStore.h"
+#include "kv/MemDBStore.h"
+#include "kv/RocksDBStore.h"
+
+namespace hf3fs::kv {
+
+std::unique_ptr<KVStore> KVStore::create(const Config &config, const Options &options) {
+  switch (options.type) {
+    case Type::LevelDB:
+      return LevelDBStore::create(config, options);
+
+    case Type::RocksDB:
+      return RocksDBStore::create(config, options);
+
+    case Type::MemDB:
+      return std::make_unique<MemDBStore>(config);
+
+    default:
+      XLOGF(ERR, "invalid KVStore type: {}", toStringView(options.type));
+  }
+  return nullptr;
+}
+
+}  // namespace hf3fs::kv

src/kv/KVStore.h

@@ -0,0 +1,132 @@
+#pragma once
+
+#include <rocksdb/options.h>
+#include <rocksdb/table.h>
+#include <string>
+#include <string_view>
+
+#include "common/serde/Serde.h"
+#include "common/utils/ConfigBase.h"
+#include "common/utils/Duration.h"
+#include "common/utils/Path.h"
+#include "common/utils/Result.h"
+#include "common/utils/Size.h"
+
+namespace hf3fs::kv {
+
+class KVStore {
+ public:
+  enum class Type { LevelDB, RocksDB, MemDB };
+  class Config : public ConfigBase<Config> {
+    CONFIG_ITEM(type, Type::LevelDB);
+    CONFIG_ITEM(create_if_missing, false);
+    CONFIG_HOT_UPDATED_ITEM(sync_when_write, true);
+
+    // for leveldb.
+    CONFIG_ITEM(leveldb_sst_file_size, 16_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
+    CONFIG_ITEM(leveldb_write_buffer_size, 16_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
+    CONFIG_ITEM(leveldb_block_cache_size, 8_GB, ConfigCheckers::checkGE<size_t, 4_MB>);
+    CONFIG_ITEM(leveldb_shared_block_cache, true);
+    CONFIG_HOT_UPDATED_ITEM(leveldb_iterator_fill_cache, true);
+    CONFIG_ITEM(integrate_leveldb_log, false);
+
+    // for rocksdb
+    CONFIG_ITEM(rocksdb_max_manifest_file_size, 64_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
+    CONFIG_ITEM(rocksdb_stats_dump_period, 2_min);
+    CONFIG_ITEM(rocksdb_enable_pipelined_write, false);
+    CONFIG_ITEM(rocksdb_unordered_write, false);
+    CONFIG_ITEM(rocksdb_avoid_flush_during_recovery, false);
+    CONFIG_ITEM(rocksdb_avoid_flush_during_shutdown, false);
+    CONFIG_ITEM(rocksdb_avoid_unnecessary_blocking_io, false);
+    CONFIG_ITEM(rocksdb_lowest_used_cache_tier, rocksdb_internal::CacheTier::kNonVolatileBlockTier);
+    CONFIG_ITEM(rocksdb_write_buffer_size, 16_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
+    CONFIG_ITEM(rocksdb_compression, rocksdb_internal::CompressionType::kNoCompression);
+    CONFIG_ITEM(rocksdb_level0_file_num_compaction_trigger, 4, ConfigCheckers::checkGE<int, 2>);
+    CONFIG_ITEM(rocksdb_enable_prefix_transform, true);
+    CONFIG_ITEM(rocksdb_enable_bloom_filter, true);
+    CONFIG_ITEM(rocksdb_bloom_filter_bits_per_key, 10);
+    CONFIG_ITEM(rocksdb_num_levels, 7, ConfigCheckers::checkGE<int, 3>);
+    CONFIG_ITEM(rocksdb_target_file_size_base, 64_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
+    CONFIG_ITEM(rocksdb_target_file_size_multiplier, 1, ConfigCheckers::checkPositive);
+    CONFIG_ITEM(rocksdb_block_cache_size, 8_GB, ConfigCheckers::checkGE<size_t, 4_MB>);
+    CONFIG_ITEM(rocksdb_shared_block_cache, true);
+    CONFIG_ITEM(rocksdb_block_size, 4_KB, ConfigCheckers::checkGE<size_t, 4_KB>);
+    CONFIG_ITEM(rocksdb_prepopulate_block_cache,
+                rocksdb_internal::BlockBasedTableOptions::PrepopulateBlockCache::kDisable);
+    CONFIG_ITEM(rocksdb_threads_num, 8u, ConfigCheckers::checkPositive);
+    CONFIG_ITEM(rocksdb_wal_recovery_mode, rocksdb_internal::WALRecoveryMode::kTolerateCorruptedTailRecords);
+    CONFIG_ITEM(rocksdb_keep_log_file_num, 10u);
+    CONFIG_HOT_UPDATED_ITEM(rocksdb_readahead_size, 2_MB);
+  };
+
+  struct Options {
+    SERDE_STRUCT_FIELD(type, Type::RocksDB);
+    SERDE_STRUCT_FIELD(path, Path{});
+    SERDE_STRUCT_FIELD(createIfMissing, false);
+  };
+
+  virtual ~KVStore() = default;
+
+  // get value corresponding to key.
+  virtual Result<std::string> get(std::string_view key) = 0;
+
+  // get the first key which is greater than input key.
+  using IterateFunc = std::function<Result<Void>(std::string_view, std::string_view)>;
+  virtual Result<Void> iterateKeysWithPrefix(std::string_view prefix,
+                                             uint32_t limit,
+                                             IterateFunc func,
+                                             std::optional<std::string> *nextValidKey = nullptr) = 0;
+
+  // put a key-value pair.
+  virtual Result<Void> put(std::string_view key, std::string_view value, bool sync = false) = 0;
+
+  // remove a key-value pair.
+  virtual Result<Void> remove(std::string_view key) = 0;
+
+  // batch operations.
+  class BatchOperations {
+   public:
+    virtual ~BatchOperations() = default;
+    // put a key-value pair.
+    virtual void put(std::string_view key, std::string_view value) = 0;
+    // remove a key.
+    virtual void remove(std::string_view key) = 0;
+    // clear a batch operations.
+    virtual void clear() = 0;
+    // commit a batch of operations.
+    virtual Result<Void> commit() = 0;
+    // destroy self.
+    virtual void destroy() = 0;
+    // deleter for std::unique_ptr.
+    struct Deleter {
+      void operator()(BatchOperations *b) { b->destroy(); }
+    };
+  };
+  using BatchOptionsPtr = std::unique_ptr<BatchOperations, BatchOperations::Deleter>;
+  virtual BatchOptionsPtr createBatchOps() = 0;
+
+  // iterator.
+  class Iterator {
+   public:
+    virtual ~Iterator() = default;
+    virtual void seek(std::string_view key) = 0;
+    virtual void seekToFirst() = 0;
+    virtual void seekToLast() = 0;
+    virtual void next() = 0;
+    virtual Result<Void> status() const = 0;
+    virtual bool valid() const = 0;
+    virtual std::string_view key() const = 0;
+    virtual std::string_view value() const = 0;
+    virtual void destroy() = 0;
+    struct Deleter {
+      void operator()(Iterator *it) { it->destroy(); }
+    };
+  };
+  using IteratorPtr = std::unique_ptr<Iterator, Iterator::Deleter>;
+  virtual IteratorPtr createIterator() = 0;
+
+  // create a KVStore instance.
+  static std::unique_ptr<KVStore> create(const Config &config, const Options &options);
+};
+
+}  // namespace hf3fs::kv

src/kv/LevelDBLogger.cpp

@@ -0,0 +1,37 @@
+#include "LevelDBLogger.h"
+
+#include <folly/logging/xlog.h>
+
+namespace hf3fs::kv {
+
+LevelDBLogger::~LevelDBLogger() = default;
+
+void LevelDBLogger::Logv(const char *format, std::va_list ap) {
+  static constexpr int fixedBufferSize = 1024;
+  thread_local char fixedBuffer[fixedBufferSize];
+
+  // ap can only be used once, we have to prepare a copy of ap for both paths
+  std::va_list apCopy;
+  va_copy(apCopy, ap);
+  auto count = std::vsnprintf(fixedBuffer, fixedBufferSize, format, apCopy);
+  va_end(apCopy);
+
+  if (count < 0) {
+    XLOGF(WARN, "[LevelDB] Print log from LevelDB failed. ret of vsnprintf is {}", count);
+  } else if (count < fixedBufferSize) {
+    // fit into fixedBuffer, note that the terminating '\0' will occupy one byte but not counted in the return value.
+    if (count >= 1 && fixedBuffer[count - 1] == '\n') {
+      --count;
+    }
+    XLOGF(INFO, "[LevelDB] {}", std::string_view{fixedBuffer, static_cast<size_t>(count)});
+  } else {
+    // not fit into fixedBuffer, allocate a dynamic buffer large enough for the content and the terminating '\0'
+    auto dynamicBuffer = std::make_unique<char[]>(count + 1);
+    std::vsnprintf(dynamicBuffer.get(), count + 1, format, ap);
+    if (count >= 1 && dynamicBuffer.get()[count - 1] == '\n') {
+      --count;
+    }
+    XLOGF(INFO, "[LevelDB] {}", std::string_view{dynamicBuffer.get(), static_cast<size_t>(count)});
+  }
+}
+}  // namespace hf3fs::kv

src/kv/LevelDBLogger.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "leveldb/env.h"
+
+namespace hf3fs::kv {
+class LevelDBLogger final : public leveldb::Logger {
+ public:
+  LevelDBLogger() = default;
+  ~LevelDBLogger() final;
+
+  void Logv(const char *format, std::va_list ap) final;
+};
+}  // namespace hf3fs::kv

src/kv/LevelDBStore.cc

@@ -0,0 +1,194 @@
+#include "kv/LevelDBStore.h"
+
+#include <folly/logging/xlog.h>
+#include <utility>
+#include <variant>
+
+#include "common/utils/ObjectPool.h"
+#include "common/utils/Size.h"
+#include "leveldb/cache.h"
+#include "leveldb/iterator.h"
+#include "leveldb/options.h"
+#include "leveldb/slice.h"
+#include "leveldb/status.h"
+#include "leveldb/write_batch.h"
+
+namespace hf3fs::kv {
+namespace {
+
+inline leveldb::Slice slice(std::string_view v) { return leveldb::Slice{v.data(), v.length()}; }
+inline std::string_view view(leveldb::Slice s) { return std::string_view{s.data(), s.size()}; }
+
+using LevelDBBatchOperationsPool = ObjectPool<LevelDBStore::LevelDBBatchOperations>;
+using LevelDBIteratorPool = ObjectPool<LevelDBStore::LevelDBIterator>;
+
+leveldb::Cache *sharedBlockCache(size_t capacity) {
+  static std::unique_ptr<leveldb::Cache> cache{leveldb::NewLRUCache(capacity)};
+  return cache.get();
+}
+
+}  // namespace
+
+// get value corresponding to key.
+Result<std::string> LevelDBStore::get(std::string_view key) {
+  leveldb::ReadOptions options;
+  std::string value;
+  auto status = db_->Get(options, slice(key), &value);
+  if (UNLIKELY(!status.ok())) {
+    if (status.IsNotFound()) {
+      return makeError(StatusCode::kKVStoreNotFound);
+    }
+    auto msg = fmt::format("LevelDB get error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreGetError, std::move(msg));
+  }
+  return Result<std::string>(std::move(value));
+}
+
+Result<Void> LevelDBStore::iterateKeysWithPrefix(std::string_view prefix,
+                                                 uint32_t limit,
+                                                 IterateFunc func,
+                                                 std::optional<std::string> *nextValidKey /* = nullptr */) {
+  leveldb::ReadOptions options;
+  options.fill_cache = config_.leveldb_iterator_fill_cache();
+  std::unique_ptr<leveldb::Iterator> iterator{db_->NewIterator(options)};
+  iterator->Seek(slice(prefix));
+  bool prefixBreak = false;
+  for (uint32_t i = 0; iterator->Valid() && i < limit; ++i, iterator->Next()) {
+    auto key = iterator->key();
+    if (key.starts_with(slice(prefix))) {
+      RETURN_ON_ERROR(func(view(key), view(iterator->value())));
+    } else {
+      prefixBreak = true;
+      break;
+    }
+  }
+  if (nextValidKey && !prefixBreak && iterator->Valid() && iterator->key().starts_with(slice(prefix))) {
+    *nextValidKey = view(iterator->key());
+  }
+  if (UNLIKELY(!iterator->status().ok())) {
+    auto msg = fmt::format("LevelDB iterate error: {}", iterator->status().ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreGetError, std::move(msg));
+  }
+  return Void{};
+}
+
+// put a key-value pair.
+Result<Void> LevelDBStore::put(std::string_view key, std::string_view value, bool sync /* = false */) {
+  leveldb::WriteOptions options;
+  options.sync = config_.sync_when_write() || sync;
+  auto status = db_->Put(options, slice(key), slice(value));
+  if (UNLIKELY(!status.ok())) {
+    auto msg = fmt::format("LevelDB put error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreSetError, std::move(msg));
+  }
+  return Void{};
+}
+
+// remove a key-value pair.
+Result<Void> LevelDBStore::remove(std::string_view key) {
+  leveldb::WriteOptions options;
+  options.sync = config_.sync_when_write();
+  auto status = db_->Delete(options, slice(key));
+  if (UNLIKELY(!status.ok())) {
+    auto msg = fmt::format("LevelDB delete error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreSetError, std::move(msg));
+  }
+  return Void{};
+}
+
+// batch operations.
+void LevelDBStore::LevelDBBatchOperations::put(std::string_view key, std::string_view value) {
+  writeBatch_.Put(slice(key), slice(value));
+}
+
+void LevelDBStore::LevelDBBatchOperations::remove(std::string_view key) { writeBatch_.Delete(slice(key)); }
+
+Result<Void> LevelDBStore::LevelDBBatchOperations::commit() {
+  if (writeBatch_.ApproximateSize() == 0) {
+    return Void{};
+  }
+  leveldb::WriteOptions options;
+  options.sync = db_.config_.sync_when_write();
+  auto status = db_.db_->Write(options, &writeBatch_);
+  if (UNLIKELY(!status.ok())) {
+    auto msg = fmt::format("LevelDB write error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreSetError, std::move(msg));
+  }
+  return Void{};
+}
+
+void LevelDBStore::LevelDBBatchOperations::destroy() { LevelDBBatchOperationsPool::Ptr{this}; }
+
+KVStore::BatchOptionsPtr LevelDBStore::createBatchOps() {
+  return BatchOptionsPtr{LevelDBBatchOperationsPool::get(*this).release()};
+}
+
+void LevelDBStore::LevelDBIterator::seek(std::string_view key) { iterator_->Seek(slice(key)); }
+void LevelDBStore::LevelDBIterator::seekToFirst() { iterator_->SeekToFirst(); }
+void LevelDBStore::LevelDBIterator::seekToLast() { iterator_->SeekToLast(); }
+void LevelDBStore::LevelDBIterator::next() { iterator_->Next(); }
+Result<Void> LevelDBStore::LevelDBIterator::status() const {
+  if (UNLIKELY(!iterator_->status().ok())) {
+    auto msg = fmt::format("LevelDB iterate error: {}", iterator_->status().ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreGetError, std::move(msg));
+  }
+  return Void{};
+}
+bool LevelDBStore::LevelDBIterator::valid() const { return iterator_->Valid(); }
+std::string_view LevelDBStore::LevelDBIterator::key() const { return view(iterator_->key()); }
+std::string_view LevelDBStore::LevelDBIterator::value() const { return view(iterator_->value()); }
+void LevelDBStore::LevelDBIterator::destroy() { LevelDBIteratorPool::Ptr{this}; }
+KVStore::IteratorPtr LevelDBStore::createIterator() {
+  leveldb::ReadOptions options;
+  options.fill_cache = config_.leveldb_iterator_fill_cache();
+  std::unique_ptr<leveldb::Iterator> it(db_->NewIterator(options));
+  if (UNLIKELY(it == nullptr)) {
+    return nullptr;
+  }
+  return IteratorPtr{LevelDBIteratorPool::get(std::move(it)).release()};
+}
+
+// initialize leveldb.
+Result<Void> LevelDBStore::init(const Options &optionsIn) {
+  leveldb::Options options;
+  options.create_if_missing = optionsIn.createIfMissing;
+  options.max_file_size = config_.leveldb_sst_file_size();
+  options.write_buffer_size = config_.leveldb_write_buffer_size();
+  if (config_.leveldb_shared_block_cache()) {
+    options.block_cache = sharedBlockCache(config_.leveldb_block_cache_size());
+  } else {
+    cache_.reset(options.block_cache = leveldb::NewLRUCache(config_.leveldb_block_cache_size()));
+  }
+  if (config_.integrate_leveldb_log()) {
+    logger_ = std::make_unique<LevelDBLogger>();
+    options.info_log = logger_.get();
+  }
+  leveldb::DB *db = nullptr;
+  auto status = leveldb::DB::Open(options, optionsIn.path.string(), &db);
+  if (UNLIKELY(!status.ok())) {
+    auto msg = fmt::format("LevelDB init error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreOpenFailed, std::move(msg));
+  }
+  db_.reset(db);
+  return Void{};
+}
+
+// create a leveldb chunk meta store object.
+std::unique_ptr<LevelDBStore> LevelDBStore::create(const Config &config, const Options &options) {
+  auto store = std::make_unique<LevelDBStore>(config);
+  auto result = store->init(options);
+  if (UNLIKELY(!result)) {
+    XLOGF(ERR, "create leveldb at {} failed: {}, config: {}", options, result.error(), config.toString());
+    return nullptr;
+  }
+  return store;
+}
+
+}  // namespace hf3fs::kv

src/kv/LevelDBStore.h

@@ -0,0 +1,89 @@
+#pragma once
+
+#include <folly/ThreadLocal.h>
+#include <leveldb/cache.h>
+#include <leveldb/db.h>
+#include <leveldb/write_batch.h>
+
+#include "LevelDBLogger.h"
+#include "kv/KVStore.h"
+
+namespace hf3fs::kv {
+
+class LevelDBStore : public KVStore {
+ public:
+  LevelDBStore(const Config &config)
+      : config_(config) {}
+
+  // get value corresponding to key.
+  Result<std::string> get(std::string_view key) final;
+
+  // get the first key which is greater than input key.
+  Result<Void> iterateKeysWithPrefix(std::string_view prefix,
+                                     uint32_t limit,
+                                     IterateFunc func,
+                                     std::optional<std::string> *nextValidKey = nullptr) final;
+
+  // put a key-value pair.
+  Result<Void> put(std::string_view key, std::string_view value, bool sync = false) final;
+
+  // remove a key-value pair.
+  Result<Void> remove(std::string_view key) final;
+
+  // batch operations.
+  class LevelDBBatchOperations : public BatchOperations {
+   public:
+    LevelDBBatchOperations(LevelDBStore &db)
+        : db_(db) {}
+    // put a key-value pair.
+    void put(std::string_view key, std::string_view value) override;
+    // remove a key.
+    void remove(std::string_view key) override;
+    // clear a batch operations.
+    void clear() override { writeBatch_.Clear(); }
+    // commit a batch of operations.
+    Result<Void> commit() override;
+    // destroy self.
+    void destroy() override;
+
+   private:
+    LevelDBStore &db_;
+    leveldb::WriteBatch writeBatch_;
+  };
+  BatchOptionsPtr createBatchOps() override;
+
+  // iterator.
+  class LevelDBIterator : public Iterator {
+   public:
+    LevelDBIterator(std::unique_ptr<leveldb::Iterator> it)
+        : iterator_(std::move(it)) {}
+    void seek(std::string_view key) override;
+    void seekToFirst() override;
+    void seekToLast() override;
+    void next() override;
+    Result<Void> status() const override;
+    bool valid() const override;
+    std::string_view key() const override;
+    std::string_view value() const override;
+    void destroy() override;
+
+   private:
+    std::unique_ptr<leveldb::Iterator> iterator_;
+  };
+  IteratorPtr createIterator() override;
+
+  // create a LevelDB instance.
+  static std::unique_ptr<LevelDBStore> create(const Config &config, const Options &options);
+
+ protected:
+  // initialize leveldb.
+  Result<Void> init(const Options &optionsIn);
+
+ private:
+  const Config &config_;
+  std::unique_ptr<leveldb::DB> db_;
+  std::unique_ptr<leveldb::Cache> cache_;
+  std::unique_ptr<LevelDBLogger> logger_;
+};
+
+}  // namespace hf3fs::kv

src/kv/MemDBStore.h

@@ -0,0 +1,125 @@
+#pragma once
+
+#include <map>
+#include <mutex>
+
+#include "kv/KVStore.h"
+
+namespace hf3fs::kv {
+
+class MemDBStore : public KVStore {
+ public:
+  MemDBStore(const Config &config)
+      : config_(config) {}
+
+  // get value corresponding to key.
+  Result<std::string> get(std::string_view key) final {
+    auto lock = std::unique_lock(mutex_);
+    auto it = map_.find(std::string{key});
+    if (it == map_.end()) {
+      return makeError(StatusCode::kKVStoreNotFound);
+    }
+    return it->second;
+  }
+
+  // get the first key which is greater than input key.
+  Result<Void> iterateKeysWithPrefix(std::string_view prefix,
+                                     uint32_t limit,
+                                     IterateFunc func,
+                                     std::optional<std::string> *nextValidKey = nullptr) final {
+    auto lock = std::unique_lock(mutex_);
+    auto it = map_.lower_bound(std::string{prefix});
+    for (auto i = 0u; i < limit && it != map_.end(); ++i, ++it) {
+      if (it->first.starts_with(prefix)) {
+        RETURN_ON_ERROR(func(it->first, it->second));
+      } else {
+        break;
+      }
+    }
+    if (nextValidKey && it != map_.end() && it->first.starts_with(prefix)) {
+      *nextValidKey = it->first;
+    }
+    return Void{};
+  }
+
+  // put a key-value pair.
+  Result<Void> put(std::string_view key, std::string_view value, bool) final {
+    auto lock = std::unique_lock(mutex_);
+    map_[std::string{key}] = value;
+    return Void{};
+  }
+
+  // remove a key-value pair.
+  Result<Void> remove(std::string_view key) final {
+    auto lock = std::unique_lock(mutex_);
+    map_.erase(std::string{key});
+    return Void{};
+  }
+
+  // batch operations.
+  class MemBatchOperations : public BatchOperations {
+   public:
+    MemBatchOperations(MemDBStore &db)
+        : db_(db) {}
+    // put a key-value pair.
+    void put(std::string_view key, std::string_view value) override {
+      writeBatch_.emplace_back(std::string{key}, std::string{value});
+    }
+    // remove a key.
+    void remove(std::string_view key) override { writeBatch_.emplace_back(std::string{key}, std::nullopt); }
+    // clear a batch operations.
+    void clear() override { writeBatch_.clear(); }
+    // commit a batch of operations.
+    Result<Void> commit() override {
+      auto lock = std::unique_lock(db_.mutex_);
+      for (auto &pair : writeBatch_) {
+        if (pair.second.has_value()) {
+          db_.map_[std::string{pair.first}] = pair.second.value();
+        } else {
+          db_.map_.erase(std::string{pair.first});
+        }
+      }
+      return Void{};
+    }
+    // destroy self.
+    void destroy() override { delete this; }
+
+   private:
+    MemDBStore &db_;
+    std::vector<std::pair<std::string, std::optional<std::string>>> writeBatch_;
+  };
+  BatchOptionsPtr createBatchOps() override {
+    return BatchOptionsPtr{std::make_unique<MemBatchOperations>(*this).release()};
+  }
+
+  // iterator.
+  class MemIterator : public Iterator {
+   public:
+    MemIterator(MemDBStore &db)
+        : lock_(db.mutex_),
+          map_(db.map_),
+          iterator_(map_.end()) {}
+    void seek(std::string_view key) override { iterator_ = map_.lower_bound(std::string{key}); }
+    void seekToFirst() override { iterator_ = map_.begin(); }
+    void seekToLast() override { iterator_ = --map_.end(); }
+    void next() override { ++iterator_; }
+    Result<Void> status() const override { return Void{}; }
+    bool valid() const override { return iterator_ != map_.end(); }
+    std::string_view key() const override { return iterator_->first; }
+    std::string_view value() const override { return iterator_->second; }
+    void destroy() override { delete this; }
+
+   private:
+    std::unique_lock<std::mutex> lock_;
+    std::map<std::string, std::string> &map_;
+    std::map<std::string, std::string>::iterator iterator_;
+  };
+  IteratorPtr createIterator() override { return IteratorPtr{std::make_unique<MemIterator>(*this).release()}; }
+
+ private:
+  [[maybe_unused]] const Config &config_;
+  std::mutex mutex_;
+  std::map<std::string, std::string> map_;
+};
+
+}  // namespace hf3fs::kv

src/kv/RocksDBStore.cc

@@ -0,0 +1,220 @@
+#include "kv/RocksDBStore.h"
+
+#include <folly/logging/xlog.h>
+#include <rocksdb/filter_policy.h>
+#include <rocksdb/slice.h>
+#include <rocksdb/slice_transform.h>
+#include <rocksdb/status.h>
+#include <rocksdb/write_batch.h>
+#include <variant>
+
+#include "common/utils/ObjectPool.h"
+#include "common/utils/Size.h"
+
+namespace hf3fs::kv {
+namespace {
+
+inline rocksdb_internal::Slice slice(std::string_view v) { return rocksdb_internal::Slice{v.data(), v.length()}; }
+inline std::string_view view(rocksdb_internal::Slice s) { return std::string_view{s.data(), s.size()}; }
+
+using RocksDBBatchOperationsPool = ObjectPool<RocksDBStore::RocksDBBatchOperations>;
+using RocksDBIteratorPool = ObjectPool<RocksDBStore::RocksDBIterator>;
+
+std::shared_ptr<rocksdb_internal::Cache> sharedBlockCache(size_t capacity) {
+  static std::shared_ptr<rocksdb_internal::Cache> cache{rocksdb_internal::NewLRUCache(capacity)};
+  return cache;
+}
+
+constexpr size_t rocksdbPrefixLen = 12;
+
+}  // namespace
+
+// get value corresponding to key.
+Result<std::string> RocksDBStore::get(std::string_view key) {
+  rocksdb_internal::ReadOptions options;
+  std::string value;
+  auto status = db_->Get(options, slice(key), &value);
+  if (UNLIKELY(!status.ok())) {
+    if (status.IsNotFound()) {
+      return makeError(StatusCode::kKVStoreNotFound);
+    }
+    auto msg = fmt::format("RocksDB get error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreGetError, std::move(msg));
+  }
+  return Result<std::string>(std::move(value));
+}
+
+Result<Void> RocksDBStore::iterateKeysWithPrefix(std::string_view prefix,
+                                                 uint32_t limit,
+                                                 IterateFunc func,
+                                                 std::optional<std::string> *nextValidKey /* = nullptr */) {
+  rocksdb_internal::ReadOptions options;
+  options.readahead_size = config_.rocksdb_readahead_size();
+  options.prefix_same_as_start = config_.rocksdb_enable_prefix_transform() && prefix.size() >= rocksdbPrefixLen;
+  std::unique_ptr<rocksdb_internal::Iterator> iterator{db_->NewIterator(options)};
+  iterator->Seek(slice(prefix));
+  bool prefixBreak = false;
+  for (uint32_t i = 0; iterator->Valid() && i < limit; ++i, iterator->Next()) {
+    auto key = iterator->key();
+    if (key.starts_with(slice(prefix))) {
+      RETURN_ON_ERROR(func(view(key), view(iterator->value())));
+    } else {
+      prefixBreak = true;
+      break;
+    }
+  }
+  if (nextValidKey && !prefixBreak && iterator->Valid() && iterator->key().starts_with(slice(prefix))) {
+    *nextValidKey = view(iterator->key());
+  }
+  if (UNLIKELY(!iterator->status().ok())) {
+    auto msg = fmt::format("RocksDB iterate error: {}", iterator->status().ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreGetError, std::move(msg));
+  }
+  return Void{};
+}
+
+// put a key-value pair.
+Result<Void> RocksDBStore::put(std::string_view key, std::string_view value, bool sync /* = false */) {
+  rocksdb_internal::WriteOptions options;
+  options.sync = config_.sync_when_write() || sync;
+  auto status = db_->Put(options, slice(key), slice(value));
+  if (UNLIKELY(!status.ok())) {
+    auto msg = fmt::format("RocksDB put error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreSetError, std::move(msg));
+  }
+  return Void{};
+}
+
+// remove a key-value pair.
+Result<Void> RocksDBStore::remove(std::string_view key) {
+  rocksdb_internal::WriteOptions options;
+  options.sync = config_.sync_when_write();
+  auto status = db_->Delete(options, slice(key));
+  if (UNLIKELY(!status.ok())) {
+    auto msg = fmt::format("RocksDB delete error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreSetError, std::move(msg));
+  }
+  return Void{};
+}
+
+// batch operations.
+void RocksDBStore::RocksDBBatchOperations::put(std::string_view key, std::string_view value) {
+  writeBatch_.Put(slice(key), slice(value));
+}
+
+void RocksDBStore::RocksDBBatchOperations::remove(std::string_view key) { writeBatch_.Delete(slice(key)); }
+
+Result<Void> RocksDBStore::RocksDBBatchOperations::commit() {
+  rocksdb_internal::WriteOptions options;
+  options.sync = db_.config_.sync_when_write();
+  auto status = db_.db_->Write(options, &writeBatch_);
+  if (UNLIKELY(!status.ok())) {
+    auto msg = fmt::format("RocksDB write error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreSetError, std::move(msg));
+  }
+  return Void{};
+}
+
+void RocksDBStore::RocksDBBatchOperations::destroy() { RocksDBBatchOperationsPool::Ptr{this}; }
+
+KVStore::BatchOptionsPtr RocksDBStore::createBatchOps() {
+  return BatchOptionsPtr{RocksDBBatchOperationsPool::get(*this).release()};
+}
+
+void RocksDBStore::RocksDBIterator::seek(std::string_view key) { iterator_->Seek(slice(key)); }
+void RocksDBStore::RocksDBIterator::seekToFirst() { iterator_->SeekToFirst(); }
+void RocksDBStore::RocksDBIterator::seekToLast() { iterator_->SeekToLast(); }
+void RocksDBStore::RocksDBIterator::next() { iterator_->Next(); }
+Result<Void> RocksDBStore::RocksDBIterator::status() const {
+  if (UNLIKELY(!iterator_->status().ok())) {
+    auto msg = fmt::format("RocksDB iterate error: {}", iterator_->status().ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreGetError, std::move(msg));
+  }
+  return Void{};
+}
+bool RocksDBStore::RocksDBIterator::valid() const { return iterator_->Valid(); }
+std::string_view RocksDBStore::RocksDBIterator::key() const { return view(iterator_->key()); }
+std::string_view RocksDBStore::RocksDBIterator::value() const { return view(iterator_->value()); }
+void RocksDBStore::RocksDBIterator::destroy() { RocksDBIteratorPool::Ptr{this}; }
+KVStore::IteratorPtr RocksDBStore::createIterator() {
+  rocksdb_internal::ReadOptions options;
+  options.readahead_size = config_.rocksdb_readahead_size();
+  std::unique_ptr<rocksdb_internal::Iterator> it(db_->NewIterator(options));
+  if (UNLIKELY(it == nullptr)) {
+    return nullptr;
+  }
+  return IteratorPtr{RocksDBIteratorPool::get(std::move(it)).release()};
+}
+
+// initialize rocksdb.
+Result<Void> RocksDBStore::init(const Options &optionsIn) {
+  rocksdb_internal::Options options;
+  rocksdb_internal::BlockBasedTableOptions table_options;
+
+  options.create_if_missing = optionsIn.createIfMissing;
+  options.max_manifest_file_size = config_.rocksdb_max_manifest_file_size();
+  options.stats_dump_period_sec =
+      std::chrono::duration_cast<std::chrono::seconds>(config_.rocksdb_stats_dump_period()).count();
+  options.enable_pipelined_write = config_.rocksdb_enable_pipelined_write();
+  options.unordered_write = config_.rocksdb_unordered_write();
+  options.avoid_flush_during_recovery = config_.rocksdb_avoid_flush_during_recovery();
+  options.avoid_flush_during_shutdown = config_.rocksdb_avoid_flush_during_shutdown();
+  options.avoid_unnecessary_blocking_io = config_.rocksdb_avoid_unnecessary_blocking_io();
+  options.lowest_used_cache_tier = config_.rocksdb_lowest_used_cache_tier();
+  options.write_buffer_size = config_.rocksdb_write_buffer_size();
+  options.compression = config_.rocksdb_compression();
+  options.level0_file_num_compaction_trigger = config_.rocksdb_level0_file_num_compaction_trigger();
+  if (config_.rocksdb_enable_prefix_transform()) {
+    options.prefix_extractor.reset(rocksdb_internal::NewFixedPrefixTransform(rocksdbPrefixLen));
+  }
+  if (config_.rocksdb_enable_bloom_filter()) {
+    table_options.filter_policy.reset(
+        rocksdb_internal::NewBloomFilterPolicy(config_.rocksdb_bloom_filter_bits_per_key()));
+  }
+  options.num_levels = config_.rocksdb_num_levels();
+  options.target_file_size_base = config_.rocksdb_target_file_size_base();
+  options.target_file_size_multiplier = config_.rocksdb_target_file_size_multiplier();
+  if (config_.rocksdb_shared_block_cache()) {
+    table_options.block_cache = sharedBlockCache(config_.rocksdb_block_cache_size());
+  } else {
+    table_options.block_cache = rocksdb_internal::NewLRUCache(config_.rocksdb_block_cache_size());
+  }
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.index_type = rocksdb_internal::BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+  table_options.block_size = config_.rocksdb_block_size();
+  table_options.prepopulate_block_cache = config_.rocksdb_prepopulate_block_cache();
+
+  options.table_factory.reset(rocksdb_internal::NewBlockBasedTableFactory(table_options));
+  options.IncreaseParallelism(config_.rocksdb_threads_num());
+  options.wal_recovery_mode = config_.rocksdb_wal_recovery_mode();
+  options.keep_log_file_num = config_.rocksdb_keep_log_file_num();
+
+  rocksdb_internal::DB *db = nullptr;
+  auto status = rocksdb_internal::DB::Open(options, optionsIn.path.string(), &db);
+  if (UNLIKELY(!status.ok())) {
+    auto msg = fmt::format("RocksDB init error: {}", status.ToString());
+    XLOG(ERR, msg);
+    return makeError(StatusCode::kKVStoreOpenFailed, std::move(msg));
+  }
+  db_.reset(db);
+  return Void{};
+}
+
+// create a rocksdb chunk meta store object.
+std::unique_ptr<RocksDBStore> RocksDBStore::create(const Config &config, const Options &options) {
+  auto store = std::make_unique<RocksDBStore>(config);
+  auto result = store->init(options);
+  if (UNLIKELY(!result)) {
+    XLOGF(ERR, "create rocksdb at {} failed: {}, config: {}", options, result.error(), config.toString());
+    return nullptr;
+  }
+  return store;
+}
+
+}  // namespace hf3fs::kv

src/kv/RocksDBStore.h

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <folly/ThreadLocal.h>
+#include <rocksdb/db.h>
+#include <rocksdb/write_batch.h>
+
+#include "kv/KVStore.h"
+
+namespace hf3fs::kv {
+
+class RocksDBStore : public KVStore {
+ public:
+  RocksDBStore(const Config &config)
+      : config_(config) {}
+
+  // get value corresponding to key.
+  Result<std::string> get(std::string_view key) final;
+
+  // get the first key which is greater than input key.
+  Result<Void> iterateKeysWithPrefix(std::string_view prefix,
+                                     uint32_t limit,
+                                     IterateFunc func,
+                                     std::optional<std::string> *nextValidKey = nullptr) final;
+
+  // put a key-value pair.
+  Result<Void> put(std::string_view key, std::string_view value, bool sync = false) final;
+
+  // remove a key-value pair.
+  Result<Void> remove(std::string_view key) final;
+
+  // batch operations.
+  class RocksDBBatchOperations : public BatchOperations {
+   public:
+    RocksDBBatchOperations(RocksDBStore &db)
+        : db_(db) {}
+    // put a key-value pair.
+    void put(std::string_view key, std::string_view value) override;
+    // remove a key.
+    void remove(std::string_view key) override;
+    // clear a batch operations.
+    void clear() override { writeBatch_.Clear(); }
+    // commit a batch of operations.
+    Result<Void> commit() override;
+    // destroy self.
+    void destroy() override;
+
+   private:
+    RocksDBStore &db_;
+    rocksdb_internal::WriteBatch writeBatch_;
+  };
+  BatchOptionsPtr createBatchOps() override;
+
+  // iterator.
+  class RocksDBIterator : public Iterator {
+   public:
+    RocksDBIterator(std::unique_ptr<rocksdb_internal::Iterator> it)
+        : iterator_(std::move(it)) {}
+    void seek(std::string_view key) override;
+    void seekToFirst() override;
+    void seekToLast() override;
+    void next() override;
+    Result<Void> status() const override;
+    bool valid() const override;
+    std::string_view key() const override;
+    std::string_view value() const override;
+    void destroy() override;
+
+   private:
+    std::unique_ptr<rocksdb_internal::Iterator> iterator_;
+  };
+  IteratorPtr createIterator() override;
+
+  // create a RocksDB instance.
+  static std::unique_ptr<RocksDBStore> create(const Config &config, const Options &options);
+
+ protected:
+  // initialize rocksdb.
+  Result<Void> init(const Options &optionsIn);
+
+ private:
+  const Config &config_;
+  std::unique_ptr<rocksdb_internal::DB> db_;
+};
+
+}  // namespace hf3fs::kv