Initial commit

tests/storage/store/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_test(test_storage_store test-fabric-lib)

tests/storage/store/TestBufferPool.cc

@@ -0,0 +1,56 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "common/serde/Serde.h"
+#include "common/utils/Reflection.h"
+#include "fbs/storage/Common.h"
+#include "fbs/storage/Service.h"
+#include "storage/aio/AioReadWorker.h"
+#include "storage/service/BufferPool.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+TEST(TestBufferPool, Normal) {
+  CPUExecutorGroup executor(8, "");
+
+  BufferPool::Config bufferPoolConfig;
+  bufferPoolConfig.set_rdmabuf_count(256);
+  bufferPoolConfig.set_big_rdmabuf_count(4);
+  AioReadWorker::Config aioReadWorkerConfig;
+  aioReadWorkerConfig.set_num_threads(8);
+  for (auto i = 0; i < 8; ++i) {
+    BufferPool pool(bufferPoolConfig);
+    ASSERT_OK(pool.init(executor));
+
+    AioReadWorker worker(aioReadWorkerConfig);
+    ASSERT_OK(worker.start({}, pool.iovecs()));
+    ASSERT_OK(worker.stopAndJoin());
+
+    pool.clear(executor);
+  }
+}
+
+TEST(TestBufferPool, BigBuffer) {
+  CPUExecutorGroup executor(8, "");
+
+  BufferPool::Config bufferPoolConfig;
+  bufferPoolConfig.set_rdmabuf_count(256);
+  bufferPoolConfig.set_rdmabuf_size(4_MB);
+  bufferPoolConfig.set_big_rdmabuf_count(4);
+  bufferPoolConfig.set_big_rdmabuf_size(64_MB);
+
+  BufferPool pool(bufferPoolConfig);
+  ASSERT_OK(pool.init(executor));
+
+  auto guard = pool.get();
+  auto result = folly::coro::blockingWait(guard.allocate(64_MB));
+  ASSERT_OK(result);
+  ASSERT_EQ(result->size(), 64_MB);
+
+  pool.clear(executor);
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/store/TestChunkEngine.cc

@@ -0,0 +1,217 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/hash/Checksum.h>
+#include <folly/logging/xlog.h>
+
+#include "chunk_engine/src/cxx.rs.h"
+#include "common/utils/Size.h"
+#include "fbs/storage/Common.h"
+#include "storage/aio/AioReadWorker.h"
+#include "storage/aio/BatchReadJob.h"
+#include "storage/service/BufferPool.h"
+#include "storage/service/TargetMap.h"
+#include "storage/store/ChunkStore.h"
+#include "storage/store/StorageTarget.h"
+#include "storage/update/UpdateJob.h"
+#include "storage/update/UpdateWorker.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+TEST(TestChunkEngine, ReadWrite) {
+  folly::test::TemporaryDirectory tmpPath;
+  CPUExecutorGroup executor(8, "");
+
+  constexpr TargetId targetId{1};
+  constexpr auto chunkSize = 512_KB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.storage_target().file_store().set_preopen_chunk_size_list({chunkSize});
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  {
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({chunkSize});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({targetId});
+
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(storageTargetConfig, targetMap);
+    ASSERT_OK(targets.create(createConfig));
+  }
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    UpdateWorker::Config updateWorkerConfig;
+    UpdateWorker updateWorker(updateWorkerConfig);
+    ASSERT_OK(updateWorker.start(1));
+
+    BufferPool::Config bufferPoolConfig;
+    bufferPoolConfig.set_rdmabuf_count(4);
+    bufferPoolConfig.set_big_rdmabuf_count(1);
+    BufferPool pool(bufferPoolConfig);
+    ASSERT_OK(pool.init(executor));
+
+    AioReadWorker::Config aioReadWorkerConfig;
+    AioReadWorker aioReadWorker(aioReadWorkerConfig);
+    ASSERT_OK(aioReadWorker.start(storageTargets.fds(), pool.iovecs()));
+
+    const auto chunkEngine = 1ul << 40;
+    for (auto high : {0ul, chunkEngine}) {
+      auto chunkId = ChunkId(high, 1);
+
+      // aio read.
+      {
+        BatchReadReq req;
+        req.payloads.emplace_back();
+        auto &aio = req.payloads.front();
+        aio.key.chunkId = chunkId;
+        aio.length = chunkSize;
+        aio.offset = 0;
+
+        BatchReadRsp rsp;
+        rsp.results.resize(1);
+        auto job = std::make_unique<BatchReadJob>(req.payloads, rsp.results, ChecksumType::NONE);
+        job->front().state().storageTarget = storageTarget.get();
+
+        ASSERT_EQ(storageTarget->aioPrepareRead(job->front()).error().code(), StorageCode::kChunkMetadataNotFound);
+      }
+
+      ChunkEngineUpdateJob updateChunk{};
+
+      // write a chunk.
+      {
+        UpdateIO writeIO;
+        writeIO.key.chunkId = chunkId;
+        writeIO.chunkSize = chunkSize;
+        writeIO.length = chunkSize;
+        writeIO.offset = 0;
+        writeIO.updateVer = ChunkVer{1};
+        writeIO.updateType = UpdateType::WRITE;
+
+        auto data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+        writeIO.checksum = ChecksumInfo::create(ChecksumType::CRC32C, data, chunkSize);
+
+        UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+        updateJob.state().data = data;
+
+        folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+        folly::coro::blockingWait(updateJob.complete());
+        ASSERT_OK(updateJob.result().lengthInfo);
+        ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+      }
+
+      // aio read.
+      {
+        BatchReadReq req;
+        req.payloads.emplace_back();
+        auto &aio = req.payloads.front();
+        aio.key.chunkId = chunkId;
+        aio.length = chunkSize;
+        aio.offset = 0;
+
+        BatchReadRsp rsp;
+        rsp.results.resize(1);
+        auto job = std::make_unique<BatchReadJob>(req.payloads, rsp.results, ChecksumType::NONE);
+        job->front().state().storageTarget = storageTarget.get();
+
+        ASSERT_TRUE(storageTarget->aioPrepareRead(job->front()).hasError());
+      }
+
+      // commit a chunk.
+      {
+        CommitIO commitIO;
+        commitIO.key.chunkId = chunkId;
+        commitIO.commitVer = ChunkVer{1};
+
+        UpdateJob commitJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+
+        folly::coro::blockingWait(updateWorker.enqueue(&commitJob));
+        folly::coro::blockingWait(commitJob.complete());
+        ASSERT_OK(commitJob.result().lengthInfo);
+      }
+
+      // aio read again.
+      {
+        BatchReadReq req;
+        req.payloads.emplace_back();
+        auto &aio = req.payloads.front();
+        aio.key.chunkId = chunkId;
+        aio.length = chunkSize;
+        aio.offset = 0;
+
+        BatchReadRsp rsp;
+        rsp.results.resize(1);
+        auto job = std::make_unique<BatchReadJob>(req.payloads, rsp.results, ChecksumType::NONE);
+        job->front().state().storageTarget = storageTarget.get();
+        auto buffer = pool.get();
+        job->front().state().localbuf = buffer.tryAllocate(chunkSize).value();
+
+        ASSERT_TRUE(storageTarget->aioPrepareRead(job->front()).hasValue());
+
+        folly::coro::blockingWait(aioReadWorker.enqueue(AioReadJobIterator(job.get())));
+        folly::coro::blockingWait(job->complete());
+        ASSERT_TRUE(job->front().result().lengthInfo);
+        ASSERT_EQ(job->front().result().lengthInfo.value(), chunkSize);
+        std::string_view out{(const char *)job->front().state().localbuf.ptr(), chunkSize};
+        ASSERT_EQ(out, dataBytes);
+      }
+
+      // query chunks.
+      {
+        ChunkIdRange chunkIdRange;
+        chunkIdRange.begin = ChunkId(high, 0);
+        chunkIdRange.end = ChunkId(high + 1, 0);
+        chunkIdRange.maxNumChunkIdsToProcess = 128;
+        auto chunks = storageTarget->queryChunks(chunkIdRange);
+        ASSERT_TRUE(chunks);
+        ASSERT_EQ(chunks->size(), 1);
+        ASSERT_EQ(chunks->front().first, chunkId);
+        auto &meta = chunks->front().second;
+        ASSERT_EQ(meta.size, chunkSize);
+
+        auto result = storageTarget->queryChunk(chunkId);
+        ASSERT_TRUE(result);
+        ASSERT_EQ(meta, *result);
+      }
+
+      // query chunks (optimize).
+      {
+        ChunkIdRange chunkIdRange;
+        chunkIdRange.begin = ChunkId(high, 1);
+        chunkIdRange.end = chunkIdRange.begin.nextChunkId();
+        chunkIdRange.maxNumChunkIdsToProcess = 128;
+        auto chunks = storageTarget->queryChunks(chunkIdRange);
+        ASSERT_TRUE(chunks);
+        ASSERT_EQ(chunks->size(), 1);
+        ASSERT_EQ(chunks->front().first, chunkId);
+        auto &meta = chunks->front().second;
+        ASSERT_EQ(meta.size, chunkSize);
+
+        auto result = storageTarget->queryChunk(chunkId);
+        ASSERT_TRUE(result);
+        ASSERT_EQ(meta, *result);
+      }
+    }
+
+    ChunkMetaVector metadataVec;
+    ASSERT_TRUE(storageTarget->getAllMetadata(metadataVec));
+    ASSERT_EQ(metadataVec.size(), 2);
+    ASSERT_GT(metadataVec.front().chunkId, metadataVec.back().chunkId);
+  }
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/store/TestChunkMetaStore.cc

@@ -0,0 +1,108 @@
+#include <chrono>
+#include <thread>
+#include <vector>
+
+#include "common/utils/Size.h"
+#include "storage/store/ChunkFileStore.h"
+#include "storage/store/ChunkMetaStore.h"
+#include "storage/store/PhysicalConfig.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+using namespace std::chrono_literals;
+
+TEST(TestChunkMetaStore, Normal) {
+  folly::test::TemporaryDirectory tmpPath;
+
+  kv::KVStore::Config config;
+  PhysicalConfig targetConfig;
+  targetConfig.path = tmpPath.path();
+  targetConfig.physical_file_count = 8;
+  targetConfig.chunk_size_list = {128_KB};
+
+  GlobalFileStore globalFileStore;
+  ChunkFileStore::Config fileStoreConfig;
+  ChunkFileStore store(fileStoreConfig, globalFileStore);
+  ASSERT_OK(store.create(targetConfig));
+
+  auto chunkSize = targetConfig.chunk_size_list.front();
+  ChunkMetaStore::Config metaConfig;
+  ChunkMetaStore metaStore(metaConfig, store);
+  ASSERT_OK(metaStore.create(config, targetConfig));
+
+  const ChunkId chunkId{0xBeef, 0xBeef};
+
+  {
+    ChunkMetadata meta;
+    auto result = metaStore.get(chunkId, meta);
+    ASSERT_FALSE(result);
+    ASSERT_EQ(result.error().code(), StorageCode::kChunkMetadataNotFound);
+  }
+
+  {
+    ChunkMetadata meta;
+    meta.innerFileId.chunkSize = chunkSize;
+    auto result = metaStore.set(chunkId, meta);
+    ASSERT_OK(result);
+  }
+
+  {
+    ChunkMetadata meta;
+    auto result = metaStore.get(chunkId, meta);
+    ASSERT_OK(result);
+    ASSERT_EQ(meta.innerFileId.chunkSize, chunkSize);
+
+    auto removeResult = metaStore.remove(chunkId, meta);
+    ASSERT_OK(removeResult);
+  }
+
+  {
+    ChunkMetadata meta;
+    auto result = metaStore.get(chunkId, meta);
+    ASSERT_FALSE(result);
+    ASSERT_EQ(result.error().code(), StorageCode::kChunkMetadataNotFound);
+  }
+}
+
+TEST(TestChunkMetaStore, Iterator) {
+  folly::test::TemporaryDirectory tmpPath;
+
+  kv::KVStore::Config config;
+  PhysicalConfig targetConfig;
+  targetConfig.path = tmpPath.path();
+  targetConfig.physical_file_count = 8;
+  targetConfig.chunk_size_list = {128_KB};
+
+  GlobalFileStore globalFileStore;
+  ChunkFileStore::Config fileStoreConfig;
+  ChunkFileStore store(fileStoreConfig, globalFileStore);
+  ASSERT_OK(store.create(targetConfig));
+
+  ChunkMetaStore::Config metaConfig;
+  ChunkMetaStore metaStore(metaConfig, store);
+  ASSERT_OK(metaStore.create(config, targetConfig));
+
+  constexpr uint64_t P = 0xBeef;
+  constexpr uint64_t N = 8;
+  for (uint64_t i = 1; i <= N; ++i) {
+    ChunkMetadata meta;
+    meta.innerFileId.chunkSize = 128_KB;
+    const ChunkId chunkId{P, i};
+    auto result = metaStore.set(chunkId, meta);
+    ASSERT_OK(result);
+  }
+
+  auto iterator = metaStore.iterator(ChunkId(P, 0).data().substr(0, 8));
+  ASSERT_OK(iterator);
+  for (uint64_t i = 0; i < N; ++i) {
+    ASSERT_TRUE(iterator->valid());
+    ASSERT_EQ(iterator->chunkId(), (ChunkId{P, N - i}));
+    iterator->next();
+  }
+  ASSERT_FALSE(iterator->valid());
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/store/TestCommonStruct.cc

@@ -0,0 +1,84 @@
+#include "common/serde/Serde.h"
+#include "common/utils/Reflection.h"
+#include "fbs/storage/Common.h"
+#include "fbs/storage/Service.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+TEST(TestCommonStruct, Normal) {
+  {
+    ChunkId ser(0xface, 0xbeef);
+    auto out = serde::serialize(ser);
+    ASSERT_EQ(out.size(), 1 + 8 + 8);
+
+    ChunkId des;
+    ASSERT_OK(serde::deserialize(des, out));
+    ASSERT_EQ(des, ser);
+
+    auto str = ser.describe();
+    auto result = ChunkId::fromString(str);
+    ASSERT_OK(result);
+    ASSERT_EQ(*result, ser);
+
+    auto next = ChunkId{0xface, 0xbef0};
+    ASSERT_EQ(ser.nextChunkId(), next);
+  }
+
+  {
+    auto id = ChunkId{0x0000, 0x007f};
+    auto next = ChunkId{0x0000, 0x0080};
+    ASSERT_EQ(id.nextChunkId(), next);
+  }
+  {
+    auto id = ChunkId{0x00ff, 0xffffffffffffffffull};
+    auto next = ChunkId{0x0100, 0x0000000000000000ull};
+    ASSERT_EQ(id.nextChunkId(), next);
+  }
+
+  {
+    std::string str = "00000000-00000346-85270000-0000000B";
+    auto chunk = ChunkId::fromString(str);
+    ASSERT_EQ(chunk->describe(), str);
+  }
+
+  {
+    ChecksumInfo ser;
+    ser.type = ChecksumType::CRC32;
+    ser.value = 0xff;
+    auto out = serde::serialize(ser);
+    ASSERT_EQ(out.size(), 1 + 1 + 4);
+
+    ChecksumInfo des;
+    ASSERT_OK(serde::deserialize(des, out));
+    ASSERT_EQ(des, ser);
+  }
+
+  {
+    BatchReadReq req;
+    req.payloads.emplace_back();
+    req.payloads.back().key.chunkId = ChunkId(1, 1);
+    req.payloads.back().rdmabuf.rkeys()[0].devId = 0;
+    auto out = serde::serialize(req);
+    XLOGF(INFO, "single read req size: {}, json: {}", out.length(), req);
+
+    auto front = req.payloads.front();
+    req.payloads.resize(16, front);
+    out = serde::serialize(req);
+    XLOGF(INFO, "10 read req size: {}", out.length());
+  }
+
+  refl::Helper::iterate<StorageSerde<>>([](auto type) {
+    using T = decltype(type);
+    XLOGF(INFO,
+          "method: {}, ID: {}\n  Req: {}\n  Rsp: {}",
+          T::name,
+          T::id,
+          serde::toJsonString(typename T::ReqType{}),
+          serde::toJsonString(typename T::RspType{}));
+  });
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/store/TestStorageTarget.cc

@@ -0,0 +1,531 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "common/utils/Size.h"
+#include "fbs/storage/Common.h"
+#include "storage/aio/AioReadWorker.h"
+#include "storage/service/TargetMap.h"
+#include "storage/store/ChunkStore.h"
+#include "storage/store/StorageTarget.h"
+#include "storage/update/UpdateWorker.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage {
+namespace {
+
+TEST(TestStorageTarget, ReadWrite) {
+  folly::test::TemporaryDirectory tmpPath;
+  CPUExecutorGroup executor(8, "");
+
+  const auto chunkId = ChunkId{12345, 12345};
+  constexpr TargetId targetId{0};
+  constexpr auto chunkSize = 1_MB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.storage_target().file_store().set_preopen_chunk_size_list({512_KB, 1_MB});
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  {
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({1_MB});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({targetId});
+
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(storageTargetConfig, targetMap);
+    ASSERT_OK(targets.create(createConfig));
+  }
+
+  for (auto loop = 0; loop < 3; ++loop) {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    ASSERT_EQ(storageTargets.fds().size(), 8 * bool(loop) + 8);
+
+    UpdateWorker::Config updateWorkerConfig;
+    UpdateWorker updateWorker(updateWorkerConfig);
+    ASSERT_OK(updateWorker.start(1));
+
+    AioReadWorker::Config aioReadWorkerConfig;
+    AioReadWorker aioReadWorker(aioReadWorkerConfig);
+    ASSERT_OK(aioReadWorker.start(storageTargets.fds(), {}));
+
+    auto chunkSize = loop == 0 ? 1_MB : Size{256_KB * loop};
+    ASSERT_OK(storageTarget->addChunkSize({chunkSize}));
+    ASSERT_OK(storageTarget->addChunkSize({chunkSize}));
+
+    ChunkEngineUpdateJob updateChunk{};
+
+    // write data to chunk.
+    {
+      UpdateIO writeIO;
+      writeIO.key.chunkId = chunkId;
+      writeIO.chunkSize = chunkSize;
+      writeIO.length = chunkSize;
+      writeIO.offset = 0;
+      writeIO.updateVer = ChunkVer{1};
+      writeIO.updateType = UpdateType::WRITE;
+
+      UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+      updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{0});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+    }
+
+    ASSERT_EQ(storageTarget->usedSize(), chunkSize);
+
+    // aio read uncommit chunk.
+    {
+      std::string buf(chunkSize, '\0');
+      BatchReadReq req;
+      req.payloads.emplace_back();
+      auto &aio = req.payloads.front();
+      aio.key.chunkId = chunkId;
+      aio.length = chunkSize;
+      aio.offset = 0;
+      // aio.addr = ...;
+
+      BatchReadRsp rsp;
+      rsp.results.resize(1);
+      auto job = std::make_unique<BatchReadJob>(req.payloads, rsp.results, ChecksumType::NONE);
+      job->front().state().storageTarget = storageTarget.get();
+      ASSERT_EQ(storageTarget->aioPrepareRead(job->front()).error().code(), StorageCode::kChunkNotCommit);
+    }
+
+    // commit chunk.
+    {
+      CommitIO commitIO;
+      commitIO.key.chunkId = chunkId;
+      commitIO.commitVer = ChunkVer{1};
+
+      auto updateJob = UpdateJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+      updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{1});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+    }
+
+    // remove chunk.
+    {
+      UpdateIO removeIO;
+      removeIO.key.chunkId = chunkId;
+      removeIO.updateType = UpdateType::REMOVE;
+
+      auto updateJob = UpdateJob(requestCtx, removeIO, {}, updateChunk, storageTarget);
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{1});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{2});
+    }
+
+    ASSERT_EQ(storageTarget->usedSize(), chunkSize);
+
+    // commit remove.
+    {
+      CommitIO commitIO;
+      commitIO.key.chunkId = chunkId;
+      commitIO.commitVer = ChunkVer{2};
+      auto updateJob = UpdateJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{2});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{2});
+    }
+
+    // remove non-existent chunk.
+    {
+      UpdateIO removeIO;
+      removeIO.key.chunkId = chunkId;
+      removeIO.updateType = UpdateType::REMOVE;
+
+      auto updateJob = UpdateJob(requestCtx, removeIO, {}, updateChunk, storageTarget);
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+    }
+
+    // create new target.
+    {
+      CreateTargetReq req;
+      req.targetId = TargetId{7};
+      req.physicalFileCount = 8;
+      req.chunkSizeList = {1_MB};
+      req.allowExistingTarget = false;
+      req.chainId = ChainId{1};
+      if (loop == 0) {
+        ASSERT_OK(storageTargets.create(req));
+      } else {
+        ASSERT_FALSE(storageTargets.create(req));
+      }
+    }
+
+    ASSERT_EQ(storageTarget->usedSize(), 0);
+  }
+}
+
+TEST(TestLocalTargetStateManager, UpdateLocalState) {
+  using LS = enum hf3fs::flat::LocalTargetState;
+  using PS = enum hf3fs::flat::PublicTargetState;
+
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::SERVING), LS::UPTODATE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::LASTSRV), LS::ONLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::SYNCING), LS::ONLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::WAITING), LS::ONLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::OFFLINE), LS::ONLINE);
+
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::SERVING), LS::UPTODATE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::LASTSRV), LS::OFFLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::SYNCING), LS::UPTODATE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::WAITING), LS::OFFLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::OFFLINE), LS::OFFLINE);
+}
+
+TEST(TestStorageTarget, Uncommitted) {
+  folly::test::TemporaryDirectory tmpPath;
+  CPUExecutorGroup executor(8, "");
+
+  const auto chunkId = ChunkId{12345, 12345};
+  constexpr TargetId targetId{0};
+  constexpr auto chunkSize = 1_MB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  ChunkEngineUpdateJob updateChunk{};
+
+  {
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({1_MB});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({targetId});
+    createConfig.set_only_chunk_engine(true);
+
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(storageTargetConfig, targetMap);
+    ASSERT_OK(targets.create(createConfig));
+
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    UpdateWorker::Config updateWorkerConfig;
+    UpdateWorker updateWorker(updateWorkerConfig);
+    ASSERT_OK(updateWorker.start(1));
+
+    // write data to chunk.
+    UpdateIO writeIO;
+    writeIO.key.chunkId = chunkId;
+    writeIO.chunkSize = chunkSize;
+    writeIO.length = chunkSize;
+    writeIO.offset = 0;
+    writeIO.updateVer = ChunkVer{1};
+    writeIO.updateType = UpdateType::WRITE;
+
+    UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+    updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+    folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+    folly::coro::blockingWait(updateJob.complete());
+    ASSERT_OK(updateJob.result().lengthInfo);
+    ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+    // ASSERT_EQ(updateJob.result().commitVer, ChunkVer{0});
+    ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+
+    // commit it.
+    CommitIO commitIO;
+    commitIO.commitVer = ChunkVer{1};
+    commitIO.key.chunkId = chunkId;
+    UpdateJob commitJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+    folly::coro::blockingWait(updateWorker.enqueue(&commitJob));
+    folly::coro::blockingWait(commitJob.complete());
+    ASSERT_OK(commitJob.result().lengthInfo);
+    ASSERT_EQ(commitJob.result().commitVer, ChunkVer{1});
+    ASSERT_EQ(commitJob.result().updateVer, ChunkVer{1});
+
+    // write again.
+    writeIO.updateVer = ChunkVer{2};
+    UpdateJob updateJob2(requestCtx, writeIO, {}, updateChunk, storageTarget);
+    updateJob2.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+    folly::coro::blockingWait(updateWorker.enqueue(&updateJob2));
+    folly::coro::blockingWait(updateJob2.complete());
+    ASSERT_OK(updateJob2.result().lengthInfo);
+    ASSERT_EQ(updateJob2.result().lengthInfo.value(), chunkSize);
+    // ASSERT_EQ(updateJob2.result().commitVer, ChunkVer{1});
+    ASSERT_EQ(updateJob2.result().updateVer, ChunkVer{2});
+  }
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    auto uncommitted = storageTarget->uncommitted().value();
+    ASSERT_EQ(uncommitted.size(), 1);
+    ASSERT_EQ(uncommitted.front(), chunkId);
+
+    ASSERT_OK(storageTarget->resetUncommitted(ChainVer{1}));
+
+    auto metaResult = storageTarget->queryChunk(chunkId);
+    ASSERT_OK(metaResult);
+    // ASSERT_EQ(metaResult->lastClientUuid, Uuid::max());
+  }
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    auto uncommitted = storageTarget->uncommitted().value();
+    ASSERT_TRUE(uncommitted.empty());
+  }
+}
+
+TEST(TestStorageTarget, UnRecycle) {
+  folly::test::TemporaryDirectory tmpPath;
+
+  const auto chunkId = ChunkId{12345, 12345};
+  constexpr TargetId targetId{0};
+  constexpr auto chunkSize = 1_MB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.storage_target().meta_store().set_removed_chunk_expiration_time(0_s);
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  StorageTargets::CreateConfig createConfig;
+  createConfig.set_chunk_size_list({1_MB});
+  createConfig.set_physical_file_count(8);
+  createConfig.set_allow_disk_without_uuid(true);
+  createConfig.set_target_ids({targetId});
+
+  AtomicallyTargetMap targetMap;
+  StorageTargets targets(storageTargetConfig, targetMap);
+  ASSERT_OK(targets.create(createConfig));
+
+  auto targetResult = targetMap.snapshot()->getTarget(targetId);
+  ASSERT_OK(targetResult);
+  auto storageTarget = (*targetResult)->storageTarget;
+
+  UpdateWorker::Config updateWorkerConfig;
+  UpdateWorker updateWorker(updateWorkerConfig);
+  ASSERT_OK(updateWorker.start(1));
+
+  ChunkEngineUpdateJob updateChunk{};
+
+  // 1. write data to chunk.
+  {
+    UpdateIO writeIO;
+    writeIO.key.chunkId = chunkId;
+    writeIO.chunkSize = chunkSize;
+    writeIO.length = chunkSize;
+    writeIO.offset = 0;
+    writeIO.updateVer = ChunkVer{1};
+    writeIO.updateType = UpdateType::WRITE;
+
+    UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+    updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+    folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+    folly::coro::blockingWait(updateJob.complete());
+    ASSERT_OK(updateJob.result().lengthInfo);
+    ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+    ASSERT_EQ(updateJob.result().commitVer, ChunkVer{0});
+    ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+
+    CommitIO commitIO;
+    commitIO.commitVer = ChunkVer{1};
+    commitIO.key.chunkId = chunkId;
+    UpdateJob commitJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+    folly::coro::blockingWait(updateWorker.enqueue(&commitJob));
+    folly::coro::blockingWait(commitJob.complete());
+    ASSERT_OK(commitJob.result().lengthInfo);
+    ASSERT_EQ(commitJob.result().commitVer, ChunkVer{1});
+    ASSERT_EQ(commitJob.result().updateVer, ChunkVer{1});
+  }
+
+  auto metaResult = storageTarget->queryChunk(chunkId);
+  ASSERT_OK(metaResult);
+  auto recycleResult = storageTarget->punchHole();
+  ASSERT_OK(recycleResult);
+  ASSERT_EQ(*recycleResult, true);
+
+  // 2. remove this chunk.
+  {
+    UpdateIO writeIO;
+    writeIO.key.chunkId = chunkId;
+    writeIO.chunkSize = chunkSize;
+    writeIO.length = chunkSize;
+    writeIO.offset = 0;
+    writeIO.updateVer = ChunkVer{2};
+    writeIO.updateType = UpdateType::REMOVE;
+
+    UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+    updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+    folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+    folly::coro::blockingWait(updateJob.complete());
+    ASSERT_OK(updateJob.result().lengthInfo);
+    ASSERT_EQ(updateJob.result().commitVer, ChunkVer{1});
+    ASSERT_EQ(updateJob.result().updateVer, ChunkVer{2});
+
+    CommitIO commitIO;
+    commitIO.commitVer = ChunkVer{2};
+    commitIO.key.chunkId = chunkId;
+    UpdateJob commitJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+    folly::coro::blockingWait(updateWorker.enqueue(&commitJob));
+    folly::coro::blockingWait(commitJob.complete());
+    ASSERT_OK(commitJob.result().lengthInfo);
+    ASSERT_EQ(commitJob.result().commitVer, ChunkVer{2});
+    ASSERT_EQ(commitJob.result().updateVer, ChunkVer{2});
+  }
+
+  recycleResult = storageTarget->punchHole();
+  ASSERT_OK(recycleResult);
+  ASSERT_EQ(*recycleResult, true);
+}
+
+TEST(TestStorageTarget, Migrate) {
+  constexpr auto N = 16u;
+  folly::test::TemporaryDirectory tmpPath;
+  CPUExecutorGroup executor(8, "");
+
+  const auto chunkId = ChunkId{12345, 12345};
+  constexpr TargetId targetId{0};
+  constexpr auto chunkSize = 1_MB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.storage_target().kv_store().set_type(kv::KVStore::Type::LevelDB);
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  ChunkEngineUpdateJob updateChunk{};
+
+  // 1. create with LevelDB.
+  {
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({1_MB});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({targetId});
+
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(storageTargetConfig, targetMap);
+    ASSERT_OK(targets.create(createConfig));
+  }
+
+  // 2. write some chunks.
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    UpdateWorker::Config updateWorkerConfig;
+    UpdateWorker updateWorker(updateWorkerConfig);
+    ASSERT_OK(updateWorker.start(1));
+
+    // write data to chunk.
+    for (auto i = 0u; i < N; ++i) {
+      UpdateIO writeIO;
+      writeIO.key.chunkId = ChunkId{0, i};
+      writeIO.chunkSize = chunkSize;
+      writeIO.length = chunkSize;
+      writeIO.offset = 0;
+      writeIO.updateVer = ChunkVer{1};
+      writeIO.updateType = UpdateType::WRITE;
+
+      UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+      updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{0});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+    }
+
+    for (auto i = 0u; i < N; ++i) {
+      CommitIO commitIO;
+      commitIO.key.chunkId = ChunkId{0, i};
+      commitIO.commitVer = ChunkVer{1};
+
+      auto updateJob = UpdateJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+      updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{1});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+    }
+
+    for (auto i = 0u; i < N; ++i) {
+      auto metaResult = storageTarget->queryChunk(ChunkId{0, i});
+      ASSERT_OK(metaResult);
+    }
+  }
+
+  // 3. migrate.
+  storageTargetConfig.storage_target().set_migrate_kv_store(true);
+  storageTargetConfig.storage_target().kv_store().set_type(kv::KVStore::Type::RocksDB);
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    for (auto i = 0u; i < N; ++i) {
+      auto metaResult = storageTarget->queryChunk(ChunkId{0, i});
+      ASSERT_OK(metaResult);
+    }
+  }
+}
+
+}  // namespace
+}  // namespace hf3fs::storage

tests/storage/store/TestStorageTargets.cc

@@ -0,0 +1,58 @@
+#include <folly/experimental/TestUtil.h>
+
+#include "common/utils/CPUExecutorGroup.h"
+#include "common/utils/SysResource.h"
+#include "storage/store/StorageTargets.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage {
+namespace {
+
+TEST(TestStorageTargets, Normal) {
+  folly::test::TemporaryDirectory tmpPath;
+
+  StorageTargets::Config config;
+  config.set_target_num_per_path(4);
+  config.set_target_paths({tmpPath.path()});
+  config.set_allow_disk_without_uuid(true);
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(config, targetMap);
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{1}));
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{5}));
+
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({1_MB});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({1, 2, 3, 4});
+    ASSERT_OK(targets.create(createConfig));
+    ASSERT_OK(targetMap.snapshot()->getTarget(TargetId{1}));
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{5}));
+  }
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(config, targetMap);
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{1}));
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{5}));
+
+    CPUExecutorGroup executor(8, "");
+    ASSERT_OK(targets.load(executor));
+    ASSERT_OK(targetMap.snapshot()->getTarget(TargetId{1}));
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{5}));
+
+    ASSERT_OK(targetMap.offlineTargets(tmpPath.path()));
+    auto target = targetMap.snapshot()->getTarget(TargetId{1});
+    ASSERT_OK(target);
+    ASSERT_TRUE((*target)->diskError);
+
+    auto result = targetMap.offlineTarget(TargetId{1});
+    ASSERT_TRUE(result.hasError());
+    ASSERT_EQ(result.error().code(), StorageCode::kTargetOffline);
+  }
+}
+
+}  // namespace
+}  // namespace hf3fs::storage