Initial commit

tests/storage/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_subdirectory(client)
+add_subdirectory(service)
+add_subdirectory(store)
+add_subdirectory(sync)
+target_add_test(test_storage test-fabric-lib)

tests/storage/client/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_test(test_storage_client test-fabric-lib)

tests/storage/client/TestFaultInjection.cc

@@ -0,0 +1,111 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+#include <folly/logging/xlog.h>
+
+#include "benchmarks/storage_bench/StorageBench.h"
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestFaultInjection : public ::testing::Test {
+ protected:
+  void SetUp() override {
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+    GTEST_SKIP() << "Skipping all tests since thread sanitizer enabled";
+#endif
+#endif
+
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+  }
+
+  void TearDown() override {}
+
+ protected:
+  test::SystemSetupConfig setupConfig_ = {
+      128_KB /*chunkSize*/,
+      3 /*numChains*/,
+      2 /*numReplicas*/,
+      3 /*numStorageNodes*/,
+      {folly::fs::temp_directory_path()} /*dataPaths*/,
+  };
+};
+
+TEST_F(TestFaultInjection, ServerError) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      300000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      true /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      true /*retryPermanentError*/,
+      true /*verifyReadData*/,
+      true /*verifyReadChecksum*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  ASSERT_EQ(StatusCode::kOK, bench.truncate());
+  ASSERT_EQ(StatusCode::kOK, bench.cleanup());
+  bench.teardown();
+}
+
+TEST_F(TestFaultInjection, ClientError) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      300000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      true /*injectRandomClientError*/,
+      true /*retryPermanentError*/,
+      true /*verifyReadData*/,
+      true /*verifyReadChecksum*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  ASSERT_EQ(StatusCode::kOK, bench.truncate());
+  ASSERT_EQ(StatusCode::kOK, bench.cleanup());
+  bench.teardown();
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageBenchmark.cc

@@ -0,0 +1,173 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+#include <folly/logging/xlog.h>
+
+#include "benchmarks/storage_bench/StorageBench.h"
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestStorageBenchmark : public ::testing::Test {
+ protected:
+  void SetUp() override {
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+    GTEST_SKIP() << "Skipping all tests since thread sanitizer enabled";
+#endif
+#endif
+
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+  }
+
+  void TearDown() override {}
+
+ protected:
+  test::SystemSetupConfig setupConfig_ = {
+      128_KB /*chunkSize*/,
+      1 /*numChains*/,
+      1 /*numReplicas*/,
+      1 /*numStorageNodes*/,
+      {folly::fs::temp_directory_path()} /*dataPaths*/,
+  };
+};
+
+TEST_F(TestStorageBenchmark, StandardTimeout) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      180000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      false /*retryPermanentError*/,
+      true /*verifyReadData*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  ASSERT_EQ(StatusCode::kOK, bench.truncate());
+  ASSERT_EQ(StatusCode::kOK, bench.cleanup());
+  bench.teardown();
+}
+
+TEST_F(TestStorageBenchmark, ShortTimeout) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      5 /*numReadSecs*/,
+      5 /*numWriteSecs*/,
+      3000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      true /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      false /*retryPermanentError*/,
+      false /*verifyReadData*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  bench.run();
+  bench.truncate();
+  bench.cleanup();
+  bench.teardown();
+}
+
+TEST_F(TestStorageBenchmark, BenchmarkNetwork) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      180000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      true /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      false /*retryPermanentError*/,
+      false /*verifyReadData*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  bench.truncate();
+  bench.cleanup();
+  bench.teardown();
+}
+
+TEST_F(TestStorageBenchmark, BenchmarkStorage) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      180000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      true /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      false /*retryPermanentError*/,
+      false /*verifyReadData*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  bench.truncate();
+  bench.cleanup();
+  bench.teardown();
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageClientFastFailover.cc

@@ -0,0 +1,115 @@
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestStorageClientFastFailover : public UnitTestFabric, public ::testing::TestWithParam<SystemSetupConfig> {
+ protected:
+  TestStorageClientFastFailover()
+      : UnitTestFabric(GetParam()) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+
+    // increase timeout to avoid comm error during data generation
+    auto retryOptions = clientConfig_.retry();
+    clientConfig_.retry().set_max_retry_time(5_s);
+
+    numChunks_ = setupConfig_.num_replicas() * 10;
+    std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+    auto result = writeToChunks(firstChainId_, ChunkId(1, 0), ChunkId(1, numChunks_), chunkData);
+    ASSERT_TRUE(result);
+    XLOGF(INFO, "Created {} test chunks", numChunks_);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+
+ protected:
+  size_t numChunks_;
+};
+
+TEST_P(TestStorageClientFastFailover, ReadChunks) {
+  client::ReadOptions options;
+  // since we have much more chunks than replicas: numChunks_ = setupConfig_.num_replicas() * 10,
+  // all replicas would be used in round robin mode
+  options.targetSelection().set_mode(client::TargetSelectionMode::RoundRobin);
+
+  std::vector<std::vector<uint8_t>> chunkData;
+  auto result = readFromChunks(firstChainId_,
+                               ChunkId(1, 0),
+                               ChunkId(1, numChunks_),
+                               chunkData,
+                               0 /*offset*/,
+                               setupConfig_.chunk_size() /*length*/,
+                               options);
+  ASSERT_TRUE(result);
+
+  // stop the head target
+  ASSERT_TRUE(stopAndRemoveStorageServer(0));
+
+  result = readFromChunks(firstChainId_,
+                          ChunkId(1, 0),
+                          ChunkId(1, numChunks_),
+                          chunkData,
+                          0 /*offset*/,
+                          setupConfig_.chunk_size() /*length*/,
+                          options);
+
+  if (setupConfig_.num_replicas() > 1) {
+    ASSERT_TRUE(result);
+  } else {
+    ASSERT_FALSE(result);
+  }
+}
+
+SystemSetupConfig singleReplica = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    1 /*numReplicas*/,
+    1 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+SystemSetupConfig twoReplicas = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    2 /*numReplicas*/,
+    2 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+SystemSetupConfig threeReplicas = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    3 /*numReplicas*/,
+    3 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+INSTANTIATE_TEST_SUITE_P(SingleReplica,
+                         TestStorageClientFastFailover,
+                         ::testing::Values(singleReplica),
+                         SystemSetupConfig::prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(TwoReplicas,
+                         TestStorageClientFastFailover,
+                         ::testing::Values(twoReplicas),
+                         SystemSetupConfig::prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(ThreeReplicas,
+                         TestStorageClientFastFailover,
+                         ::testing::Values(threeReplicas),
+                         SystemSetupConfig::prettyPrintConfig);
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageClientHCStress.cc

@@ -0,0 +1,383 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+struct ClientTestConfig {
+  hf3fs::Duration initWaitTime;
+  hf3fs::Duration maxWaitTime;
+  hf3fs::Duration maxRetryTime;
+  size_t batchSize;
+  size_t maxBatchSize;
+  size_t numConcurrentReqs;
+  size_t maxConcurrentReqs;
+};
+
+// high-concurrency stress test
+class TestStorageClientHCStress : public UnitTestFabric, public ::testing::TestWithParam<ClientTestConfig> {
+ protected:
+  TestStorageClientHCStress()
+      : UnitTestFabric(SystemSetupConfig{128_KB /*chunkSize*/,
+                                         3 /*numChains*/,
+                                         2 /*numReplicas*/,
+                                         3 /*numStorageNodes*/,
+                                         {folly::fs::temp_directory_path()}}) {}
+
+  void SetUp() override {
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+    GTEST_SKIP() << "Skipping all tests since thread sanitizer enabled";
+#endif
+#endif
+
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+
+    auto testConfig = GetParam();
+
+    clientConfig_.traffic_control().read().set_max_batch_size(testConfig.maxBatchSize);
+    clientConfig_.traffic_control().write().set_max_batch_size(testConfig.maxBatchSize);
+    clientConfig_.traffic_control().remove().set_max_batch_size(testConfig.maxBatchSize);
+    clientConfig_.traffic_control().truncate().set_max_batch_size(testConfig.maxBatchSize);
+    clientConfig_.traffic_control().read().set_max_concurrent_requests(testConfig.maxConcurrentReqs);
+    clientConfig_.traffic_control().write().set_max_concurrent_requests(testConfig.maxConcurrentReqs);
+    clientConfig_.traffic_control().remove().set_max_concurrent_requests(testConfig.maxConcurrentReqs);
+    clientConfig_.traffic_control().truncate().set_max_concurrent_requests(testConfig.maxConcurrentReqs);
+
+    batchSize_ = testConfig.batchSize;
+    numConcurrentReqs_ = testConfig.numConcurrentReqs;
+    numChunks_ = batchSize_ * numConcurrentReqs_;
+
+    ASSERT_TRUE(setUpStorageSystem());
+
+    // increase timeout to avoid comm error during data generation
+    clientConfig_.retry().set_init_wait_time(3_s);
+    clientConfig_.retry().set_max_wait_time(10_s);
+    clientConfig_.retry().set_max_retry_time(300_s);
+
+    // create chunks
+    std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+    for (auto chainId : chainIds_) {
+      auto result = writeToChunks(chainId, ChunkId(1, 0), ChunkId(1, numChunks_), chunkData);
+      ASSERT_TRUE(result);
+      XLOGF(INFO, "Created {} test chunks on chain {}", numChunks_, chainId);
+    }
+
+    // set timeout for test
+    clientConfig_.retry().set_init_wait_time(testConfig.initWaitTime);
+    clientConfig_.retry().set_max_wait_time(testConfig.maxWaitTime);
+    clientConfig_.retry().set_max_retry_time(testConfig.maxRetryTime);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+
+ protected:
+  size_t numConcurrentReqs_;
+  size_t numChunks_;
+  size_t batchSize_;
+};
+
+TEST_P(TestStorageClientHCStress, BatchRead) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * numChunks_, 0);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  // create read IOs
+
+  std::vector<ReadIO> readIOs;
+  size_t chunkIndex = 0;
+
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    for (size_t ioIndex = 0; ioIndex < batchSize_; ioIndex++, chunkIndex++) {
+      ChunkId chunkId(1 /*high*/, chunkIndex /*low*/);
+      ChainId chainId = chainIds_[chunkIndex % chainIds_.size()];
+      auto readIO = storageClient_->createReadIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 &memoryBlock[chunkIndex * setupConfig_.chunk_size()],
+                                                 &ioBuffer);
+      readIOs.push_back(std::move(readIO));
+    }
+  }
+
+  flat::UserInfo dummyUserInfo{};
+  ReadOptions options;
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+
+  // start many coroutines to hit the concurrency limit
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    auto ios = std::span<ReadIO>(&readIOs[batchSize_ * reqIndex], &readIOs[batchSize_ * (reqIndex + 1)]);
+    auto task = storageClient_->batchRead(ios, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+
+  for (const auto &readIO : readIOs) {
+    ASSERT_OK(readIO.result.lengthInfo);
+    ASSERT_EQ(readIO.length, readIO.result.lengthInfo.value());
+    ASSERT_EQ(1, readIO.result.commitVer);
+  }
+
+  for (unsigned char &byte : memoryBlock) ASSERT_EQ(0xFF, byte);
+}
+
+TEST_P(TestStorageClientHCStress, BatchWrite) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * numChunks_, 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  // create write IOs
+
+  std::vector<WriteIO> writeIOs;
+  size_t chunkIndex = 0;
+
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    for (size_t ioIndex = 0; ioIndex < batchSize_; ioIndex++, chunkIndex++) {
+      ChunkId chunkId(1 /*high*/, chunkIndex /*low*/);
+      ChainId chainId = chainIds_[chunkIndex % chainIds_.size()];
+      auto writeIO = storageClient_->createWriteIO(chainId,
+                                                   chunkId,
+                                                   0 /*offset*/,
+                                                   setupConfig_.chunk_size() /*length*/,
+                                                   setupConfig_.chunk_size() /*chunkSize*/,
+                                                   &memoryBlock[chunkIndex * setupConfig_.chunk_size()],
+                                                   &ioBuffer);
+      writeIOs.push_back(std::move(writeIO));
+    }
+  }
+
+  flat::UserInfo dummyUserInfo{};
+  WriteOptions options;
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+
+  // start many coroutines to hit the concurrency limit
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    auto ios = std::span<WriteIO>(&writeIOs[batchSize_ * reqIndex], &writeIOs[batchSize_ * (reqIndex + 1)]);
+    auto task = storageClient_->batchWrite(ios, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+
+  for (const auto &writeIO : writeIOs) {
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+    ASSERT_EQ(2, writeIO.result.commitVer);
+  }
+}
+
+TEST_P(TestStorageClientHCStress, BatchTruncate) {
+  // create truncate ops
+
+  std::vector<TruncateChunkOp> truncateOps;
+  size_t chunkIndex = 0;
+
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    for (size_t ioIndex = 0; ioIndex < batchSize_; ioIndex++, chunkIndex++) {
+      ChunkId chunkId(1 /*high*/, chunkIndex /*low*/);
+      ChainId chainId = chainIds_[chunkIndex % chainIds_.size()];
+      auto truncateOp = storageClient_->createTruncateOp(chainId,
+                                                         chunkId,
+                                                         setupConfig_.chunk_size() / 2 /*length*/,
+                                                         setupConfig_.chunk_size() /*chunkSize*/);
+      truncateOps.push_back(std::move(truncateOp));
+    }
+  }
+
+  flat::UserInfo dummyUserInfo{};
+  WriteOptions options;
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+
+  // start many coroutines to hit the concurrency limit and use all channel ids
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    auto ops =
+        std::span<TruncateChunkOp>(&truncateOps[batchSize_ * reqIndex], &truncateOps[batchSize_ * (reqIndex + 1)]);
+    auto task = storageClient_->truncateChunks(ops, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+
+  for (const auto &truncateOp : truncateOps) {
+    XLOGF(DBG5, "Truncate chunk {} to length {}/{}", truncateOp.chunkId, truncateOp.chunkLen, truncateOp.chunkSize);
+    ASSERT_OK(truncateOp.result.lengthInfo);
+    ASSERT_EQ(truncateOp.chunkLen, truncateOp.result.lengthInfo.value());
+    ASSERT_EQ(2, truncateOp.result.commitVer);
+  }
+}
+
+TEST_P(TestStorageClientHCStress, BatchRemove) {
+  // create remove ops
+
+  std::vector<RemoveChunksOp> removeOps;
+
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    ChunkId chunkBegin(1 /*high*/, batchSize_ * reqIndex /*low*/);
+    ChunkId chunkEnd(1 /*high*/, batchSize_ * (reqIndex + 1) /*low*/);
+    ChainId chainId = chainIds_[reqIndex % chainIds_.size()];
+    auto removeOp = storageClient_->createRemoveOp(chainId, chunkBegin, chunkEnd, batchSize_);
+    removeOps.push_back(std::move(removeOp));
+  }
+
+  flat::UserInfo dummyUserInfo{};
+  WriteOptions options;
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+
+  // start many coroutines to hit the concurrency limit and use all channel ids
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    auto ops = std::span<RemoveChunksOp>(&removeOps[reqIndex], 1);
+    auto task = storageClient_->removeChunks(ops, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+
+  for (const auto &removeOp : removeOps) {
+    XLOGF(DBG5, "Remove range [{}, {})", removeOp.range.begin, removeOp.range.end);
+    ASSERT_OK(removeOp.result.statusCode);
+    ASSERT_LE(removeOp.result.numChunksRemoved, batchSize_);
+
+    // check if chunks are removed
+    auto queryOp =
+        storageClient_->createQueryOp(removeOp.routingTarget.chainId, removeOp.range.begin, removeOp.range.end);
+    folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+    XLOGF(DBG5, "#chunks in range [{}, {}): {}", queryOp.range.begin, queryOp.range.end, queryOp.result.totalNumChunks);
+    ASSERT_OK(queryOp.result.statusCode);
+    ASSERT_EQ(0, queryOp.result.totalNumChunks);
+    ASSERT_EQ(0, queryOp.result.totalChunkLen);
+  }
+}
+
+TEST_P(TestStorageClientHCStress, ConcurrentReadRemove) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * numChunks_, 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  uint32_t maxLoops = chainIds_.size() * 2;
+
+  for (uint32_t testLoop = 1; testLoop <= maxLoops; testLoop++) {
+    // create chunks
+    std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+    ChainId chainId = chainIds_[testLoop % chainIds_.size()];
+    auto result = writeToChunks(chainId, ChunkId(1, 0), ChunkId(1, numChunks_), chunkData);
+    ASSERT_TRUE(result);
+
+    // create remove ops
+
+    std::vector<RemoveChunksOp> removeOps;
+
+    for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+      ChunkId chunkBegin(1 /*high*/, batchSize_ * reqIndex /*low*/);
+      ChunkId chunkEnd(1 /*high*/, batchSize_ * (reqIndex + 1) /*low*/);
+      auto removeOp = storageClient_->createRemoveOp(chainId, chunkBegin, chunkEnd, batchSize_);
+      removeOps.push_back(std::move(removeOp));
+    }
+
+    // create read IOs
+
+    std::vector<ReadIO> readIOs;
+    size_t chunkIndex = 0;
+
+    for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+      for (size_t ioIndex = 0; ioIndex < batchSize_; ioIndex++, chunkIndex++) {
+        ChunkId chunkId(1 /*high*/, chunkIndex /*low*/);
+        auto readIO = storageClient_->createReadIO(chainId,
+                                                   chunkId,
+                                                   0 /*offset*/,
+                                                   setupConfig_.chunk_size() /*length*/,
+                                                   &memoryBlock[chunkIndex * setupConfig_.chunk_size()],
+                                                   &ioBuffer);
+        readIOs.push_back(std::move(readIO));
+      }
+    }
+
+    flat::UserInfo dummyUserInfo{};
+
+    // start read tasks
+    std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> readTasks;
+    ReadOptions readOptions;
+
+    for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+      auto ios = std::span<ReadIO>(&readIOs[batchSize_ * reqIndex], &readIOs[batchSize_ * (reqIndex + 1)]);
+      auto task = storageClient_->batchRead(ios, dummyUserInfo, readOptions).scheduleOn(&requestExe_).start();
+      readTasks.push_back(std::move(task));
+    }
+
+    // start remove tasks
+    std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> removeTasks;
+    WriteOptions removeOptions;
+
+    for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+      auto ops = std::span<RemoveChunksOp>(&removeOps[reqIndex], 1);
+      auto task = storageClient_->removeChunks(ops, dummyUserInfo, removeOptions).scheduleOn(&requestExe_).start();
+      removeTasks.push_back(std::move(task));
+    }
+
+    folly::coro::blockingWait(folly::coro::collectAllRange(std::move(readTasks)));
+    folly::coro::blockingWait(folly::coro::collectAllRange(std::move(removeTasks)));
+
+    for (const auto &removeOp : removeOps) {
+      XLOGF(DBG5, "Remove chunks in {}", removeOp.range);
+      ASSERT_OK(removeOp.result.statusCode);
+      ASSERT_LE(removeOp.result.numChunksRemoved, batchSize_);
+    }
+  }
+}
+
+ClientTestConfig standardTimeout = {
+    3_s /*initWaitTime*/,
+    10_s /*maxWaitTime*/,
+    300_s /*maxRetryTime*/,
+    16U /*batchSize*/,
+    8U /*maxBatchSize*/,
+    32U /*numConcurrentReqs*/,
+    16U /*maxConcurrentReqs*/,
+};
+
+ClientTestConfig onecoroBusyRetry = {
+    1_ms /*initWaitTime*/,
+    10_s /*maxWaitTime*/,
+    300_s /*maxRetryTime*/,
+    16U /*batchSize*/,
+    16U /*maxBatchSize*/,
+    1U /*numConcurrentReqs*/,
+    1U /*maxConcurrentReqs*/,
+};
+
+ClientTestConfig multicoroBusyRetry = {
+    1_ms /*initWaitTime*/,
+    10_s /*maxWaitTime*/,
+    300_s /*maxRetryTime*/,
+    16U /*batchSize*/,
+    8U /*maxBatchSize*/,
+    32U /*numConcurrentReqs*/,
+    16U /*maxConcurrentReqs*/,
+};
+
+INSTANTIATE_TEST_SUITE_P(StandardTimeout, TestStorageClientHCStress, ::testing::Values(standardTimeout));
+
+INSTANTIATE_TEST_SUITE_P(OneCoroBusyRetry, TestStorageClientHCStress, ::testing::Values(onecoroBusyRetry));
+
+INSTANTIATE_TEST_SUITE_P(MultiCoroBusyRetry, TestStorageClientHCStress, ::testing::Values(multicoroBusyRetry));
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageClientInterface.cc

@@ -0,0 +1,790 @@
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestStorageClientInterface : public UnitTestFabric, public ::testing::TestWithParam<SystemSetupConfig> {
+ protected:
+  TestStorageClientInterface()
+      : UnitTestFabric(GetParam()) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+
+    clientConfig_.retry().set_init_wait_time(30_s);
+    clientConfig_.retry().set_max_wait_time(30_s);
+    clientConfig_.retry().set_max_retry_time(30_s);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+};
+
+TEST_P(TestStorageClientInterface, Write) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create write IO
+
+  auto ioBuffer = std::move(*regRes);
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+
+  for (bool bypassDiskIO : {true, false}) {
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+
+    WriteOptions options;
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+  }
+
+  {
+    QueryChunkReq req;
+    req.chainId = firstChainId_;
+    req.chunkId = chunkId;
+    auto result = folly::coro::blockingWait(storageClient_->queryChunk(req));
+    ASSERT_OK(result);
+    XLOGF(INFO, "chunk info {}", serde::toJsonString(*result, false, true));
+  }
+}
+
+TEST_P(TestStorageClientInterface, BatchWrite) {
+  // register a block of memory
+  size_t numWriteIOs = 3;
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * numWriteIOs, 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create write IOs
+
+  auto ioBuffer = std::move(*regRes);
+
+  for (bool bypassDiskIO : {true, false}) {
+    std::vector<WriteIO> writeIOs;
+
+    for (size_t writeIndex = 0; writeIndex < numWriteIOs; writeIndex++) {
+      auto chainId = firstChainId_;
+      ChunkId chunkId(1 /*high*/, writeIndex /*low*/);
+      auto writeIO = storageClient_->createWriteIO(chainId,
+                                                   chunkId,
+                                                   0 /*offset*/,
+                                                   setupConfig_.chunk_size() /*length*/,
+                                                   setupConfig_.chunk_size() /*chunkSize*/,
+                                                   &memoryBlock[writeIndex * setupConfig_.chunk_size()],
+                                                   &ioBuffer);
+      writeIOs.push_back(std::move(writeIO));
+    }
+
+    WriteOptions options;
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->batchWrite(writeIOs, flat::UserInfo(), options));
+
+    for (const auto &writeIO : writeIOs) {
+      ASSERT_OK(writeIO.result.lengthInfo);
+      ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, Read) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create read IO
+
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_OK(ioResult.lengthInfo);
+  ASSERT_EQ(chunkData.size(), ioResult.lengthInfo.value());
+
+  auto ioBuffer = std::move(*regRes);
+  auto readIO = storageClient_->createReadIO(chainId,
+                                             chunkId /*chunkId*/,
+                                             0 /*offset*/,
+                                             setupConfig_.chunk_size() /*length*/,
+                                             &memoryBlock[0],
+                                             &ioBuffer);
+
+  ReadOptions options;
+
+  for (bool bypassDiskIO : {true, false}) {
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), options));
+
+    ASSERT_OK(readIO.result.lengthInfo);
+    ASSERT_EQ(readIO.length, readIO.result.lengthInfo.value());
+
+    if (!bypassDiskIO) {
+      for (size_t i = 0; i < chunkData.size(); i++) ASSERT_EQ(chunkData[i], memoryBlock[i]);
+    }
+  };
+}
+
+TEST_P(TestStorageClientInterface, BatchRead) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create read IOs
+
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_OK(ioResult.lengthInfo);
+  ASSERT_EQ(chunkData.size(), ioResult.lengthInfo.value());
+
+  auto ioBuffer = std::move(*regRes);
+  std::vector<ReadIO> readIOs;
+
+  for (size_t offset = 0; offset < setupConfig_.chunk_size(); offset += setupConfig_.chunk_size() / 4) {
+    auto readIO = storageClient_->createReadIO(chainId,
+                                               chunkId,
+                                               offset /*offset*/,
+                                               setupConfig_.chunk_size() / 4 /*length*/,
+                                               &memoryBlock[offset],
+                                               &ioBuffer);
+    readIOs.push_back(std::move(readIO));
+  }
+
+  ReadOptions options;
+
+  for (bool bypassDiskIO : {true, false}) {
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo(), options));
+
+    for (const auto &readIO : readIOs) {
+      ASSERT_OK(readIO.result.lengthInfo);
+      ASSERT_EQ(readIO.length, readIO.result.lengthInfo.value());
+    }
+
+    if (!bypassDiskIO) {
+      for (size_t i = 0; i < chunkData.size(); i++) ASSERT_EQ(chunkData[i], memoryBlock[i]);
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, BatchReadUnaligned) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create read IOs
+
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+  for (auto &ch : chunkData) {
+    ch = rand() & 0xFF;
+  }
+
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_OK(ioResult.lengthInfo);
+  ASSERT_EQ(chunkData.size(), ioResult.lengthInfo.value());
+
+  auto ioBuffer = std::move(*regRes);
+  std::vector<ReadIO> readIOs;
+
+  for (size_t offset = 0; offset < setupConfig_.chunk_size(); offset += setupConfig_.chunk_size() / 4) {
+    auto headSize = 1 + rand() % 17;
+    auto tailSize = 1 + rand() % 17;
+    auto unalignedOffset = offset + headSize;
+    auto unalignedLength = setupConfig_.chunk_size() / 4 - headSize - tailSize;
+    XLOGF(INFO,
+          "Unaligned offset {} length {}, chunk size {} chunk id {}",
+          unalignedOffset,
+          unalignedLength,
+          setupConfig_.chunk_size(),
+          chunkId);
+    auto readIO = storageClient_->createReadIO(chainId,
+                                               chunkId,
+                                               unalignedOffset /*offset*/,
+                                               unalignedLength /*length*/,
+                                               &memoryBlock[unalignedOffset],
+                                               &ioBuffer);
+    readIOs.push_back(std::move(readIO));
+  }
+
+  ReadOptions options;
+
+  for (bool bypassDiskIO : {true, false}) {
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo(), options));
+
+    for (const auto &readIO : readIOs) {
+      ASSERT_OK(readIO.result.lengthInfo);
+      ASSERT_EQ(readIO.length, readIO.result.lengthInfo.value());
+      if (!bypassDiskIO) {
+        ASSERT_TRUE(std::memcmp(&chunkData[readIO.offset], &memoryBlock[readIO.offset], readIO.length) == 0);
+      }
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, ReadWrite) {
+  // register a block of memory
+  std::vector<uint8_t> writeData(setupConfig_.chunk_size() / 2, 0xFF);
+  auto regWriteDataRes = storageClient_->registerIOBuffer(&writeData[0], writeData.size());
+  ASSERT_OK(regWriteDataRes);
+
+  auto writeBuffer = std::move(*regWriteDataRes);
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+
+  // write data to chunk
+  auto writeIO = storageClient_->createWriteIO(chainId,
+                                               chunkId,
+                                               0 /*offset*/,
+                                               writeData.size() /*length*/,
+                                               setupConfig_.chunk_size(),
+                                               &writeData[0],
+                                               &writeBuffer);
+
+  folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+
+  ASSERT_OK(writeIO.result.lengthInfo);
+  ASSERT_EQ(writeData.size(), writeIO.result.lengthInfo.value());
+
+  // read the data back
+  std::vector<uint8_t> readData(setupConfig_.chunk_size(), 0);
+  auto regReadDataRes = storageClient_->registerIOBuffer(&readData[0], readData.size());
+  ASSERT_OK(regReadDataRes);
+
+  auto readBuffer = std::move(*regReadDataRes);
+  auto readIO = storageClient_->createReadIO(chainId,
+                                             chunkId,
+                                             0 /*offset*/,
+                                             setupConfig_.chunk_size() /*length*/,
+                                             &readData[0],
+                                             &readBuffer);
+
+  folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo()));
+
+  ASSERT_OK(readIO.result.lengthInfo);
+  ASSERT_EQ(writeIO.result.lengthInfo.value(), readIO.result.lengthInfo.value());
+  for (size_t i = 0; i < writeData.size(); i++) ASSERT_EQ(writeData[i], readData[i]);
+}
+
+TEST_P(TestStorageClientInterface, BatchReadWrite) {
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+
+  // write data to chunk
+  size_t numChunks = 3;
+  std::vector<uint8_t> writeData(setupConfig_.chunk_size() * numChunks, 0xFF);
+  auto regWriteDataRes = storageClient_->registerIOBuffer(&writeData[0], writeData.size());
+  ASSERT_OK(regWriteDataRes);
+  auto writeBuffer = std::move(*regWriteDataRes);
+
+  std::vector<WriteIO> writeIOs;
+
+  for (size_t writeIndex = 0; writeIndex < numChunks; writeIndex++) {
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 ChunkId(chunkId, writeIndex),
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 &writeData[writeIndex * setupConfig_.chunk_size()],
+                                                 &writeBuffer);
+    writeIOs.push_back(std::move(writeIO));
+  }
+
+  folly::coro::blockingWait(storageClient_->batchWrite(writeIOs, flat::UserInfo()));
+
+  for (const auto &writeIO : writeIOs) {
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(setupConfig_.chunk_size(), writeIO.result.lengthInfo.value());
+  }
+
+  // read the data back
+  std::vector<uint8_t> readData(setupConfig_.chunk_size() * numChunks, 0);
+  auto regReadDataRes = storageClient_->registerIOBuffer(&readData[0], readData.size());
+  ASSERT_OK(regReadDataRes);
+  auto readBuffer = std::move(*regReadDataRes);
+
+  std::vector<ReadIO> readIOs;
+
+  for (size_t readIndex = 0; readIndex < numChunks; readIndex++) {
+    auto readIO = storageClient_->createReadIO(chainId,
+                                               ChunkId(chunkId, readIndex),
+                                               0 /*offset*/,
+                                               setupConfig_.chunk_size() /*length*/,
+                                               &readData[readIndex * setupConfig_.chunk_size()],
+                                               &readBuffer);
+    readIOs.push_back(std::move(readIO));
+  }
+
+  folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo()));
+
+  for (const auto &readIO : readIOs) {
+    ASSERT_OK(readIO.result.lengthInfo);
+    ASSERT_EQ(setupConfig_.chunk_size(), readIO.result.lengthInfo.value());
+  }
+
+  for (size_t i = 0; i < writeData.size(); i++) ASSERT_EQ(writeData[i], readData[i]) << "i " << i;
+}
+
+TEST_P(TestStorageClientInterface, VerifyChecksum) {
+  if (setupConfig_.client_impl_type() == StorageClient::ImplementationType::InMem) return;
+
+  ChainId chainId = firstChainId_;
+
+  std::vector<uint8_t> readData(setupConfig_.chunk_size(), 0);
+  std::vector<uint8_t> writeData(setupConfig_.chunk_size(), 0xFF);
+
+  auto regReadBuf = storageClient_->registerIOBuffer(&readData[0], readData.size());
+  ASSERT_OK(regReadBuf);
+  auto readBuffer = std::move(*regReadBuf);
+
+  auto regWriteBuf = storageClient_->registerIOBuffer(&writeData[0], writeData.size());
+  ASSERT_OK(regWriteBuf);
+  auto writeBuffer = std::move(*regWriteBuf);
+
+  // enable checksum for read/write IO
+  clientConfig_.set_chunk_checksum_type(ChecksumType::CRC32C);
+  client::WriteOptions writeOptions;
+  writeOptions.set_enableChecksum(true);
+  client::ReadOptions readOptions;
+  readOptions.set_enableChecksum(true);
+  readOptions.targetSelection().set_mode(TargetSelectionMode::RoundRobin);
+
+  enum WritePattern {
+    SEQWRITE = 1,
+    JUMPWRITE,
+    RANDWRITE,
+  };
+
+  for (WritePattern pattern : {SEQWRITE, JUMPWRITE, RANDWRITE}) {
+    ChunkId chunkId(1 /*high*/, pattern /*low*/);
+    std::vector<uint8_t> chunkData;
+    size_t offset = 0;
+    size_t length = 0;
+
+    for (size_t writeIndex = 1; writeIndex <= 100; writeIndex++) {
+      switch (pattern) {
+        case SEQWRITE:
+          offset += length;
+          break;
+        case JUMPWRITE:
+          offset += length + folly::Random::rand64(0, length / 2);
+          break;
+        case RANDWRITE:
+          offset = folly::Random::rand64(0, setupConfig_.chunk_size());
+          break;
+      }
+
+      if (offset + 1 >= setupConfig_.chunk_size()) continue;
+      length = folly::Random::rand64(1, (setupConfig_.chunk_size() - offset) / 2);
+
+      XLOGF(INFO,
+            "Verify {} checksum #{}: offset {} length {} chunk size {}",
+            magic_enum::enum_name(pattern),
+            writeIndex,
+            offset,
+            length,
+            chunkData.size());
+
+      // generate random chunk data
+      for (size_t byteIndex = 0; byteIndex + sizeof(uint64_t) < length; byteIndex += sizeof(uint64_t)) {
+        auto dataPtr = reinterpret_cast<uint64_t *>(&writeData[byteIndex]);
+        *dataPtr = folly::Random::rand64();
+      }
+
+      // write to a random offset in the chunk
+      auto writeIO =
+          storageClient_
+              ->createWriteIO(chainId, chunkId, offset, length, setupConfig_.chunk_size(), &writeData[0], &writeBuffer);
+      folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), writeOptions));
+      ASSERT_RESULT_EQ(writeIO.length, writeIO.result.lengthInfo);
+
+      // verify checksum of the write data
+      ASSERT_EQ(folly::crc32c(&writeData[0], *writeIO.result.lengthInfo), writeIO.localChecksum().value);
+      // update chunk data and compute the chunk checksum
+      if (offset + length > chunkData.size()) chunkData.resize(offset + length);
+      std::memcpy(&chunkData[offset], &writeData[0], *writeIO.result.lengthInfo);
+      ASSERT_EQ(folly::crc32c(&chunkData[0], chunkData.size()), writeIO.result.checksum.value);
+
+      {
+        // read back the write data
+        auto readIO = storageClient_->createReadIO(chainId, chunkId, offset, length, &readData[0], &readBuffer);
+        folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), readOptions));
+        ASSERT_RESULT_EQ(readIO.length, readIO.result.lengthInfo);
+
+        // verify checksum of the read data
+        ASSERT_EQ(folly::crc32c(&readData[0], *readIO.result.lengthInfo), readIO.result.checksum.value);
+        // compare with checksum of write data
+        ASSERT_EQ(writeIO.localChecksum().value, readIO.result.checksum.value);
+      }
+
+      {
+        // read the entire chunk
+        auto readIO =
+            storageClient_->createReadIO(chainId, chunkId, 0, setupConfig_.chunk_size(), &readData[0], &readBuffer);
+        folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), readOptions));
+        ASSERT_OK(readIO.result.lengthInfo);
+
+        // verify checksum of the read data
+        ASSERT_EQ(folly::crc32c(&readData[0], *readIO.result.lengthInfo), readIO.result.checksum.value);
+        // compare with chunk checksum
+        ASSERT_EQ(writeIO.result.checksum.value, readIO.result.checksum.value);
+      }
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, QueryRemoveTruncateChunks) {
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 0 /*low*/);
+  const uint32_t maxNumResultsPerQuery = serverConfigs_.front().storage().max_num_results_per_query();
+
+  for (uint32_t numChunks : {1U,
+                             maxNumResultsPerQuery - 1,
+                             maxNumResultsPerQuery,
+                             maxNumResultsPerQuery + 1,
+                             2 * maxNumResultsPerQuery - 2,
+                             2 * maxNumResultsPerQuery,
+                             2 * maxNumResultsPerQuery + 2,
+                             folly::Random::rand32(1, 3 * maxNumResultsPerQuery)}) {
+    uint32_t writeLen = setupConfig_.chunk_size() / 2;
+    std::vector<uint8_t> writeData(writeLen, 0xFF);
+
+    auto result = writeToChunks(chainId, ChunkId(1, 0), ChunkId(1, numChunks), writeData);
+    ASSERT_TRUE(result);
+
+    {
+      // truncate chunks
+      std::vector<TruncateChunkOp> truncateOps;
+      for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+        auto op = storageClient_->createTruncateOp(chainId,
+                                                   ChunkId(chunkId, chunkIndex),
+                                                   setupConfig_.chunk_size(),
+                                                   setupConfig_.chunk_size());
+        truncateOps.push_back(std::move(op));
+      }
+
+      folly::coro::blockingWait(storageClient_->truncateChunks(truncateOps, flat::UserInfo()));
+
+      for (const auto &op : truncateOps) {
+        ASSERT_OK(op.result.lengthInfo);
+        ASSERT_EQ(op.chunkLen, op.result.lengthInfo.value());
+      }
+    }
+
+    {
+      // read chunks created by truncate ops
+      for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+        std::vector<uint8_t> readData(setupConfig_.chunk_size(), 0x0);
+        auto result = readFromChunk(chainId, ChunkId(chunkId, chunkIndex), readData);
+        ASSERT_OK(result.lengthInfo);
+        ASSERT_EQ(setupConfig_.chunk_size(), result.lengthInfo.value());
+
+        for (size_t index = 0; index < readData.size(); index++) {
+          if (index < writeLen)
+            ASSERT_EQ(0xFF, readData[index]);  // the written part
+          else
+            ASSERT_EQ(0x00, readData[index]);  // the extended part
+        }
+      }
+    }
+
+    {
+      // query only last chunk
+      auto queryOp = storageClient_->createQueryOp(chainId,
+                                                   ChunkId(chunkId, 0),
+                                                   ChunkId(chunkId, numChunks),
+                                                   1 /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(queryOp.result.statusCode);
+      ASSERT_EQ(ChunkId(chunkId, numChunks - 1).data(), queryOp.result.lastChunkId.data());
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.lastChunkLen);
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.totalChunkLen);
+      ASSERT_EQ(1, queryOp.result.totalNumChunks);
+      ASSERT_TRUE(queryOp.result.moreChunksInRange || numChunks <= 1);
+    }
+
+    if (numChunks > 1) {
+      // query half of the chunks
+      auto halfNumChunks = numChunks / 2;
+      auto queryOp = storageClient_->createQueryOp(chainId,
+                                                   ChunkId(chunkId, 0),
+                                                   ChunkId(chunkId, numChunks),
+                                                   halfNumChunks /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(queryOp.result.statusCode);
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.lastChunkLen);
+      ASSERT_EQ(setupConfig_.chunk_size() * halfNumChunks, queryOp.result.totalChunkLen);
+      ASSERT_EQ(halfNumChunks, queryOp.result.totalNumChunks);
+      ASSERT_TRUE(queryOp.result.moreChunksInRange);
+    }
+
+    {
+      // query all chunks
+      auto queryOp = storageClient_->createQueryOp(chainId,
+                                                   ChunkId(chunkId, 0),
+                                                   ChunkId(chunkId, numChunks),
+                                                   numChunks /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(queryOp.result.statusCode);
+      ASSERT_EQ(ChunkId(chunkId, numChunks - 1).data(), queryOp.result.lastChunkId.data());
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.lastChunkLen);
+      ASSERT_EQ(setupConfig_.chunk_size() * numChunks, queryOp.result.totalChunkLen);
+      ASSERT_EQ(numChunks, queryOp.result.totalNumChunks);
+      ASSERT_FALSE(queryOp.result.moreChunksInRange);
+    }
+
+    {
+      // query the max range
+      auto queryOp = storageClient_->createQueryOp(chainId,
+                                                   ChunkId(chunkId, 0),
+                                                   ChunkId(chunkId, UINT32_MAX),
+                                                   UINT32_MAX /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(queryOp.result.statusCode);
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.lastChunkLen);
+      ASSERT_EQ(setupConfig_.chunk_size() * numChunks, queryOp.result.totalChunkLen);
+      ASSERT_EQ(numChunks, queryOp.result.totalNumChunks);
+      ASSERT_FALSE(queryOp.result.moreChunksInRange);
+    }
+
+    {
+      // remove the last chunk
+      auto removeOp = storageClient_->createRemoveOp(chainId,
+                                                     ChunkId(chunkId, 0),
+                                                     ChunkId(chunkId, numChunks),
+                                                     1 /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(removeOp.result.statusCode);
+      ASSERT_EQ(1, removeOp.result.numChunksRemoved);
+      ASSERT_TRUE(removeOp.result.moreChunksInRange || numChunks <= 1);
+    }
+
+    {
+      // remove the remaining chunks
+      auto removeOp = storageClient_->createRemoveOp(chainId,
+                                                     ChunkId(chunkId, 0),
+                                                     ChunkId(chunkId, numChunks),
+                                                     numChunks /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(removeOp.result.statusCode);
+      ASSERT_EQ(numChunks - 1, removeOp.result.numChunksRemoved);
+      ASSERT_FALSE(removeOp.result.moreChunksInRange);
+    }
+
+    {
+      // check if chunks are removed
+      auto queryOp = storageClient_->createQueryOp(chainId, ChunkId(chunkId, 0), ChunkId(chunkId, numChunks));
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_TRUE(queryOp.result.statusCode);
+      ASSERT_EQ(ChunkId().data(), queryOp.result.lastChunkId.data());
+      ASSERT_EQ(0, queryOp.result.lastChunkLen);
+      ASSERT_EQ(0, queryOp.result.totalChunkLen);
+      ASSERT_EQ(0, queryOp.result.totalNumChunks);
+      ASSERT_FALSE(queryOp.result.moreChunksInRange);
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, TruncateExtendChunks) {
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 0 /*low*/);
+  uint32_t numChunks = 3;
+  uint32_t writeLen = setupConfig_.chunk_size() / 2;
+
+  std::vector<uint8_t> writeData(writeLen, 0xFF);
+  auto result = writeToChunks(chainId, ChunkId(1, 0), ChunkId(1, numChunks), writeData);
+  ASSERT_TRUE(result);
+
+  {
+    // extend op does not reduce chunk length
+    std::vector<TruncateChunkOp> truncateOps;
+    for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+      auto op = storageClient_->createTruncateOp(chainId,
+                                                 ChunkId(chunkId, chunkIndex),
+                                                 writeLen / 2 /* try to reduce chunk length */,
+                                                 setupConfig_.chunk_size(),
+                                                 true /*onlyExtendChunk*/);
+      truncateOps.push_back(std::move(op));
+    }
+
+    folly::coro::blockingWait(storageClient_->truncateChunks(truncateOps, flat::UserInfo()));
+
+    for (const auto &op : truncateOps) {
+      ASSERT_OK(op.result.lengthInfo);
+      ASSERT_EQ(writeLen, op.result.lengthInfo.value());  // still get the original length
+    }
+  }
+
+  {
+    // extend to full chunks
+    std::vector<TruncateChunkOp> truncateOps;
+    for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+      auto op = storageClient_->createTruncateOp(chainId,
+                                                 ChunkId(chunkId, chunkIndex),
+                                                 setupConfig_.chunk_size() /* full chunk size */,
+                                                 setupConfig_.chunk_size(),
+                                                 true /*onlyExtendChunk*/);
+      truncateOps.push_back(std::move(op));
+    }
+
+    folly::coro::blockingWait(storageClient_->truncateChunks(truncateOps, flat::UserInfo()));
+
+    for (const auto &op : truncateOps) {
+      ASSERT_OK(op.result.lengthInfo);
+      ASSERT_EQ(setupConfig_.chunk_size(), op.result.lengthInfo.value());
+    }
+  }
+
+  {
+    // read the extended chunks
+    for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+      std::vector<uint8_t> readData(setupConfig_.chunk_size(), 0xAA);
+      auto result = readFromChunk(chainId, ChunkId(chunkId, chunkIndex), readData);
+      ASSERT_OK(result.lengthInfo);
+      ASSERT_EQ(setupConfig_.chunk_size(), result.lengthInfo.value());
+
+      for (size_t index = 0; index < readData.size(); index++) {
+        if (index < writeLen)
+          ASSERT_EQ(0xFF, readData[index]);  // the written part
+        else
+          ASSERT_EQ(0x00, readData[index]);  // the extended part
+      }
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, QuerySpaceInfo) {
+  auto result = folly::coro::blockingWait(storageClient_->querySpaceInfo(NodeId{1}));
+  ASSERT_OK(result);
+  ASSERT_TRUE(!result->spaceInfos.empty());
+}
+
+TEST_P(TestStorageClientInterface, CreateTarget) {
+  CreateTargetReq req;
+  req.targetId = TargetId{255};
+  req.chainId = ChainId{255};
+  req.diskIndex = 0;
+  req.onlyChunkEngine = true;
+  auto result = folly::coro::blockingWait(storageClient_->createTarget(NodeId{1}, req));
+  ASSERT_OK(result);
+}
+
+TEST_P(TestStorageClientInterface, GetAllChunkMetadata) {
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 0 /*low*/);
+  uint32_t numChunks = 3;
+
+  std::vector<uint8_t> writeData(setupConfig_.chunk_size(), 0xFF);
+  auto result = writeToChunks(chainId, chunkId, ChunkId(chunkId, numChunks), writeData);
+  ASSERT_TRUE(result);
+
+  auto routingInfo = getRoutingInfo();
+  ASSERT_TRUE(routingInfo);
+  auto chainInfo = routingInfo->getChain(chainId);
+  ASSERT_TRUE(chainInfo);
+
+  for (const auto targetInfo : chainInfo->targets) {
+    auto chunkMetaVec = folly::coro::blockingWait(storageClient_->getAllChunkMetadata(chainId, targetInfo.targetId));
+    ASSERT_OK(chunkMetaVec);
+
+    std::set<ChunkId> uniqChunkIds;
+    for (const auto &chunkMeta : *chunkMetaVec) {
+      ASSERT_EQ(ChunkVer{1}, chunkMeta.commitVer);
+      ASSERT_EQ(setupConfig_.chunk_size(), chunkMeta.length);
+      uniqChunkIds.insert(chunkMeta.chunkId);
+    }
+
+    ASSERT_EQ(numChunks, uniqChunkIds.size());
+    ASSERT_EQ(chunkId, *uniqChunkIds.begin());
+    ASSERT_EQ(ChunkId(chunkId, numChunks - 1), *uniqChunkIds.rbegin());
+  }
+}
+
+SystemSetupConfig testInMemClient = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    1 /*numReplicas*/,
+    1 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+    hf3fs::Path() /*clientConfig*/,
+    hf3fs::Path() /*serverConfig*/,
+    {} /*storageEndpoints*/,
+    0 /*serviceLevel*/,
+    0 /*listenPort*/,
+    StorageClient::ImplementationType::InMem /*clientImplType*/,
+};
+
+SystemSetupConfig testRpcClient = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    2 /*numReplicas*/,
+    2 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+SystemSetupConfig testSmallChunk = {
+    512 /*chunkSize*/,
+    1 /*numChains*/,
+    2 /*numReplicas*/,
+    2 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+INSTANTIATE_TEST_SUITE_P(InMemClient,
+                         TestStorageClientInterface,
+                         ::testing::Values(testInMemClient),
+                         SystemSetupConfig::prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(RpcClient,
+                         TestStorageClientInterface,
+                         ::testing::Values(testRpcClient),
+                         SystemSetupConfig::prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(SmallChunk,
+                         TestStorageClientInterface,
+                         ::testing::Values(testSmallChunk),
+                         SystemSetupConfig::prettyPrintConfig);
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageClientSideError.cc

@@ -0,0 +1,401 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/logging/xlog.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestStorageClientSideError : public UnitTestFabric, public ::testing::Test {
+ protected:
+  TestStorageClientSideError()
+      : UnitTestFabric(SystemSetupConfig{128_KB /*chunkSize*/,
+                                         1 /*numChains*/,
+                                         1 /*numReplicas*/,
+                                         1 /*numStorageNodes*/,
+                                         {folly::fs::temp_directory_path()}}) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+    clientConfig_.retry().set_max_retry_time(2_s);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+
+  template <typename Op>
+  void checkFailedOpsPtrs(const std::vector<Op> &ops, const std::vector<Op *> &failedOps) {
+    bool ok = std::all_of(failedOps.cbegin(), failedOps.cend(), [&ops](const Op *failedOp) -> bool {
+      for (const auto &op : ops) {
+        if (failedOp == &op) return true;
+      }
+      XLOGF(ERR,
+            "Address of failed op {} not in range: [{}, {})",
+            fmt::ptr(failedOp),
+            fmt::ptr(&ops[0]),
+            fmt::ptr(&ops[ops.size() - 1]));
+      return false;
+    });
+    ASSERT_TRUE(ok);
+  }
+};
+
+TEST_F(TestStorageClientSideError, GetReplicationChainError) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &chainTable = *routingInfo.getChainTable(kTableId());
+    auto &chainInfo = routingInfo.chains[chainTable.chains.front()];
+
+    // first target offline
+    chainInfo.targets.begin()->publicState = hf3fs::flat::PublicTargetState::OFFLINE;
+  });
+
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+  auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNotAvailable);
+
+  // inconsistent chain id
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &chainTable = *routingInfo.getChainTable(kTableId());
+    auto &chainInfo = routingInfo.chains[chainTable.chains.front()];
+    chainInfo.chainId = flat::ChainId(0);
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+
+  // empty chain table
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &chainTable = *routingInfo.getChainTable(kTableId());
+    chainTable.chains.clear();
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+}
+
+TEST_F(TestStorageClientSideError, GetStorageTargetError) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &targetInfo = routingInfo.targets.begin()->second;
+
+    // host node id not unknown
+    targetInfo.nodeId = std::nullopt;
+  });
+
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+  auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNotAvailable);
+
+  // target offline
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &targetInfo = routingInfo.targets.begin()->second;
+    targetInfo.publicState = hf3fs::flat::PublicTargetState::OFFLINE;
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNotAvailable);
+
+  // inconsistent target id
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &targetInfo = routingInfo.targets.begin()->second;
+    targetInfo.targetId = flat::TargetId(0);
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+
+  // empty target list
+  updateRoutingInfo([&](auto &routingInfo) { routingInfo.targets.clear(); });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+}
+
+TEST_F(TestStorageClientSideError, GetStorageNodeError) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+
+    // not storage node
+    nodeInfo.type = hf3fs::flat::NodeType::META;
+  });
+
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+  auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+
+  // inconsistent node id
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+    nodeInfo.app.nodeId = hf3fs::flat::NodeId(0);
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+
+  // empty node list
+  updateRoutingInfo([&](auto &routingInfo) { routingInfo.nodes.clear(); });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+}
+
+TEST_F(TestStorageClientSideError, WrongServerAddress) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+
+    // wrong port
+    nodeInfo.app.serviceGroups.front().endpoints.front().port = 0;
+  });
+
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+  auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_TRUE(!ioResult.lengthInfo);
+  ASSERT_TRUE(ioResult.lengthInfo.error().code() == StorageClientCode::kTimeout ||
+              ioResult.lengthInfo.error().code() == StorageClientCode::kCommError);
+
+  ioResult = readFromChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_TRUE(!ioResult.lengthInfo);
+  ASSERT_TRUE(ioResult.lengthInfo.error().code() == StorageClientCode::kTimeout ||
+              ioResult.lengthInfo.error().code() == StorageClientCode::kCommError);
+
+  // empty address list
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+    nodeInfo.app.serviceGroups.front().endpoints.clear();
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNoRDMAInterface);
+
+  ioResult = readFromChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNoRDMAInterface);
+}
+
+TEST_F(TestStorageClientSideError, DifferentTrafficZone) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+    // set traffic zone of the host
+    nodeInfo.tags = {flat::TagPair{flat::kTrafficZoneTagKey, "TEST_ZONE0"}};
+  });
+
+  {
+    std::vector<uint8_t> userData(1_KB, 0xFF);
+    auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+    ASSERT_RESULT_EQ(userData.size(), ioResult.lengthInfo);
+
+    ReadOptions options;
+    // client is in the same traffic zone as the storage node
+    options.targetSelection().set_trafficZone("TEST_ZONE0");
+    ioResult = readFromChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+    ASSERT_RESULT_EQ(userData.size(), ioResult.lengthInfo);
+  }
+
+  {
+    std::vector<uint8_t> userData(1_KB, 0xFF);
+    auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+    ASSERT_RESULT_EQ(userData.size(), ioResult.lengthInfo);
+
+    ReadOptions options;
+    // client is in a different traffic zone from the storage node
+    options.targetSelection().set_trafficZone("TEST_ZONE1");
+    ioResult = readFromChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size(), options);
+    ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNotAvailable);
+  }
+}
+
+TEST_F(TestStorageClientSideError, InvalidDataRange) {
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+
+  {
+    // overlapping data buffers
+    std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * 2, 0xFF);
+    auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    std::vector<ReadIO> readIOs;
+
+    for (size_t offset = 0; offset < setupConfig_.chunk_size(); offset += setupConfig_.chunk_size() / 4) {
+      auto readIO = storageClient_->createReadIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 offset /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 &memoryBlock[offset],
+                                                 &ioBuffer);
+      readIOs.push_back(std::move(readIO));
+    }
+
+    std::vector<ReadIO *> failedIOs;
+    folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo(), ReadOptions(), &failedIOs));
+
+    for (const auto &readIO : readIOs) {
+      ASSERT_FALSE(readIO.result.lengthInfo);
+      ASSERT_EQ(StorageClientCode::kInvalidArg, readIO.result.lengthInfo.error().code());
+    }
+
+    checkFailedOpsPtrs(readIOs, failedIOs);
+    std::set<ReadIO *> uniqueFailedIOs(failedIOs.begin(), failedIOs.end());
+    ASSERT_EQ(readIOs.size(), uniqueFailedIOs.size());
+  }
+
+  {
+    // duplicate IOs
+    std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
+    auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    std::vector<ReadIO> readIOs;
+
+    for (int i = 0; i < 2; i++) {
+      auto readIO = storageClient_->createReadIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+      readIOs.push_back(std::move(readIO));
+    }
+
+    std::vector<ReadIO *> failedIOs;
+    folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo(), ReadOptions(), &failedIOs));
+
+    for (const auto &readIO : readIOs) {
+      ASSERT_FALSE(readIO.result.lengthInfo);
+      ASSERT_EQ(StorageClientCode::kInvalidArg, readIO.result.lengthInfo.error().code());
+    }
+
+    checkFailedOpsPtrs(readIOs, failedIOs);
+    std::set<ReadIO *> uniqueFailedIOs(failedIOs.begin(), failedIOs.end());
+    ASSERT_EQ(readIOs.size(), uniqueFailedIOs.size());
+  }
+
+  {
+    // null io buffer pointer
+    auto writeIO = storageClient_->createWriteIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 userData.size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 &userData[0],
+                                                 nullptr);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, writeIO.result.lengthInfo.error().code());
+  }
+
+  {
+    // null data pointer
+    auto regRes = storageClient_->registerIOBuffer(&userData[0], userData.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    auto writeIO = storageClient_->createWriteIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 userData.size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 nullptr,
+                                                 &ioBuffer);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, writeIO.result.lengthInfo.error().code());
+  }
+
+  {
+    // write range is out of the chunk boundary
+    std::vector<uint8_t> largeArray(setupConfig_.chunk_size() * 2, 0xFF);
+    auto regRes = storageClient_->registerIOBuffer(&largeArray[0], largeArray.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    auto writeIO = storageClient_->createWriteIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 largeArray.size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 &largeArray[0],
+                                                 &ioBuffer);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, writeIO.result.lengthInfo.error().code());
+  }
+
+  {
+    // write range is out of the io buffer
+    auto regRes = storageClient_->registerIOBuffer(&userData[0], userData.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    auto writeIO = storageClient_->createWriteIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 2 * userData.size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 &userData[0],
+                                                 &ioBuffer);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, writeIO.result.lengthInfo.error().code());
+  }
+}
+
+TEST_F(TestStorageClientSideError, InvalidChunkIdRange) {
+  ChunkId largeChunkId(1 /*high*/, 3 /*low*/);
+  ChunkId smallChunkId(1 /*high*/, 1 /*low*/);
+  ASSERT_GT(largeChunkId, smallChunkId);
+
+  {
+    // invalid begin/end chunk ids for a range query
+    auto queryOp =
+        storageClient_->createQueryOp(firstChainId_, largeChunkId, smallChunkId, 1 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+    ASSERT_FALSE(queryOp.result.statusCode);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, queryOp.result.statusCode.error().code());
+  }
+  {
+    // invalid maxNumChunkIdsToProcess for a range query
+    auto queryOp =
+        storageClient_->createQueryOp(firstChainId_, smallChunkId, largeChunkId, 0 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+    ASSERT_FALSE(queryOp.result.statusCode);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, queryOp.result.statusCode.error().code());
+  }
+
+  {
+    // invalid begin/end chunk ids for a batch remove
+    auto removeOp =
+        storageClient_->createRemoveOp(firstChainId_, largeChunkId, smallChunkId, 1 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo()));
+    ASSERT_FALSE(removeOp.result.statusCode);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, removeOp.result.statusCode.error().code());
+  }
+  {
+    // invalid maxNumChunkIdsToProcess for a batch remove
+    auto removeOp =
+        storageClient_->createRemoveOp(firstChainId_, smallChunkId, largeChunkId, 0 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo()));
+    ASSERT_FALSE(removeOp.result.statusCode);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, removeOp.result.statusCode.error().code());
+  }
+}
+
+TEST_F(TestStorageClientSideError, InvalidChunkSize) {
+  auto truncateOp = storageClient_->createTruncateOp(firstChainId_,
+                                                     ChunkId(1 /*high*/, 1 /*low*/),
+                                                     setupConfig_.chunk_size() * 2 /*chunkLen*/,
+                                                     setupConfig_.chunk_size());
+  folly::coro::blockingWait(storageClient_->truncateChunks(std::span(&truncateOp, 1), flat::UserInfo()));
+  ASSERT_FALSE(truncateOp.result.lengthInfo);
+  ASSERT_EQ(StorageClientCode::kInvalidArg, truncateOp.result.lengthInfo.error().code());
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestTargetSelectionStrategy.cc

@@ -0,0 +1,155 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+
+#include "client/storage/TargetSelection.h"
+#include "common/utils/Duration.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+TEST(TargetSelectionStrategy, LoadDistribution) {
+  const uint64_t numReplicas = 3;
+  const uint64_t numNodes = 180;
+  const uint64_t numTargets = numNodes * 16 * 16;
+  const uint64_t numChains = numTargets / 3;
+  const uint64_t numBatches = 100;
+
+  uint64_t nodeSeqNum = 0;
+  uint64_t targetId = 0;
+  std::unordered_map<NodeId, uint64_t> numTargetsOnNode;
+  std::unordered_map<ChainId, SlimChainInfo> chainMap;
+
+  for (uint64_t chainId = 1; chainId <= numChains; chainId++) {
+    auto &chainInfo = chainMap[ChainId(chainId)];
+    chainInfo.chainId = ChainId(chainId);
+    chainInfo.totalNumTargets = numReplicas;
+
+    for (uint64_t replica = 0; replica < numReplicas; replica++) {
+      NodeId nodeId((nodeSeqNum++ % numNodes) + 1);
+      chainInfo.servingTargets.push_back({TargetId(++targetId), nodeId});
+      numTargetsOnNode[nodeId]++;
+      ASSERT_LE(numTargetsOnNode[nodeId], numTargets / numNodes);
+      ASSERT_LE(targetId, numTargets);
+    }
+  }
+
+  auto computeIODist = [](const std::unordered_map<NodeId, uint64_t> &numIOsOnNode) {
+    std::vector<uint64_t> numIOs;
+    uint64_t sumIOs = 0;
+
+    for (const auto &item : numIOsOnNode) {
+      numIOs.push_back(item.second);
+      sumIOs += item.second;
+    }
+
+    std::sort(numIOs.begin(), numIOs.end());
+    return std::vector{
+        numIOs.front(),
+        numIOs.back(),
+        numIOs[numIOs.size() / 2],
+        sumIOs / numIOsOnNode.size(),
+        numIOsOnNode.size(),
+    };
+  };
+
+  for (uint64_t numIOsPerBatch : {100, 200, 400, 1000}) {
+    for (uint64_t mode = 1; mode < TargetSelectionMode::EndOfMode; mode++) {
+      std::unordered_map<NodeId, uint64_t> totalIOsOnNode;
+      std::vector<uint64_t> avgBatchDist(5);
+
+      for (uint64_t k = 0; k < numBatches; k++) {
+        TargetSelectionOptions options;
+        options.set_mode(static_cast<TargetSelectionMode>(mode));
+
+        auto strategy = TargetSelectionStrategy::create(options);
+        if (k == 0) strategy->reset();
+
+        std::unordered_map<NodeId, uint64_t> batchIOsOnNode;
+
+        for (uint64_t i = 0; i < numIOsPerBatch; i++) {
+          ChainId chainId(folly::Random::rand32(1, numChains + 1));
+          ASSERT_TRUE(!chainMap[chainId].servingTargets.empty()) << "chainId " << chainId;
+          auto selectedTarget = strategy->selectTarget(chainMap[chainId]);
+          ASSERT_TRUE(std::find(chainMap[chainId].servingTargets.cbegin(),
+                                chainMap[chainId].servingTargets.cend(),
+                                *selectedTarget) != chainMap[chainId].servingTargets.end());
+          totalIOsOnNode[selectedTarget->nodeId]++;
+          batchIOsOnNode[selectedTarget->nodeId]++;
+        }
+
+        auto dist = computeIODist(batchIOsOnNode);
+        for (uint64_t i = 0; i < dist.size(); i++) {
+          avgBatchDist[i] += dist[i];
+        }
+      }
+
+      auto overallDist = computeIODist(totalIOsOnNode);
+
+      fmt::print("\n=== numIOsPerBatch {}, TargetSelectionMode {} ===\n", numIOsPerBatch, mode);
+      fmt::print("   overall :  min {:<5d} max {:<5d} median {:<5d} avg {:<5d} #nodes {:<5d}\n",
+                 overallDist[0],
+                 overallDist[1],
+                 overallDist[2],
+                 overallDist[3],
+                 overallDist[4]);
+      fmt::print(" per batch :  min {:<5d} max {:<5d} median {:<5d} avg {:<5d} #nodes {:<5d}\n",
+                 avgBatchDist[0] / numBatches,
+                 avgBatchDist[1] / numBatches,
+                 avgBatchDist[2] / numBatches,
+                 avgBatchDist[3] / numBatches,
+                 avgBatchDist[4] / numBatches);
+    }
+  }
+}
+
+TEST(TargetSelectionStrategy, MultiThreads) {
+  constexpr uint64_t numReplicas = 2;
+  constexpr uint64_t numNodes = 180;
+  constexpr uint64_t numTargets = numNodes * 16 * 16;
+  constexpr uint64_t numChains = numTargets / numReplicas;
+  constexpr uint64_t numIOsPerBatch = 1024;
+  constexpr uint32_t numThreads = 32;
+
+  uint64_t nodeSeqNum = 0;
+  uint64_t targetId = 0;
+  std::unordered_map<NodeId, uint64_t> numTargetsOnNode;
+  std::unordered_map<ChainId, SlimChainInfo> chainMap;
+
+  for (uint64_t chainId = 1; chainId <= numChains; chainId++) {
+    auto &chainInfo = chainMap[ChainId(chainId)];
+    chainInfo.chainId = ChainId(chainId);
+    chainInfo.totalNumTargets = numReplicas;
+
+    for (uint64_t replica = 0; replica < numReplicas; replica++) {
+      NodeId nodeId((nodeSeqNum++ % numNodes) + 1);
+      chainInfo.servingTargets.push_back({TargetId(++targetId), nodeId});
+      numTargetsOnNode[nodeId]++;
+    }
+  }
+
+  folly::CPUThreadPoolExecutor executor(numThreads);
+  auto startTime = RelativeTime::now();
+  std::atomic<uint64_t> finished;
+  for (auto i = 0u; i < numThreads; ++i) {
+    executor.add([&] {
+      while (startTime + 1_s >= RelativeTime::now()) {
+        TargetSelectionOptions options;
+        options.set_mode(TargetSelectionMode::RoundRobin);
+        auto strategy = TargetSelectionStrategy::create(options);
+
+        for (auto j = 0u; j < numIOsPerBatch; ++j) {
+          ChainId chainId{folly::Random::rand32(1, numChains + 1)};
+          ASSERT_TRUE(!chainMap[chainId].servingTargets.empty()) << "chainId " << chainId;
+          auto selectedTarget = strategy->selectTarget(chainMap[chainId]);
+          ASSERT_OK(selectedTarget);
+        }
+        finished += numIOsPerBatch;
+      }
+    });
+  }
+  executor.join();
+  XLOGF(WARNING, "select target {} times", finished.load());
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/service/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_test(test_storage_service test-fabric-lib)

tests/storage/service/TestDumpMeta.cc

@@ -0,0 +1,80 @@
+#include <boost/filesystem/operations.hpp>
+#include <thread>
+
+#include "common/serde/Serde.h"
+#include "common/utils/FileUtils.h"
+#include "kv/KVStore.h"
+#include "kv/MemDBStore.h"
+#include "storage/service/StorageServer.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/Helper.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestDumpMeta : public UnitTestFabric, public ::testing::Test {
+ protected:
+  TestDumpMeta()
+      : UnitTestFabric(SystemSetupConfig{128_KB /*chunkSize*/,
+                                         1 /*numChains*/,
+                                         1 /*numReplicas*/,
+                                         1 /*numStorageNodes*/,
+                                         {folly::fs::temp_directory_path()} /*dataPaths*/,
+                                         hf3fs::Path() /*clientConfig*/,
+                                         hf3fs::Path() /*serverConfig*/,
+                                         {} /*storageEndpoints*/,
+                                         0 /*serviceLevel*/,
+                                         0 /*listenPort*/,
+                                         client::StorageClient::ImplementationType::RPC,
+                                         kv::KVStore::Type::RocksDB,
+                                         false,
+                                         true}) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+};
+
+TEST_F(TestDumpMeta, Normal) {
+  auto chunkId = ChunkId{0u, 0u};
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+  auto writeRes = writeToChunk(chainIds_.front(), chunkId, chunkData);
+  ASSERT_OK(writeRes.lengthInfo);
+
+  folly::test::TemporaryDirectory tmpPath;
+  serverConfigs_[0].dump_worker().set_dump_root_path(tmpPath.path());
+  serverConfigs_[0].dump_worker().set_dump_interval(100_ms);
+
+  std::this_thread::sleep_for(2_s);
+  stopAndRemoveStorageServer(0);
+
+  std::vector<Path> files;
+  for (auto &filePath : boost::filesystem::recursive_directory_iterator(tmpPath.path())) {
+    if (boost::filesystem::is_regular_file(filePath)) {
+      files.push_back(filePath);
+    }
+  }
+  ASSERT_EQ(files.size(), 1);
+  XLOGF(CRITICAL, "dump files: {}", serde::toJsonString(files));
+
+  auto readResult = loadFile(files.front());
+  ASSERT_OK(readResult);
+
+  std::map<ChunkId, ChunkMetadata> metas;
+  ASSERT_TRUE(serde::deserialize(metas, *readResult));
+  ASSERT_EQ(metas.size(), 1);
+  ASSERT_EQ(metas.begin()->first, chunkId);
+  XLOGF(CRITICAL, "dump metas: {}", serde::toJsonString(metas));
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/service/TestIncorrectRoutingInfo.cc

@@ -0,0 +1,209 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/lib/Helper.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestIncorrectRoutingInfo : public UnitTestFabric, public ::testing::Test {
+ protected:
+  TestIncorrectRoutingInfo()
+      : UnitTestFabric(SystemSetupConfig{
+            128_KB /*chunkSize*/,
+            1 /*numChains*/,
+            3 /*numReplicas*/,
+            3 /*numStorageNodes*/,
+            {folly::fs::temp_directory_path()} /*dataPaths*/,
+            hf3fs::Path() /*clientConfig*/,
+            hf3fs::Path() /*serverConfig*/,
+            {} /*storageEndpoints*/,
+            0 /*serviceLevel*/,
+            0 /*listenPort*/,
+            client::StorageClient::ImplementationType::RPC,
+            kv::KVStore::Type::RocksDB,
+            true /*useFakeMgmtdClient*/,
+        }) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+    clientConfig_.retry().set_max_retry_time(10_s);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+};
+
+TEST_F(TestIncorrectRoutingInfo, MismatchChainVersion) {
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  // create a test chunk
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_RESULT_EQ(chunkData.size(), ioResult.lengthInfo);
+
+  // get the first chain
+  auto newRoutingInfo = copyRoutingInfo();
+  auto &chainTable = *newRoutingInfo->getChainTable(kTableId());
+  auto &firstChain = newRoutingInfo->chains[chainTable.chains.front()];
+  ASSERT_EQ(chainId, firstChain.chainId);
+  auto chainVersion = firstChain.chainVersion;
+
+  for (int32_t deltaVer : {-1, 1}) {
+    // change version of the first chain
+    firstChain.chainVersion = flat::ChainVersion(chainVersion + deltaVer);
+    // only update client's routing info
+    auto fakeClient = dynamic_cast<FakeMgmtdClient *>(mgmtdForClient_.get());
+    newRoutingInfo->routingInfoVersion++;
+    fakeClient->setRoutingInfo(newRoutingInfo);
+    ASSERT_OK(folly::coro::blockingWait(mgmtdForClient_->refreshRoutingInfo(/*force=*/false)));
+    // try to read/write with mismatch chain version
+    auto readRes = readFromChunk(chainId, chunkId, chunkData, 0, chunkData.size());
+    ASSERT_ERROR(readRes.lengthInfo, StorageClientCode::kRoutingVersionMismatch);
+    auto writeRes = writeToChunk(chainId, chunkId, chunkData, 0, chunkData.size());
+    ASSERT_ERROR(writeRes.lengthInfo, StorageClientCode::kRoutingVersionMismatch);
+  }
+}
+
+TEST_F(TestIncorrectRoutingInfo, WriteNotSentToHeadTarget) {
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  // send write to the second target on chain
+  client::WriteOptions options;
+  options.targetSelection().set_mode(client::TargetSelectionMode::ManualMode);
+  options.targetSelection().set_targetIndex(1);
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData, 0, chunkData.size(), options);
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+}
+
+TEST_F(TestIncorrectRoutingInfo, RemoveNotSentToHeadTarget) {
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  // create a test chunk
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_RESULT_EQ(chunkData.size(), ioResult.lengthInfo);
+
+  // send remove to the second target on chain
+  client::WriteOptions options;
+  options.targetSelection().set_mode(client::TargetSelectionMode::ManualMode);
+  options.targetSelection().set_targetIndex(1);
+  auto removeOp = storageClient_->createRemoveOp(chainId,
+                                                 ChunkId(chunkId, 0),
+                                                 ChunkId(chunkId, 0xFF),
+                                                 1 /*maxNumChunkIdsToProcess*/);
+  folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo(), options));
+  ASSERT_ERROR(removeOp.result.statusCode, StorageClientCode::kRoutingError);
+}
+
+TEST_F(TestIncorrectRoutingInfo, WorkingHeadDetectedAsOffline) {
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  size_t numWriteIOs = 100;
+  std::vector<client::WriteIO> writeIOs;
+
+  // create write IOs to one chunk
+  for (size_t writeIndex = 0; writeIndex < numWriteIOs; writeIndex++) {
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 chunkData.size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &chunkData[0],
+                                                 &ioBuffer);
+    writeIOs.push_back(std::move(writeIO));
+  }
+
+  // issue the write requests
+  flat::UserInfo dummyUserInfo{};
+  auto options = client::WriteOptions();
+
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> writeTasks;
+
+  for (auto &writeIO : writeIOs) {
+    auto writeTask = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    writeTasks.push_back(std::move(writeTask));
+  }
+
+  // set head target to offline state but it's actually working
+  auto newRoutingInfo = copyRoutingInfo();
+  setTargetOffline(*newRoutingInfo, 0);
+
+  // let all storage servers except the host of head targets know the latest routing info
+  for (size_t serverIndex = 1; serverIndex < storageServers_.size(); serverIndex++) {
+    auto &storageServer = storageServers_[serverIndex];
+    auto client = RoutingStoreHelper::getMgmtdClient(*storageServer);
+    auto fakeClient = dynamic_cast<FakeMgmtdClient *>(client.get());
+    fakeClient->setRoutingInfo(newRoutingInfo);
+    RoutingStoreHelper::refreshRoutingInfo(*storageServer);
+  }
+
+  // client does not konw the head is marked offline yet, so let it retry for a short time
+  std::this_thread::sleep_for(500_ms);
+
+#if defined(__has_feature)
+#if !__has_feature(thread_sanitizer)
+  for (auto &writeIO : writeIOs) {
+    // some writes are completed but others are still under retry
+    if (writeIO.result.lengthInfo) {
+      ASSERT_RESULT_EQ(writeIO.length, writeIO.result.lengthInfo);
+    } else {
+      switch (writeIO.statusCode()) {
+        case StorageClientCode::kCommError:
+        case StorageClientCode::kTimeout:
+        case StorageClientCode::kRoutingVersionMismatch:
+          XLOGF(INFO, "Write IO {} length info: {}", fmt::ptr(&writeIO), writeIO.result.lengthInfo);
+          break;
+        default:
+          ASSERT_EQ(StorageClientCode::kNotInitialized, writeIO.result.lengthInfo.error().code())
+              << fmt::format("Write IO {} length info: {}", fmt::ptr(&writeIO), writeIO.result.lengthInfo);
+      }
+    }
+  }
+#endif
+#endif
+
+  // let client know the latest routing info
+  auto fakeClient = dynamic_cast<FakeMgmtdClient *>(mgmtdForClient_.get());
+  fakeClient->setRoutingInfo(newRoutingInfo);
+  ASSERT_OK(folly::coro::blockingWait(mgmtdForClient_->refreshRoutingInfo(/*force=*/false)));
+
+  // wait until all write tasks completed
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));
+
+  // check all write IOs have succeeded
+  std::set<ChunkVer> updateVersions;
+  for (const auto &writeIO : writeIOs) {
+    ASSERT_OK(writeIO.result.lengthInfo);
+    // check commit version == update version
+    ASSERT_EQ(writeIO.result.commitVer, writeIO.result.updateVer);
+    // check the update versions are in range [1..numWriteIOs]
+    ASSERT_LE(1, writeIO.result.updateVer);
+    ASSERT_LE(writeIO.result.updateVer, numWriteIOs);
+    updateVersions.insert(writeIO.result.updateVer);
+  }
+  // check the update versions are unique
+  ASSERT_EQ(numWriteIOs, updateVersions.size());
+}
+
+}  // namespace
+}  // namespace hf3fs::storage

tests/storage/service/TestSingleProcessCluster.cc

@@ -0,0 +1,679 @@
+#include <filesystem>
+#include <folly/experimental/coro/Collect.h>
+
+#include "common/serde/Serde.h"
+#include "common/utils/FileUtils.h"
+#include "common/utils/SysResource.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/Helper.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestSingleProcessCluster : public UnitTestFabric, public ::testing::TestWithParam<SystemSetupConfig> {
+ protected:
+  TestSingleProcessCluster()
+      : UnitTestFabric(GetParam()) {}
+
+  void SetUp() override {
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+    GTEST_SKIP();
+#endif
+#endif
+
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+
+    ASSERT_TRUE(setUpStorageSystem());
+
+    for (auto nodeId : storageNodeIds_) {
+      waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED);
+      waitStorageTargetStatus(nodeTargets_[nodeId], flat::LocalTargetState::UPTODATE, flat::PublicTargetState::SERVING);
+    }
+
+    mgmtdServer_.config.service().set_check_status_interval(200_ms);
+    mgmtdServer_.config.service().set_heartbeat_fail_interval(1_s);
+
+    clientConfig_.retry().set_init_wait_time(200_ms);
+    clientConfig_.retry().set_max_wait_time(5_s);
+    clientConfig_.retry().set_max_retry_time(60_s);
+    ASSERT_LT(clientConfig_.retry().init_wait_time(), clientConfig_.retry().max_wait_time());
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+
+  void checkStorageNodeStatus(const std::vector<flat::NodeId> &nodeIds, flat::NodeStatus status) {
+    for (const auto &nodeId : nodeIds) {
+      ASSERT_TRUE(nodeTargets_.count(nodeId));
+      auto nodeInfo = rawRoutingInfo_->getNode(nodeId);
+      ASSERT_NE(nullptr, nodeInfo);
+      ASSERT_EQ(status, nodeInfo->status);
+    }
+  }
+
+  void checkStorageTargetStatus(const std::vector<flat::TargetId> &targetIds,
+                                flat::LocalTargetState localStatus,
+                                flat::PublicTargetState publicStatus) {
+    for (const auto &targetId : targetIds) {
+      auto targetInfo = rawRoutingInfo_->getTarget(targetId);
+      ASSERT_NE(nullptr, targetInfo);
+      ASSERT_EQ(localStatus, targetInfo->localState);
+      ASSERT_EQ(publicStatus, targetInfo->publicState);
+    }
+  }
+
+  void waitStorageNodeStatus(const std::vector<flat::NodeId> &nodeIds,
+                             flat::NodeStatus status,
+                             size_t maxRetries = 200) {
+    flat::NodeInfo *nodeInfo = nullptr;
+
+    XLOGF(INFO,
+          "# Waiting storage nodes {} moving to status {}",
+          serde::toJsonString(nodeIds),
+          magic_enum::enum_name(status));
+
+    for (size_t retry = 0; retry < maxRetries; retry++) {
+      if (retry > 0) std::this_thread::sleep_for(mgmtdServer_.config.service().check_status_interval());
+
+      auto routingInfo = getRoutingInfo();
+
+      if (routingInfo) {
+        bool allMatch = true;
+
+        for (const auto &nodeId : nodeIds) {
+          ASSERT_TRUE(nodeTargets_.count(nodeId));
+          nodeInfo = routingInfo->getNode(nodeId);
+          ASSERT_NE(nullptr, nodeInfo);
+          if (nodeInfo->status != status) {
+            XLOGF(INFO,
+                  "#{} Waiting storage node {} moving to status {}, current status {}, routing info: {}",
+                  retry,
+                  nodeId,
+                  magic_enum::enum_name(status),
+                  magic_enum::enum_name(nodeInfo->status),
+                  routingInfo->routingInfoVersion);
+            allMatch = false;
+            break;
+          }
+        }
+
+        if (allMatch) return;
+      }
+    }
+
+    ASSERT_NE(nullptr, nodeInfo);
+    ASSERT_EQ(status, nodeInfo->status) << "node id " << nodeInfo->app.nodeId;
+  }
+
+  void waitStorageTargetStatus(const std::vector<flat::TargetId> &targetIds,
+                               flat::LocalTargetState localStatus,
+                               flat::PublicTargetState publicStatus,
+                               size_t maxRetries = 500) {
+    flat::TargetInfo *targetInfo = nullptr;
+
+    XLOGF(INFO,
+          "# wait storage targets {} moving to status {}/{}",
+          serde::toJsonString(targetIds),
+          magic_enum::enum_name(localStatus),
+          magic_enum::enum_name(publicStatus));
+
+    for (size_t retry = 0; retry < maxRetries; retry++) {
+      if (retry > 0) std::this_thread::sleep_for(mgmtdServer_.config.service().check_status_interval());
+
+      auto routingInfo = getRoutingInfo();
+
+      if (routingInfo) {
+        bool allMatch = true;
+
+        for (const auto &targetId : targetIds) {
+          targetInfo = routingInfo->getTarget(targetId);
+          ASSERT_NE(nullptr, targetInfo);
+          if (localStatus != targetInfo->localState || publicStatus != targetInfo->publicState) {
+            XLOGF(INFO,
+                  "#{} Waiting storage target {} moving to status {}/{}, current status {}/{}, routing info: {}",
+                  retry,
+                  targetId,
+                  magic_enum::enum_name(localStatus),
+                  magic_enum::enum_name(publicStatus),
+                  magic_enum::enum_name(targetInfo->localState),
+                  magic_enum::enum_name(targetInfo->publicState),
+                  routingInfo->routingInfoVersion);
+            allMatch = false;
+            break;
+          }
+        }
+
+        if (allMatch) return;
+      }
+    }
+
+    ASSERT_NE(nullptr, targetInfo);
+    ASSERT_EQ(localStatus, targetInfo->localState) << "target id " << targetInfo->targetId;
+    ASSERT_EQ(publicStatus, targetInfo->publicState) << "target id " << targetInfo->targetId;
+  }
+
+  void readAndCompareAllReplicas(const ChainId &chainId,
+                                 const ChunkId &chunkBegin,
+                                 size_t numChunks,
+                                 const std::vector<IOResult> &expectedResults) {
+    const ChunkId chunkEnd(chunkBegin, numChunks);
+    std::vector<std::vector<std::vector<uint8_t>>> chunkData(setupConfig_.num_replicas());
+    std::vector<std::vector<IOResult>> ioResults(setupConfig_.num_replicas());
+
+    for (size_t replicaIndex = 0; replicaIndex < setupConfig_.num_replicas(); replicaIndex++) {
+      client::ReadOptions options;
+      options.targetSelection().set_mode(client::TargetSelectionMode::ManualMode);
+      options.targetSelection().set_targetIndex(replicaIndex);
+
+      auto readRes = readFromChunks(chainId,
+                                    chunkBegin,
+                                    chunkEnd,
+                                    chunkData[replicaIndex],
+                                    0,
+                                    setupConfig_.chunk_size(),
+                                    options,
+                                    &ioResults[replicaIndex]);
+      ASSERT_TRUE(readRes);
+    }
+
+    for (size_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+      for (size_t replicaIndex = 0; replicaIndex < setupConfig_.num_replicas(); replicaIndex++) {
+        ASSERT_EQ(chunkData[0][chunkIndex], chunkData[replicaIndex][chunkIndex]);
+        ASSERT_EQ(expectedResults[chunkIndex].commitVer, ioResults[replicaIndex][chunkIndex].commitVer);
+        ASSERT_EQ(expectedResults[chunkIndex].updateVer, ioResults[replicaIndex][chunkIndex].updateVer);
+        ASSERT_EQ(expectedResults[chunkIndex].checksum, ioResults[replicaIndex][chunkIndex].checksum);
+        ASSERT_EQ(expectedResults[chunkIndex].commitChainVer, ioResults[replicaIndex][chunkIndex].commitChainVer);
+      }
+    }
+  }
+};
+
+TEST_P(TestSingleProcessCluster, StorageFailureDetected) {
+  // stop the first storage server
+  flat::NodeId firstNodeId{1};
+  stopAndRemoveStorageServer(firstNodeId);
+
+  waitStorageTargetStatus(nodeTargets_[firstNodeId], flat::LocalTargetState::OFFLINE, flat::PublicTargetState::OFFLINE);
+  waitStorageNodeStatus({firstNodeId}, flat::NodeStatus::HEARTBEAT_FAILED);
+}
+
+TEST_P(TestSingleProcessCluster, StorageFailureDetectedBeforeWrite) {
+  // stop the first storage server
+  flat::NodeId firstNodeId{1};
+  stopAndRemoveStorageServer(firstNodeId);
+
+  waitStorageTargetStatus(nodeTargets_[firstNodeId], flat::LocalTargetState::OFFLINE, flat::PublicTargetState::OFFLINE);
+
+  size_t numChunks = 10;
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  for (const auto &chainId : chainIds_) {
+    storage::ChunkId chunkBegin(1, 0);
+    storage::ChunkId chunkEnd(1, numChunks);
+    auto writeRes = writeToChunks(chainId, chunkBegin, chunkEnd, chunkData);
+    ASSERT_TRUE(writeRes);
+  }
+}
+
+#if false
+TEST_P(TestSingleProcessCluster, DISABLED_StorageWriteFailed) {
+  clientConfig_.retry().set_init_wait_time(1_s);
+  clientConfig_.retry().set_max_wait_time(1_s);
+  clientConfig_.retry().set_max_retry_time(5_s);
+
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+  auto writeRes = writeToChunk(chainIds_.front(), ChunkId{0u, 0u}, chunkData);
+  ASSERT_OK(writeRes.lengthInfo);
+
+  auto flagPath = fmt::format("/tmp/storage_main_write_failed.{}", SysResource::pid());
+  ASSERT_OK(hf3fs::storeToFile(flagPath, "100"));
+  auto guard = folly::makeGuard([&] { std::filesystem::remove(flagPath); });
+
+  // write the special chunk and receive fail.
+  writeRes = writeToChunk(chainIds_.front(), ChunkId{0u, 0u}, chunkData);
+  ASSERT_FALSE(writeRes.lengthInfo);
+
+  waitStorageTargetStatus(nodeTargets_[flat::NodeId{1}],
+                          flat::LocalTargetState::OFFLINE,
+                          flat::PublicTargetState::OFFLINE);
+}
+#endif
+
+TEST_P(TestSingleProcessCluster, StorageFailAndRestart) {
+  // stop the first storage server
+  uint32_t nodeIndex = 0;
+  auto firstNodeId = storageNodeIds_[nodeIndex];
+  stopAndRemoveStorageServer(nodeIndex);
+
+  waitStorageNodeStatus({firstNodeId}, flat::NodeStatus::HEARTBEAT_FAILED);
+  waitStorageTargetStatus(nodeTargets_[firstNodeId], flat::LocalTargetState::OFFLINE, flat::PublicTargetState::OFFLINE);
+
+  size_t numChunks = 10;
+  storage::ChunkId chunkBegin(1, 0);
+  storage::ChunkId chunkEnd(1, numChunks);
+  std::vector<std::vector<IOResult>> writeResults(chainIds_.size());
+
+  for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
+    uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size());
+    std::vector<uint8_t> chunkData(folly::Random::rand32(1, setupConfig_.chunk_size() - offset + 1),
+                                   (uint8_t)folly::Random::rand32());
+    auto writeRes = writeToChunks(chainIds_[chainIndex],
+                                  chunkBegin,
+                                  chunkEnd,
+                                  chunkData,
+                                  offset,
+                                  chunkData.size(),
+                                  client::WriteOptions(),
+                                  &writeResults[chainIndex]);
+    ASSERT_TRUE(writeRes);
+  }
+
+  // restart storage server
+  auto storageServer = createStorageServer(nodeIndex);
+  ASSERT_NE(storageServer, nullptr);
+  storageServers_.insert(storageServers_.begin() + nodeIndex, std::move(storageServer));
+
+  waitStorageNodeStatus({firstNodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED);
+  waitStorageTargetStatus(nodeTargets_[firstNodeId],
+                          flat::LocalTargetState::UPTODATE,
+                          flat::PublicTargetState::SERVING);
+
+  // read chunk data from all replicas
+  for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
+    readAndCompareAllReplicas(chainIds_[chainIndex], chunkBegin, numChunks, writeResults[chainIndex]);
+  }
+}
+
+TEST_P(TestSingleProcessCluster, StorageRepeatedFailAndRestart) {
+  uint32_t maxLoops = setupConfig_.num_storage_nodes();
+
+  for (uint32_t testLoop = 1; testLoop <= maxLoops; testLoop++) {
+    uint64_t chunkIdPrefix = testLoop;
+    size_t numChunks = 10;
+    storage::ChunkId chunkBegin(chunkIdPrefix, 0);
+    storage::ChunkId chunkEnd(chunkIdPrefix, numChunks);
+    std::vector<std::vector<IOResult>> writeResults(chainIds_.size());
+
+    uint32_t nodeIndex = folly::Random::rand32(0, setupConfig_.num_storage_nodes());
+    auto nodeId = storageNodeIds_[nodeIndex];
+    XLOGF(INFO, "Test loop {}, stopping #{} {}", testLoop, nodeIndex, nodeId);
+    stopAndRemoveStorageServer(nodeIndex);
+
+    waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_FAILED, 100 /*maxRetries*/);
+    waitStorageTargetStatus(nodeTargets_[nodeId],
+                            flat::LocalTargetState::OFFLINE,
+                            flat::PublicTargetState::OFFLINE,
+                            500 /*maxRetries*/);
+
+    XLOGF(INFO, "Test loop {}, writing chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
+
+    for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
+      uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size());
+      std::vector<uint8_t> chunkData(folly::Random::rand32(1, setupConfig_.chunk_size() - offset + 1),
+                                     (uint8_t)folly::Random::rand32());
+      auto writeRes = writeToChunks(chainIds_[chainIndex],
+                                    chunkBegin,
+                                    chunkEnd,
+                                    chunkData,
+                                    offset,
+                                    chunkData.size(),
+                                    client::WriteOptions(),
+                                    &writeResults[chainIndex]);
+      ASSERT_TRUE(writeRes);
+    }
+
+    // restart storage server
+    XLOGF(INFO, "Test loop {}, restarting #{} {}", testLoop, nodeIndex, nodeId);
+    auto storageServer = createStorageServer(nodeIndex);
+    ASSERT_NE(storageServer, nullptr);
+    storageServers_.insert(storageServers_.begin() + nodeIndex, std::move(storageServer));
+
+    waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED, 200 /*maxRetries*/);
+    waitStorageTargetStatus(nodeTargets_[nodeId],
+                            flat::LocalTargetState::UPTODATE,
+                            flat::PublicTargetState::SERVING,
+                            1000 /*maxRetries*/);
+
+    XLOGF(INFO, "Test loop {}, reading chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
+
+    // read chunk data from all replicas
+    for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
+      readAndCompareAllReplicas(chainIds_[chainIndex], chunkBegin, numChunks, writeResults[chainIndex]);
+    }
+  }
+}
+
+TEST_P(TestSingleProcessCluster, MultiStoragesRepeatedFailAndRestart) {
+  uint32_t nodeIndexStart = folly::Random::rand32(0, setupConfig_.num_storage_nodes());
+  uint32_t maxLoops = setupConfig_.num_storage_nodes();
+
+  for (uint32_t testLoop = 1; testLoop <= maxLoops; testLoop++) {
+    uint64_t chunkIdPrefix = testLoop;
+    size_t numChunks = 10;
+    storage::ChunkId chunkBegin(chunkIdPrefix, 0);
+    storage::ChunkId chunkEnd(chunkIdPrefix, numChunks);
+
+    uint32_t nodeIndex = (nodeIndexStart + testLoop) % setupConfig_.num_storage_nodes();
+    auto nodeId = storageNodeIds_[nodeIndex];
+
+    auto routingInfo = getRoutingInfo();
+    ASSERT_NE(nullptr, routingInfo);
+    auto nodeInfo = routingInfo->getNode(nodeId);
+    ASSERT_NE(nullptr, nodeInfo);
+
+    bool stoppedStorage = false;
+    bool writeFailed = false;
+
+    if (nodeInfo->status == flat::NodeStatus::HEARTBEAT_CONNECTED) {
+      XLOGF(INFO, "Test loop {}, stopping #{} {}", testLoop, nodeIndex, nodeId);
+      stoppedStorage = stopAndRemoveStorageServer(nodeId);
+    }
+
+    XLOGF(INFO, "Test loop {}, writing chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
+
+    for (const auto &chainId : chainIds_) {
+      uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size());
+      std::vector<uint8_t> chunkData(folly::Random::rand32(1, setupConfig_.chunk_size() - offset + 1),
+                                     (uint8_t)folly::Random::rand32());
+      auto writeRes = writeToChunks(chainId, chunkBegin, chunkEnd, chunkData, offset, chunkData.size());
+
+      auto routingInfo = getRoutingInfo();
+      auto chainInfo = routingInfo->getChain(chainId);
+      ASSERT_NE(nullptr, chainInfo);
+
+      if (chainInfo->targets.front().publicState == flat::PublicTargetState::SERVING) {
+        // check write result if the chain has serving target
+        ASSERT_TRUE(writeRes);
+      } else {
+        XLOGF(INFO, "Test loop {}, failed to write to chain: {:?}", testLoop, *chainInfo);
+        writeFailed = true;
+        break;
+      }
+    }
+
+    // restart storage server
+    if (stoppedStorage) {
+      XLOGF(INFO, "Test loop {}, restarting #{} {}", testLoop, nodeIndex, nodeId);
+      auto storageServer = createStorageServer(nodeIndex);
+      ASSERT_NE(storageServer, nullptr);
+      storageServers_.insert(storageServers_.begin() + nodeIndex, std::move(storageServer));
+    }
+
+    if (writeFailed) {
+      // read chunks from last test loop if write in current loop failed
+      chunkBegin = storage::ChunkId(chunkIdPrefix - 1, 0);
+      chunkEnd = storage::ChunkId(chunkIdPrefix - 1, numChunks);
+    }
+
+    // read chunk data from all serving replicas
+    XLOGF(INFO, "Test loop {}, reading chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
+
+    for (const auto &chainId : chainIds_) {
+      for (size_t retryIndex = 0;; retryIndex++) {
+        XLOGF(INFO,
+              "Test loop {}, #{} retry, reading chunks in range {} - {}",
+              testLoop,
+              retryIndex,
+              chunkBegin,
+              chunkEnd);
+
+        std::vector<std::vector<uint8_t>> chunkData;
+        storage::client::ReadOptions options;
+        options.targetSelection().set_mode(storage::client::TargetSelectionMode::RoundRobin);
+        auto readRes = readFromChunks(chainId, chunkBegin, chunkEnd, chunkData, 0, setupConfig_.chunk_size(), options);
+
+        auto chainInfo = routingInfo->getChain(chainId);
+        ASSERT_NE(nullptr, chainInfo);
+
+        if (chainInfo->targets.front().publicState == flat::PublicTargetState::SERVING) {
+          // check read result if the chain has serving target
+          ASSERT_TRUE(readRes);
+          break;
+        } else {
+          // otherwise wait the storage server restart and retry
+          waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED, 200 /*maxRetries*/);
+          waitStorageTargetStatus(nodeTargets_[nodeId],
+                                  flat::LocalTargetState::UPTODATE,
+                                  flat::PublicTargetState::SERVING,
+                                  1000 /*maxRetries*/);
+        }
+      }
+    }
+  }
+}
+
+TEST_P(TestSingleProcessCluster, ConcurrentStorageSyncAndWrite) {
+  uint32_t nodeIndexStart = folly::Random::rand32(0, setupConfig_.num_storage_nodes());
+  uint32_t maxLoops = setupConfig_.num_storage_nodes();
+
+  for (uint32_t testLoop = 1; testLoop <= maxLoops; testLoop++) {
+    uint64_t chunkIdPrefix = testLoop;
+    size_t numChunks = 20;
+    storage::ChunkId chunkBegin(chunkIdPrefix, 0);
+    storage::ChunkId chunkEnd(chunkIdPrefix, numChunks);
+
+    uint32_t nodeIndex = (nodeIndexStart + testLoop) % setupConfig_.num_storage_nodes();
+    auto nodeId = storageNodeIds_[nodeIndex];
+
+    XLOGF(INFO, "Test loop {}, writing chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
+    std::atomic<bool> runWrite = true;
+    std::atomic<bool> writeOK = true;
+
+    std::vector<std::jthread> backgroundWorkers;
+    backgroundWorkers.reserve(chainIds_.size());
+    std::vector<std::vector<IOResult>> writeResults(chainIds_.size());
+
+    for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
+      const auto &chainId = chainIds_[chainIndex];
+      auto &ioResults = writeResults[chainIndex];
+      backgroundWorkers.emplace_back([&, this]() {
+        for (uint32_t chunkVer = 1; runWrite && writeOK; chunkVer++) {
+          uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size());
+          std::vector<uint8_t> chunkData(folly::Random::rand32(1, setupConfig_.chunk_size() - offset + 1),
+                                         (uint8_t)folly::Random::rand32());
+
+          ioResults.clear();
+          auto writeRes = writeToChunks(chainId,
+                                        chunkBegin,
+                                        chunkEnd,
+                                        chunkData,
+                                        offset,
+                                        chunkData.size(),
+                                        client::WriteOptions(),
+                                        &ioResults);
+          if (!writeRes) writeOK = false;
+          ASSERT_TRUE(writeRes);
+
+          for (const auto &result : ioResults) {
+            writeRes = result.lengthInfo && *result.lengthInfo == chunkData.size() && chunkVer == result.commitVer;
+            if (!writeRes) writeOK = false;
+            ASSERT_TRUE(writeRes);
+          }
+        }
+      });
+    }
+
+    // write a few chunks before restart
+    auto waitTime = mgmtdServer_.config.service().heartbeat_fail_interval() * 2;
+    std::this_thread::sleep_for(waitTime);
+    ASSERT_TRUE(writeOK);
+
+    auto routingInfo = getRoutingInfo();
+    ASSERT_NE(nullptr, routingInfo);
+    auto nodeInfo = routingInfo->getNode(nodeId);
+    ASSERT_NE(nullptr, nodeInfo);
+
+    if (nodeInfo->status == flat::NodeStatus::HEARTBEAT_CONNECTED) {
+      XLOGF(INFO, "Test loop {}, try to stop #{} {}", testLoop, nodeIndex, nodeId);
+      auto stoppedStorage = stopAndRemoveStorageServer(nodeId);
+      ASSERT_TRUE(writeOK);
+
+      if (stoppedStorage) {
+        if (folly::Random::randBool(0.5)) {  // wait until mgmtd knows the failure
+          waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_FAILED);
+          XLOGF(INFO, "Test loop {}, stopped #{} {}", testLoop, nodeIndex, nodeId);
+          ASSERT_TRUE(writeOK);
+        }
+
+        // restart the storage service
+        XLOGF(INFO, "Test loop {}, try to start #{} {}", testLoop, nodeIndex, nodeId);
+        auto storageServer = createStorageServer(nodeIndex);
+        ASSERT_NE(storageServer, nullptr);
+        storageServers_.insert(storageServers_.begin() + nodeIndex, std::move(storageServer));
+        ASSERT_TRUE(writeOK);
+
+        waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED);
+        XLOGF(INFO, "Test loop {}, started #{} {}", testLoop, nodeIndex, nodeId);
+        ASSERT_TRUE(writeOK);
+      }
+    }
+
+    std::this_thread::sleep_for(waitTime);
+    ASSERT_TRUE(writeOK);
+
+    runWrite = false;
+    backgroundWorkers.clear();
+    ASSERT_TRUE(writeOK);
+
+    // read chunk data from the serving replicas
+
+    for (const auto &chainId : chainIds_) {
+      XLOGF(INFO, "Test loop {}, chain {}, reading chunks in range {} - {}", testLoop, chainId, chunkBegin, chunkEnd);
+      std::vector<std::vector<uint8_t>> chunkData;
+      auto readRes = readFromChunks(chainId, chunkBegin, chunkEnd, chunkData, 0, setupConfig_.chunk_size());
+      ASSERT_TRUE(readRes);
+    }
+
+    waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED, 200 /*maxRetries*/);
+    waitStorageTargetStatus(nodeTargets_[nodeId],
+                            flat::LocalTargetState::UPTODATE,
+                            flat::PublicTargetState::SERVING,
+                            1000 /*maxRetries*/);
+
+    // read chunk data from all replicas
+    for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
+      readAndCompareAllReplicas(chainIds_[chainIndex], chunkBegin, numChunks, writeResults[chainIndex]);
+    }
+  }
+}
+
+TEST_P(TestSingleProcessCluster, StorageResetUncommittedChunks) {
+  size_t numChunks = 5;
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size() - 1, 0x86);
+  auto chainId = chainIds_.front();
+
+  auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  // create write IOs
+  std::vector<client::WriteIO> writeIOs;
+  for (size_t i = 0; i < numChunks; ++i) {
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 ChunkId{1, i},
+                                                 1 /*offset*/,
+                                                 chunkData.size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &chunkData[0],
+                                                 &ioBuffer);
+    writeIOs.push_back(std::move(writeIO));
+  }
+
+  clientConfig_.retry().set_init_wait_time(1_s);
+  clientConfig_.retry().set_max_wait_time(1_s);
+  clientConfig_.retry().set_max_retry_time(1_s);
+  flat::UserInfo dummyUserInfo{};
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+  storage::client::WriteOptions options;
+  for (auto &writeIO : writeIOs) {
+    auto task = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  // stop tail server.
+  auto lastNodeId = storageNodeIds_.back();
+  stopAndRemoveStorageServer(lastNodeId);
+  std::this_thread::sleep_for(100_ms);
+
+  // restart head server.
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+  auto succ = std::count_if(writeIOs.begin(), writeIOs.end(), [](auto &w) { return bool(w.result.lengthInfo); });
+  XLOGF(INFO, "write chunks succ: {}", succ);
+  ASSERT_LE(succ, numChunks);
+  auto firstNodeId = storageNodeIds_.front();
+  stopAndRemoveStorageServer(firstNodeId);
+
+  auto storageServer = createStorageServer(0);
+  ASSERT_NE(storageServer, nullptr);
+  storageServers_.insert(storageServers_.end(), std::move(storageServer));
+  waitStorageTargetStatus(nodeTargets_[flat::NodeId{1}],
+                          flat::LocalTargetState::UPTODATE,
+                          flat::PublicTargetState::SERVING,
+                          1000);
+
+  clientConfig_.retry().set_init_wait_time(1_s);
+  clientConfig_.retry().set_max_wait_time(1_s);
+  clientConfig_.retry().set_max_retry_time(5_s);
+  std::vector<std::vector<uint8_t>> readResult;
+  ASSERT_TRUE(readFromChunks(chainId, ChunkId{1, 0}, ChunkId{1, numChunks}, readResult, 1, chunkData.size()));
+  ASSERT_EQ(readResult.size(), numChunks);
+  for (auto &line : readResult) {
+    ASSERT_EQ(line, chunkData);
+  }
+
+  // write again.
+  ASSERT_TRUE(writeToChunks(chainId, ChunkId{1, 0}, ChunkId{1, numChunks}, chunkData));
+}
+
+SystemSetupConfig testTwoReplicas = {
+    32_KB /*chunkSize*/,
+    16 /*numChains*/,
+    2 /*numReplicas*/,
+    2 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+    "" /*clientConfig*/,
+    "" /*serverConfig*/,
+    {} /*storageEndpoints*/,
+    0 /*serviceLevel*/,
+    0 /*listenPort*/,
+    storage::client::StorageClient::ImplementationType::RPC /*clientImplType*/,
+    kv::KVStore::Type::LevelDB /*metaStoreType*/,
+    false /*useFakeMgmtdClient*/,
+    true /*startStorageServer*/,
+};
+
+SystemSetupConfig testThreeReplicas = {
+    32_KB /*chunkSize*/,
+    16 /*numChains*/,
+    3 /*numReplicas*/,
+    3 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+    "" /*clientConfig*/,
+    "" /*serverConfig*/,
+    {} /*storageEndpoints*/,
+    0 /*serviceLevel*/,
+    0 /*listenPort*/,
+    storage::client::StorageClient::ImplementationType::RPC /*clientImplType*/,
+    kv::KVStore::Type::LevelDB /*metaStoreType*/,
+    false /*useFakeMgmtdClient*/,
+    true /*startStorageServer*/,
+};
+
+INSTANTIATE_TEST_SUITE_P(TwoReplicas,
+                         TestSingleProcessCluster,
+                         ::testing::Values(testTwoReplicas),
+                         SystemSetupConfig::prettyPrintConfig);
+
+INSTANTIATE_TEST_SUITE_P(ThreeReplicas,
+                         TestSingleProcessCluster,
+                         ::testing::Values(testThreeReplicas),
+                         SystemSetupConfig::prettyPrintConfig);
+
+}  // namespace
+}  // namespace hf3fs::storage

tests/storage/service/TestStorageService.cc

@@ -0,0 +1,165 @@
+#include <algorithm>
+#include <chrono>
+#include <thread>
+
+#include "client/storage/StorageClient.h"
+#include "common/kv/mem/MemKVEngine.h"
+#include "common/net/Client.h"
+#include "common/serde/Serde.h"
+#include "fbs/mgmtd/RoutingInfo.h"
+#include "fbs/storage/Service.h"
+#include "storage/service/StorageOperator.h"
+#include "storage/service/StorageServer.h"
+#include "storage/store/ChunkMetadata.h"
+#include "tests/GtestHelpers.h"
+#include "tests/client/ClientWithConfig.h"
+#include "tests/client/ServerWithConfig.h"
+#include "tests/lib/Helper.h"
+#include "tests/lib/UnitTestFabric.h"
+#include "tests/mgmtd/MgmtdTestHelper.h"
+
+namespace hf3fs::storage {
+namespace {
+
+using namespace hf3fs::test;
+class TestStorageForward : public UnitTestFabric, public ::testing::Test {
+ protected:
+  TestStorageForward()
+      : UnitTestFabric(SystemSetupConfig{
+            128_KB /*chunkSize*/,
+            1 /*numChains*/,
+            3 /*numReplicas*/,
+            3 /*numStorageNodes*/,
+            {folly::fs::temp_directory_path()} /*dataPaths*/,
+            hf3fs::Path() /*clientConfig*/,
+            hf3fs::Path() /*serverConfig*/,
+            {} /*storageEndpoints*/,
+            0 /*serviceLevel*/,
+            0 /*listenPort*/,
+            client::StorageClient::ImplementationType::RPC /*clientImplType*/,
+            kv::KVStore::Type::RocksDB /*metaStoreType*/,
+            true /*useFakeMgmtdClient*/,
+        }) {}
+
+  void SetUp() override {
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+};
+
+TEST_F(TestStorageForward, Write) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create write IO
+
+  auto ioBuffer = std::move(*regRes);
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  auto writeIO = storageClient_->createWriteIO(chainId,
+                                               chunkId,
+                                               0 /*offset*/,
+                                               setupConfig_.chunk_size() /*length*/,
+                                               setupConfig_.chunk_size() /*chunkSize*/,
+                                               &memoryBlock[0],
+                                               &ioBuffer);
+
+  client::WriteOptions options;
+
+  storageServers_.back()->stopAndJoin();
+  storageServers_.pop_back();
+
+  ASSERT_TRUE(updateRoutingInfo([&](auto &routingInfo) {
+    setTargetOffline(routingInfo, 0);
+    setTargetOffline(routingInfo, 1);
+  }));
+
+  folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+  ASSERT_OK(writeIO.result.lengthInfo);
+  ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+}
+
+TEST_F(TestStorageForward, WriteFailed) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create write IO
+
+  auto ioBuffer = std::move(*regRes);
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  auto writeIO = storageClient_->createWriteIO(chainId,
+                                               chunkId,
+                                               0 /*offset*/,
+                                               setupConfig_.chunk_size() /*length*/,
+                                               setupConfig_.chunk_size() /*chunkSize*/,
+                                               &memoryBlock[0],
+                                               &ioBuffer);
+
+  client::WriteOptions options;
+
+  storageServers_.back()->stopAndJoin();
+  storageServers_.pop_back();
+
+  clientConfig_.retry().set_max_retry_time(10_s);
+  folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+  ASSERT_FALSE(writeIO.result.lengthInfo);
+}
+
+TEST_F(TestStorageForward, WriteAndRead) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
+  memoryBlock[0] = 0x01;
+  memoryBlock[1] = 0x02;
+  memoryBlock[2] = 0x03;
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create write IO
+
+  auto ioBuffer = std::move(*regRes);
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  auto writeIO = storageClient_->createWriteIO(chainId,
+                                               chunkId,
+                                               0 /*offset*/,
+                                               3,
+                                               setupConfig_.chunk_size() /*chunkSize*/,
+                                               &memoryBlock[0],
+                                               &ioBuffer);
+
+  client::WriteOptions options;
+  folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+  ASSERT_OK(writeIO.result.lengthInfo);
+  ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+
+  {
+    auto readIO = storageClient_->createReadIO(chainId, chunkId, 0, 4096, &memoryBlock[0], &ioBuffer);
+    folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), client::ReadOptions{}));
+    ASSERT_OK(readIO.result.lengthInfo);
+    ASSERT_EQ(*readIO.result.lengthInfo, 3);
+    ASSERT_EQ(memoryBlock[0], 0x01);
+    ASSERT_EQ(memoryBlock[1], 0x02);
+    ASSERT_EQ(memoryBlock[2], 0x03);
+  }
+
+  {
+    auto readIO = storageClient_->createReadIO(chainId, chunkId, 1, 4096, &memoryBlock[0], &ioBuffer);
+    folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), client::ReadOptions{}));
+    ASSERT_OK(readIO.result.lengthInfo);
+    ASSERT_EQ(*readIO.result.lengthInfo, 2);
+    ASSERT_EQ(memoryBlock[0], 0x02);
+    ASSERT_EQ(memoryBlock[1], 0x03);
+  }
+}
+
+}  // namespace
+}  // namespace hf3fs::storage

tests/storage/service/TestStorageServiceFailStop.cc

@@ -0,0 +1,302 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage {
+namespace {
+
+using namespace hf3fs::test;
+
+using SystemFailureConfig =
+    std::tuple<uint32_t /*numReplicas*/, uint32_t /*failedTargetIndex*/, bool /*useFakeMgmtdClient*/>;
+
+std::string prettyPrintConfig(const testing::TestParamInfo<SystemFailureConfig> &info) {
+  return fmt::format("{}of{}failed_fakemgmtd{}",
+                     std::get<1>(info.param) + 1,
+                     std::get<0>(info.param),
+                     std::get<2>(info.param));
+}
+
+class TestStorageServiceFailStop : public UnitTestFabric, public ::testing::TestWithParam<SystemFailureConfig> {
+ protected:
+  TestStorageServiceFailStop()
+      : UnitTestFabric(SystemSetupConfig{
+            128_KB /*chunkSize*/,
+            1 /*numChains*/,
+            std::get<0>(GetParam()) /*numReplicas*/,
+            std::get<0>(GetParam()) /*numStorageNodes*/,
+            {folly::fs::temp_directory_path()} /*dataPaths*/,
+            hf3fs::Path() /*clientConfig*/,
+            hf3fs::Path() /*serverConfig*/,
+            {} /*storageEndpoints*/,
+            0 /*serviceLevel*/,
+            0 /*listenPort*/,
+            client::StorageClient::ImplementationType::RPC,
+            kv::KVStore::Type::RocksDB,
+            std::get<2>(GetParam()) /*useFakeMgmtdClient*/,
+        }),
+        failedTargetIndex_(std::get<1>(GetParam())) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+
+ protected:
+  uint32_t failedTargetIndex_;
+};
+
+TEST_P(TestStorageServiceFailStop, FailureUndetected) {
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  // stop the target
+  ASSERT_TRUE(stopAndRemoveStorageServer(failedTargetIndex_));
+
+  // write fails after the target stops undetectedly
+  clientConfig_.retry().set_max_retry_time(10_s);
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_TRUE(ioResult.lengthInfo.hasError());
+  if (failedTargetIndex_ == 0) {
+    switch (ioResult.lengthInfo.error().code()) {
+      case StorageClientCode::kCommError:
+      case StorageClientCode::kTimeout:
+        break;
+      default:
+        ASSERT_TRUE(ioResult.lengthInfo.error().code());
+    }
+  } else
+    ASSERT_EQ(StorageClientCode::kResourceBusy, ioResult.lengthInfo.error().code());
+}
+
+TEST_P(TestStorageServiceFailStop, FailureDetectedBeforeWrite) {
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  // stop the target
+  ASSERT_TRUE(stopAndRemoveStorageServer(failedTargetIndex_));
+
+  // set the target to offline state
+  ASSERT_TRUE(updateRoutingInfo([&](auto &routingInfo) { setTargetOffline(routingInfo, failedTargetIndex_); }));
+
+  // write succeeds after the failure is detected
+  if (storageServers_.empty()) clientConfig_.retry().set_max_retry_time(10_s);
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData, 0 /*offset*/, chunkData.size());
+
+  if (storageServers_.empty()) {
+    // only single replica
+    ASSERT_TRUE(ioResult.lengthInfo.hasError());
+    ASSERT_EQ(StorageClientCode::kNotAvailable, ioResult.lengthInfo.error().code());
+  } else {
+    // multiple replicas
+    ASSERT_OK(ioResult.lengthInfo);
+    ASSERT_EQ(chunkData.size(), ioResult.lengthInfo.value());
+  }
+}
+
+TEST_P(TestStorageServiceFailStop, FailureDetectedDuringRetry) {
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
+  ASSERT_OK(regRes);
+
+  // create write IO
+  auto ioBuffer = std::move(*regRes);
+  auto writeIO = storageClient_->createWriteIO(chainId,
+                                               chunkId,
+                                               0 /*offset*/,
+                                               chunkData.size() /*length*/,
+                                               setupConfig_.chunk_size() /*chunkSize*/,
+                                               &chunkData[0],
+                                               &ioBuffer);
+
+  // stop the target
+  ASSERT_TRUE(stopAndRemoveStorageServer(failedTargetIndex_));
+
+  // issue the write request
+  flat::UserInfo dummyUserInfo{};
+  auto options = client::WriteOptions();
+  options.retry().set_max_retry_time(5_s);
+  auto writeTask = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+
+  // retry for a short time
+  std::this_thread::sleep_for(2000_ms);
+  // check the request failed with communication error
+#if defined(__has_feature)
+#if !__has_feature(thread_sanitizer)
+  ASSERT_FALSE(writeIO.result.lengthInfo);
+  if (failedTargetIndex_ == 0) {
+    switch (writeIO.result.lengthInfo.error().code()) {
+      case StorageClientCode::kCommError:
+      case StorageClientCode::kTimeout:
+        break;
+      default:
+        ASSERT_TRUE(writeIO.result.lengthInfo.error().code());
+    }
+  } else
+    ASSERT_EQ(StorageClientCode::kResourceBusy, writeIO.result.lengthInfo.error().code());
+#endif
+#endif
+  // set the target to offline state
+  ASSERT_TRUE(updateRoutingInfo([&](auto &routingInfo) { setTargetOffline(routingInfo, failedTargetIndex_); }));
+
+  writeTask.wait();
+
+  if (storageServers_.empty()) {
+    // only single replica
+    ASSERT_TRUE(writeIO.result.lengthInfo.hasError());
+    ASSERT_EQ(StorageClientCode::kNotAvailable, writeIO.result.lengthInfo.error().code());
+  } else {
+    // multiple replicas
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(chunkData.size(), writeIO.result.lengthInfo.value());
+
+    // read and check chunk data
+    std::vector<uint8_t> readData(chunkData.size());
+    auto readRes = readFromChunk(chainId, chunkId, readData);
+    ASSERT_OK(readRes.lengthInfo);
+    ASSERT_EQ(1, readRes.commitVer);
+    ASSERT_EQ(1, readRes.updateVer);
+    ASSERT_EQ(chunkData.size(), *readRes.lengthInfo);
+    ASSERT_EQ(chunkData, readData);
+  }
+}
+
+TEST_P(TestStorageServiceFailStop, ConcurrentWrites) {
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  size_t numWriteIOs = 10;
+  std::vector<client::WriteIO> writeIOs;
+
+  for (size_t writeIndex = 0; writeIndex < numWriteIOs; writeIndex++) {
+    // create write IO
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 chunkData.size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &chunkData[0],
+                                                 &ioBuffer);
+    writeIOs.push_back(std::move(writeIO));
+  }
+
+  // stop the target
+  ASSERT_TRUE(stopAndRemoveStorageServer(failedTargetIndex_));
+
+  // issue the write request
+  flat::UserInfo dummyUserInfo{};
+  auto options = client::WriteOptions();
+  options.retry().set_max_retry_time(10_s);
+
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> writeTasks;
+
+  for (auto &writeIO : writeIOs) {
+    auto writeTask = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    writeTasks.push_back(std::move(writeTask));
+  }
+
+  // retry for a short time
+  std::this_thread::sleep_for(1500_ms);
+  // check the request failed with communication error
+#if defined(__has_feature)
+#if !__has_feature(thread_sanitizer)
+  for (auto &writeIO : writeIOs) {
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    if (failedTargetIndex_ == 0) {
+      switch (writeIO.result.lengthInfo.error().code()) {
+        case StorageClientCode::kCommError:
+        case StorageClientCode::kTimeout:
+          break;
+        default:
+          ASSERT_TRUE(writeIO.result.lengthInfo.error().code());
+      }
+    } else
+      ASSERT_EQ(StorageClientCode::kResourceBusy, writeIO.result.lengthInfo.error().code());
+  }
+#endif
+#endif
+
+  // set the target to offline state
+  ASSERT_TRUE(updateRoutingInfo([&](auto &routingInfo) { setTargetOffline(routingInfo, failedTargetIndex_); }));
+
+  // wait until all write IOs completed
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));
+
+  if (!storageServers_.empty()) {
+    std::set<ChunkVer> updateVersions;
+    for (const auto &writeIO : writeIOs) {
+      ASSERT_OK(writeIO.result.lengthInfo);
+      // check commit version == update version
+      ASSERT_EQ(writeIO.result.commitVer, writeIO.result.updateVer);
+      // check the update versions are in range [1..numWriteIOs]
+      ASSERT_LE(1, writeIO.result.updateVer);
+      ASSERT_LE(writeIO.result.updateVer, numWriteIOs);
+      updateVersions.insert(writeIO.result.updateVer);
+    }
+    // check the update versions are unique
+    ASSERT_EQ(numWriteIOs, updateVersions.size());
+  }
+
+  for (auto &writeIO : writeIOs) {
+    if (storageServers_.empty()) {
+      // only single replica
+      ASSERT_TRUE(writeIO.result.lengthInfo.hasError());
+      ASSERT_EQ(StorageClientCode::kNotAvailable, writeIO.result.lengthInfo.error().code());
+    } else {
+      // multiple replicas
+      ASSERT_OK(writeIO.result.lengthInfo);
+      ASSERT_EQ(chunkData.size(), writeIO.result.lengthInfo.value());
+
+      // read and check chunk data
+      std::vector<uint8_t> readData(chunkData.size());
+      auto readRes = readFromChunk(chainId, chunkId, readData);
+      ASSERT_OK(readRes.lengthInfo);
+      ASSERT_EQ(numWriteIOs, readRes.commitVer);
+      ASSERT_EQ(numWriteIOs, readRes.updateVer);
+      ASSERT_EQ(chunkData.size(), *readRes.lengthInfo);
+      ASSERT_EQ(chunkData, readData);
+    }
+  }
+}
+
+INSTANTIATE_TEST_SUITE_P(SingleReplica,
+                         TestStorageServiceFailStop,
+                         ::testing::Combine(::testing::Values(1) /*numReplicas*/,
+                                            ::testing::Values(0) /*failedTargetIndex*/,
+                                            ::testing::Values(true /*useFakeMgmtdClient*/)),
+                         prettyPrintConfig);
+
+INSTANTIATE_TEST_SUITE_P(TwoReplicas,
+                         TestStorageServiceFailStop,
+                         ::testing::Combine(::testing::Values(2) /*numReplicas*/,
+                                            ::testing::Values(0, 1) /*failedTargetIndex*/,
+                                            ::testing::Values(true /*useFakeMgmtdClient*/)),
+                         prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(ThreeReplicas,
+                         TestStorageServiceFailStop,
+                         ::testing::Combine(::testing::Values(3) /*numReplicas*/,
+                                            ::testing::Values(0, 1, 2) /*failedTargetIndex*/,
+                                            ::testing::Values(true /*useFakeMgmtdClient*/)),
+                         prettyPrintConfig);
+
+}  // namespace
+}  // namespace hf3fs::storage

tests/storage/store/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_test(test_storage_store test-fabric-lib)

tests/storage/store/TestBufferPool.cc

@@ -0,0 +1,56 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "common/serde/Serde.h"
+#include "common/utils/Reflection.h"
+#include "fbs/storage/Common.h"
+#include "fbs/storage/Service.h"
+#include "storage/aio/AioReadWorker.h"
+#include "storage/service/BufferPool.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+TEST(TestBufferPool, Normal) {
+  CPUExecutorGroup executor(8, "");
+
+  BufferPool::Config bufferPoolConfig;
+  bufferPoolConfig.set_rdmabuf_count(256);
+  bufferPoolConfig.set_big_rdmabuf_count(4);
+  AioReadWorker::Config aioReadWorkerConfig;
+  aioReadWorkerConfig.set_num_threads(8);
+  for (auto i = 0; i < 8; ++i) {
+    BufferPool pool(bufferPoolConfig);
+    ASSERT_OK(pool.init(executor));
+
+    AioReadWorker worker(aioReadWorkerConfig);
+    ASSERT_OK(worker.start({}, pool.iovecs()));
+    ASSERT_OK(worker.stopAndJoin());
+
+    pool.clear(executor);
+  }
+}
+
+TEST(TestBufferPool, BigBuffer) {
+  CPUExecutorGroup executor(8, "");
+
+  BufferPool::Config bufferPoolConfig;
+  bufferPoolConfig.set_rdmabuf_count(256);
+  bufferPoolConfig.set_rdmabuf_size(4_MB);
+  bufferPoolConfig.set_big_rdmabuf_count(4);
+  bufferPoolConfig.set_big_rdmabuf_size(64_MB);
+
+  BufferPool pool(bufferPoolConfig);
+  ASSERT_OK(pool.init(executor));
+
+  auto guard = pool.get();
+  auto result = folly::coro::blockingWait(guard.allocate(64_MB));
+  ASSERT_OK(result);
+  ASSERT_EQ(result->size(), 64_MB);
+
+  pool.clear(executor);
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/store/TestChunkEngine.cc

@@ -0,0 +1,217 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/hash/Checksum.h>
+#include <folly/logging/xlog.h>
+
+#include "chunk_engine/src/cxx.rs.h"
+#include "common/utils/Size.h"
+#include "fbs/storage/Common.h"
+#include "storage/aio/AioReadWorker.h"
+#include "storage/aio/BatchReadJob.h"
+#include "storage/service/BufferPool.h"
+#include "storage/service/TargetMap.h"
+#include "storage/store/ChunkStore.h"
+#include "storage/store/StorageTarget.h"
+#include "storage/update/UpdateJob.h"
+#include "storage/update/UpdateWorker.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+TEST(TestChunkEngine, ReadWrite) {
+  folly::test::TemporaryDirectory tmpPath;
+  CPUExecutorGroup executor(8, "");
+
+  constexpr TargetId targetId{1};
+  constexpr auto chunkSize = 512_KB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.storage_target().file_store().set_preopen_chunk_size_list({chunkSize});
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  {
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({chunkSize});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({targetId});
+
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(storageTargetConfig, targetMap);
+    ASSERT_OK(targets.create(createConfig));
+  }
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    UpdateWorker::Config updateWorkerConfig;
+    UpdateWorker updateWorker(updateWorkerConfig);
+    ASSERT_OK(updateWorker.start(1));
+
+    BufferPool::Config bufferPoolConfig;
+    bufferPoolConfig.set_rdmabuf_count(4);
+    bufferPoolConfig.set_big_rdmabuf_count(1);
+    BufferPool pool(bufferPoolConfig);
+    ASSERT_OK(pool.init(executor));
+
+    AioReadWorker::Config aioReadWorkerConfig;
+    AioReadWorker aioReadWorker(aioReadWorkerConfig);
+    ASSERT_OK(aioReadWorker.start(storageTargets.fds(), pool.iovecs()));
+
+    const auto chunkEngine = 1ul << 40;
+    for (auto high : {0ul, chunkEngine}) {
+      auto chunkId = ChunkId(high, 1);
+
+      // aio read.
+      {
+        BatchReadReq req;
+        req.payloads.emplace_back();
+        auto &aio = req.payloads.front();
+        aio.key.chunkId = chunkId;
+        aio.length = chunkSize;
+        aio.offset = 0;
+
+        BatchReadRsp rsp;
+        rsp.results.resize(1);
+        auto job = std::make_unique<BatchReadJob>(req.payloads, rsp.results, ChecksumType::NONE);
+        job->front().state().storageTarget = storageTarget.get();
+
+        ASSERT_EQ(storageTarget->aioPrepareRead(job->front()).error().code(), StorageCode::kChunkMetadataNotFound);
+      }
+
+      ChunkEngineUpdateJob updateChunk{};
+
+      // write a chunk.
+      {
+        UpdateIO writeIO;
+        writeIO.key.chunkId = chunkId;
+        writeIO.chunkSize = chunkSize;
+        writeIO.length = chunkSize;
+        writeIO.offset = 0;
+        writeIO.updateVer = ChunkVer{1};
+        writeIO.updateType = UpdateType::WRITE;
+
+        auto data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+        writeIO.checksum = ChecksumInfo::create(ChecksumType::CRC32C, data, chunkSize);
+
+        UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+        updateJob.state().data = data;
+
+        folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+        folly::coro::blockingWait(updateJob.complete());
+        ASSERT_OK(updateJob.result().lengthInfo);
+        ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+      }
+
+      // aio read.
+      {
+        BatchReadReq req;
+        req.payloads.emplace_back();
+        auto &aio = req.payloads.front();
+        aio.key.chunkId = chunkId;
+        aio.length = chunkSize;
+        aio.offset = 0;
+
+        BatchReadRsp rsp;
+        rsp.results.resize(1);
+        auto job = std::make_unique<BatchReadJob>(req.payloads, rsp.results, ChecksumType::NONE);
+        job->front().state().storageTarget = storageTarget.get();
+
+        ASSERT_TRUE(storageTarget->aioPrepareRead(job->front()).hasError());
+      }
+
+      // commit a chunk.
+      {
+        CommitIO commitIO;
+        commitIO.key.chunkId = chunkId;
+        commitIO.commitVer = ChunkVer{1};
+
+        UpdateJob commitJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+
+        folly::coro::blockingWait(updateWorker.enqueue(&commitJob));
+        folly::coro::blockingWait(commitJob.complete());
+        ASSERT_OK(commitJob.result().lengthInfo);
+      }
+
+      // aio read again.
+      {
+        BatchReadReq req;
+        req.payloads.emplace_back();
+        auto &aio = req.payloads.front();
+        aio.key.chunkId = chunkId;
+        aio.length = chunkSize;
+        aio.offset = 0;
+
+        BatchReadRsp rsp;
+        rsp.results.resize(1);
+        auto job = std::make_unique<BatchReadJob>(req.payloads, rsp.results, ChecksumType::NONE);
+        job->front().state().storageTarget = storageTarget.get();
+        auto buffer = pool.get();
+        job->front().state().localbuf = buffer.tryAllocate(chunkSize).value();
+
+        ASSERT_TRUE(storageTarget->aioPrepareRead(job->front()).hasValue());
+
+        folly::coro::blockingWait(aioReadWorker.enqueue(AioReadJobIterator(job.get())));
+        folly::coro::blockingWait(job->complete());
+        ASSERT_TRUE(job->front().result().lengthInfo);
+        ASSERT_EQ(job->front().result().lengthInfo.value(), chunkSize);
+        std::string_view out{(const char *)job->front().state().localbuf.ptr(), chunkSize};
+        ASSERT_EQ(out, dataBytes);
+      }
+
+      // query chunks.
+      {
+        ChunkIdRange chunkIdRange;
+        chunkIdRange.begin = ChunkId(high, 0);
+        chunkIdRange.end = ChunkId(high + 1, 0);
+        chunkIdRange.maxNumChunkIdsToProcess = 128;
+        auto chunks = storageTarget->queryChunks(chunkIdRange);
+        ASSERT_TRUE(chunks);
+        ASSERT_EQ(chunks->size(), 1);
+        ASSERT_EQ(chunks->front().first, chunkId);
+        auto &meta = chunks->front().second;
+        ASSERT_EQ(meta.size, chunkSize);
+
+        auto result = storageTarget->queryChunk(chunkId);
+        ASSERT_TRUE(result);
+        ASSERT_EQ(meta, *result);
+      }
+
+      // query chunks (optimize).
+      {
+        ChunkIdRange chunkIdRange;
+        chunkIdRange.begin = ChunkId(high, 1);
+        chunkIdRange.end = chunkIdRange.begin.nextChunkId();
+        chunkIdRange.maxNumChunkIdsToProcess = 128;
+        auto chunks = storageTarget->queryChunks(chunkIdRange);
+        ASSERT_TRUE(chunks);
+        ASSERT_EQ(chunks->size(), 1);
+        ASSERT_EQ(chunks->front().first, chunkId);
+        auto &meta = chunks->front().second;
+        ASSERT_EQ(meta.size, chunkSize);
+
+        auto result = storageTarget->queryChunk(chunkId);
+        ASSERT_TRUE(result);
+        ASSERT_EQ(meta, *result);
+      }
+    }
+
+    ChunkMetaVector metadataVec;
+    ASSERT_TRUE(storageTarget->getAllMetadata(metadataVec));
+    ASSERT_EQ(metadataVec.size(), 2);
+    ASSERT_GT(metadataVec.front().chunkId, metadataVec.back().chunkId);
+  }
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/store/TestChunkMetaStore.cc

@@ -0,0 +1,108 @@
+#include <chrono>
+#include <thread>
+#include <vector>
+
+#include "common/utils/Size.h"
+#include "storage/store/ChunkFileStore.h"
+#include "storage/store/ChunkMetaStore.h"
+#include "storage/store/PhysicalConfig.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+using namespace std::chrono_literals;
+
+TEST(TestChunkMetaStore, Normal) {
+  folly::test::TemporaryDirectory tmpPath;
+
+  kv::KVStore::Config config;
+  PhysicalConfig targetConfig;
+  targetConfig.path = tmpPath.path();
+  targetConfig.physical_file_count = 8;
+  targetConfig.chunk_size_list = {128_KB};
+
+  GlobalFileStore globalFileStore;
+  ChunkFileStore::Config fileStoreConfig;
+  ChunkFileStore store(fileStoreConfig, globalFileStore);
+  ASSERT_OK(store.create(targetConfig));
+
+  auto chunkSize = targetConfig.chunk_size_list.front();
+  ChunkMetaStore::Config metaConfig;
+  ChunkMetaStore metaStore(metaConfig, store);
+  ASSERT_OK(metaStore.create(config, targetConfig));
+
+  const ChunkId chunkId{0xBeef, 0xBeef};
+
+  {
+    ChunkMetadata meta;
+    auto result = metaStore.get(chunkId, meta);
+    ASSERT_FALSE(result);
+    ASSERT_EQ(result.error().code(), StorageCode::kChunkMetadataNotFound);
+  }
+
+  {
+    ChunkMetadata meta;
+    meta.innerFileId.chunkSize = chunkSize;
+    auto result = metaStore.set(chunkId, meta);
+    ASSERT_OK(result);
+  }
+
+  {
+    ChunkMetadata meta;
+    auto result = metaStore.get(chunkId, meta);
+    ASSERT_OK(result);
+    ASSERT_EQ(meta.innerFileId.chunkSize, chunkSize);
+
+    auto removeResult = metaStore.remove(chunkId, meta);
+    ASSERT_OK(removeResult);
+  }
+
+  {
+    ChunkMetadata meta;
+    auto result = metaStore.get(chunkId, meta);
+    ASSERT_FALSE(result);
+    ASSERT_EQ(result.error().code(), StorageCode::kChunkMetadataNotFound);
+  }
+}
+
+TEST(TestChunkMetaStore, Iterator) {
+  folly::test::TemporaryDirectory tmpPath;
+
+  kv::KVStore::Config config;
+  PhysicalConfig targetConfig;
+  targetConfig.path = tmpPath.path();
+  targetConfig.physical_file_count = 8;
+  targetConfig.chunk_size_list = {128_KB};
+
+  GlobalFileStore globalFileStore;
+  ChunkFileStore::Config fileStoreConfig;
+  ChunkFileStore store(fileStoreConfig, globalFileStore);
+  ASSERT_OK(store.create(targetConfig));
+
+  ChunkMetaStore::Config metaConfig;
+  ChunkMetaStore metaStore(metaConfig, store);
+  ASSERT_OK(metaStore.create(config, targetConfig));
+
+  constexpr uint64_t P = 0xBeef;
+  constexpr uint64_t N = 8;
+  for (uint64_t i = 1; i <= N; ++i) {
+    ChunkMetadata meta;
+    meta.innerFileId.chunkSize = 128_KB;
+    const ChunkId chunkId{P, i};
+    auto result = metaStore.set(chunkId, meta);
+    ASSERT_OK(result);
+  }
+
+  auto iterator = metaStore.iterator(ChunkId(P, 0).data().substr(0, 8));
+  ASSERT_OK(iterator);
+  for (uint64_t i = 0; i < N; ++i) {
+    ASSERT_TRUE(iterator->valid());
+    ASSERT_EQ(iterator->chunkId(), (ChunkId{P, N - i}));
+    iterator->next();
+  }
+  ASSERT_FALSE(iterator->valid());
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/store/TestCommonStruct.cc

@@ -0,0 +1,84 @@
+#include "common/serde/Serde.h"
+#include "common/utils/Reflection.h"
+#include "fbs/storage/Common.h"
+#include "fbs/storage/Service.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+TEST(TestCommonStruct, Normal) {
+  {
+    ChunkId ser(0xface, 0xbeef);
+    auto out = serde::serialize(ser);
+    ASSERT_EQ(out.size(), 1 + 8 + 8);
+
+    ChunkId des;
+    ASSERT_OK(serde::deserialize(des, out));
+    ASSERT_EQ(des, ser);
+
+    auto str = ser.describe();
+    auto result = ChunkId::fromString(str);
+    ASSERT_OK(result);
+    ASSERT_EQ(*result, ser);
+
+    auto next = ChunkId{0xface, 0xbef0};
+    ASSERT_EQ(ser.nextChunkId(), next);
+  }
+
+  {
+    auto id = ChunkId{0x0000, 0x007f};
+    auto next = ChunkId{0x0000, 0x0080};
+    ASSERT_EQ(id.nextChunkId(), next);
+  }
+  {
+    auto id = ChunkId{0x00ff, 0xffffffffffffffffull};
+    auto next = ChunkId{0x0100, 0x0000000000000000ull};
+    ASSERT_EQ(id.nextChunkId(), next);
+  }
+
+  {
+    std::string str = "00000000-00000346-85270000-0000000B";
+    auto chunk = ChunkId::fromString(str);
+    ASSERT_EQ(chunk->describe(), str);
+  }
+
+  {
+    ChecksumInfo ser;
+    ser.type = ChecksumType::CRC32;
+    ser.value = 0xff;
+    auto out = serde::serialize(ser);
+    ASSERT_EQ(out.size(), 1 + 1 + 4);
+
+    ChecksumInfo des;
+    ASSERT_OK(serde::deserialize(des, out));
+    ASSERT_EQ(des, ser);
+  }
+
+  {
+    BatchReadReq req;
+    req.payloads.emplace_back();
+    req.payloads.back().key.chunkId = ChunkId(1, 1);
+    req.payloads.back().rdmabuf.rkeys()[0].devId = 0;
+    auto out = serde::serialize(req);
+    XLOGF(INFO, "single read req size: {}, json: {}", out.length(), req);
+
+    auto front = req.payloads.front();
+    req.payloads.resize(16, front);
+    out = serde::serialize(req);
+    XLOGF(INFO, "10 read req size: {}", out.length());
+  }
+
+  refl::Helper::iterate<StorageSerde<>>([](auto type) {
+    using T = decltype(type);
+    XLOGF(INFO,
+          "method: {}, ID: {}\n  Req: {}\n  Rsp: {}",
+          T::name,
+          T::id,
+          serde::toJsonString(typename T::ReqType{}),
+          serde::toJsonString(typename T::RspType{}));
+  });
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/store/TestStorageTarget.cc

@@ -0,0 +1,531 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "common/utils/Size.h"
+#include "fbs/storage/Common.h"
+#include "storage/aio/AioReadWorker.h"
+#include "storage/service/TargetMap.h"
+#include "storage/store/ChunkStore.h"
+#include "storage/store/StorageTarget.h"
+#include "storage/update/UpdateWorker.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage {
+namespace {
+
+TEST(TestStorageTarget, ReadWrite) {
+  folly::test::TemporaryDirectory tmpPath;
+  CPUExecutorGroup executor(8, "");
+
+  const auto chunkId = ChunkId{12345, 12345};
+  constexpr TargetId targetId{0};
+  constexpr auto chunkSize = 1_MB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.storage_target().file_store().set_preopen_chunk_size_list({512_KB, 1_MB});
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  {
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({1_MB});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({targetId});
+
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(storageTargetConfig, targetMap);
+    ASSERT_OK(targets.create(createConfig));
+  }
+
+  for (auto loop = 0; loop < 3; ++loop) {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    ASSERT_EQ(storageTargets.fds().size(), 8 * bool(loop) + 8);
+
+    UpdateWorker::Config updateWorkerConfig;
+    UpdateWorker updateWorker(updateWorkerConfig);
+    ASSERT_OK(updateWorker.start(1));
+
+    AioReadWorker::Config aioReadWorkerConfig;
+    AioReadWorker aioReadWorker(aioReadWorkerConfig);
+    ASSERT_OK(aioReadWorker.start(storageTargets.fds(), {}));
+
+    auto chunkSize = loop == 0 ? 1_MB : Size{256_KB * loop};
+    ASSERT_OK(storageTarget->addChunkSize({chunkSize}));
+    ASSERT_OK(storageTarget->addChunkSize({chunkSize}));
+
+    ChunkEngineUpdateJob updateChunk{};
+
+    // write data to chunk.
+    {
+      UpdateIO writeIO;
+      writeIO.key.chunkId = chunkId;
+      writeIO.chunkSize = chunkSize;
+      writeIO.length = chunkSize;
+      writeIO.offset = 0;
+      writeIO.updateVer = ChunkVer{1};
+      writeIO.updateType = UpdateType::WRITE;
+
+      UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+      updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{0});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+    }
+
+    ASSERT_EQ(storageTarget->usedSize(), chunkSize);
+
+    // aio read uncommit chunk.
+    {
+      std::string buf(chunkSize, '\0');
+      BatchReadReq req;
+      req.payloads.emplace_back();
+      auto &aio = req.payloads.front();
+      aio.key.chunkId = chunkId;
+      aio.length = chunkSize;
+      aio.offset = 0;
+      // aio.addr = ...;
+
+      BatchReadRsp rsp;
+      rsp.results.resize(1);
+      auto job = std::make_unique<BatchReadJob>(req.payloads, rsp.results, ChecksumType::NONE);
+      job->front().state().storageTarget = storageTarget.get();
+      ASSERT_EQ(storageTarget->aioPrepareRead(job->front()).error().code(), StorageCode::kChunkNotCommit);
+    }
+
+    // commit chunk.
+    {
+      CommitIO commitIO;
+      commitIO.key.chunkId = chunkId;
+      commitIO.commitVer = ChunkVer{1};
+
+      auto updateJob = UpdateJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+      updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{1});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+    }
+
+    // remove chunk.
+    {
+      UpdateIO removeIO;
+      removeIO.key.chunkId = chunkId;
+      removeIO.updateType = UpdateType::REMOVE;
+
+      auto updateJob = UpdateJob(requestCtx, removeIO, {}, updateChunk, storageTarget);
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{1});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{2});
+    }
+
+    ASSERT_EQ(storageTarget->usedSize(), chunkSize);
+
+    // commit remove.
+    {
+      CommitIO commitIO;
+      commitIO.key.chunkId = chunkId;
+      commitIO.commitVer = ChunkVer{2};
+      auto updateJob = UpdateJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{2});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{2});
+    }
+
+    // remove non-existent chunk.
+    {
+      UpdateIO removeIO;
+      removeIO.key.chunkId = chunkId;
+      removeIO.updateType = UpdateType::REMOVE;
+
+      auto updateJob = UpdateJob(requestCtx, removeIO, {}, updateChunk, storageTarget);
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+    }
+
+    // create new target.
+    {
+      CreateTargetReq req;
+      req.targetId = TargetId{7};
+      req.physicalFileCount = 8;
+      req.chunkSizeList = {1_MB};
+      req.allowExistingTarget = false;
+      req.chainId = ChainId{1};
+      if (loop == 0) {
+        ASSERT_OK(storageTargets.create(req));
+      } else {
+        ASSERT_FALSE(storageTargets.create(req));
+      }
+    }
+
+    ASSERT_EQ(storageTarget->usedSize(), 0);
+  }
+}
+
+TEST(TestLocalTargetStateManager, UpdateLocalState) {
+  using LS = enum hf3fs::flat::LocalTargetState;
+  using PS = enum hf3fs::flat::PublicTargetState;
+
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::SERVING), LS::UPTODATE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::LASTSRV), LS::ONLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::SYNCING), LS::ONLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::WAITING), LS::ONLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::ONLINE, PS::OFFLINE), LS::ONLINE);
+
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::SERVING), LS::UPTODATE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::LASTSRV), LS::OFFLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::SYNCING), LS::UPTODATE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::WAITING), LS::OFFLINE);
+  ASSERT_EQ(TargetMap::updateLocalState(TargetId{0}, LS::UPTODATE, PS::OFFLINE), LS::OFFLINE);
+}
+
+TEST(TestStorageTarget, Uncommitted) {
+  folly::test::TemporaryDirectory tmpPath;
+  CPUExecutorGroup executor(8, "");
+
+  const auto chunkId = ChunkId{12345, 12345};
+  constexpr TargetId targetId{0};
+  constexpr auto chunkSize = 1_MB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  ChunkEngineUpdateJob updateChunk{};
+
+  {
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({1_MB});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({targetId});
+    createConfig.set_only_chunk_engine(true);
+
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(storageTargetConfig, targetMap);
+    ASSERT_OK(targets.create(createConfig));
+
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    UpdateWorker::Config updateWorkerConfig;
+    UpdateWorker updateWorker(updateWorkerConfig);
+    ASSERT_OK(updateWorker.start(1));
+
+    // write data to chunk.
+    UpdateIO writeIO;
+    writeIO.key.chunkId = chunkId;
+    writeIO.chunkSize = chunkSize;
+    writeIO.length = chunkSize;
+    writeIO.offset = 0;
+    writeIO.updateVer = ChunkVer{1};
+    writeIO.updateType = UpdateType::WRITE;
+
+    UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+    updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+    folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+    folly::coro::blockingWait(updateJob.complete());
+    ASSERT_OK(updateJob.result().lengthInfo);
+    ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+    // ASSERT_EQ(updateJob.result().commitVer, ChunkVer{0});
+    ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+
+    // commit it.
+    CommitIO commitIO;
+    commitIO.commitVer = ChunkVer{1};
+    commitIO.key.chunkId = chunkId;
+    UpdateJob commitJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+    folly::coro::blockingWait(updateWorker.enqueue(&commitJob));
+    folly::coro::blockingWait(commitJob.complete());
+    ASSERT_OK(commitJob.result().lengthInfo);
+    ASSERT_EQ(commitJob.result().commitVer, ChunkVer{1});
+    ASSERT_EQ(commitJob.result().updateVer, ChunkVer{1});
+
+    // write again.
+    writeIO.updateVer = ChunkVer{2};
+    UpdateJob updateJob2(requestCtx, writeIO, {}, updateChunk, storageTarget);
+    updateJob2.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+    folly::coro::blockingWait(updateWorker.enqueue(&updateJob2));
+    folly::coro::blockingWait(updateJob2.complete());
+    ASSERT_OK(updateJob2.result().lengthInfo);
+    ASSERT_EQ(updateJob2.result().lengthInfo.value(), chunkSize);
+    // ASSERT_EQ(updateJob2.result().commitVer, ChunkVer{1});
+    ASSERT_EQ(updateJob2.result().updateVer, ChunkVer{2});
+  }
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    auto uncommitted = storageTarget->uncommitted().value();
+    ASSERT_EQ(uncommitted.size(), 1);
+    ASSERT_EQ(uncommitted.front(), chunkId);
+
+    ASSERT_OK(storageTarget->resetUncommitted(ChainVer{1}));
+
+    auto metaResult = storageTarget->queryChunk(chunkId);
+    ASSERT_OK(metaResult);
+    // ASSERT_EQ(metaResult->lastClientUuid, Uuid::max());
+  }
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    auto uncommitted = storageTarget->uncommitted().value();
+    ASSERT_TRUE(uncommitted.empty());
+  }
+}
+
+TEST(TestStorageTarget, UnRecycle) {
+  folly::test::TemporaryDirectory tmpPath;
+
+  const auto chunkId = ChunkId{12345, 12345};
+  constexpr TargetId targetId{0};
+  constexpr auto chunkSize = 1_MB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.storage_target().meta_store().set_removed_chunk_expiration_time(0_s);
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  StorageTargets::CreateConfig createConfig;
+  createConfig.set_chunk_size_list({1_MB});
+  createConfig.set_physical_file_count(8);
+  createConfig.set_allow_disk_without_uuid(true);
+  createConfig.set_target_ids({targetId});
+
+  AtomicallyTargetMap targetMap;
+  StorageTargets targets(storageTargetConfig, targetMap);
+  ASSERT_OK(targets.create(createConfig));
+
+  auto targetResult = targetMap.snapshot()->getTarget(targetId);
+  ASSERT_OK(targetResult);
+  auto storageTarget = (*targetResult)->storageTarget;
+
+  UpdateWorker::Config updateWorkerConfig;
+  UpdateWorker updateWorker(updateWorkerConfig);
+  ASSERT_OK(updateWorker.start(1));
+
+  ChunkEngineUpdateJob updateChunk{};
+
+  // 1. write data to chunk.
+  {
+    UpdateIO writeIO;
+    writeIO.key.chunkId = chunkId;
+    writeIO.chunkSize = chunkSize;
+    writeIO.length = chunkSize;
+    writeIO.offset = 0;
+    writeIO.updateVer = ChunkVer{1};
+    writeIO.updateType = UpdateType::WRITE;
+
+    UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+    updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+    folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+    folly::coro::blockingWait(updateJob.complete());
+    ASSERT_OK(updateJob.result().lengthInfo);
+    ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+    ASSERT_EQ(updateJob.result().commitVer, ChunkVer{0});
+    ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+
+    CommitIO commitIO;
+    commitIO.commitVer = ChunkVer{1};
+    commitIO.key.chunkId = chunkId;
+    UpdateJob commitJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+    folly::coro::blockingWait(updateWorker.enqueue(&commitJob));
+    folly::coro::blockingWait(commitJob.complete());
+    ASSERT_OK(commitJob.result().lengthInfo);
+    ASSERT_EQ(commitJob.result().commitVer, ChunkVer{1});
+    ASSERT_EQ(commitJob.result().updateVer, ChunkVer{1});
+  }
+
+  auto metaResult = storageTarget->queryChunk(chunkId);
+  ASSERT_OK(metaResult);
+  auto recycleResult = storageTarget->punchHole();
+  ASSERT_OK(recycleResult);
+  ASSERT_EQ(*recycleResult, true);
+
+  // 2. remove this chunk.
+  {
+    UpdateIO writeIO;
+    writeIO.key.chunkId = chunkId;
+    writeIO.chunkSize = chunkSize;
+    writeIO.length = chunkSize;
+    writeIO.offset = 0;
+    writeIO.updateVer = ChunkVer{2};
+    writeIO.updateType = UpdateType::REMOVE;
+
+    UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+    updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+    folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+    folly::coro::blockingWait(updateJob.complete());
+    ASSERT_OK(updateJob.result().lengthInfo);
+    ASSERT_EQ(updateJob.result().commitVer, ChunkVer{1});
+    ASSERT_EQ(updateJob.result().updateVer, ChunkVer{2});
+
+    CommitIO commitIO;
+    commitIO.commitVer = ChunkVer{2};
+    commitIO.key.chunkId = chunkId;
+    UpdateJob commitJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+    folly::coro::blockingWait(updateWorker.enqueue(&commitJob));
+    folly::coro::blockingWait(commitJob.complete());
+    ASSERT_OK(commitJob.result().lengthInfo);
+    ASSERT_EQ(commitJob.result().commitVer, ChunkVer{2});
+    ASSERT_EQ(commitJob.result().updateVer, ChunkVer{2});
+  }
+
+  recycleResult = storageTarget->punchHole();
+  ASSERT_OK(recycleResult);
+  ASSERT_EQ(*recycleResult, true);
+}
+
+TEST(TestStorageTarget, Migrate) {
+  constexpr auto N = 16u;
+  folly::test::TemporaryDirectory tmpPath;
+  CPUExecutorGroup executor(8, "");
+
+  const auto chunkId = ChunkId{12345, 12345};
+  constexpr TargetId targetId{0};
+  constexpr auto chunkSize = 1_MB;
+  std::string dataBytes(chunkSize, 'B');
+  ServiceRequestContext requestCtx;
+
+  StorageTargets::Config storageTargetConfig;
+  storageTargetConfig.set_target_num_per_path(1);
+  storageTargetConfig.set_target_paths({tmpPath.path()});
+  storageTargetConfig.storage_target().kv_store().set_type(kv::KVStore::Type::LevelDB);
+  storageTargetConfig.set_allow_disk_without_uuid(true);
+
+  ChunkEngineUpdateJob updateChunk{};
+
+  // 1. create with LevelDB.
+  {
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({1_MB});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({targetId});
+
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(storageTargetConfig, targetMap);
+    ASSERT_OK(targets.create(createConfig));
+  }
+
+  // 2. write some chunks.
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    UpdateWorker::Config updateWorkerConfig;
+    UpdateWorker updateWorker(updateWorkerConfig);
+    ASSERT_OK(updateWorker.start(1));
+
+    // write data to chunk.
+    for (auto i = 0u; i < N; ++i) {
+      UpdateIO writeIO;
+      writeIO.key.chunkId = ChunkId{0, i};
+      writeIO.chunkSize = chunkSize;
+      writeIO.length = chunkSize;
+      writeIO.offset = 0;
+      writeIO.updateVer = ChunkVer{1};
+      writeIO.updateType = UpdateType::WRITE;
+
+      UpdateJob updateJob(requestCtx, writeIO, {}, updateChunk, storageTarget);
+      updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().lengthInfo.value(), chunkSize);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{0});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+    }
+
+    for (auto i = 0u; i < N; ++i) {
+      CommitIO commitIO;
+      commitIO.key.chunkId = ChunkId{0, i};
+      commitIO.commitVer = ChunkVer{1};
+
+      auto updateJob = UpdateJob(requestCtx, commitIO, {}, updateChunk, storageTarget);
+      updateJob.state().data = reinterpret_cast<const uint8_t *>(dataBytes.data());
+
+      folly::coro::blockingWait(updateWorker.enqueue(&updateJob));
+      folly::coro::blockingWait(updateJob.complete());
+      ASSERT_OK(updateJob.result().lengthInfo);
+      ASSERT_EQ(updateJob.result().commitVer, ChunkVer{1});
+      ASSERT_EQ(updateJob.result().updateVer, ChunkVer{1});
+    }
+
+    for (auto i = 0u; i < N; ++i) {
+      auto metaResult = storageTarget->queryChunk(ChunkId{0, i});
+      ASSERT_OK(metaResult);
+    }
+  }
+
+  // 3. migrate.
+  storageTargetConfig.storage_target().set_migrate_kv_store(true);
+  storageTargetConfig.storage_target().kv_store().set_type(kv::KVStore::Type::RocksDB);
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets storageTargets(storageTargetConfig, targetMap);
+    ASSERT_OK(storageTargets.load(executor));
+    auto targetResult = targetMap.snapshot()->getTarget(targetId);
+    ASSERT_OK(targetResult);
+    auto storageTarget = (*targetResult)->storageTarget;
+
+    for (auto i = 0u; i < N; ++i) {
+      auto metaResult = storageTarget->queryChunk(ChunkId{0, i});
+      ASSERT_OK(metaResult);
+    }
+  }
+}
+
+}  // namespace
+}  // namespace hf3fs::storage

tests/storage/store/TestStorageTargets.cc

@@ -0,0 +1,58 @@
+#include <folly/experimental/TestUtil.h>
+
+#include "common/utils/CPUExecutorGroup.h"
+#include "common/utils/SysResource.h"
+#include "storage/store/StorageTargets.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage {
+namespace {
+
+TEST(TestStorageTargets, Normal) {
+  folly::test::TemporaryDirectory tmpPath;
+
+  StorageTargets::Config config;
+  config.set_target_num_per_path(4);
+  config.set_target_paths({tmpPath.path()});
+  config.set_allow_disk_without_uuid(true);
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(config, targetMap);
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{1}));
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{5}));
+
+    StorageTargets::CreateConfig createConfig;
+    createConfig.set_chunk_size_list({1_MB});
+    createConfig.set_physical_file_count(8);
+    createConfig.set_allow_disk_without_uuid(true);
+    createConfig.set_target_ids({1, 2, 3, 4});
+    ASSERT_OK(targets.create(createConfig));
+    ASSERT_OK(targetMap.snapshot()->getTarget(TargetId{1}));
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{5}));
+  }
+
+  {
+    AtomicallyTargetMap targetMap;
+    StorageTargets targets(config, targetMap);
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{1}));
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{5}));
+
+    CPUExecutorGroup executor(8, "");
+    ASSERT_OK(targets.load(executor));
+    ASSERT_OK(targetMap.snapshot()->getTarget(TargetId{1}));
+    ASSERT_FALSE(targetMap.snapshot()->getTarget(TargetId{5}));
+
+    ASSERT_OK(targetMap.offlineTargets(tmpPath.path()));
+    auto target = targetMap.snapshot()->getTarget(TargetId{1});
+    ASSERT_OK(target);
+    ASSERT_TRUE((*target)->diskError);
+
+    auto result = targetMap.offlineTarget(TargetId{1});
+    ASSERT_TRUE(result.hasError());
+    ASSERT_EQ(result.error().code(), StorageCode::kTargetOffline);
+  }
+}
+
+}  // namespace
+}  // namespace hf3fs::storage

tests/storage/sync/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_test(test_storage_sync test-fabric-lib)

tests/storage/sync/TestSyncForward.cc

@@ -0,0 +1,348 @@
+#include <thread>
+
+#include "client/storage/TargetSelection.h"
+#include "common/serde/Serde.h"
+#include "kv/KVStore.h"
+#include "kv/MemDBStore.h"
+#include "storage/service/StorageServer.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/Helper.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestSyncForward : public UnitTestFabric, public ::testing::Test {
+ protected:
+  TestSyncForward()
+      : UnitTestFabric(SystemSetupConfig{
+            128_KB /*chunkSize*/,
+            1 /*numChains*/,
+            3 /*numReplicas*/,
+            3 /*numStorageNodes*/,
+            {folly::fs::temp_directory_path()} /*dataPaths*/,
+            hf3fs::Path() /*clientConfig*/,
+            hf3fs::Path() /*serverConfig*/,
+            {} /*storageEndpoints*/,
+            0 /*serviceLevel*/,
+            0 /*listenPort*/,
+            client::StorageClient::ImplementationType::RPC /*clientImplType*/,
+            kv::KVStore::Type::RocksDB /*metaStoreType*/,
+            true /*useFakeMgmtdClient*/,
+        }) {}
+
+  void SetUp() override {
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+};
+
+TEST_F(TestSyncForward, ForwardToSyncingTarget) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    for (auto &[id, chain] : routingInfo.chains) {
+      chain.targets.back().publicState = flat::PublicTargetState::OFFLINE;
+      chain.chainVersion = flat::ChainVersion{1};
+      routingInfo.targets[chain.targets.back().targetId].publicState = flat::PublicTargetState::OFFLINE;
+    }
+  });
+
+  // 1. register a block of memory.
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+  std::fill(memoryBlock.begin(), memoryBlock.end(), 0x01);
+
+  // 2. create write io.
+  auto ioBuffer = std::move(*regRes);
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  auto writeIO = storageClient_->createWriteIO(chainId,
+                                               chunkId,
+                                               0 /*offset*/,
+                                               setupConfig_.chunk_size() /*length*/,
+                                               setupConfig_.chunk_size() /*chunkSize*/,
+                                               &memoryBlock[0],
+                                               &ioBuffer);
+
+  // 3. first write.
+  client::WriteOptions options;
+  folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+  ASSERT_OK(writeIO.result.lengthInfo);
+  ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+
+  // 4. update routing info.
+  updateRoutingInfo([&](auto &routingInfo) {
+    for (auto &[id, chain] : routingInfo.chains) {
+      chain.targets.back().publicState = flat::PublicTargetState::SYNCING;
+      chain.chainVersion = flat::ChainVersion{2};
+      routingInfo.targets[chain.targets.back().targetId].publicState = flat::PublicTargetState::SYNCING;
+    }
+  });
+
+  std::this_thread::sleep_for(2000_ms);  // wait sync start.
+  RoutingStoreHelper::refreshRoutingInfo(*storageServers_.back());
+
+  // 5. second write.
+  {
+    std::fill(memoryBlock.begin(), memoryBlock.end(), 0x02);
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 ChunkId(1, 2),
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+  }
+
+  for (auto waitTimes = 0_s; waitTimes < 10_s; waitTimes += 1_s) {
+    std::this_thread::sleep_for(1_s);  // wait sync done.
+    if (TargetMapHelper::checkLocalTargetState(*storageServers_.back(), flat::LocalTargetState::UPTODATE)) {
+      break;
+    }
+  }
+  ASSERT_TRUE(TargetMapHelper::checkLocalTargetState(*storageServers_.back(), flat::LocalTargetState::UPTODATE));
+
+  // 6. update routing info.
+  updateRoutingInfo([&](auto &routingInfo) {
+    for (auto &[id, chain] : routingInfo.chains) {
+      chain.targets.back().publicState = flat::PublicTargetState::SERVING;
+      chain.chainVersion = flat::ChainVersion{3};
+      routingInfo.targets[chain.targets.back().targetId].publicState = flat::PublicTargetState::SERVING;
+    }
+  });
+
+  client::ReadOptions readOptions;
+  readOptions.targetSelection().set_mode(client::TargetSelectionMode::TailTarget);
+  std::vector<uint8_t> readBlock(setupConfig_.chunk_size(), 0xFF);
+  auto readResult = readFromChunk(chainId, ChunkId(1, 1), readBlock, 0, 0, readOptions);
+  ASSERT_OK(readResult.lengthInfo);
+  ASSERT_EQ(*readResult.lengthInfo, readBlock.size());
+  auto isAllOne = std::all_of(readBlock.begin(), readBlock.end(), [](uint8_t c) { return c == 1; });
+  if (!isAllOne) {
+    for (auto i = 0u; i < readBlock.size(); ++i) {
+      if (readBlock[i] != 1) {
+        XLOGF(ERR, "index {} value {}", i, readBlock[i]);
+        ASSERT_EQ(readBlock[i], 1);
+      }
+    }
+  }
+
+  readResult = readFromChunk(chainId, ChunkId(1, 2), readBlock, 0, 0, readOptions);
+  ASSERT_OK(readResult.lengthInfo);
+  ASSERT_EQ(*readResult.lengthInfo, readBlock.size());
+  auto isAllTwo = std::all_of(readBlock.begin(), readBlock.end(), [](uint8_t c) { return c == 2; });
+  if (!isAllTwo) {
+    for (auto i = 0u; i < readBlock.size(); ++i) {
+      if (readBlock[i] != 2) {
+        XLOGF(ERR, "index {} value {}", i, readBlock[i]);
+        ASSERT_EQ(readBlock[i], 2);
+        break;
+      }
+    }
+  }
+
+  // 7. third write.
+  {
+    std::fill(memoryBlock.begin(), memoryBlock.end(), 0x03);
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 ChunkId(1, 3),
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+  }
+
+  readResult = readFromChunk(chainId, ChunkId(1, 3), readBlock, 0, 0, readOptions);
+  ASSERT_OK(readResult.lengthInfo);
+  ASSERT_EQ(*readResult.lengthInfo, readBlock.size());
+  ASSERT_TRUE(std::all_of(readBlock.begin(), readBlock.end(), [](uint8_t c) { return c == 3; }));
+}
+
+TEST_F(TestSyncForward, SyncingBatch) {
+  constexpr auto kChunkNum = 16u;
+
+  XLOGF(WARNING, "1. register a block of memory.");
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0x01);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+  auto chainId = firstChainId_;
+
+  XLOGF(WARNING, "2. write a batch of chunks.");
+  for (auto i = 0u; i < kChunkNum; ++i) {
+    std::fill(memoryBlock.begin(), memoryBlock.end(), 0x10 + i);
+    ChunkId chunkId(1 /*high*/, i /*low*/);
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+
+    client::WriteOptions options;
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+  }
+
+  XLOGF(WARNING, "3. check chunk content.");
+  client::ReadOptions readOptions;
+  readOptions.targetSelection().set_mode(client::TargetSelectionMode::TailTarget);
+  std::vector<uint8_t> readBlock(setupConfig_.chunk_size(), 0xFF);
+  for (auto i = 0u; i < kChunkNum; ++i) {
+    ChunkId chunkId(1 /*high*/, i /*low*/);
+    auto readResult = readFromChunk(chainId, chunkId, readBlock, 0, 0, readOptions);
+    ASSERT_OK(readResult.lengthInfo);
+    ASSERT_EQ(*readResult.lengthInfo, readBlock.size());
+    auto expect = 0x10 + i;
+    for (auto j = 0u; j < readBlock.size(); ++j) {
+      if (readBlock[j] != expect) {
+        XLOGF(ERR, "i: {} j: {} value: {}, expect: {}", i, j, readBlock[j], expect);
+        ASSERT_TRUE(false);
+      }
+    }
+  }
+
+  XLOGF(WARNING, "4. offline the last server.");
+  updateRoutingInfo([&](auto &routingInfo) {
+    for (auto &[id, chain] : routingInfo.chains) {
+      chain.targets.back().publicState = flat::PublicTargetState::OFFLINE;
+      chain.chainVersion = flat::ChainVersion{2};
+      routingInfo.targets[chain.targets.back().targetId].publicState = flat::PublicTargetState::OFFLINE;
+    }
+  });
+
+  XLOGF(WARNING, "5. second write [0, 1/4).");
+  for (auto i = 0u; i < kChunkNum / 4; ++i) {
+    std::fill(memoryBlock.begin(), memoryBlock.end(), 0x20 + i);
+    ChunkId chunkId(1 /*high*/, i /*low*/);
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+
+    client::WriteOptions options;
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+  }
+
+  XLOGF(WARNING, "6. remove chunks [1/2, 1).");
+  for (auto i = kChunkNum / 2; i < kChunkNum; ++i) {
+    auto removeIO =
+        storageClient_->createRemoveOp(chainId, ChunkId{1, i}, ChunkId{1, i + 1}, 1 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeIO, 1), flat::UserInfo()));
+    ASSERT_OK(removeIO.result.statusCode);
+    ASSERT_LE(removeIO.result.numChunksRemoved, 1);
+  }
+
+  XLOGF(WARNING, "7. check content.");
+  readOptions.targetSelection().set_mode(client::TargetSelectionMode::Default);
+  for (auto i = 0u; i < kChunkNum / 2; ++i) {
+    ChunkId chunkId(1 /*high*/, i /*low*/);
+    auto readResult = readFromChunk(chainId, chunkId, readBlock, 0, 0, readOptions);
+    ASSERT_OK(readResult.lengthInfo);
+    ASSERT_EQ(*readResult.lengthInfo, readBlock.size());
+    auto expect = i < kChunkNum / 4 ? 0x20 + i : 0x10 + i;
+    for (auto j = 0u; j < readBlock.size(); ++j) {
+      if (readBlock[j] != expect) {
+        XLOGF(ERR, "i: {} j: {} value: {}, expect: {}", i, j, readBlock[j], expect);
+        ASSERT_TRUE(false);
+      }
+    }
+  }
+  for (auto i = kChunkNum / 2; i < kChunkNum; ++i) {
+    ChunkId chunkId(1 /*high*/, i /*low*/);
+    auto readResult = readFromChunk(chainId, chunkId, readBlock, 0, 0, readOptions);
+    ASSERT_FALSE(readResult.lengthInfo);
+    ASSERT_EQ(readResult.lengthInfo.error().code(), StorageClientCode::kChunkNotFound);
+  }
+
+  XLOGF(WARNING, "8. syncing the last server.");
+  updateRoutingInfo([&](auto &routingInfo) {
+    for (auto &[id, chain] : routingInfo.chains) {
+      chain.targets.back().publicState = flat::PublicTargetState::SYNCING;
+      chain.chainVersion = flat::ChainVersion{2};
+      routingInfo.targets[chain.targets.back().targetId].publicState = flat::PublicTargetState::SYNCING;
+    }
+  });
+
+  for (auto i = kChunkNum / 8; i < kChunkNum * 3 / 8; ++i) {
+    std::fill(memoryBlock.begin(), memoryBlock.end(), 0x30 + i);
+    ChunkId chunkId(1 /*high*/, i /*low*/);
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+
+    client::WriteOptions options;
+    options.retry().set_init_wait_time(3000_ms);
+    options.retry().set_max_wait_time(3000_ms);
+    options.retry().set_max_retry_time(5000_ms);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+  }
+
+  for (auto waitTimes = 0_s; waitTimes < 30_s; waitTimes += 1_s) {
+    std::this_thread::sleep_for(1_s);  // wait sync done.
+    if (TargetMapHelper::checkLocalTargetState(*storageServers_.back(), flat::LocalTargetState::UPTODATE)) {
+      break;
+    }
+  }
+  ASSERT_TRUE(TargetMapHelper::checkLocalTargetState(*storageServers_.back(), flat::LocalTargetState::UPTODATE));
+
+  XLOGF(WARNING, "9. online the last server.");
+  updateRoutingInfo([&](auto &routingInfo) {
+    for (auto &[id, chain] : routingInfo.chains) {
+      chain.targets.back().publicState = flat::PublicTargetState::SERVING;
+      chain.chainVersion = flat::ChainVersion{3};
+      routingInfo.targets[chain.targets.back().targetId].publicState = flat::PublicTargetState::SERVING;
+    }
+  });
+
+  XLOGF(WARNING, "10. check content.");
+  readOptions.targetSelection().set_mode(client::TargetSelectionMode::TailTarget);
+  for (auto i = 0u; i < kChunkNum / 2; ++i) {
+    ChunkId chunkId(1 /*high*/, i /*low*/);
+    auto readResult = readFromChunk(chainId, chunkId, readBlock, 0, 0, readOptions);
+    ASSERT_OK(readResult.lengthInfo);
+    ASSERT_EQ(*readResult.lengthInfo, readBlock.size());
+    auto expect = (i < kChunkNum / 8 ? 0x20 : i < kChunkNum * 3 / 8 ? 0x30 : 0x10) + i;
+    for (auto j = 0u; j < readBlock.size(); ++j) {
+      if (readBlock[j] != expect) {
+        XLOGF(ERR, "i: {} j: {} value: {}, expect: {}", i, j, readBlock[j], expect);
+        ASSERT_TRUE(false);
+      }
+    }
+  }
+  for (auto i = kChunkNum / 2; i < kChunkNum; ++i) {
+    ChunkId chunkId(1 /*high*/, i /*low*/);
+    auto readResult = readFromChunk(chainId, chunkId, readBlock, 0, 0, readOptions);
+    ASSERT_FALSE(readResult.lengthInfo);
+    ASSERT_EQ(readResult.lengthInfo.error().code(), StorageClientCode::kChunkNotFound);
+  }
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test

tests/storage/sync/TestSyncStartAndDone.cc

@@ -0,0 +1,80 @@
+#include <thread>
+
+#include "common/serde/Serde.h"
+#include "kv/KVStore.h"
+#include "kv/MemDBStore.h"
+#include "storage/service/StorageServer.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/Helper.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::test {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestSyncStart : public UnitTestFabric, public ::testing::Test {
+ protected:
+  TestSyncStart()
+      : UnitTestFabric(SystemSetupConfig{128_KB /*chunkSize*/,
+                                         1 /*numChains*/,
+                                         1 /*numReplicas*/,
+                                         1 /*numStorageNodes*/,
+                                         {folly::fs::temp_directory_path()} /*dataPaths*/,
+                                         hf3fs::Path() /*clientConfig*/,
+                                         hf3fs::Path() /*serverConfig*/,
+                                         {} /*storageEndpoints*/,
+                                         0 /*serviceLevel*/,
+                                         0 /*listenPort*/,
+                                         client::StorageClient::ImplementationType::RPC,
+                                         kv::KVStore::Type::RocksDB,
+                                         false,
+                                         true}) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+};
+
+TEST_F(TestSyncStart, DISABLED_Normal) {
+  net::Client::Config config;
+  client::StorageMessenger messenger(config);
+  ASSERT_TRUE(messenger.start());
+
+  auto addr = storageServers_.back()->groups().front()->addressList().front();
+
+  std::optional<hf3fs::flat::ChainInfo> firstChain;
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto *chainTable = routingInfo.getChainTable(kTableId());
+    firstChain = routingInfo.chains[chainTable->chains.front()];
+
+    for (auto &[id, chain] : routingInfo.chains) {
+      chain.targets.back().publicState = flat::PublicTargetState::SYNCING;
+      chain.chainVersion = flat::ChainVersion{2};
+      routingInfo.targets[chain.targets.back().targetId].localState = flat::LocalTargetState::ONLINE;
+      routingInfo.targets[chain.targets.back().targetId].publicState = flat::PublicTargetState::SYNCING;
+    }
+  });
+  std::this_thread::sleep_for(1000_ms);
+
+  SyncStartReq syncStartReq;
+  syncStartReq.vChainId.chainId = firstChain->chainId;
+  syncStartReq.vChainId.chainVer = firstChain->chainVersion;
+  auto syncStartResult = folly::coro::blockingWait(messenger.syncStart(addr, syncStartReq));
+  ASSERT_OK(syncStartResult);
+
+  SyncDoneReq syncDoneReq;
+  syncDoneReq.vChainId.chainId = firstChain->chainId;
+  syncDoneReq.vChainId.chainVer = firstChain->chainVersion;
+  auto syncDoneResult = folly::coro::blockingWait(messenger.syncDone(addr, syncDoneReq));
+  ASSERT_OK(syncDoneResult);
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::test