Initial commit

tests/storage/client/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_test(test_storage_client test-fabric-lib)

tests/storage/client/TestFaultInjection.cc

@@ -0,0 +1,111 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+#include <folly/logging/xlog.h>
+
+#include "benchmarks/storage_bench/StorageBench.h"
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestFaultInjection : public ::testing::Test {
+ protected:
+  void SetUp() override {
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+    GTEST_SKIP() << "Skipping all tests since thread sanitizer enabled";
+#endif
+#endif
+
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+  }
+
+  void TearDown() override {}
+
+ protected:
+  test::SystemSetupConfig setupConfig_ = {
+      128_KB /*chunkSize*/,
+      3 /*numChains*/,
+      2 /*numReplicas*/,
+      3 /*numStorageNodes*/,
+      {folly::fs::temp_directory_path()} /*dataPaths*/,
+  };
+};
+
+TEST_F(TestFaultInjection, ServerError) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      300000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      true /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      true /*retryPermanentError*/,
+      true /*verifyReadData*/,
+      true /*verifyReadChecksum*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  ASSERT_EQ(StatusCode::kOK, bench.truncate());
+  ASSERT_EQ(StatusCode::kOK, bench.cleanup());
+  bench.teardown();
+}
+
+TEST_F(TestFaultInjection, ClientError) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      300000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      true /*injectRandomClientError*/,
+      true /*retryPermanentError*/,
+      true /*verifyReadData*/,
+      true /*verifyReadChecksum*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  ASSERT_EQ(StatusCode::kOK, bench.truncate());
+  ASSERT_EQ(StatusCode::kOK, bench.cleanup());
+  bench.teardown();
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageBenchmark.cc

@@ -0,0 +1,173 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+#include <folly/logging/xlog.h>
+
+#include "benchmarks/storage_bench/StorageBench.h"
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestStorageBenchmark : public ::testing::Test {
+ protected:
+  void SetUp() override {
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+    GTEST_SKIP() << "Skipping all tests since thread sanitizer enabled";
+#endif
+#endif
+
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+  }
+
+  void TearDown() override {}
+
+ protected:
+  test::SystemSetupConfig setupConfig_ = {
+      128_KB /*chunkSize*/,
+      1 /*numChains*/,
+      1 /*numReplicas*/,
+      1 /*numStorageNodes*/,
+      {folly::fs::temp_directory_path()} /*dataPaths*/,
+  };
+};
+
+TEST_F(TestStorageBenchmark, StandardTimeout) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      180000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      false /*retryPermanentError*/,
+      true /*verifyReadData*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  ASSERT_EQ(StatusCode::kOK, bench.truncate());
+  ASSERT_EQ(StatusCode::kOK, bench.cleanup());
+  bench.teardown();
+}
+
+TEST_F(TestStorageBenchmark, ShortTimeout) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      5 /*numReadSecs*/,
+      5 /*numWriteSecs*/,
+      3000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      true /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      false /*retryPermanentError*/,
+      false /*verifyReadData*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  bench.run();
+  bench.truncate();
+  bench.cleanup();
+  bench.teardown();
+}
+
+TEST_F(TestStorageBenchmark, BenchmarkNetwork) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      180000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      true /*benchmarkNetwork*/,
+      false /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      false /*retryPermanentError*/,
+      false /*verifyReadData*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  bench.truncate();
+  bench.cleanup();
+  bench.teardown();
+}
+
+TEST_F(TestStorageBenchmark, BenchmarkStorage) {
+  benchmark::StorageBench::Options benchOptions{
+      32 /*numChunks*/,
+      64_KB /*readSize*/,
+      64_KB /*writeSize*/,
+      16 /*batchSize*/,
+      3 /*numReadSecs*/,
+      3 /*numWriteSecs*/,
+      180000 /*clientTimeoutMS*/,
+      64 /*numCoroutines*/,
+      32 /*numTestThreads*/,
+      0 /*randSeed*/,
+      0xFFFF /*chunkIdPrefix*/,
+      false /*benchmarkNetwork*/,
+      true /*benchmarkStorage*/,
+      false /*ignoreIOError*/,
+      false /*injectRandomServerError*/,
+      false /*injectRandomClientError*/,
+      false /*retryPermanentError*/,
+      false /*verifyReadData*/,
+  };
+
+  benchmark::StorageBench bench(setupConfig_, benchOptions);
+  ASSERT_TRUE(bench.setup());
+  bench.generateChunkIds();
+
+  ASSERT_TRUE(bench.run());
+  bench.truncate();
+  bench.cleanup();
+  bench.teardown();
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageClientFastFailover.cc

@@ -0,0 +1,115 @@
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestStorageClientFastFailover : public UnitTestFabric, public ::testing::TestWithParam<SystemSetupConfig> {
+ protected:
+  TestStorageClientFastFailover()
+      : UnitTestFabric(GetParam()) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+
+    // increase timeout to avoid comm error during data generation
+    auto retryOptions = clientConfig_.retry();
+    clientConfig_.retry().set_max_retry_time(5_s);
+
+    numChunks_ = setupConfig_.num_replicas() * 10;
+    std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+    auto result = writeToChunks(firstChainId_, ChunkId(1, 0), ChunkId(1, numChunks_), chunkData);
+    ASSERT_TRUE(result);
+    XLOGF(INFO, "Created {} test chunks", numChunks_);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+
+ protected:
+  size_t numChunks_;
+};
+
+TEST_P(TestStorageClientFastFailover, ReadChunks) {
+  client::ReadOptions options;
+  // since we have much more chunks than replicas: numChunks_ = setupConfig_.num_replicas() * 10,
+  // all replicas would be used in round robin mode
+  options.targetSelection().set_mode(client::TargetSelectionMode::RoundRobin);
+
+  std::vector<std::vector<uint8_t>> chunkData;
+  auto result = readFromChunks(firstChainId_,
+                               ChunkId(1, 0),
+                               ChunkId(1, numChunks_),
+                               chunkData,
+                               0 /*offset*/,
+                               setupConfig_.chunk_size() /*length*/,
+                               options);
+  ASSERT_TRUE(result);
+
+  // stop the head target
+  ASSERT_TRUE(stopAndRemoveStorageServer(0));
+
+  result = readFromChunks(firstChainId_,
+                          ChunkId(1, 0),
+                          ChunkId(1, numChunks_),
+                          chunkData,
+                          0 /*offset*/,
+                          setupConfig_.chunk_size() /*length*/,
+                          options);
+
+  if (setupConfig_.num_replicas() > 1) {
+    ASSERT_TRUE(result);
+  } else {
+    ASSERT_FALSE(result);
+  }
+}
+
+SystemSetupConfig singleReplica = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    1 /*numReplicas*/,
+    1 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+SystemSetupConfig twoReplicas = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    2 /*numReplicas*/,
+    2 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+SystemSetupConfig threeReplicas = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    3 /*numReplicas*/,
+    3 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+INSTANTIATE_TEST_SUITE_P(SingleReplica,
+                         TestStorageClientFastFailover,
+                         ::testing::Values(singleReplica),
+                         SystemSetupConfig::prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(TwoReplicas,
+                         TestStorageClientFastFailover,
+                         ::testing::Values(twoReplicas),
+                         SystemSetupConfig::prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(ThreeReplicas,
+                         TestStorageClientFastFailover,
+                         ::testing::Values(threeReplicas),
+                         SystemSetupConfig::prettyPrintConfig);
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageClientHCStress.cc

@@ -0,0 +1,383 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+struct ClientTestConfig {
+  hf3fs::Duration initWaitTime;
+  hf3fs::Duration maxWaitTime;
+  hf3fs::Duration maxRetryTime;
+  size_t batchSize;
+  size_t maxBatchSize;
+  size_t numConcurrentReqs;
+  size_t maxConcurrentReqs;
+};
+
+// high-concurrency stress test
+class TestStorageClientHCStress : public UnitTestFabric, public ::testing::TestWithParam<ClientTestConfig> {
+ protected:
+  TestStorageClientHCStress()
+      : UnitTestFabric(SystemSetupConfig{128_KB /*chunkSize*/,
+                                         3 /*numChains*/,
+                                         2 /*numReplicas*/,
+                                         3 /*numStorageNodes*/,
+                                         {folly::fs::temp_directory_path()}}) {}
+
+  void SetUp() override {
+#if defined(__has_feature)
+#if __has_feature(thread_sanitizer)
+    GTEST_SKIP() << "Skipping all tests since thread sanitizer enabled";
+#endif
+#endif
+
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+
+    auto testConfig = GetParam();
+
+    clientConfig_.traffic_control().read().set_max_batch_size(testConfig.maxBatchSize);
+    clientConfig_.traffic_control().write().set_max_batch_size(testConfig.maxBatchSize);
+    clientConfig_.traffic_control().remove().set_max_batch_size(testConfig.maxBatchSize);
+    clientConfig_.traffic_control().truncate().set_max_batch_size(testConfig.maxBatchSize);
+    clientConfig_.traffic_control().read().set_max_concurrent_requests(testConfig.maxConcurrentReqs);
+    clientConfig_.traffic_control().write().set_max_concurrent_requests(testConfig.maxConcurrentReqs);
+    clientConfig_.traffic_control().remove().set_max_concurrent_requests(testConfig.maxConcurrentReqs);
+    clientConfig_.traffic_control().truncate().set_max_concurrent_requests(testConfig.maxConcurrentReqs);
+
+    batchSize_ = testConfig.batchSize;
+    numConcurrentReqs_ = testConfig.numConcurrentReqs;
+    numChunks_ = batchSize_ * numConcurrentReqs_;
+
+    ASSERT_TRUE(setUpStorageSystem());
+
+    // increase timeout to avoid comm error during data generation
+    clientConfig_.retry().set_init_wait_time(3_s);
+    clientConfig_.retry().set_max_wait_time(10_s);
+    clientConfig_.retry().set_max_retry_time(300_s);
+
+    // create chunks
+    std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+    for (auto chainId : chainIds_) {
+      auto result = writeToChunks(chainId, ChunkId(1, 0), ChunkId(1, numChunks_), chunkData);
+      ASSERT_TRUE(result);
+      XLOGF(INFO, "Created {} test chunks on chain {}", numChunks_, chainId);
+    }
+
+    // set timeout for test
+    clientConfig_.retry().set_init_wait_time(testConfig.initWaitTime);
+    clientConfig_.retry().set_max_wait_time(testConfig.maxWaitTime);
+    clientConfig_.retry().set_max_retry_time(testConfig.maxRetryTime);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+
+ protected:
+  size_t numConcurrentReqs_;
+  size_t numChunks_;
+  size_t batchSize_;
+};
+
+TEST_P(TestStorageClientHCStress, BatchRead) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * numChunks_, 0);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  // create read IOs
+
+  std::vector<ReadIO> readIOs;
+  size_t chunkIndex = 0;
+
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    for (size_t ioIndex = 0; ioIndex < batchSize_; ioIndex++, chunkIndex++) {
+      ChunkId chunkId(1 /*high*/, chunkIndex /*low*/);
+      ChainId chainId = chainIds_[chunkIndex % chainIds_.size()];
+      auto readIO = storageClient_->createReadIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 &memoryBlock[chunkIndex * setupConfig_.chunk_size()],
+                                                 &ioBuffer);
+      readIOs.push_back(std::move(readIO));
+    }
+  }
+
+  flat::UserInfo dummyUserInfo{};
+  ReadOptions options;
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+
+  // start many coroutines to hit the concurrency limit
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    auto ios = std::span<ReadIO>(&readIOs[batchSize_ * reqIndex], &readIOs[batchSize_ * (reqIndex + 1)]);
+    auto task = storageClient_->batchRead(ios, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+
+  for (const auto &readIO : readIOs) {
+    ASSERT_OK(readIO.result.lengthInfo);
+    ASSERT_EQ(readIO.length, readIO.result.lengthInfo.value());
+    ASSERT_EQ(1, readIO.result.commitVer);
+  }
+
+  for (unsigned char &byte : memoryBlock) ASSERT_EQ(0xFF, byte);
+}
+
+TEST_P(TestStorageClientHCStress, BatchWrite) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * numChunks_, 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  // create write IOs
+
+  std::vector<WriteIO> writeIOs;
+  size_t chunkIndex = 0;
+
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    for (size_t ioIndex = 0; ioIndex < batchSize_; ioIndex++, chunkIndex++) {
+      ChunkId chunkId(1 /*high*/, chunkIndex /*low*/);
+      ChainId chainId = chainIds_[chunkIndex % chainIds_.size()];
+      auto writeIO = storageClient_->createWriteIO(chainId,
+                                                   chunkId,
+                                                   0 /*offset*/,
+                                                   setupConfig_.chunk_size() /*length*/,
+                                                   setupConfig_.chunk_size() /*chunkSize*/,
+                                                   &memoryBlock[chunkIndex * setupConfig_.chunk_size()],
+                                                   &ioBuffer);
+      writeIOs.push_back(std::move(writeIO));
+    }
+  }
+
+  flat::UserInfo dummyUserInfo{};
+  WriteOptions options;
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+
+  // start many coroutines to hit the concurrency limit
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    auto ios = std::span<WriteIO>(&writeIOs[batchSize_ * reqIndex], &writeIOs[batchSize_ * (reqIndex + 1)]);
+    auto task = storageClient_->batchWrite(ios, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+
+  for (const auto &writeIO : writeIOs) {
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+    ASSERT_EQ(2, writeIO.result.commitVer);
+  }
+}
+
+TEST_P(TestStorageClientHCStress, BatchTruncate) {
+  // create truncate ops
+
+  std::vector<TruncateChunkOp> truncateOps;
+  size_t chunkIndex = 0;
+
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    for (size_t ioIndex = 0; ioIndex < batchSize_; ioIndex++, chunkIndex++) {
+      ChunkId chunkId(1 /*high*/, chunkIndex /*low*/);
+      ChainId chainId = chainIds_[chunkIndex % chainIds_.size()];
+      auto truncateOp = storageClient_->createTruncateOp(chainId,
+                                                         chunkId,
+                                                         setupConfig_.chunk_size() / 2 /*length*/,
+                                                         setupConfig_.chunk_size() /*chunkSize*/);
+      truncateOps.push_back(std::move(truncateOp));
+    }
+  }
+
+  flat::UserInfo dummyUserInfo{};
+  WriteOptions options;
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+
+  // start many coroutines to hit the concurrency limit and use all channel ids
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    auto ops =
+        std::span<TruncateChunkOp>(&truncateOps[batchSize_ * reqIndex], &truncateOps[batchSize_ * (reqIndex + 1)]);
+    auto task = storageClient_->truncateChunks(ops, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+
+  for (const auto &truncateOp : truncateOps) {
+    XLOGF(DBG5, "Truncate chunk {} to length {}/{}", truncateOp.chunkId, truncateOp.chunkLen, truncateOp.chunkSize);
+    ASSERT_OK(truncateOp.result.lengthInfo);
+    ASSERT_EQ(truncateOp.chunkLen, truncateOp.result.lengthInfo.value());
+    ASSERT_EQ(2, truncateOp.result.commitVer);
+  }
+}
+
+TEST_P(TestStorageClientHCStress, BatchRemove) {
+  // create remove ops
+
+  std::vector<RemoveChunksOp> removeOps;
+
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    ChunkId chunkBegin(1 /*high*/, batchSize_ * reqIndex /*low*/);
+    ChunkId chunkEnd(1 /*high*/, batchSize_ * (reqIndex + 1) /*low*/);
+    ChainId chainId = chainIds_[reqIndex % chainIds_.size()];
+    auto removeOp = storageClient_->createRemoveOp(chainId, chunkBegin, chunkEnd, batchSize_);
+    removeOps.push_back(std::move(removeOp));
+  }
+
+  flat::UserInfo dummyUserInfo{};
+  WriteOptions options;
+  std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
+
+  // start many coroutines to hit the concurrency limit and use all channel ids
+  for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+    auto ops = std::span<RemoveChunksOp>(&removeOps[reqIndex], 1);
+    auto task = storageClient_->removeChunks(ops, dummyUserInfo, options).scheduleOn(&requestExe_).start();
+    ioTasks.push_back(std::move(task));
+  }
+
+  folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
+
+  for (const auto &removeOp : removeOps) {
+    XLOGF(DBG5, "Remove range [{}, {})", removeOp.range.begin, removeOp.range.end);
+    ASSERT_OK(removeOp.result.statusCode);
+    ASSERT_LE(removeOp.result.numChunksRemoved, batchSize_);
+
+    // check if chunks are removed
+    auto queryOp =
+        storageClient_->createQueryOp(removeOp.routingTarget.chainId, removeOp.range.begin, removeOp.range.end);
+    folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+    XLOGF(DBG5, "#chunks in range [{}, {}): {}", queryOp.range.begin, queryOp.range.end, queryOp.result.totalNumChunks);
+    ASSERT_OK(queryOp.result.statusCode);
+    ASSERT_EQ(0, queryOp.result.totalNumChunks);
+    ASSERT_EQ(0, queryOp.result.totalChunkLen);
+  }
+}
+
+TEST_P(TestStorageClientHCStress, ConcurrentReadRemove) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * numChunks_, 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+  auto ioBuffer = std::move(*regRes);
+
+  uint32_t maxLoops = chainIds_.size() * 2;
+
+  for (uint32_t testLoop = 1; testLoop <= maxLoops; testLoop++) {
+    // create chunks
+    std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+    ChainId chainId = chainIds_[testLoop % chainIds_.size()];
+    auto result = writeToChunks(chainId, ChunkId(1, 0), ChunkId(1, numChunks_), chunkData);
+    ASSERT_TRUE(result);
+
+    // create remove ops
+
+    std::vector<RemoveChunksOp> removeOps;
+
+    for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+      ChunkId chunkBegin(1 /*high*/, batchSize_ * reqIndex /*low*/);
+      ChunkId chunkEnd(1 /*high*/, batchSize_ * (reqIndex + 1) /*low*/);
+      auto removeOp = storageClient_->createRemoveOp(chainId, chunkBegin, chunkEnd, batchSize_);
+      removeOps.push_back(std::move(removeOp));
+    }
+
+    // create read IOs
+
+    std::vector<ReadIO> readIOs;
+    size_t chunkIndex = 0;
+
+    for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+      for (size_t ioIndex = 0; ioIndex < batchSize_; ioIndex++, chunkIndex++) {
+        ChunkId chunkId(1 /*high*/, chunkIndex /*low*/);
+        auto readIO = storageClient_->createReadIO(chainId,
+                                                   chunkId,
+                                                   0 /*offset*/,
+                                                   setupConfig_.chunk_size() /*length*/,
+                                                   &memoryBlock[chunkIndex * setupConfig_.chunk_size()],
+                                                   &ioBuffer);
+        readIOs.push_back(std::move(readIO));
+      }
+    }
+
+    flat::UserInfo dummyUserInfo{};
+
+    // start read tasks
+    std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> readTasks;
+    ReadOptions readOptions;
+
+    for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+      auto ios = std::span<ReadIO>(&readIOs[batchSize_ * reqIndex], &readIOs[batchSize_ * (reqIndex + 1)]);
+      auto task = storageClient_->batchRead(ios, dummyUserInfo, readOptions).scheduleOn(&requestExe_).start();
+      readTasks.push_back(std::move(task));
+    }
+
+    // start remove tasks
+    std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> removeTasks;
+    WriteOptions removeOptions;
+
+    for (size_t reqIndex = 0; reqIndex < numConcurrentReqs_; reqIndex++) {
+      auto ops = std::span<RemoveChunksOp>(&removeOps[reqIndex], 1);
+      auto task = storageClient_->removeChunks(ops, dummyUserInfo, removeOptions).scheduleOn(&requestExe_).start();
+      removeTasks.push_back(std::move(task));
+    }
+
+    folly::coro::blockingWait(folly::coro::collectAllRange(std::move(readTasks)));
+    folly::coro::blockingWait(folly::coro::collectAllRange(std::move(removeTasks)));
+
+    for (const auto &removeOp : removeOps) {
+      XLOGF(DBG5, "Remove chunks in {}", removeOp.range);
+      ASSERT_OK(removeOp.result.statusCode);
+      ASSERT_LE(removeOp.result.numChunksRemoved, batchSize_);
+    }
+  }
+}
+
+ClientTestConfig standardTimeout = {
+    3_s /*initWaitTime*/,
+    10_s /*maxWaitTime*/,
+    300_s /*maxRetryTime*/,
+    16U /*batchSize*/,
+    8U /*maxBatchSize*/,
+    32U /*numConcurrentReqs*/,
+    16U /*maxConcurrentReqs*/,
+};
+
+ClientTestConfig onecoroBusyRetry = {
+    1_ms /*initWaitTime*/,
+    10_s /*maxWaitTime*/,
+    300_s /*maxRetryTime*/,
+    16U /*batchSize*/,
+    16U /*maxBatchSize*/,
+    1U /*numConcurrentReqs*/,
+    1U /*maxConcurrentReqs*/,
+};
+
+ClientTestConfig multicoroBusyRetry = {
+    1_ms /*initWaitTime*/,
+    10_s /*maxWaitTime*/,
+    300_s /*maxRetryTime*/,
+    16U /*batchSize*/,
+    8U /*maxBatchSize*/,
+    32U /*numConcurrentReqs*/,
+    16U /*maxConcurrentReqs*/,
+};
+
+INSTANTIATE_TEST_SUITE_P(StandardTimeout, TestStorageClientHCStress, ::testing::Values(standardTimeout));
+
+INSTANTIATE_TEST_SUITE_P(OneCoroBusyRetry, TestStorageClientHCStress, ::testing::Values(onecoroBusyRetry));
+
+INSTANTIATE_TEST_SUITE_P(MultiCoroBusyRetry, TestStorageClientHCStress, ::testing::Values(multicoroBusyRetry));
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageClientInterface.cc

@@ -0,0 +1,790 @@
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/GtestHelpers.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestStorageClientInterface : public UnitTestFabric, public ::testing::TestWithParam<SystemSetupConfig> {
+ protected:
+  TestStorageClientInterface()
+      : UnitTestFabric(GetParam()) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+
+    clientConfig_.retry().set_init_wait_time(30_s);
+    clientConfig_.retry().set_max_wait_time(30_s);
+    clientConfig_.retry().set_max_retry_time(30_s);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+};
+
+TEST_P(TestStorageClientInterface, Write) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create write IO
+
+  auto ioBuffer = std::move(*regRes);
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+
+  for (bool bypassDiskIO : {true, false}) {
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 chunkId,
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size() /*chunkSize*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+
+    WriteOptions options;
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
+
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+  }
+
+  {
+    QueryChunkReq req;
+    req.chainId = firstChainId_;
+    req.chunkId = chunkId;
+    auto result = folly::coro::blockingWait(storageClient_->queryChunk(req));
+    ASSERT_OK(result);
+    XLOGF(INFO, "chunk info {}", serde::toJsonString(*result, false, true));
+  }
+}
+
+TEST_P(TestStorageClientInterface, BatchWrite) {
+  // register a block of memory
+  size_t numWriteIOs = 3;
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * numWriteIOs, 0xFF);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create write IOs
+
+  auto ioBuffer = std::move(*regRes);
+
+  for (bool bypassDiskIO : {true, false}) {
+    std::vector<WriteIO> writeIOs;
+
+    for (size_t writeIndex = 0; writeIndex < numWriteIOs; writeIndex++) {
+      auto chainId = firstChainId_;
+      ChunkId chunkId(1 /*high*/, writeIndex /*low*/);
+      auto writeIO = storageClient_->createWriteIO(chainId,
+                                                   chunkId,
+                                                   0 /*offset*/,
+                                                   setupConfig_.chunk_size() /*length*/,
+                                                   setupConfig_.chunk_size() /*chunkSize*/,
+                                                   &memoryBlock[writeIndex * setupConfig_.chunk_size()],
+                                                   &ioBuffer);
+      writeIOs.push_back(std::move(writeIO));
+    }
+
+    WriteOptions options;
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->batchWrite(writeIOs, flat::UserInfo(), options));
+
+    for (const auto &writeIO : writeIOs) {
+      ASSERT_OK(writeIO.result.lengthInfo);
+      ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, Read) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create read IO
+
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_OK(ioResult.lengthInfo);
+  ASSERT_EQ(chunkData.size(), ioResult.lengthInfo.value());
+
+  auto ioBuffer = std::move(*regRes);
+  auto readIO = storageClient_->createReadIO(chainId,
+                                             chunkId /*chunkId*/,
+                                             0 /*offset*/,
+                                             setupConfig_.chunk_size() /*length*/,
+                                             &memoryBlock[0],
+                                             &ioBuffer);
+
+  ReadOptions options;
+
+  for (bool bypassDiskIO : {true, false}) {
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), options));
+
+    ASSERT_OK(readIO.result.lengthInfo);
+    ASSERT_EQ(readIO.length, readIO.result.lengthInfo.value());
+
+    if (!bypassDiskIO) {
+      for (size_t i = 0; i < chunkData.size(); i++) ASSERT_EQ(chunkData[i], memoryBlock[i]);
+    }
+  };
+}
+
+TEST_P(TestStorageClientInterface, BatchRead) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create read IOs
+
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_OK(ioResult.lengthInfo);
+  ASSERT_EQ(chunkData.size(), ioResult.lengthInfo.value());
+
+  auto ioBuffer = std::move(*regRes);
+  std::vector<ReadIO> readIOs;
+
+  for (size_t offset = 0; offset < setupConfig_.chunk_size(); offset += setupConfig_.chunk_size() / 4) {
+    auto readIO = storageClient_->createReadIO(chainId,
+                                               chunkId,
+                                               offset /*offset*/,
+                                               setupConfig_.chunk_size() / 4 /*length*/,
+                                               &memoryBlock[offset],
+                                               &ioBuffer);
+    readIOs.push_back(std::move(readIO));
+  }
+
+  ReadOptions options;
+
+  for (bool bypassDiskIO : {true, false}) {
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo(), options));
+
+    for (const auto &readIO : readIOs) {
+      ASSERT_OK(readIO.result.lengthInfo);
+      ASSERT_EQ(readIO.length, readIO.result.lengthInfo.value());
+    }
+
+    if (!bypassDiskIO) {
+      for (size_t i = 0; i < chunkData.size(); i++) ASSERT_EQ(chunkData[i], memoryBlock[i]);
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, BatchReadUnaligned) {
+  // register a block of memory
+  std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0);
+  auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+  ASSERT_OK(regRes);
+
+  // create read IOs
+
+  auto chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+  std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
+  for (auto &ch : chunkData) {
+    ch = rand() & 0xFF;
+  }
+
+  auto ioResult = writeToChunk(chainId, chunkId, chunkData);
+  ASSERT_OK(ioResult.lengthInfo);
+  ASSERT_EQ(chunkData.size(), ioResult.lengthInfo.value());
+
+  auto ioBuffer = std::move(*regRes);
+  std::vector<ReadIO> readIOs;
+
+  for (size_t offset = 0; offset < setupConfig_.chunk_size(); offset += setupConfig_.chunk_size() / 4) {
+    auto headSize = 1 + rand() % 17;
+    auto tailSize = 1 + rand() % 17;
+    auto unalignedOffset = offset + headSize;
+    auto unalignedLength = setupConfig_.chunk_size() / 4 - headSize - tailSize;
+    XLOGF(INFO,
+          "Unaligned offset {} length {}, chunk size {} chunk id {}",
+          unalignedOffset,
+          unalignedLength,
+          setupConfig_.chunk_size(),
+          chunkId);
+    auto readIO = storageClient_->createReadIO(chainId,
+                                               chunkId,
+                                               unalignedOffset /*offset*/,
+                                               unalignedLength /*length*/,
+                                               &memoryBlock[unalignedOffset],
+                                               &ioBuffer);
+    readIOs.push_back(std::move(readIO));
+  }
+
+  ReadOptions options;
+
+  for (bool bypassDiskIO : {true, false}) {
+    options.debug().set_bypass_disk_io(bypassDiskIO);
+    folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo(), options));
+
+    for (const auto &readIO : readIOs) {
+      ASSERT_OK(readIO.result.lengthInfo);
+      ASSERT_EQ(readIO.length, readIO.result.lengthInfo.value());
+      if (!bypassDiskIO) {
+        ASSERT_TRUE(std::memcmp(&chunkData[readIO.offset], &memoryBlock[readIO.offset], readIO.length) == 0);
+      }
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, ReadWrite) {
+  // register a block of memory
+  std::vector<uint8_t> writeData(setupConfig_.chunk_size() / 2, 0xFF);
+  auto regWriteDataRes = storageClient_->registerIOBuffer(&writeData[0], writeData.size());
+  ASSERT_OK(regWriteDataRes);
+
+  auto writeBuffer = std::move(*regWriteDataRes);
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+
+  // write data to chunk
+  auto writeIO = storageClient_->createWriteIO(chainId,
+                                               chunkId,
+                                               0 /*offset*/,
+                                               writeData.size() /*length*/,
+                                               setupConfig_.chunk_size(),
+                                               &writeData[0],
+                                               &writeBuffer);
+
+  folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+
+  ASSERT_OK(writeIO.result.lengthInfo);
+  ASSERT_EQ(writeData.size(), writeIO.result.lengthInfo.value());
+
+  // read the data back
+  std::vector<uint8_t> readData(setupConfig_.chunk_size(), 0);
+  auto regReadDataRes = storageClient_->registerIOBuffer(&readData[0], readData.size());
+  ASSERT_OK(regReadDataRes);
+
+  auto readBuffer = std::move(*regReadDataRes);
+  auto readIO = storageClient_->createReadIO(chainId,
+                                             chunkId,
+                                             0 /*offset*/,
+                                             setupConfig_.chunk_size() /*length*/,
+                                             &readData[0],
+                                             &readBuffer);
+
+  folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo()));
+
+  ASSERT_OK(readIO.result.lengthInfo);
+  ASSERT_EQ(writeIO.result.lengthInfo.value(), readIO.result.lengthInfo.value());
+  for (size_t i = 0; i < writeData.size(); i++) ASSERT_EQ(writeData[i], readData[i]);
+}
+
+TEST_P(TestStorageClientInterface, BatchReadWrite) {
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 1 /*low*/);
+
+  // write data to chunk
+  size_t numChunks = 3;
+  std::vector<uint8_t> writeData(setupConfig_.chunk_size() * numChunks, 0xFF);
+  auto regWriteDataRes = storageClient_->registerIOBuffer(&writeData[0], writeData.size());
+  ASSERT_OK(regWriteDataRes);
+  auto writeBuffer = std::move(*regWriteDataRes);
+
+  std::vector<WriteIO> writeIOs;
+
+  for (size_t writeIndex = 0; writeIndex < numChunks; writeIndex++) {
+    auto writeIO = storageClient_->createWriteIO(chainId,
+                                                 ChunkId(chunkId, writeIndex),
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 &writeData[writeIndex * setupConfig_.chunk_size()],
+                                                 &writeBuffer);
+    writeIOs.push_back(std::move(writeIO));
+  }
+
+  folly::coro::blockingWait(storageClient_->batchWrite(writeIOs, flat::UserInfo()));
+
+  for (const auto &writeIO : writeIOs) {
+    ASSERT_OK(writeIO.result.lengthInfo);
+    ASSERT_EQ(setupConfig_.chunk_size(), writeIO.result.lengthInfo.value());
+  }
+
+  // read the data back
+  std::vector<uint8_t> readData(setupConfig_.chunk_size() * numChunks, 0);
+  auto regReadDataRes = storageClient_->registerIOBuffer(&readData[0], readData.size());
+  ASSERT_OK(regReadDataRes);
+  auto readBuffer = std::move(*regReadDataRes);
+
+  std::vector<ReadIO> readIOs;
+
+  for (size_t readIndex = 0; readIndex < numChunks; readIndex++) {
+    auto readIO = storageClient_->createReadIO(chainId,
+                                               ChunkId(chunkId, readIndex),
+                                               0 /*offset*/,
+                                               setupConfig_.chunk_size() /*length*/,
+                                               &readData[readIndex * setupConfig_.chunk_size()],
+                                               &readBuffer);
+    readIOs.push_back(std::move(readIO));
+  }
+
+  folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo()));
+
+  for (const auto &readIO : readIOs) {
+    ASSERT_OK(readIO.result.lengthInfo);
+    ASSERT_EQ(setupConfig_.chunk_size(), readIO.result.lengthInfo.value());
+  }
+
+  for (size_t i = 0; i < writeData.size(); i++) ASSERT_EQ(writeData[i], readData[i]) << "i " << i;
+}
+
+TEST_P(TestStorageClientInterface, VerifyChecksum) {
+  if (setupConfig_.client_impl_type() == StorageClient::ImplementationType::InMem) return;
+
+  ChainId chainId = firstChainId_;
+
+  std::vector<uint8_t> readData(setupConfig_.chunk_size(), 0);
+  std::vector<uint8_t> writeData(setupConfig_.chunk_size(), 0xFF);
+
+  auto regReadBuf = storageClient_->registerIOBuffer(&readData[0], readData.size());
+  ASSERT_OK(regReadBuf);
+  auto readBuffer = std::move(*regReadBuf);
+
+  auto regWriteBuf = storageClient_->registerIOBuffer(&writeData[0], writeData.size());
+  ASSERT_OK(regWriteBuf);
+  auto writeBuffer = std::move(*regWriteBuf);
+
+  // enable checksum for read/write IO
+  clientConfig_.set_chunk_checksum_type(ChecksumType::CRC32C);
+  client::WriteOptions writeOptions;
+  writeOptions.set_enableChecksum(true);
+  client::ReadOptions readOptions;
+  readOptions.set_enableChecksum(true);
+  readOptions.targetSelection().set_mode(TargetSelectionMode::RoundRobin);
+
+  enum WritePattern {
+    SEQWRITE = 1,
+    JUMPWRITE,
+    RANDWRITE,
+  };
+
+  for (WritePattern pattern : {SEQWRITE, JUMPWRITE, RANDWRITE}) {
+    ChunkId chunkId(1 /*high*/, pattern /*low*/);
+    std::vector<uint8_t> chunkData;
+    size_t offset = 0;
+    size_t length = 0;
+
+    for (size_t writeIndex = 1; writeIndex <= 100; writeIndex++) {
+      switch (pattern) {
+        case SEQWRITE:
+          offset += length;
+          break;
+        case JUMPWRITE:
+          offset += length + folly::Random::rand64(0, length / 2);
+          break;
+        case RANDWRITE:
+          offset = folly::Random::rand64(0, setupConfig_.chunk_size());
+          break;
+      }
+
+      if (offset + 1 >= setupConfig_.chunk_size()) continue;
+      length = folly::Random::rand64(1, (setupConfig_.chunk_size() - offset) / 2);
+
+      XLOGF(INFO,
+            "Verify {} checksum #{}: offset {} length {} chunk size {}",
+            magic_enum::enum_name(pattern),
+            writeIndex,
+            offset,
+            length,
+            chunkData.size());
+
+      // generate random chunk data
+      for (size_t byteIndex = 0; byteIndex + sizeof(uint64_t) < length; byteIndex += sizeof(uint64_t)) {
+        auto dataPtr = reinterpret_cast<uint64_t *>(&writeData[byteIndex]);
+        *dataPtr = folly::Random::rand64();
+      }
+
+      // write to a random offset in the chunk
+      auto writeIO =
+          storageClient_
+              ->createWriteIO(chainId, chunkId, offset, length, setupConfig_.chunk_size(), &writeData[0], &writeBuffer);
+      folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), writeOptions));
+      ASSERT_RESULT_EQ(writeIO.length, writeIO.result.lengthInfo);
+
+      // verify checksum of the write data
+      ASSERT_EQ(folly::crc32c(&writeData[0], *writeIO.result.lengthInfo), writeIO.localChecksum().value);
+      // update chunk data and compute the chunk checksum
+      if (offset + length > chunkData.size()) chunkData.resize(offset + length);
+      std::memcpy(&chunkData[offset], &writeData[0], *writeIO.result.lengthInfo);
+      ASSERT_EQ(folly::crc32c(&chunkData[0], chunkData.size()), writeIO.result.checksum.value);
+
+      {
+        // read back the write data
+        auto readIO = storageClient_->createReadIO(chainId, chunkId, offset, length, &readData[0], &readBuffer);
+        folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), readOptions));
+        ASSERT_RESULT_EQ(readIO.length, readIO.result.lengthInfo);
+
+        // verify checksum of the read data
+        ASSERT_EQ(folly::crc32c(&readData[0], *readIO.result.lengthInfo), readIO.result.checksum.value);
+        // compare with checksum of write data
+        ASSERT_EQ(writeIO.localChecksum().value, readIO.result.checksum.value);
+      }
+
+      {
+        // read the entire chunk
+        auto readIO =
+            storageClient_->createReadIO(chainId, chunkId, 0, setupConfig_.chunk_size(), &readData[0], &readBuffer);
+        folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), readOptions));
+        ASSERT_OK(readIO.result.lengthInfo);
+
+        // verify checksum of the read data
+        ASSERT_EQ(folly::crc32c(&readData[0], *readIO.result.lengthInfo), readIO.result.checksum.value);
+        // compare with chunk checksum
+        ASSERT_EQ(writeIO.result.checksum.value, readIO.result.checksum.value);
+      }
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, QueryRemoveTruncateChunks) {
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 0 /*low*/);
+  const uint32_t maxNumResultsPerQuery = serverConfigs_.front().storage().max_num_results_per_query();
+
+  for (uint32_t numChunks : {1U,
+                             maxNumResultsPerQuery - 1,
+                             maxNumResultsPerQuery,
+                             maxNumResultsPerQuery + 1,
+                             2 * maxNumResultsPerQuery - 2,
+                             2 * maxNumResultsPerQuery,
+                             2 * maxNumResultsPerQuery + 2,
+                             folly::Random::rand32(1, 3 * maxNumResultsPerQuery)}) {
+    uint32_t writeLen = setupConfig_.chunk_size() / 2;
+    std::vector<uint8_t> writeData(writeLen, 0xFF);
+
+    auto result = writeToChunks(chainId, ChunkId(1, 0), ChunkId(1, numChunks), writeData);
+    ASSERT_TRUE(result);
+
+    {
+      // truncate chunks
+      std::vector<TruncateChunkOp> truncateOps;
+      for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+        auto op = storageClient_->createTruncateOp(chainId,
+                                                   ChunkId(chunkId, chunkIndex),
+                                                   setupConfig_.chunk_size(),
+                                                   setupConfig_.chunk_size());
+        truncateOps.push_back(std::move(op));
+      }
+
+      folly::coro::blockingWait(storageClient_->truncateChunks(truncateOps, flat::UserInfo()));
+
+      for (const auto &op : truncateOps) {
+        ASSERT_OK(op.result.lengthInfo);
+        ASSERT_EQ(op.chunkLen, op.result.lengthInfo.value());
+      }
+    }
+
+    {
+      // read chunks created by truncate ops
+      for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+        std::vector<uint8_t> readData(setupConfig_.chunk_size(), 0x0);
+        auto result = readFromChunk(chainId, ChunkId(chunkId, chunkIndex), readData);
+        ASSERT_OK(result.lengthInfo);
+        ASSERT_EQ(setupConfig_.chunk_size(), result.lengthInfo.value());
+
+        for (size_t index = 0; index < readData.size(); index++) {
+          if (index < writeLen)
+            ASSERT_EQ(0xFF, readData[index]);  // the written part
+          else
+            ASSERT_EQ(0x00, readData[index]);  // the extended part
+        }
+      }
+    }
+
+    {
+      // query only last chunk
+      auto queryOp = storageClient_->createQueryOp(chainId,
+                                                   ChunkId(chunkId, 0),
+                                                   ChunkId(chunkId, numChunks),
+                                                   1 /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(queryOp.result.statusCode);
+      ASSERT_EQ(ChunkId(chunkId, numChunks - 1).data(), queryOp.result.lastChunkId.data());
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.lastChunkLen);
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.totalChunkLen);
+      ASSERT_EQ(1, queryOp.result.totalNumChunks);
+      ASSERT_TRUE(queryOp.result.moreChunksInRange || numChunks <= 1);
+    }
+
+    if (numChunks > 1) {
+      // query half of the chunks
+      auto halfNumChunks = numChunks / 2;
+      auto queryOp = storageClient_->createQueryOp(chainId,
+                                                   ChunkId(chunkId, 0),
+                                                   ChunkId(chunkId, numChunks),
+                                                   halfNumChunks /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(queryOp.result.statusCode);
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.lastChunkLen);
+      ASSERT_EQ(setupConfig_.chunk_size() * halfNumChunks, queryOp.result.totalChunkLen);
+      ASSERT_EQ(halfNumChunks, queryOp.result.totalNumChunks);
+      ASSERT_TRUE(queryOp.result.moreChunksInRange);
+    }
+
+    {
+      // query all chunks
+      auto queryOp = storageClient_->createQueryOp(chainId,
+                                                   ChunkId(chunkId, 0),
+                                                   ChunkId(chunkId, numChunks),
+                                                   numChunks /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(queryOp.result.statusCode);
+      ASSERT_EQ(ChunkId(chunkId, numChunks - 1).data(), queryOp.result.lastChunkId.data());
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.lastChunkLen);
+      ASSERT_EQ(setupConfig_.chunk_size() * numChunks, queryOp.result.totalChunkLen);
+      ASSERT_EQ(numChunks, queryOp.result.totalNumChunks);
+      ASSERT_FALSE(queryOp.result.moreChunksInRange);
+    }
+
+    {
+      // query the max range
+      auto queryOp = storageClient_->createQueryOp(chainId,
+                                                   ChunkId(chunkId, 0),
+                                                   ChunkId(chunkId, UINT32_MAX),
+                                                   UINT32_MAX /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(queryOp.result.statusCode);
+      ASSERT_EQ(setupConfig_.chunk_size(), queryOp.result.lastChunkLen);
+      ASSERT_EQ(setupConfig_.chunk_size() * numChunks, queryOp.result.totalChunkLen);
+      ASSERT_EQ(numChunks, queryOp.result.totalNumChunks);
+      ASSERT_FALSE(queryOp.result.moreChunksInRange);
+    }
+
+    {
+      // remove the last chunk
+      auto removeOp = storageClient_->createRemoveOp(chainId,
+                                                     ChunkId(chunkId, 0),
+                                                     ChunkId(chunkId, numChunks),
+                                                     1 /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(removeOp.result.statusCode);
+      ASSERT_EQ(1, removeOp.result.numChunksRemoved);
+      ASSERT_TRUE(removeOp.result.moreChunksInRange || numChunks <= 1);
+    }
+
+    {
+      // remove the remaining chunks
+      auto removeOp = storageClient_->createRemoveOp(chainId,
+                                                     ChunkId(chunkId, 0),
+                                                     ChunkId(chunkId, numChunks),
+                                                     numChunks /*maxNumChunkIdsToProcess*/);
+
+      folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo()));
+
+      ASSERT_OK(removeOp.result.statusCode);
+      ASSERT_EQ(numChunks - 1, removeOp.result.numChunksRemoved);
+      ASSERT_FALSE(removeOp.result.moreChunksInRange);
+    }
+
+    {
+      // check if chunks are removed
+      auto queryOp = storageClient_->createQueryOp(chainId, ChunkId(chunkId, 0), ChunkId(chunkId, numChunks));
+
+      folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+
+      ASSERT_TRUE(queryOp.result.statusCode);
+      ASSERT_EQ(ChunkId().data(), queryOp.result.lastChunkId.data());
+      ASSERT_EQ(0, queryOp.result.lastChunkLen);
+      ASSERT_EQ(0, queryOp.result.totalChunkLen);
+      ASSERT_EQ(0, queryOp.result.totalNumChunks);
+      ASSERT_FALSE(queryOp.result.moreChunksInRange);
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, TruncateExtendChunks) {
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 0 /*low*/);
+  uint32_t numChunks = 3;
+  uint32_t writeLen = setupConfig_.chunk_size() / 2;
+
+  std::vector<uint8_t> writeData(writeLen, 0xFF);
+  auto result = writeToChunks(chainId, ChunkId(1, 0), ChunkId(1, numChunks), writeData);
+  ASSERT_TRUE(result);
+
+  {
+    // extend op does not reduce chunk length
+    std::vector<TruncateChunkOp> truncateOps;
+    for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+      auto op = storageClient_->createTruncateOp(chainId,
+                                                 ChunkId(chunkId, chunkIndex),
+                                                 writeLen / 2 /* try to reduce chunk length */,
+                                                 setupConfig_.chunk_size(),
+                                                 true /*onlyExtendChunk*/);
+      truncateOps.push_back(std::move(op));
+    }
+
+    folly::coro::blockingWait(storageClient_->truncateChunks(truncateOps, flat::UserInfo()));
+
+    for (const auto &op : truncateOps) {
+      ASSERT_OK(op.result.lengthInfo);
+      ASSERT_EQ(writeLen, op.result.lengthInfo.value());  // still get the original length
+    }
+  }
+
+  {
+    // extend to full chunks
+    std::vector<TruncateChunkOp> truncateOps;
+    for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+      auto op = storageClient_->createTruncateOp(chainId,
+                                                 ChunkId(chunkId, chunkIndex),
+                                                 setupConfig_.chunk_size() /* full chunk size */,
+                                                 setupConfig_.chunk_size(),
+                                                 true /*onlyExtendChunk*/);
+      truncateOps.push_back(std::move(op));
+    }
+
+    folly::coro::blockingWait(storageClient_->truncateChunks(truncateOps, flat::UserInfo()));
+
+    for (const auto &op : truncateOps) {
+      ASSERT_OK(op.result.lengthInfo);
+      ASSERT_EQ(setupConfig_.chunk_size(), op.result.lengthInfo.value());
+    }
+  }
+
+  {
+    // read the extended chunks
+    for (uint32_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
+      std::vector<uint8_t> readData(setupConfig_.chunk_size(), 0xAA);
+      auto result = readFromChunk(chainId, ChunkId(chunkId, chunkIndex), readData);
+      ASSERT_OK(result.lengthInfo);
+      ASSERT_EQ(setupConfig_.chunk_size(), result.lengthInfo.value());
+
+      for (size_t index = 0; index < readData.size(); index++) {
+        if (index < writeLen)
+          ASSERT_EQ(0xFF, readData[index]);  // the written part
+        else
+          ASSERT_EQ(0x00, readData[index]);  // the extended part
+      }
+    }
+  }
+}
+
+TEST_P(TestStorageClientInterface, QuerySpaceInfo) {
+  auto result = folly::coro::blockingWait(storageClient_->querySpaceInfo(NodeId{1}));
+  ASSERT_OK(result);
+  ASSERT_TRUE(!result->spaceInfos.empty());
+}
+
+TEST_P(TestStorageClientInterface, CreateTarget) {
+  CreateTargetReq req;
+  req.targetId = TargetId{255};
+  req.chainId = ChainId{255};
+  req.diskIndex = 0;
+  req.onlyChunkEngine = true;
+  auto result = folly::coro::blockingWait(storageClient_->createTarget(NodeId{1}, req));
+  ASSERT_OK(result);
+}
+
+TEST_P(TestStorageClientInterface, GetAllChunkMetadata) {
+  ChainId chainId = firstChainId_;
+  ChunkId chunkId(1 /*high*/, 0 /*low*/);
+  uint32_t numChunks = 3;
+
+  std::vector<uint8_t> writeData(setupConfig_.chunk_size(), 0xFF);
+  auto result = writeToChunks(chainId, chunkId, ChunkId(chunkId, numChunks), writeData);
+  ASSERT_TRUE(result);
+
+  auto routingInfo = getRoutingInfo();
+  ASSERT_TRUE(routingInfo);
+  auto chainInfo = routingInfo->getChain(chainId);
+  ASSERT_TRUE(chainInfo);
+
+  for (const auto targetInfo : chainInfo->targets) {
+    auto chunkMetaVec = folly::coro::blockingWait(storageClient_->getAllChunkMetadata(chainId, targetInfo.targetId));
+    ASSERT_OK(chunkMetaVec);
+
+    std::set<ChunkId> uniqChunkIds;
+    for (const auto &chunkMeta : *chunkMetaVec) {
+      ASSERT_EQ(ChunkVer{1}, chunkMeta.commitVer);
+      ASSERT_EQ(setupConfig_.chunk_size(), chunkMeta.length);
+      uniqChunkIds.insert(chunkMeta.chunkId);
+    }
+
+    ASSERT_EQ(numChunks, uniqChunkIds.size());
+    ASSERT_EQ(chunkId, *uniqChunkIds.begin());
+    ASSERT_EQ(ChunkId(chunkId, numChunks - 1), *uniqChunkIds.rbegin());
+  }
+}
+
+SystemSetupConfig testInMemClient = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    1 /*numReplicas*/,
+    1 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+    hf3fs::Path() /*clientConfig*/,
+    hf3fs::Path() /*serverConfig*/,
+    {} /*storageEndpoints*/,
+    0 /*serviceLevel*/,
+    0 /*listenPort*/,
+    StorageClient::ImplementationType::InMem /*clientImplType*/,
+};
+
+SystemSetupConfig testRpcClient = {
+    128_KB /*chunkSize*/,
+    1 /*numChains*/,
+    2 /*numReplicas*/,
+    2 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+SystemSetupConfig testSmallChunk = {
+    512 /*chunkSize*/,
+    1 /*numChains*/,
+    2 /*numReplicas*/,
+    2 /*numStorageNodes*/,
+    {folly::fs::temp_directory_path()} /*dataPaths*/,
+};
+
+INSTANTIATE_TEST_SUITE_P(InMemClient,
+                         TestStorageClientInterface,
+                         ::testing::Values(testInMemClient),
+                         SystemSetupConfig::prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(RpcClient,
+                         TestStorageClientInterface,
+                         ::testing::Values(testRpcClient),
+                         SystemSetupConfig::prettyPrintConfig);
+INSTANTIATE_TEST_SUITE_P(SmallChunk,
+                         TestStorageClientInterface,
+                         ::testing::Values(testSmallChunk),
+                         SystemSetupConfig::prettyPrintConfig);
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestStorageClientSideError.cc

@@ -0,0 +1,401 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/logging/xlog.h>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/net/Client.h"
+#include "tests/lib/UnitTestFabric.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+using namespace hf3fs::test;
+
+class TestStorageClientSideError : public UnitTestFabric, public ::testing::Test {
+ protected:
+  TestStorageClientSideError()
+      : UnitTestFabric(SystemSetupConfig{128_KB /*chunkSize*/,
+                                         1 /*numChains*/,
+                                         1 /*numReplicas*/,
+                                         1 /*numStorageNodes*/,
+                                         {folly::fs::temp_directory_path()}}) {}
+
+  void SetUp() override {
+    // init ib device
+    net::IBDevice::Config ibConfig;
+    auto ibResult = net::IBManager::start(ibConfig);
+    ASSERT_OK(ibResult);
+    ASSERT_TRUE(setUpStorageSystem());
+    clientConfig_.retry().set_max_retry_time(2_s);
+  }
+
+  void TearDown() override { tearDownStorageSystem(); }
+
+  template <typename Op>
+  void checkFailedOpsPtrs(const std::vector<Op> &ops, const std::vector<Op *> &failedOps) {
+    bool ok = std::all_of(failedOps.cbegin(), failedOps.cend(), [&ops](const Op *failedOp) -> bool {
+      for (const auto &op : ops) {
+        if (failedOp == &op) return true;
+      }
+      XLOGF(ERR,
+            "Address of failed op {} not in range: [{}, {})",
+            fmt::ptr(failedOp),
+            fmt::ptr(&ops[0]),
+            fmt::ptr(&ops[ops.size() - 1]));
+      return false;
+    });
+    ASSERT_TRUE(ok);
+  }
+};
+
+TEST_F(TestStorageClientSideError, GetReplicationChainError) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &chainTable = *routingInfo.getChainTable(kTableId());
+    auto &chainInfo = routingInfo.chains[chainTable.chains.front()];
+
+    // first target offline
+    chainInfo.targets.begin()->publicState = hf3fs::flat::PublicTargetState::OFFLINE;
+  });
+
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+  auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNotAvailable);
+
+  // inconsistent chain id
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &chainTable = *routingInfo.getChainTable(kTableId());
+    auto &chainInfo = routingInfo.chains[chainTable.chains.front()];
+    chainInfo.chainId = flat::ChainId(0);
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+
+  // empty chain table
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &chainTable = *routingInfo.getChainTable(kTableId());
+    chainTable.chains.clear();
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+}
+
+TEST_F(TestStorageClientSideError, GetStorageTargetError) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &targetInfo = routingInfo.targets.begin()->second;
+
+    // host node id not unknown
+    targetInfo.nodeId = std::nullopt;
+  });
+
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+  auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNotAvailable);
+
+  // target offline
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &targetInfo = routingInfo.targets.begin()->second;
+    targetInfo.publicState = hf3fs::flat::PublicTargetState::OFFLINE;
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNotAvailable);
+
+  // inconsistent target id
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &targetInfo = routingInfo.targets.begin()->second;
+    targetInfo.targetId = flat::TargetId(0);
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+
+  // empty target list
+  updateRoutingInfo([&](auto &routingInfo) { routingInfo.targets.clear(); });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+}
+
+TEST_F(TestStorageClientSideError, GetStorageNodeError) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+
+    // not storage node
+    nodeInfo.type = hf3fs::flat::NodeType::META;
+  });
+
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+  auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+
+  // inconsistent node id
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+    nodeInfo.app.nodeId = hf3fs::flat::NodeId(0);
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+
+  // empty node list
+  updateRoutingInfo([&](auto &routingInfo) { routingInfo.nodes.clear(); });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
+}
+
+TEST_F(TestStorageClientSideError, WrongServerAddress) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+
+    // wrong port
+    nodeInfo.app.serviceGroups.front().endpoints.front().port = 0;
+  });
+
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+  auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_TRUE(!ioResult.lengthInfo);
+  ASSERT_TRUE(ioResult.lengthInfo.error().code() == StorageClientCode::kTimeout ||
+              ioResult.lengthInfo.error().code() == StorageClientCode::kCommError);
+
+  ioResult = readFromChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_TRUE(!ioResult.lengthInfo);
+  ASSERT_TRUE(ioResult.lengthInfo.error().code() == StorageClientCode::kTimeout ||
+              ioResult.lengthInfo.error().code() == StorageClientCode::kCommError);
+
+  // empty address list
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+    nodeInfo.app.serviceGroups.front().endpoints.clear();
+  });
+
+  ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNoRDMAInterface);
+
+  ioResult = readFromChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+  ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNoRDMAInterface);
+}
+
+TEST_F(TestStorageClientSideError, DifferentTrafficZone) {
+  updateRoutingInfo([&](auto &routingInfo) {
+    auto &nodeInfo = routingInfo.nodes.begin()->second;
+    // set traffic zone of the host
+    nodeInfo.tags = {flat::TagPair{flat::kTrafficZoneTagKey, "TEST_ZONE0"}};
+  });
+
+  {
+    std::vector<uint8_t> userData(1_KB, 0xFF);
+    auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+    ASSERT_RESULT_EQ(userData.size(), ioResult.lengthInfo);
+
+    ReadOptions options;
+    // client is in the same traffic zone as the storage node
+    options.targetSelection().set_trafficZone("TEST_ZONE0");
+    ioResult = readFromChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+    ASSERT_RESULT_EQ(userData.size(), ioResult.lengthInfo);
+  }
+
+  {
+    std::vector<uint8_t> userData(1_KB, 0xFF);
+    auto ioResult = writeToChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size());
+    ASSERT_RESULT_EQ(userData.size(), ioResult.lengthInfo);
+
+    ReadOptions options;
+    // client is in a different traffic zone from the storage node
+    options.targetSelection().set_trafficZone("TEST_ZONE1");
+    ioResult = readFromChunk(firstChainId_, ChunkId(1, 1), userData, 0 /*offset*/, userData.size(), options);
+    ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kNotAvailable);
+  }
+}
+
+TEST_F(TestStorageClientSideError, InvalidDataRange) {
+  std::vector<uint8_t> userData(1_KB, 0xFF);
+
+  {
+    // overlapping data buffers
+    std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size() * 2, 0xFF);
+    auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    std::vector<ReadIO> readIOs;
+
+    for (size_t offset = 0; offset < setupConfig_.chunk_size(); offset += setupConfig_.chunk_size() / 4) {
+      auto readIO = storageClient_->createReadIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 offset /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 &memoryBlock[offset],
+                                                 &ioBuffer);
+      readIOs.push_back(std::move(readIO));
+    }
+
+    std::vector<ReadIO *> failedIOs;
+    folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo(), ReadOptions(), &failedIOs));
+
+    for (const auto &readIO : readIOs) {
+      ASSERT_FALSE(readIO.result.lengthInfo);
+      ASSERT_EQ(StorageClientCode::kInvalidArg, readIO.result.lengthInfo.error().code());
+    }
+
+    checkFailedOpsPtrs(readIOs, failedIOs);
+    std::set<ReadIO *> uniqueFailedIOs(failedIOs.begin(), failedIOs.end());
+    ASSERT_EQ(readIOs.size(), uniqueFailedIOs.size());
+  }
+
+  {
+    // duplicate IOs
+    std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
+    auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    std::vector<ReadIO> readIOs;
+
+    for (int i = 0; i < 2; i++) {
+      auto readIO = storageClient_->createReadIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 setupConfig_.chunk_size() /*length*/,
+                                                 &memoryBlock[0],
+                                                 &ioBuffer);
+      readIOs.push_back(std::move(readIO));
+    }
+
+    std::vector<ReadIO *> failedIOs;
+    folly::coro::blockingWait(storageClient_->batchRead(readIOs, flat::UserInfo(), ReadOptions(), &failedIOs));
+
+    for (const auto &readIO : readIOs) {
+      ASSERT_FALSE(readIO.result.lengthInfo);
+      ASSERT_EQ(StorageClientCode::kInvalidArg, readIO.result.lengthInfo.error().code());
+    }
+
+    checkFailedOpsPtrs(readIOs, failedIOs);
+    std::set<ReadIO *> uniqueFailedIOs(failedIOs.begin(), failedIOs.end());
+    ASSERT_EQ(readIOs.size(), uniqueFailedIOs.size());
+  }
+
+  {
+    // null io buffer pointer
+    auto writeIO = storageClient_->createWriteIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 userData.size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 &userData[0],
+                                                 nullptr);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, writeIO.result.lengthInfo.error().code());
+  }
+
+  {
+    // null data pointer
+    auto regRes = storageClient_->registerIOBuffer(&userData[0], userData.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    auto writeIO = storageClient_->createWriteIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 userData.size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 nullptr,
+                                                 &ioBuffer);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, writeIO.result.lengthInfo.error().code());
+  }
+
+  {
+    // write range is out of the chunk boundary
+    std::vector<uint8_t> largeArray(setupConfig_.chunk_size() * 2, 0xFF);
+    auto regRes = storageClient_->registerIOBuffer(&largeArray[0], largeArray.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    auto writeIO = storageClient_->createWriteIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 largeArray.size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 &largeArray[0],
+                                                 &ioBuffer);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, writeIO.result.lengthInfo.error().code());
+  }
+
+  {
+    // write range is out of the io buffer
+    auto regRes = storageClient_->registerIOBuffer(&userData[0], userData.size());
+    ASSERT_TRUE(regRes);
+    auto ioBuffer = std::move(*regRes);
+
+    auto writeIO = storageClient_->createWriteIO(firstChainId_,
+                                                 ChunkId(1, 1),
+                                                 0 /*offset*/,
+                                                 2 * userData.size() /*length*/,
+                                                 setupConfig_.chunk_size(),
+                                                 &userData[0],
+                                                 &ioBuffer);
+    folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo()));
+    ASSERT_FALSE(writeIO.result.lengthInfo);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, writeIO.result.lengthInfo.error().code());
+  }
+}
+
+TEST_F(TestStorageClientSideError, InvalidChunkIdRange) {
+  ChunkId largeChunkId(1 /*high*/, 3 /*low*/);
+  ChunkId smallChunkId(1 /*high*/, 1 /*low*/);
+  ASSERT_GT(largeChunkId, smallChunkId);
+
+  {
+    // invalid begin/end chunk ids for a range query
+    auto queryOp =
+        storageClient_->createQueryOp(firstChainId_, largeChunkId, smallChunkId, 1 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+    ASSERT_FALSE(queryOp.result.statusCode);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, queryOp.result.statusCode.error().code());
+  }
+  {
+    // invalid maxNumChunkIdsToProcess for a range query
+    auto queryOp =
+        storageClient_->createQueryOp(firstChainId_, smallChunkId, largeChunkId, 0 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->queryLastChunk(std::span(&queryOp, 1), flat::UserInfo()));
+    ASSERT_FALSE(queryOp.result.statusCode);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, queryOp.result.statusCode.error().code());
+  }
+
+  {
+    // invalid begin/end chunk ids for a batch remove
+    auto removeOp =
+        storageClient_->createRemoveOp(firstChainId_, largeChunkId, smallChunkId, 1 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo()));
+    ASSERT_FALSE(removeOp.result.statusCode);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, removeOp.result.statusCode.error().code());
+  }
+  {
+    // invalid maxNumChunkIdsToProcess for a batch remove
+    auto removeOp =
+        storageClient_->createRemoveOp(firstChainId_, smallChunkId, largeChunkId, 0 /*maxNumChunkIdsToProcess*/);
+    folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo()));
+    ASSERT_FALSE(removeOp.result.statusCode);
+    ASSERT_EQ(StorageClientCode::kInvalidArg, removeOp.result.statusCode.error().code());
+  }
+}
+
+TEST_F(TestStorageClientSideError, InvalidChunkSize) {
+  auto truncateOp = storageClient_->createTruncateOp(firstChainId_,
+                                                     ChunkId(1 /*high*/, 1 /*low*/),
+                                                     setupConfig_.chunk_size() * 2 /*chunkLen*/,
+                                                     setupConfig_.chunk_size());
+  folly::coro::blockingWait(storageClient_->truncateChunks(std::span(&truncateOp, 1), flat::UserInfo()));
+  ASSERT_FALSE(truncateOp.result.lengthInfo);
+  ASSERT_EQ(StorageClientCode::kInvalidArg, truncateOp.result.lengthInfo.error().code());
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client

tests/storage/client/TestTargetSelectionStrategy.cc

@@ -0,0 +1,155 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+
+#include "client/storage/TargetSelection.h"
+#include "common/utils/Duration.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+TEST(TargetSelectionStrategy, LoadDistribution) {
+  const uint64_t numReplicas = 3;
+  const uint64_t numNodes = 180;
+  const uint64_t numTargets = numNodes * 16 * 16;
+  const uint64_t numChains = numTargets / 3;
+  const uint64_t numBatches = 100;
+
+  uint64_t nodeSeqNum = 0;
+  uint64_t targetId = 0;
+  std::unordered_map<NodeId, uint64_t> numTargetsOnNode;
+  std::unordered_map<ChainId, SlimChainInfo> chainMap;
+
+  for (uint64_t chainId = 1; chainId <= numChains; chainId++) {
+    auto &chainInfo = chainMap[ChainId(chainId)];
+    chainInfo.chainId = ChainId(chainId);
+    chainInfo.totalNumTargets = numReplicas;
+
+    for (uint64_t replica = 0; replica < numReplicas; replica++) {
+      NodeId nodeId((nodeSeqNum++ % numNodes) + 1);
+      chainInfo.servingTargets.push_back({TargetId(++targetId), nodeId});
+      numTargetsOnNode[nodeId]++;
+      ASSERT_LE(numTargetsOnNode[nodeId], numTargets / numNodes);
+      ASSERT_LE(targetId, numTargets);
+    }
+  }
+
+  auto computeIODist = [](const std::unordered_map<NodeId, uint64_t> &numIOsOnNode) {
+    std::vector<uint64_t> numIOs;
+    uint64_t sumIOs = 0;
+
+    for (const auto &item : numIOsOnNode) {
+      numIOs.push_back(item.second);
+      sumIOs += item.second;
+    }
+
+    std::sort(numIOs.begin(), numIOs.end());
+    return std::vector{
+        numIOs.front(),
+        numIOs.back(),
+        numIOs[numIOs.size() / 2],
+        sumIOs / numIOsOnNode.size(),
+        numIOsOnNode.size(),
+    };
+  };
+
+  for (uint64_t numIOsPerBatch : {100, 200, 400, 1000}) {
+    for (uint64_t mode = 1; mode < TargetSelectionMode::EndOfMode; mode++) {
+      std::unordered_map<NodeId, uint64_t> totalIOsOnNode;
+      std::vector<uint64_t> avgBatchDist(5);
+
+      for (uint64_t k = 0; k < numBatches; k++) {
+        TargetSelectionOptions options;
+        options.set_mode(static_cast<TargetSelectionMode>(mode));
+
+        auto strategy = TargetSelectionStrategy::create(options);
+        if (k == 0) strategy->reset();
+
+        std::unordered_map<NodeId, uint64_t> batchIOsOnNode;
+
+        for (uint64_t i = 0; i < numIOsPerBatch; i++) {
+          ChainId chainId(folly::Random::rand32(1, numChains + 1));
+          ASSERT_TRUE(!chainMap[chainId].servingTargets.empty()) << "chainId " << chainId;
+          auto selectedTarget = strategy->selectTarget(chainMap[chainId]);
+          ASSERT_TRUE(std::find(chainMap[chainId].servingTargets.cbegin(),
+                                chainMap[chainId].servingTargets.cend(),
+                                *selectedTarget) != chainMap[chainId].servingTargets.end());
+          totalIOsOnNode[selectedTarget->nodeId]++;
+          batchIOsOnNode[selectedTarget->nodeId]++;
+        }
+
+        auto dist = computeIODist(batchIOsOnNode);
+        for (uint64_t i = 0; i < dist.size(); i++) {
+          avgBatchDist[i] += dist[i];
+        }
+      }
+
+      auto overallDist = computeIODist(totalIOsOnNode);
+
+      fmt::print("\n=== numIOsPerBatch {}, TargetSelectionMode {} ===\n", numIOsPerBatch, mode);
+      fmt::print("   overall :  min {:<5d} max {:<5d} median {:<5d} avg {:<5d} #nodes {:<5d}\n",
+                 overallDist[0],
+                 overallDist[1],
+                 overallDist[2],
+                 overallDist[3],
+                 overallDist[4]);
+      fmt::print(" per batch :  min {:<5d} max {:<5d} median {:<5d} avg {:<5d} #nodes {:<5d}\n",
+                 avgBatchDist[0] / numBatches,
+                 avgBatchDist[1] / numBatches,
+                 avgBatchDist[2] / numBatches,
+                 avgBatchDist[3] / numBatches,
+                 avgBatchDist[4] / numBatches);
+    }
+  }
+}
+
+TEST(TargetSelectionStrategy, MultiThreads) {
+  constexpr uint64_t numReplicas = 2;
+  constexpr uint64_t numNodes = 180;
+  constexpr uint64_t numTargets = numNodes * 16 * 16;
+  constexpr uint64_t numChains = numTargets / numReplicas;
+  constexpr uint64_t numIOsPerBatch = 1024;
+  constexpr uint32_t numThreads = 32;
+
+  uint64_t nodeSeqNum = 0;
+  uint64_t targetId = 0;
+  std::unordered_map<NodeId, uint64_t> numTargetsOnNode;
+  std::unordered_map<ChainId, SlimChainInfo> chainMap;
+
+  for (uint64_t chainId = 1; chainId <= numChains; chainId++) {
+    auto &chainInfo = chainMap[ChainId(chainId)];
+    chainInfo.chainId = ChainId(chainId);
+    chainInfo.totalNumTargets = numReplicas;
+
+    for (uint64_t replica = 0; replica < numReplicas; replica++) {
+      NodeId nodeId((nodeSeqNum++ % numNodes) + 1);
+      chainInfo.servingTargets.push_back({TargetId(++targetId), nodeId});
+      numTargetsOnNode[nodeId]++;
+    }
+  }
+
+  folly::CPUThreadPoolExecutor executor(numThreads);
+  auto startTime = RelativeTime::now();
+  std::atomic<uint64_t> finished;
+  for (auto i = 0u; i < numThreads; ++i) {
+    executor.add([&] {
+      while (startTime + 1_s >= RelativeTime::now()) {
+        TargetSelectionOptions options;
+        options.set_mode(TargetSelectionMode::RoundRobin);
+        auto strategy = TargetSelectionStrategy::create(options);
+
+        for (auto j = 0u; j < numIOsPerBatch; ++j) {
+          ChainId chainId{folly::Random::rand32(1, numChains + 1)};
+          ASSERT_TRUE(!chainMap[chainId].servingTargets.empty()) << "chainId " << chainId;
+          auto selectedTarget = strategy->selectTarget(chainMap[chainId]);
+          ASSERT_OK(selectedTarget);
+        }
+        finished += numIOsPerBatch;
+      }
+    });
+  }
+  executor.join();
+  XLOGF(WARNING, "select target {} times", finished.load());
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client