Initial commit

tests/meta/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_test(test_meta meta fdb mgmtd)

tests/meta/MetaTestBase.cc

@@ -0,0 +1,348 @@
+#include "MetaTestBase.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdlib>
+#include <fmt/core.h>
+#include <folly/Conv.h>
+#include <folly/Math.h>
+#include <folly/Random.h>
+#include <folly/Unit.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/CurrentExecutor.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <memory>
+#include <set>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <type_traits>
+#include <unistd.h>
+
+#include "client/storage/StorageClient.h"
+#include "common/app/ClientId.h"
+#include "common/kv/IKVEngine.h"
+#include "common/kv/ITransaction.h"
+#include "common/kv/mem/MemKV.h"
+#include "common/kv/mem/MemKVEngine.h"
+#include "common/kv/mem/MemTransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/MagicEnum.hpp"
+#include "common/utils/Path.h"
+#include "common/utils/Result.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/core/user/User.h"
+#include "fbs/meta/FileOperation.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/meta/Service.h"
+#include "fbs/mgmtd/ChainRef.h"
+#include "fbs/mgmtd/ChainSetting.h"
+#include "fdb/FDB.h"
+#include "fdb/FDBKVEngine.h"
+#include "fdb/FDBTransaction.h"
+#include "foundationdb/fdb_c_types.h"
+#include "meta/service/MetaOperator.h"
+#include "meta/service/MockMeta.h"
+#include "meta/store/MetaStore.h"
+#include "stubs/mgmtd/MgmtdServiceStub.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::meta::server {
+
+namespace {
+auto randomAcl() {
+  return Acl(Uid(folly::Random::rand32()), Gid(folly::Random::rand32()), Permission(folly::Random::rand32()));
+}
+}  // namespace
+
+Inode MetaTestHelper::randomInode() {
+  switch (folly::Random::rand32() % 3) {
+    case 0:
+      return randomFile();
+    case 1:
+      return randomDirectory();
+    default: {
+      auto acl = randomAcl();
+      auto name = std::to_string(folly::Random::rand32());
+      return Inode::newSymlink(randomInodeId(), name, acl.uid, acl.gid, UtcClock::now().castGranularity(1_s));
+    }
+  }
+}
+
+Inode MetaTestHelper::randomFile() {
+  return Inode::newFile(randomInodeId(), randomAcl(), randomLayout(), UtcClock::now().castGranularity(1_s));
+}
+
+Inode MetaTestHelper::randomDirectory() {
+  return Inode::newDirectory(randomInodeId(),
+                             randomInodeId(),
+                             folly::to<std::string>(folly::Random::rand32()),
+                             randomAcl(),
+                             randomEmptyLayout(),
+                             UtcClock::now().castGranularity(1_s));
+}
+
+DirEntry MetaTestHelper::randomDirEntry() {
+  InodeId parent = MetaTestHelper::randomInodeId();
+  String name = fmt::format("entry-{}", parent, folly::Random::rand64());
+  InodeId id = MetaTestHelper::randomInodeId();
+  auto type = InodeType(rand() % magic_enum::enum_count<InodeType>());
+  DirEntry entry;
+  switch (type) {
+    case InodeType::File:
+      entry = DirEntry::newFile(parent, name, id);
+      break;
+    case InodeType::Directory:
+      entry = DirEntry::newDirectory(
+          parent,
+          name,
+          id,
+          Acl(Uid(folly::Random::rand32()), Gid(folly::Random::rand32()), Permission(folly::Random::rand32())));
+      break;
+    case InodeType::Symlink:
+      entry = DirEntry::newSymlink(parent, name, id);
+      break;
+  }
+  return entry;
+}
+
+template <typename KV>
+KvTestBase<KV>::KvTestBase()
+    : skip_(true),
+      engine_(nullptr) {
+  if constexpr (std::is_same<KV, mem::MemKV>::value) {
+    skip_ = false;
+  } else if constexpr (std::is_same<KV, fdb::DB>::value) {
+    auto cluster = std::getenv("FDB_UNITTEST_CLUSTER");
+    if (cluster) {
+      skip_ = false;
+    }
+  } else {
+    XLOGF(FATAL, "Invalid KV type {}", std::type_identity<KV>::type);
+  }
+}
+
+template <typename KV>
+std::shared_ptr<IKVEngine> KvTestBase<KV>::createEngine() {
+  if (skip_) {
+    throw std::runtime_error("Shouldn't reach here!!!");
+  }
+
+  if constexpr (std::is_same<KV, mem::MemKV>::value) {
+    return std::shared_ptr<IKVEngine>(new MemKVEngine());
+  } else if constexpr (std::is_same<KV, fdb::DB>::value) {
+    auto cluster = std::getenv("FDB_UNITTEST_CLUSTER");
+    fdb::DB db(cluster, false);
+    if (db.error()) {
+      throw std::runtime_error(fmt::format("Failed to create fdb, error {}", db.error()));
+    }
+    return std::shared_ptr<IKVEngine>(new FDBKVEngine(std::move(db)));
+  } else {
+    XLOGF(FATAL, "Invalid KV type {}", std::type_identity<KV>::type);
+  }
+}
+
+template <typename KV>
+void KvTestBase<KV>::SetUp() {
+  if (skip_) {
+    GTEST_SKIP_("FDB_UNITTEST_CLUSTER not set");
+  }
+
+  // create KV engine
+  engine_ = createEngine();
+  if (dynamic_cast<FDBKVEngine *>(engine_.get())) {
+    // this is FDB test, remove all kvs from FDB
+    folly::coro::blockingWait([&]() -> CoTask<void> {
+      auto txn = engine_->createReadWriteTransaction();
+      auto clearResult = co_await dynamic_cast<FDBTransaction *>(txn.get())->clearRange("", "\xff");
+      CO_ASSERT_OK(clearResult);
+      auto result = co_await txn->commit();
+      CO_ASSERT_OK(result);
+      XLOGF(INFO, "FDB cleared, result {}", result.hasError() ? result.error().describe() : "ok");
+    }());
+  }
+}
+
+template <typename KV>
+void KvTestBase<KV>::TearDown() {
+  // do nothing
+}
+
+template class KvTestBase<mem::MemKV>;
+template class KvTestBase<fdb::DB>;
+
+static const String clusterId = "mock-cluster";
+
+void MockCluster::SetUp() {
+  ReqBase::currentClientId() = ClientId(Uuid::random(), "meta_test");
+  exec_ = std::make_unique<CPUExecutorGroup>(4, clusterId);
+
+  std::vector<std::tuple<flat::ChainTableId, size_t, size_t>> tables;
+  for (unsigned i = 0; i < config_.chain_tables_length(); i++) {
+    auto table = flat::ChainTableId(config_.chain_tables(i).table_id());
+    auto numChains = config_.chain_tables(i).num_chains();
+    auto numReplica = config_.chain_tables(i).num_replica();
+    tables.emplace_back(table, numChains, numReplica);
+  }
+  mgmtdClient_ = hf3fs::tests::FakeMgmtdClient::create(tables, config_.num_meta(), 10);
+
+  auto meta = folly::coro::blockingWait(MockMeta::create(config_.mock_meta(), engine_, mgmtdClient_));
+  ASSERT_OK(meta);
+  meta_ = std::move(*meta);
+  meta_->start(*exec_);
+
+  std::this_thread::sleep_for(std::chrono::milliseconds(200));
+}
+
+MockCluster::~MockCluster() {
+  ReqBase::currentClientId() = std::nullopt;
+  if (meta_) {
+    meta_->stop();
+  }
+}
+
+CoTask<void> randomWrite(MetaOperator &meta,
+                         storage::client::StorageClient &storage,
+                         const Inode &inode,
+                         uint64_t offset,
+                         uint64_t length) {
+  auto stripe = std::min((uint32_t)folly::divCeil(offset + length, (uint64_t)inode.asFile().layout.chunkSize),
+                         inode.asFile().layout.stripeSize);
+  if (inode.asFile().dynStripe && inode.asFile().dynStripe < stripe) {
+    auto result = co_await meta.setAttr(SetAttrReq::extendStripe(flat::UserInfo{}, inode.id, stripe));
+    CO_ASSERT_OK(result);
+  }
+
+  uint64_t chunkSize = inode.asFile().layout.chunkSize;
+  std::vector<uint8_t> writeData(chunkSize, 0x00);
+  std::vector<folly::SemiFuture<folly::Unit>> tasks;
+  while (length) {
+    auto offsetInChunk = offset % chunkSize;
+    auto lengthInChunk = std::min(length, chunkSize - offsetInChunk);
+    auto chunkId = inode.asFile().getChunkId(inode.id, offset);
+    auto routingInfo = storage.getMgmtdClient().getRoutingInfo()->raw();
+    XLOGF_IF(FATAL, !routingInfo, "No routingInfo");
+    auto chainId = inode.asFile().getChainId(inode, offset, *routingInfo);
+    XLOGF_IF(FATAL, !chainId, "resolve chainId failed: {}", chainId);
+
+    auto task = [=, &storage, &writeData]() -> CoTask<void> {
+      auto writeIO = storage.createWriteIO(storage::ChainId(*chainId),
+                                           storage::ChunkId(chunkId->pack()),
+                                           offsetInChunk,
+                                           lengthInChunk,
+                                           chunkSize,
+                                           writeData.data(),
+                                           nullptr);
+      XLOGF(DBG, "write {} offset {}, offsetInChunk {} length {}", chunkId, offset, offsetInChunk, lengthInChunk);
+      auto result = co_await storage.write(writeIO, flat::UserInfo());
+      CO_ASSERT_FALSE(result.hasError()) << result.error().describe();
+      CO_ASSERT_FALSE(writeIO.result.lengthInfo.hasError()) << writeIO.result.lengthInfo.error().describe();
+      CO_ASSERT_EQ(*writeIO.result.lengthInfo, lengthInChunk);
+    };
+
+    tasks.push_back(folly::coro::co_invoke(task).scheduleOn(co_await folly::coro::co_current_executor).start());
+
+    offset += lengthInChunk;
+    length -= lengthInChunk;
+  }
+
+  co_await folly::collectAll(tasks.begin(), tasks.end());
+}
+
+CoTask<void> truncate(MetaOperator &meta,
+                      storage::client::StorageClient &storage,
+                      const Inode &inode,
+                      uint64_t length) {
+  auto stripe = std::min((uint32_t)folly::divCeil(length, (uint64_t)inode.asFile().layout.chunkSize),
+                         inode.asFile().layout.stripeSize);
+  if (inode.asFile().dynStripe && inode.asFile().dynStripe < stripe) {
+    auto result = co_await meta.setAttr(SetAttrReq::extendStripe(flat::UserInfo{}, inode.id, stripe));
+    CO_ASSERT_OK(result);
+  }
+
+  CO_ASSERT_TRUE(inode.isFile()) << fmt::format("{}", inode);
+  flat::UserInfo user{};
+  auto routing = storage.getMgmtdClient().getRoutingInfo();
+  auto fop = FileOperation(storage, *routing->raw(), user, inode);
+  bool more = true;
+  uint64_t removed = 0;
+  while (more) {
+    auto remove = co_await fop.removeChunks(length, 32, false, {});
+    CO_ASSERT_OK(remove);
+    removed += remove->first;
+    more = remove->second;
+    XLOGF(INFO, "truncate {} to {}, removed {}, more {}", inode.id, length, removed, more);
+    CO_ASSERT_FALSE((!remove->first && more));
+  }
+
+  CO_ASSERT_OK(co_await fop.truncateChunk(length));
+  CO_ASSERT_OK(co_await meta.sync({SUPER_USER, inode.id, true, std::nullopt, std::nullopt, true, std::nullopt}));
+}
+
+CoTryTask<size_t> queryTotalChunks(storage::client::StorageClient &storage, client::ICommonMgmtdClient &mgmtd) {
+  ChunkId begin(InodeId(0), 0, 0), end(InodeId(-1), 0, 0);
+  auto routing = mgmtd.getRoutingInfo();
+  EXPECT_TRUE(routing);
+  size_t total = 0;
+  for (const auto &chain : routing->raw()->chains) {
+    auto query =
+        storage.createQueryOp(chain.first, storage::ChunkId(begin.pack()), storage::ChunkId(end.pack()), UINT32_MAX);
+    CO_RETURN_ON_ERROR(co_await storage.queryLastChunk(std::span(&query, 1), {}));
+    total += query.result.totalNumChunks;
+  }
+  co_return total;
+}
+
+CoTryTask<void> printTree(MetaOperator &meta) {
+  auto queue = std::queue<std::pair<InodeId, Path>>();
+  queue.push({InodeId::root(), "/"});
+  auto map = std::map<Path, std::string>();
+  while (!queue.empty()) {
+    auto [inodeId, path] = queue.front();
+    queue.pop();
+    auto prev = std::string();
+    while (true) {
+      auto res = co_await meta.list({SUPER_USER, PathAt(inodeId), prev, 256, true});
+      CO_RETURN_ON_ERROR(res);
+      for (auto idx = 0ul; idx < res->entries.size(); idx++) {
+        auto &entry = res->entries[idx];
+        auto &inode = res->inodes[idx];
+        prev = entry.name;
+        auto entryPath = path / entry.name;
+        switch (entry.type) {
+          case InodeType::File:
+            map[entryPath] = "File";
+            break;
+          case InodeType::Directory:
+            map[entryPath] = "";
+            queue.push({entry.id, entryPath});
+            break;
+          case InodeType::Symlink: {
+            map[entryPath] = inode.asSymlink().target.string();
+            break;
+          }
+        }
+      }
+      if (!res->more) {
+        break;
+      }
+    }
+  }
+
+  for (auto &iter : map) {
+    auto [path, msg] = iter;
+    if (msg.empty()) {
+      fmt::println("{}", path);
+    } else {
+      fmt::println("- {} -> {}", path, msg);
+    }
+  }
+
+  co_return Void{};
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/MetaTestBase.h

@@ -0,0 +1,460 @@
+#pragma once
+
+#include <algorithm>
+#include <experimental/array>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/Synchronized.h>
+#include <folly/concurrency/ConcurrentHashMap.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/logging/xlog.h>
+#include <functional>
+#include <gtest/gtest.h>
+#include <iostream>
+#include <limits>
+#include <map>
+#include <memory>
+#include <optional>
+#include <queue>
+#include <stack>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <unordered_set>
+#include <vector>
+
+#include "client/mgmtd/MgmtdClientForServer.h"
+#include "common/kv/IKVEngine.h"
+#include "common/kv/ITransaction.h"
+#include "common/kv/mem/MemKV.h"
+#include "common/kv/mem/MemKVEngine.h"
+#include "common/kv/mem/MemTransaction.h"
+#include "common/utils/ConfigBase.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Result.h"
+#include "common/utils/Status.h"
+#include "common/utils/StatusCode.h"
+#include "common/utils/String.h"
+#include "common/utils/Uuid.h"
+#include "fbs/meta/Common.h"
+#include "fbs/mgmtd/ChainRef.h"
+#include "fbs/mgmtd/ChainSetting.h"
+#include "fdb/FDB.h"
+#include "fdb/FDBKVEngine.h"
+#include "fdb/FDBTransaction.h"
+#include "meta/base/Config.h"
+#include "meta/components/ChainAllocator.h"
+#include "meta/components/InodeIdAllocator.h"
+#include "meta/components/SessionManager.h"
+#include "meta/service/MetaOperator.h"
+#include "meta/service/MockMeta.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "mgmtd/service/MgmtdOperator.h"
+#include "mgmtd/service/MockMgmtd.h"
+#include "tests/FakeMgmtdClient.h"
+#include "tests/GtestHelpers.h"
+#include "tests/fdb/KvTraits.h"
+#include "tests/fdb/SetupFDB.h"
+
+#define RANDOM_SLEEP_MS(ms)                                                 \
+  do {                                                                      \
+    auto dur = std::chrono::microseconds(folly::Random::rand32((ms)*1000)); \
+    co_await folly::coro::sleep(dur);                                       \
+  } while (0)
+
+#define READ_ONLY_TRANSACTION(code_to_run)                    \
+  do {                                                        \
+    auto txn = this->kvEngine()->createReadonlyTransaction(); \
+    code_to_run;                                              \
+  } while (0)
+
+#define READ_WRITE_TRANSACTION_ERROR(code_to_run, expected_code) \
+  do {                                                           \
+    auto txn = this->kvEngine()->createReadWriteTransaction();   \
+    { code_to_run; }                                             \
+    auto txnResult = co_await txn->commit();                     \
+    CO_ASSERT_ERROR(txnResult, expected_code);                   \
+  } while (0)
+
+#define READ_WRITE_TRANSACTION_OK(code_to_run)                                     \
+  do {                                                                             \
+    auto txn = this->kvEngine()->createReadWriteTransaction();                     \
+    { code_to_run; }                                                               \
+    auto txnResult = co_await txn->commit();                                       \
+    CO_ASSERT_FALSE(txnResult.hasError()) << fmt::format("{}", txnResult.error()); \
+  } while (0)
+
+#define READ_WRITE_TRANSACTION_NO_COMMIT(code_to_run)          \
+  do {                                                         \
+    auto txn = this->kvEngine()->createReadWriteTransaction(); \
+    { code_to_run; }                                           \
+  } while (0)
+
+#define LOAD_INODE(name, inodeId)                   \
+  Inode name(inodeId);                              \
+  READ_ONLY_TRANSACTION({                           \
+    auto result = co_await name.snapshotLoad(*txn); \
+    CO_ASSERT_OK(result);                           \
+    CO_ASSERT_TRUE(result->has_value());            \
+  });
+
+#define CO_ASSERT_DIRENTRY_EXISTS(parent, name)                            \
+  READ_ONLY_TRANSACTION({                                                  \
+    auto loadResult = co_await DirEntry::snapshotLoad(*txn, parent, name); \
+    CO_ASSERT_OK(loadResult);                                              \
+    CO_ASSERT_TRUE(loadResult->has_value());                               \
+  })
+
+#define CO_ASSERT_INODE_EXISTS(inodeId)                            \
+  READ_ONLY_TRANSACTION({                                          \
+    auto loadResult = co_await Inode::snapshotLoad(*txn, inodeId); \
+    CO_ASSERT_OK(loadResult);                                      \
+    CO_ASSERT_TRUE(loadResult->has_value());                       \
+  })
+
+#define CO_ASSERT_INODE_NOT_EXISTS(inodeId)                        \
+  READ_ONLY_TRANSACTION({                                          \
+    auto loadResult = co_await Inode::snapshotLoad(*txn, inodeId); \
+    CO_ASSERT_OK(loadResult);                                      \
+    CO_ASSERT_FALSE(loadResult->has_value());                      \
+  })
+
+#define GET_KV_CNTS(val, begin, end) \
+  size_t val = 0;                    \
+  READ_ONLY_TRANSACTION({ val = co_await MetaTestHelper::getKeyCountInRange(*txn, {begin, false}, {end, false}); })
+
+#define GET_INODE_CNTS(val) GET_KV_CNTS(val, "INOD", "INOE")
+#define GET_DIRENTRY_CNTS(val) GET_KV_CNTS(val, "DENT", "DENU")
+#define GET_SESSION_CNTS(val) GET_KV_CNTS(val, "INOS", "INOT")
+
+#define CO_ASSERT_KV_CNTS(val, begin, end) \
+  do {                                     \
+    GET_KV_CNTS(actual, begin, end);       \
+    CO_ASSERT_EQ(actual, val);             \
+  } while (0)
+#define CO_ASSERT_INODE_CNTS(val) CO_ASSERT_KV_CNTS(val, "INOD", "INOE")
+#define CO_ASSERT_DIRENTRY_CNTS(val) CO_ASSERT_KV_CNTS(val, "DENT", "DENU")
+#define CO_ASSERT_SESSION_CNTS(val) CO_ASSERT_KV_CNTS(val, "INOS", "INOT")
+
+#define CHECK_CONFLICT_SET(op, readSet, writeSet, exactly)                        \
+  do {                                                                            \
+    auto txn = this->kvEngine()->createReadWriteTransaction();                    \
+    co_await op(*txn);                                                            \
+    auto ok = MetaTestHelper::checkConflictSet(*txn, readSet, writeSet, exactly); \
+    CO_ASSERT_TRUE(ok);                                                           \
+  } while (0)
+
+// check two operations not conflict
+#define CO_ASSERT_NO_CONFLICT(op1, op2)                                                       \
+  do {                                                                                        \
+    auto txn1 = this->kvEngine()->createReadWriteTransaction();                               \
+    auto txn2 = this->kvEngine()->createReadWriteTransaction();                               \
+    co_await op1(*txn1);                                                                      \
+    co_await op2(*txn2);                                                                      \
+    auto *first = txn1.get(), *second = txn2.get();                                           \
+    if (folly::Random::oneIn(2)) {                                                            \
+      std::swap(first, second);                                                               \
+    }                                                                                         \
+    if (dynamic_cast<MemTransaction *>(first) && dynamic_cast<MemTransaction *>(second)) {    \
+      auto conflict = MemTransaction::checkConflict(*dynamic_cast<MemTransaction *>(first),   \
+                                                    *dynamic_cast<MemTransaction *>(second)); \
+      CO_ASSERT_FALSE(conflict);                                                              \
+      auto &memkv = dynamic_cast<MemTransaction *>(first)->kv();                              \
+      memkv.backup();                                                                         \
+      auto secondResult = co_await second->commit();                                          \
+      auto firstResult = co_await first->commit();                                            \
+      CO_ASSERT_OK(firstResult);                                                              \
+      CO_ASSERT_OK(secondResult);                                                             \
+      memkv.restore();                                                                        \
+    }                                                                                         \
+    auto firstResult = co_await first->commit();                                              \
+    auto secondResult = co_await second->commit();                                            \
+    CO_ASSERT_OK(firstResult);                                                                \
+    CO_ASSERT_OK(secondResult);                                                               \
+  } while (0)
+
+// op1 commit first, op2 should fail with conflict
+#define CO_ASSERT_CONFLICT(op1, op2)                                                                \
+  do {                                                                                              \
+    auto txn1 = this->kvEngine()->createReadWriteTransaction();                                     \
+    auto txn2 = this->kvEngine()->createReadWriteTransaction();                                     \
+    co_await op1(*txn1);                                                                            \
+    co_await op2(*txn2);                                                                            \
+    if (dynamic_cast<MemTransaction *>(txn1.get()) && dynamic_cast<MemTransaction *>(txn2.get())) { \
+      auto conflict = MemTransaction::checkConflict(*dynamic_cast<MemTransaction *>(txn1.get()),    \
+                                                    *dynamic_cast<MemTransaction *>(txn2.get()));   \
+      CO_ASSERT_TRUE(conflict);                                                                     \
+    }                                                                                               \
+    auto firstResult = co_await txn1->commit();                                                     \
+    auto secondResult = co_await txn2->commit();                                                    \
+    CO_ASSERT_OK(firstResult);                                                                      \
+    CO_ASSERT_ERROR(secondResult, TransactionCode::kConflict);                                      \
+  } while (0)
+
+#define FAULT_INJECTION_CHECK(result)                                     \
+  do {                                                                    \
+    if (result.hasError()) {                                              \
+      bool ok = !TransactionHelper::isRetryable(result.error(), false) && \
+                TransactionHelper::isTransactionError(result.error());    \
+      CO_ASSERT_TRUE(ok) << result.error().describe();                    \
+    }                                                                     \
+  } while (0)
+
+#define FAULT_INJECTION_CHECK_ERROR(result, expected)                   \
+  do {                                                                  \
+    CO_ASSERT_TRUE(result.hasError());                                  \
+    if (result.error().code() != expected) {                            \
+      bool ok = !TransactionHelper::isRetryable(result.error(), false); \
+      CO_ASSERT_TRUE(ok) << result.error().describe();                  \
+    }                                                                   \
+  } while (0)
+
+#define REQ(...) \
+  { __VA_ARGS__ }
+
+#define SUPER_USER \
+  flat::UserInfo { Uid(0), Gid(0), String() }
+
+namespace hf3fs::meta::server {
+using namespace ::hf3fs::kv;
+
+class MetaTestHelper {
+ public:
+  static String getInodeKey(const InodeId &id) {
+    Inode inode(id);
+    return inode.packKey();
+  }
+
+  static String getDirEntryKey(const InodeId &id, std::string_view name) {
+    DirEntry entry(id, std::string(name));
+    return entry.packKey();
+  }
+
+  static InodeId randomInodeId() {
+    static folly::ConcurrentHashMap<uint64_t, Void> map;
+    do {
+      auto id = folly::Random::rand64(1 << 16, InodeId::normalMax().u64());
+      auto [iter, insert] = map.insert(id, Void{});
+      if (insert) {
+        return InodeId(id);
+      }
+    } while (true);
+  }
+
+  static SessionInfo randomSession(std::optional<Uuid> client = {}) {
+    return SessionInfo(ClientId(client.value_or(Uuid::random())), Uuid::random());
+  }
+
+  static auto randomEmptyLayout() {
+    return Layout::newEmpty(ChainTableId(1),
+                            ChainTableVersion(1),
+                            1 << folly::Random::rand32(10, 32),
+                            folly::Random::rand32(1, 256));
+  }
+
+  static auto randomLayout() {
+    auto layout = Layout::newEmpty(ChainTableId(1),
+                                   ChainTableVersion(1),
+                                   1 << folly::Random::rand32(10, 32),
+                                   folly::Random::rand32(1, 256));
+    if (folly::Random::oneIn(2)) {
+      layout.chains = Layout::ChainRange(folly::Random::rand32(1, 512),
+                                         Layout::ChainRange::STD_SHUFFLE_MT19937,
+                                         folly::Random::rand64());
+    } else {
+      std::vector<uint32_t> chains;
+      for (size_t i = 0; i < layout.stripeSize; i++) {
+        chains.push_back(folly::Random::rand32(1, std::numeric_limits<uint32_t>::max()));
+      }
+      layout.chains = Layout::ChainList{chains};
+    }
+    return layout;
+  }
+
+  static Inode randomInode();
+  static Inode randomFile();
+  static Inode randomDirectory();
+
+  static DirEntry randomDirEntry();
+
+  static bool checkConflictSet(IReadOnlyTransaction &txn,
+                               const std::vector<String> &readConflict,
+                               const std::vector<String> &writeConflict,
+                               bool exactly) {
+    auto memTxn = dynamic_cast<MemTransaction *>(&txn);
+    if (memTxn) {
+      auto ok = memTxn->checkConflictSet(readConflict, writeConflict, exactly);
+      return ok;
+    }
+    return true;
+  }
+
+  static String getInodeSessionKey(const FileSession &session) {
+    return FileSession::packKey(session.inodeId, session.sessionId);
+  }
+
+  // static String getClientSessionKey(const FileSession &session) {
+  //   return FileSession::packKey(session.clientId.uuid, session.sessionId);
+  // }
+
+  [[nodiscard]] static CoTask<size_t> getKeyCountInRange(IReadOnlyTransaction &txn,
+                                                         IReadOnlyTransaction::KeySelector begin,
+                                                         const IReadOnlyTransaction::KeySelector end) {
+    size_t total = 0;
+    std::string prev;
+    while (true) {
+      auto range = co_await txn.snapshotGetRange(begin, end, 65535);
+      EXPECT_TRUE(!range.hasError()) << range.error().describe();
+      if (range.hasError()) break;
+      total += range->kvs.size();
+      if (range->hasMore) {
+        prev = range->kvs.rbegin()->key;
+        begin = {prev, false};
+      } else {
+        break;
+      }
+    }
+    co_return total;
+  }
+};
+
+template <typename KV>
+class KvTestBase : public testing::SetupFDB {
+ public:
+  KvTestBase();
+  ~KvTestBase() override = default;
+
+  std::shared_ptr<IKVEngine> &kvEngine() { return engine_; }
+
+ protected:
+  void SetUp() override;
+  void TearDown() override;
+
+  std::shared_ptr<IKVEngine> createEngine();
+
+  bool skip_;
+  std::shared_ptr<IKVEngine> engine_;
+};
+
+class MockCluster {
+ public:
+  struct Config : ConfigBase<Config> {
+    struct ChainTableConfig : ConfigBase<ChainTableConfig> {
+      CONFIG_ITEM(table_id, 1u);
+      CONFIG_ITEM(num_chains, 128u);
+      CONFIG_ITEM(num_replica, 1u);
+    };
+
+    CONFIG_ITEM(num_meta, 3);
+    CONFIG_OBJ(mock_meta, meta::server::Config, [](meta::server::Config &cfg) {
+      cfg.set_iflags_chain_allocation(true);
+      cfg.set_inodeId_check_unique(true);
+      cfg.set_inodeId_abort_on_duplicate(true);
+      cfg.set_check_file_hole(false);
+      cfg.set_otrunc_replace_file(true);
+      cfg.set_otrunc_replace_file_threshold(1_GB);
+      cfg.session_manager().set_enable(false);
+      cfg.session_manager().set_scan_interval(10_ms);
+      cfg.gc().set_check_session(true);
+      cfg.gc().set_scan_interval(10_ms);
+      cfg.gc().set_retry_delay(100_ms);
+      cfg.distributor().set_update_interval(50_ms);
+      cfg.set_dynamic_stripe(true);
+      cfg.set_enable_new_chunk_engine(folly::Random::oneIn(2));
+      cfg.set_idempotent_rename(true);
+      cfg.event_trace_log().set_enabled(true);
+      cfg.event_trace_log().set_dump_interval(5_s);
+      cfg.event_trace_log().set_max_num_writers(8);
+    });
+    CONFIG_OBJ(mock_mgmtd, mgmtd::MockMgmtd::Config);
+    CONFIG_OBJ(mgmtd_client, client::MgmtdClientForServer::Config, [](client::MgmtdClientForServer::Config &cfg) {
+      cfg.set_enable_auto_heartbeat(false);  // currently heartbeat is not needed in meta test
+      cfg.set_mgmtd_server_addresses({net::Address::from("TCP://127.0.0.1:8000").value()});  // just a fake TCP address.
+    });
+
+    CONFIG_OBJ_ARRAY(chain_tables, ChainTableConfig, 64, [](std::array<ChainTableConfig, 64> &cfg) {
+      cfg[0].set_table_id(1);
+      cfg[0].set_num_chains(100 * 16);
+      cfg[0].set_num_replica(1);
+      cfg[1].set_table_id(2);
+      cfg[1].set_num_chains(64 * 16);
+      cfg[1].set_num_replica(2);
+      cfg[2].set_table_id(3);
+      cfg[2].set_num_chains(180 * 16);
+      cfg[2].set_num_replica(2);
+      return 3;
+    });
+  };
+
+  static MockCluster create(std::shared_ptr<IKVEngine> engine, const Config &config) {
+    MockCluster cluster(std::move(engine), config);
+    cluster.SetUp();
+    return cluster;
+  }
+
+  ~MockCluster();
+
+  const Config &config() { return config_; }
+
+  std::shared_ptr<hf3fs::tests::FakeMgmtdClient> mgmtdClient() { return mgmtdClient_; }
+
+  MockMeta &meta() { return *meta_; }
+
+  std::shared_ptr<IKVEngine> kvEngine() { return engine_; }
+
+ private:
+  MockCluster(std::shared_ptr<IKVEngine> engine, const Config &config)
+      : config_(config),
+        engine_(engine) {}
+  MockCluster(MockCluster &&) = default;
+
+  void SetUp();
+
+  const Config &config_;
+  std::shared_ptr<IKVEngine> engine_;
+
+  std::unique_ptr<CPUExecutorGroup> exec_;
+  std::shared_ptr<hf3fs::tests::FakeMgmtdClient> mgmtdClient_;
+  std::unique_ptr<MockMeta> meta_;
+};
+
+template <typename KV>
+class MetaTestBase : public KvTestBase<KV> {
+ public:
+  MockCluster createMockCluster() {
+    static MockCluster::Config config = {};
+    return MockCluster::create(this->kvEngine(), config);
+  }
+
+  MockCluster createMockCluster(const MockCluster::Config &config) {
+    return MockCluster::create(this->kvEngine(), config);
+  }
+};
+
+CoTask<void> randomWrite(MetaOperator &meta,
+                         storage::client::StorageClient &storage,
+                         const Inode &inode,
+                         uint64_t offset,
+                         uint64_t length);
+CoTask<void> truncate(MetaOperator &meta, storage::client::StorageClient &storage, const Inode &inode, uint64_t length);
+
+CoTryTask<size_t> queryTotalChunks(storage::client::StorageClient &storage, client::ICommonMgmtdClient &mgmtd);
+
+CoTryTask<void> printTree(MetaOperator &meta);
+
+inline const Uid rootUid{0};
+inline const Gid rootGid{0};
+
+inline const Permission p777{0777};
+inline const Permission p755{0755};
+inline const Permission p700{0700};
+inline const Permission p644{0644};
+
+inline const Acl rootp777{rootUid, rootGid, p777};
+inline const Acl rootp755{rootUid, rootGid, p755};
+inline const Acl rootp700{rootUid, rootGid, p700};
+inline const Acl rootp644{rootUid, rootGid, p644};
+
+}  // namespace hf3fs::meta::server

tests/meta/TestCommon.cc

@@ -0,0 +1,172 @@
+#include <cstdint>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/Utility.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <gflags/gflags.h>
+#include <gtest/gtest.h>
+#include <iterator>
+#include <optional>
+#include <type_traits>
+#include <typeinfo>
+
+#include "common/serde/Serde.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Path.h"
+#include "common/utils/Result.h"
+#include "common/utils/Status.h"
+#include "common/utils/Uuid.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/meta/Service.h"
+#include "fbs/meta/Utils.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/Operation.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+DEFINE_string(path, "/a", "path to print");
+
+namespace hf3fs::meta::server {
+
+using hf3fs::meta::InodeId;
+
+static void print(const Path &path) {
+  fmt::print("path {}\n", path);
+  fmt::print("abs {}, complete {}, relative {}\n", path.is_absolute(), path.is_complete(), path.is_relative());
+  fmt::print("has root {}, {}\n", path.has_root_path(), path.root_path());
+  fmt::print("has relative {}, {}\n", path.has_relative_path(), path.relative_path());
+  fmt::print("has branch {}, {}\n", path.has_parent_path(), path.branch_path());
+  fmt::print("has stem {}, {}\n", path.has_stem(), path.stem());
+  fmt::print("has filename {}, {}, is dot {}, is dot dot {}\n",
+             path.has_filename(),
+             path.filename(),
+             path.filename_is_dot(),
+             path.filename_is_dot_dot());
+  fmt::print("has leaf {}, {}", path.has_leaf(), path.leaf());
+  fmt::print("size {}, components {}, first {}, last {}\n",
+             path.size(),
+             std::distance(path.begin(), path.end()),
+             *path.begin(),
+             *path.rbegin());
+}
+
+TEST(TestMetaCommon, Json) {
+  fmt::print("{}\n", StatReq(flat::UserInfo(), Path("random-path"), AtFlags(AT_SYMLINK_FOLLOW)));
+  fmt::print("{}\n", StatReq(flat::UserInfo(), Path("path"), AtFlags(AT_SYMLINK_FOLLOW)));
+  fmt::print("{}\n", OpenReq(flat::UserInfo(), Path("path"), std::nullopt, 0));
+  fmt::print("{}\n", OpenReq(flat::UserInfo(), Path("path"), SessionInfo(ClientId::random(), Uuid::random()), 0));
+  fmt::print("{}\n", DirEntry::newSymlink(InodeId::root(), "symlink", InodeId(folly::Random::rand64())));
+}
+
+TEST(TestMetaCommon, PathAppend) {}
+
+TEST(TestMetaCommon, PrintPath) {
+  Path path(FLAGS_path);
+  print(path);
+}
+
+TEST(TestMetaCommon, Path) {
+  ASSERT_OK(PathAt("a").validForCreate());
+  ASSERT_OK(PathAt("a/b").validForCreate());
+  ASSERT_OK(PathAt("a/./b").validForCreate());
+  ASSERT_OK(PathAt("a/b/../c").validForCreate());
+
+  ASSERT_ERROR(PathAt("/").validForCreate(), StatusCode::kInvalidArg);
+  ASSERT_ERROR(PathAt("").validForCreate(), StatusCode::kInvalidArg);
+  ASSERT_ERROR(PathAt(".").validForCreate(), StatusCode::kInvalidArg);
+  ASSERT_ERROR(PathAt("..").validForCreate(), StatusCode::kInvalidArg);
+  ASSERT_ERROR(PathAt("a/").validForCreate(), StatusCode::kInvalidArg);
+  ASSERT_ERROR(PathAt("a/.").validForCreate(), StatusCode::kInvalidArg);
+  ASSERT_ERROR(PathAt("a/..").validForCreate(), StatusCode::kInvalidArg);
+  ASSERT_ERROR(PathAt("a/b/../.").validForCreate(), StatusCode::kInvalidArg);
+
+  for (auto [path, parent, ppath, fname] :
+       std::vector<std::tuple<Path, bool, Path, std::string>>{{"/a", true, "/", "a"},
+                                                              {"/", false, "", "/"},
+                                                              {"/a/", true, "/a", "."},
+                                                              {"/a/..", true, "/a", ".."},
+                                                              {"/a/../", true, "/a/..", "."},
+                                                              {"a/.././b/c", true, "a/.././b", "c"}}) {
+    ASSERT_EQ(path.has_parent_path(), parent);
+    ASSERT_EQ(path.parent_path(), ppath);
+    ASSERT_EQ(path.filename(), fname);
+  }
+}
+
+TEST(TestMetaCommon, ChunkSize) {
+  auto chunk = Layout::ChunkSize(1 << 20);
+  auto length = 1 * chunk;
+  static_assert(std::is_same_v<uint64_t, decltype(length)>);
+  auto gb = Layout::ChunkSize(1 << 30);
+  auto tb = (1 << 10) * gb;
+  auto pb = (1 << 20) * gb;
+  auto eb = (1 << 30) * gb;
+  ASSERT_EQ(gb, 1ull << 30);
+  ASSERT_EQ(tb, 1ull << 40);
+  ASSERT_EQ(pb, 1ull << 50);
+  ASSERT_EQ(eb, 1ull << 60);
+
+  for (size_t i = 0; i < 10; i++) {
+    auto val = folly::Random::rand32();
+    auto chunk = Layout::ChunkSize(val);
+    auto ser1 = serde::serialize(val);
+    auto ser2 = serde::serialize(chunk);
+    ASSERT_EQ(ser1, ser2);
+    auto des1 = Layout::ChunkSize();
+    serde::deserialize(des1, ser1);
+    ASSERT_EQ(des1, chunk);
+    auto des2 = Layout::ChunkSize();
+    serde::deserialize(des2, ser2);
+    ASSERT_EQ(des2, chunk);
+  }
+}
+
+TEST(TestMetaCommon, Compare) {
+  EXPECT_LT('\xff', '0');
+  EXPECT_GT(std::string("\xff"), std::string("0"));
+}
+
+meta::VersionedLength mergeAll(const std::vector<VersionedLength> &vec) {
+  auto hint = meta::VersionedLength{0, 0};
+  for (auto l : vec) {
+    hint = *VersionedLength::mergeHint(hint, l);
+  }
+  return hint;
+}
+
+TEST(TestMetaCommon, VersionedLength) {
+  for (size_t i = 0; i < 100; i++) {
+    auto l = VersionedLength{folly::Random::rand32(), folly::Random::rand32()};
+    ASSERT_EQ(VersionedLength::mergeHint(l, VersionedLength{0, 0}), l);
+    ASSERT_EQ(VersionedLength::mergeHint(l, std::nullopt), std::nullopt);
+    ASSERT_EQ(VersionedLength::mergeHint(VersionedLength{0, 0}, l), l);
+    ASSERT_EQ(VersionedLength::mergeHint(std::nullopt, l), std::nullopt);
+  }
+
+  for (size_t i = 0; i < 100; i++) {
+    auto l1 = VersionedLength{folly::Random::rand32(), folly::Random::rand32()};
+    auto l2 = VersionedLength{folly::Random::rand32(), folly::Random::rand32()};
+    ASSERT_EQ(VersionedLength::mergeHint(l1, l2), VersionedLength::mergeHint(l2, l1));
+  }
+
+  auto l1 = VersionedLength{34, 0};
+  auto l2 = VersionedLength{100, 0};
+  auto l3 = VersionedLength{60, 1};
+  auto l4 = VersionedLength{200, 3};
+  auto l5 = VersionedLength{32, 4};
+  ASSERT_EQ(VersionedLength::mergeHint(l1, l2), l2);
+  ASSERT_EQ(VersionedLength::mergeHint(l1, l3), l3);
+  ASSERT_EQ(VersionedLength::mergeHint(l1, l5), l1);
+  ASSERT_EQ(mergeAll({l1, l2, l3}), l2);
+  ASSERT_EQ(mergeAll({l2, l3}), l2);
+  ASSERT_EQ(mergeAll({l1, l2, l3, l4}), l4);
+  ASSERT_EQ(mergeAll({l2, l3, l4}), l4);
+  ASSERT_EQ(mergeAll({l1, l2, l3, l4, l5}), l4);
+}
+
+TEST(TestMetaCommon, RetryThrottled) { ASSERT_TRUE(ErrorHandling::retryable(Status(TransactionCode::kThrottled))); }
+
+}  // namespace hf3fs::meta::server

tests/meta/components/TestAclCache.cc

@@ -0,0 +1,80 @@
+#include <atomic>
+#include <folly/Random.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <gtest/gtest.h>
+#include <map>
+#include <thread>
+#include <vector>
+
+#include "common/utils/Coroutine.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/meta/Schema.h"
+#include "meta/components/AclCache.h"
+
+namespace hf3fs::meta::server {
+
+TEST(TestAclCache, basic) {
+  std::map<InodeId, Acl> map;
+  AclCache cache(1 << 20);
+  for (size_t i = 0; i < (64 << 10); i++) {
+    InodeId inode{folly::Random::rand64()};
+    Acl acl{flat::Uid(folly::Random::rand32()),
+            flat::Gid(folly::Random::rand32()),
+            meta::Permission(folly::Random::rand32())};
+    map[inode] = acl;
+    cache.set(inode, acl);
+  }
+
+  for (auto [inode, acl] : map) {
+    ASSERT_EQ(cache.get(inode, 0_s), std::nullopt);
+    ASSERT_EQ(cache.get(inode, 1_h), acl);
+    cache.invalid(inode);
+    ASSERT_EQ(cache.get(inode, 1_h), std::nullopt);
+    Acl newacl{flat::Uid(folly::Random::rand32()),
+               flat::Gid(folly::Random::rand32()),
+               meta::Permission(folly::Random::rand32())};
+    cache.set(inode, newacl);
+    ASSERT_EQ(cache.get(inode, 1_h), newacl);
+  }
+}
+
+TEST(TestAclCache, benchmark) {
+  std::vector<InodeId> inodes;
+  AclCache cache(4 << 20);
+  for (size_t i = 0; i < (1 << 20); i++) {
+    InodeId inode{folly::Random::rand64(1 << 20)};
+    inodes.emplace_back(inode);
+    cache.set(inode, Acl());
+  }
+
+  std::atomic<size_t> total;
+  std::vector<std::jthread> threads;
+  for (size_t i = 0; i < 8; i++) {
+    threads.emplace_back([&]() {
+      size_t ops = 0;
+      auto now = SteadyClock::now();
+      while (SteadyClock::now() - now < 1_s) {
+        auto inode = inodes.at(folly::Random::rand64(inodes.size()));
+        if (folly::Random::oneIn(4)) {
+          // set
+          cache.set(inode, Acl());
+        } else if (folly::Random::oneIn(4)) {
+          cache.invalid(inode);
+        } else {
+          // get
+          cache.get(inode, 1_h);
+        }
+        ops += 1;
+      }
+      total += ops;
+    });
+  }
+
+  for (auto &th : threads) {
+    th.join();
+  }
+
+  fmt::print("total {} ops in 1s\n", total.load());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/components/TestChainAllocator.cc

@@ -0,0 +1,123 @@
+#include <folly/Random.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <gflags/gflags.h>
+#include <gtest/gtest-param-test.h>
+#include <map>
+
+#include "common/utils/Coroutine.h"
+#include "fbs/meta/Common.h"
+#include "fbs/mgmtd/ChainRef.h"
+#include "fbs/mgmtd/MgmtdTypes.h"
+#include "gtest/gtest.h"
+#include "meta/components/ChainAllocator.h"
+#include "tests/meta/MetaTestBase.h"
+
+DEFINE_int64(chain_alloc_test, 50000, "files in test chain allocator");
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestChainAllocator : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestChainAllocator, KVTypes);
+
+TYPED_TEST(TestChainAllocator, basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto mgmtd = cluster.mgmtdClient();
+    ChainAllocator alloc(mgmtd);
+
+    auto empty = Layout::newEmpty(flat::ChainTableId(1), flat::ChainTableVersion(0), 512 << 10, 16);
+    CO_ASSERT_OK(co_await alloc.checkLayoutValid(empty));
+    CO_ASSERT_OK(co_await alloc.allocateChainsForLayout(empty));
+    CO_ASSERT_TRUE(empty.valid(false));
+  }());
+}
+
+TYPED_TEST(TestChainAllocator, perDirCounter) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+    auto mgmtd = cluster.mgmtdClient();
+    ChainAllocator alloc(mgmtd);
+
+    CO_ASSERT_OK(
+        co_await meta.setAttr(SetAttrReq::setIFlags(SUPER_USER, InodeId::root(), IFlags(FS_CHAIN_ALLOCATION_FL))));
+
+    auto routing = mgmtd->getRoutingInfo();
+    auto chainCount = routing->raw()->getChainTable(flat::ChainTableId(1), flat::ChainTableVersion(0))->chains.size();
+
+    std::optional<size_t> prevChain;
+    for (size_t i = 0; i < 1000; i++) {
+      auto result = co_await meta.create({SUPER_USER, folly::to<std::string>(i), std::nullopt, OpenFlags(), p644});
+      CO_ASSERT_OK(result);
+      auto layout = result->stat.asFile().layout;
+      CO_ASSERT_OK(co_await alloc.checkLayoutValid(layout));
+      auto chain = std::get<Layout::ChainRange>(layout.chains).baseIndex;
+      if (prevChain) {
+        CO_ASSERT_EQ(chain % chainCount, (*prevChain + layout.stripeSize) % chainCount);
+      }
+      prevChain = chain;
+    }
+  }());
+}
+
+TYPED_TEST(TestChainAllocator, balance) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto mgmtd = cluster.mgmtdClient();
+    ChainAllocator alloc(mgmtd);
+
+    for (auto tableId :
+         std::vector<flat::ChainTableId>{flat::ChainTableId(1), flat::ChainTableId(2), flat::ChainTableId(3)}) {
+      auto routing = mgmtd->getRoutingInfo();
+      auto chainCount = routing->raw()->getChainTable(tableId, flat::ChainTableVersion(0))->chains.size();
+
+      std::map<flat::ChainId, size_t> map;
+      std::vector<size_t> stripeSizes{1, 2, 4, 8, 30, 32, 60, 64, 120, 128, 240, 256, 512};
+      for (int i = 0; i < FLAGS_chain_alloc_test; i++) {
+        auto stripe = 1000000u;
+        while (stripe > chainCount) {
+          stripe = stripeSizes.at(folly::Random::rand64(stripeSizes.size()));
+        }
+        auto empty = Layout::newEmpty(tableId, 512 << 10, stripe);
+        auto emptyCheck = co_await alloc.checkLayoutValid(empty);
+        CO_ASSERT_OK(emptyCheck);
+        CO_ASSERT_OK(co_await alloc.allocateChainsForLayout(empty));
+        CO_ASSERT_TRUE(empty.valid(false));
+        CO_ASSERT_OK(co_await alloc.checkLayoutValid(empty));
+
+        size_t begin = chainCount;
+        std::set<flat::ChainId> set;
+        for (auto chain : empty.getChainIndexList()) {
+          auto index = chain % chainCount ? chain % chainCount : chainCount;
+          auto ref = flat::ChainRef(empty.tableId, empty.tableVersion, index);
+          auto id = routing->raw()->getChainId(ref);
+          CO_ASSERT_TRUE(id.has_value());
+          CO_ASSERT_FALSE(set.contains(*id));
+          set.emplace(*id);
+          map[*id]++;
+          begin = std::min(index, begin);
+        }
+
+        if (chainCount % stripe == 0 && stripe != 1) {
+          CO_ASSERT_EQ(begin % stripe, 1)
+              << fmt::format("{} {} {}", stripe, begin, fmt::join(set.begin(), set.end(), ","));
+        }
+      }
+
+      size_t min = map.begin()->second, max = map.begin()->second;
+      for (auto [id, cnt] : map) {
+        (void)id;
+        min = std::min(cnt, min);
+        max = std::max(cnt, max);
+      }
+      CO_ASSERT_EQ(map.size(), chainCount);
+      CO_ASSERT_GE(min + stripeSizes.size(), max);
+    }
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/components/TestDistributor.cc

@@ -0,0 +1,215 @@
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Invoke.h>
+#include <folly/logging/xlog.h>
+#include <gflags/gflags.h>
+#include <gtest/gtest.h>
+#include <limits>
+#include <map>
+#include <memory>
+#include <set>
+#include <thread>
+#include <unistd.h>
+#include <vector>
+
+#include "common/app/NodeId.h"
+#include "common/kv/ITransaction.h"
+#include "common/utils/BackgroundRunner.h"
+#include "common/utils/CPUExecutorGroup.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Duration.h"
+#include "common/utils/MurmurHash3.h"
+#include "common/utils/Result.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Utils.h"
+#include "gtest/gtest.h"
+#include "meta/components/Distributor.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+DEFINE_uint64(distributor_iters, 10, "");
+
+namespace hf3fs::meta::server {
+
+#define CHECK_DIST(txn, dist, inodeId, expected)                                        \
+  do {                                                                                  \
+    auto result = co_await dist.checkOnServer((txn), (inodeId), (expected));            \
+    CO_ASSERT_OK(result);                                                               \
+    auto actual = (dist).getServer((inodeId));                                          \
+    auto msg = fmt::format("{} not on {}, actual {}", (inodeId), (expected), (actual)); \
+    CO_ASSERT_TRUE(result->first) << msg;                                               \
+    CO_ASSERT_EQ((actual), (expected)) << msg;                                          \
+  } while (0)
+
+#define CHECK_DIST_AT(dist, inodeId, expected)                                 \
+  do {                                                                         \
+    READ_WRITE_TRANSACTION_OK({ CHECK_DIST(*txn, dist, inodeId, expected); }); \
+  } while (0)
+
+#define CHECK_ALL_DIST_AT(dist, expected)                  \
+  do {                                                     \
+    for (size_t i = 0; i < 10; i++) {                      \
+      READ_WRITE_TRANSACTION_OK({                          \
+        for (size_t j = 0; j < 1; j++) {                   \
+          auto inodeId = InodeId(folly::Random::rand64()); \
+          CHECK_DIST(*txn, dist, inodeId, expected);       \
+        }                                                  \
+      });                                                  \
+    }                                                      \
+  } while (0)
+
+template <typename KV>
+class TestDistributor : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestDistributor, KVTypes);
+
+TYPED_TEST(TestDistributor, basic) {
+  auto config = Distributor::Config();
+  config.set_update_interval(1_s);
+  config.set_timeout(5_s);
+  auto kvEngine = this->kvEngine();
+  auto exec = CPUExecutorGroup(2, "test");
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto dist = Distributor(config, flat::NodeId(50), kvEngine);
+    CHECK_ALL_DIST_AT(dist, flat::NodeId());
+    dist.start(exec);
+    for (size_t i = 0; i < 5; i++) {
+      CHECK_ALL_DIST_AT(dist, flat::NodeId(50));
+      std::this_thread::sleep_for(std::chrono::milliseconds(500));
+    }
+
+    std::vector<std::unique_ptr<IReadWriteTransaction>> txns;
+    for (size_t i = 0; i < 100; i++) {
+      auto txn = kvEngine->createReadWriteTransaction();
+      auto result = co_await dist.checkOnServer(*txn, InodeId(0));
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(result->first);
+      auto dummy = std::string(8, '\0');
+      CO_ASSERT_OK(co_await txn->clear(dummy));
+      txns.push_back(std::move(txn));
+    }
+
+    dist.stopAndJoin();
+    for (auto &txn : txns) {
+      CO_ASSERT_ERROR(co_await txn->commit(), TransactionCode::kConflict);
+    }
+    CHECK_ALL_DIST_AT(dist, flat::NodeId());
+  }());
+
+  exec.join();
+}
+
+TYPED_TEST(TestDistributor, balance) {
+  std::map<flat::NodeId, size_t> cnts;
+  std::map<InodeId, flat::NodeId> map;
+  std::vector<flat::NodeId> nodes;
+
+  auto servers = folly::Random::rand32(3, 10);
+  for (size_t i = 0; i < servers; i++) {
+    auto nodeId = flat::NodeId(50 + i);
+    nodes.push_back(nodeId);
+  }
+  for (size_t i = 0; i < 10000; i++) {
+    auto inodeId = InodeId(folly::Random::rand64());
+    auto server = Weight::select(nodes, inodeId);
+    cnts[server]++;
+    map[inodeId] = server;
+  }
+
+  auto max = std::numeric_limits<size_t>::min();
+  auto min = std::numeric_limits<size_t>::max();
+  for (auto &[node, cnt] : cnts) {
+    max = std::max(max, cnt);
+    min = std::min(min, cnt);
+  }
+  fmt::print("min {}, max {}, avg {}\n", min, max, map.size() / nodes.size());
+
+  nodes.erase(nodes.begin() + folly::Random::rand32(servers));
+
+  size_t changed = 0;
+  for (auto &[inodeId, server] : map) {
+    auto newServer = Weight::select(nodes, inodeId);
+    if (server != newServer) {
+      changed++;
+    }
+  }
+  fmt::print("changed {}, expected {}\n", changed, map.size() / (nodes.size() + 1));
+}
+
+TYPED_TEST(TestDistributor, multiple) {
+  auto config = Distributor::Config();
+  config.set_update_interval(100_ms);
+  config.set_timeout(1_s);
+  auto kvEngine = this->kvEngine();
+  auto exec = CPUExecutorGroup(4, "distributor");
+  auto exec2 = CPUExecutorGroup(4, "worker");
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    std::map<flat::NodeId, std::unique_ptr<Distributor>> distMap;
+    std::map<flat::NodeId, bool> activeMap;
+
+    for (size_t i = 0; i < 10; i++) {
+      auto nodeId = flat::NodeId(50 + i);
+      distMap[nodeId] = std::make_unique<Distributor>(config, nodeId, kvEngine);
+      activeMap[nodeId] = false;
+    }
+
+    for (size_t i = 0; i < FLAGS_distributor_iters; i++) {
+      std::vector<flat::NodeId> activeNodes;
+      for (auto &iter : activeMap) {
+        auto nodeId = iter.first;
+        auto &active = iter.second;
+        auto &dist = distMap[nodeId];
+        if (folly::Random::oneIn(2)) {
+          if (active) {
+            auto update = folly::Random::oneIn(2);
+            XLOGF(CRITICAL, "stop {}, update {}", nodeId, update);
+            dist->stopAndJoin(update);
+          } else {
+            XLOGF(CRITICAL, "start {}", nodeId);
+            dist->start(exec);
+          }
+          active = !active;
+        }
+        if (active) {
+          activeNodes.push_back(nodeId);
+        }
+      }
+
+      if (activeNodes.empty()) {
+        continue;
+      }
+
+      std::this_thread::sleep_for(5_s);
+      std::vector<folly::SemiFuture<Void>> tasks;
+      for (size_t i = 0; i < 16; i++) {
+        auto task = folly::coro::co_invoke([&]() -> CoTask<void> {
+          for (size_t i = 0; i < 100; i++) {
+            auto inodeId = InodeId(folly::Random::rand64());
+            auto expected = Weight::select(activeNodes, inodeId);
+            for (auto &[nodeId, dist] : distMap) {
+              READ_WRITE_TRANSACTION_OK({ CHECK_DIST(*txn, (*dist), inodeId, expected); });
+            }
+            co_await folly::coro::co_current_executor;
+          }
+        });
+        tasks.push_back(std::move(task).scheduleOn(&exec2.pickNext()).start());
+      }
+      co_await folly::coro::collectAllRange(std::move(tasks));
+    }
+  }());
+
+  XLOGF(INFO, "test finished");
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/components/TestFileHelper.cc

@@ -0,0 +1,138 @@
+#include <chrono>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/Math.h>
+#include <folly/Random.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Invoke.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/futures/Future.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest-param-test.h>
+#include <gtest/gtest.h>
+#include <limits>
+#include <map>
+#include <vector>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/Result.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "gtest/gtest.h"
+#include "meta/components/GcManager.h"
+#include "meta/components/SessionManager.h"
+#include "meta/store/Inode.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestFileHelper : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestFileHelper, KVTypes);
+
+TYPED_TEST(TestFileHelper, sync) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+    auto &fileHelper = cluster.meta().getFileHelper();
+    auto &storage = cluster.meta().getStorageClient();
+
+    for (size_t i = 0; i < 100; i++) {
+      auto result = co_await meta.create({SUPER_USER, fmt::format("file-{}", i), std::nullopt, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+
+      auto inode = Inode(result->stat);
+      auto offset = folly::Random::rand64(100ULL << 30);
+      auto length = folly::Random::rand64(16ULL << 20);
+      co_await randomWrite(meta, storage, inode, offset, length);
+      size_t total = offset + length;
+
+      FAULT_INJECTION_SET(10, 10);
+      while (true) {
+        auto stat = co_await meta.stat({SUPER_USER, inode.id, AtFlags()});
+        CO_ASSERT_OK(stat);
+        auto result = co_await fileHelper.queryLength({}, stat->stat, nullptr);
+        if (!result.hasError()) {
+          CO_ASSERT_EQ(*result, total);
+          break;
+        }
+      }
+    }
+  }());
+}
+TYPED_TEST(TestFileHelper, remove) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+    auto &fileHelper = cluster.meta().getFileHelper();
+    auto &storage = cluster.meta().getStorageClient();
+    auto mgmtd = cluster.mgmtdClient();
+
+    struct File {
+      Inode inode;
+      size_t chunks = 0;
+      size_t length = 0;
+    };
+    std::map<size_t, File> files;
+    for (size_t i = 0; i < 10; i++) {
+      auto result = co_await meta.create({SUPER_USER, fmt::format("file-{}", i), std::nullopt, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      File file{result->stat};
+      file.inode.asFile().length = folly::Random::rand64(100ULL << 30);
+      file.inode.asFile().truncateVer = folly::Random::rand64();
+      std::set<size_t> chunks;
+      for (size_t i = 0; i < 10; i++) {
+        auto offset = folly::Random::rand64(100ULL << 30);
+        auto length = folly::Random::rand64(16ULL << 20);
+        co_await randomWrite(meta, storage, file.inode, offset, length);
+        size_t firstChunk = offset / file.inode.asFile().layout.chunkSize.u64();
+        size_t lastChunk = folly::divCeil(offset + length, file.inode.asFile().layout.chunkSize.u64());
+        for (size_t chunk = firstChunk; chunk < lastChunk; chunk++) {
+          if (chunks.contains(chunk)) {
+            continue;
+          }
+          chunks.insert(chunk);
+          file.chunks++;
+        }
+        file.length = std::max(file.length, offset + length);
+      }
+      files[i] = file;
+    }
+
+    size_t total = 0;
+    for (auto &[i, file] : files) {
+      total += file.chunks;
+    }
+    auto query = co_await queryTotalChunks(storage, *mgmtd);
+    CO_ASSERT_OK(query);
+    CO_ASSERT_EQ(*query, total);
+
+    for (size_t i = 0; i < 10; i++) {
+      if (folly::Random::oneIn(2)) {
+        continue;
+      }
+      while (true) {
+        FAULT_INJECTION_SET(20, 10);
+        files[i].inode.asFile().dynStripe = 0;
+        auto result = co_await fileHelper.remove({}, files[i].inode, {}, folly::Random::rand32(8, 16));
+        if (!result.hasError()) {
+          break;
+        }
+      }
+      total -= files[i].chunks;
+      auto query = co_await queryTotalChunks(storage, *mgmtd);
+      CO_ASSERT_OK(query);
+      CO_ASSERT_EQ(*query, total);
+    }
+  }());
+}
+}  // namespace hf3fs::meta::server

tests/meta/components/TestGcManager.cc

@@ -0,0 +1,326 @@
+#include <chrono>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Invoke.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/futures/Future.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest-param-test.h>
+#include <limits>
+#include <map>
+#include <vector>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "client/storage/StorageClientInMem.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Result.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/core/user/User.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "gtest/gtest.h"
+#include "meta/components/GcManager.h"
+#include "meta/components/SessionManager.h"
+#include "meta/store/Inode.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestGcManager : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestGcManager, KVTypes);
+
+TYPED_TEST(TestGcManager, entry) {
+  fmt::print("{}\n", GcManager::formatGcEntry('d', UtcClock::now(), InodeId(folly::Random::rand64())));
+  for (size_t i = 0; i < 10000; i++) {
+    auto a = UtcTime::fromMicroseconds(folly::Random::rand64(std::numeric_limits<int64_t>::max()));
+    auto b = UtcTime::fromMicroseconds(folly::Random::rand64(std::numeric_limits<int64_t>::max()));
+    if (a == b) continue;
+    auto entryA = GcManager::formatGcEntry('f', a, InodeId::gcRoot());
+    auto entryB = GcManager::formatGcEntry('f', b, InodeId::gcRoot());
+    ASSERT_EQ(a < b, entryA < entryB) << fmt::format("a {}, b {}", entryA, entryB);
+    auto parsed = GcManager::parseGcEntry(entryA);
+    ASSERT_OK(parsed);
+    ASSERT_EQ(parsed->first, a);
+    ASSERT_EQ(parsed->second, InodeId::gcRoot());
+  }
+}
+
+TYPED_TEST(TestGcManager, basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_remove_chunks_batch_size(1);
+    config.mock_meta().gc().set_gc_directory_delay(2_s);
+    config.mock_meta().gc().set_gc_file_delay(4_s);
+    config.mock_meta().gc().set_scan_batch(64);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &storage = cluster.meta().getStorageClient();
+    auto mgmtd = cluster.mgmtdClient();
+
+    std::vector<std::pair<InodeId, SessionInfo>> sessions;
+    for (int i = 0; i < 10; i++) {
+      auto session = MetaTestHelper::randomSession();
+      auto result = co_await meta.create({SUPER_USER, std::to_string(i) + ".writing", session, O_RDWR, p644});
+      CO_ASSERT_OK(result);
+      co_await randomWrite(meta, storage, result->stat, folly::Random::rand64(100ULL << 30), 1ULL << 20);
+      CO_ASSERT_OK(co_await meta.sync({SUPER_USER, result->stat.id, true, std::nullopt, std::nullopt}));
+      sessions.push_back({result->stat.id, session});
+    }
+
+    for (int i = 0; i < 512; i++) {
+      auto result = co_await meta.create({SUPER_USER, std::to_string(i) + ".file", std::nullopt, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      co_await randomWrite(meta, storage, result->stat, folly::Random::rand64(100ULL << 30), 1ULL << 20);
+      CO_ASSERT_OK(co_await meta.sync({SUPER_USER, result->stat.id, true, std::nullopt, std::nullopt}));
+    }
+
+    GET_INODE_CNTS(numInodes);
+    GET_DIRENTRY_CNTS(numDirentries);
+    CO_ASSERT_SESSION_CNTS(10);
+
+    auto query = co_await queryTotalChunks(storage, *mgmtd);
+    CO_ASSERT_OK(query);
+    CO_ASSERT_NE(*query, 0);
+
+    auto begin = UtcClock::now();
+    for (int i = 0; i < 10; i++) {
+      CO_ASSERT_OK(co_await meta.remove({SUPER_USER, std::to_string(i) + ".writing", AtFlags(), false}));
+    }
+    for (int i = 0; i < 512; i++) {
+      CO_ASSERT_OK(co_await meta.remove({SUPER_USER, std::to_string(i) + ".file", AtFlags(), false}));
+    }
+
+    auto start = std::chrono::steady_clock::now();
+    bool finish = false;
+    while (!finish) {
+      CO_ASSERT_LE(std::chrono::steady_clock::now() - start, std::chrono::seconds(60));
+      co_await folly::coro::sleep(std::chrono::seconds(1));
+      GET_INODE_CNTS(cnts);
+      if (cnts == numInodes - 512) {
+        finish = true;
+      } else {
+        if (cnts < numInodes) {
+          CO_ASSERT_GE(UtcClock::now(), begin + 4_s);
+        }
+        fmt::print("{} inodes left\n", cnts);
+      }
+    }
+
+    CO_ASSERT_INODE_CNTS(numInodes - 512);
+    CO_ASSERT_DIRENTRY_CNTS(numDirentries - 512);
+
+    for (auto [inode, session] : sessions) {
+      CO_ASSERT_OK(co_await meta.close({SUPER_USER, inode, session, true, {}, {}}));
+    }
+
+    start = std::chrono::steady_clock::now();
+    finish = false;
+    while (!finish) {
+      CO_ASSERT_LE(std::chrono::steady_clock::now() - start, std::chrono::seconds(10));
+      co_await folly::coro::sleep(std::chrono::seconds(1));
+      GET_INODE_CNTS(cnts);
+      if (cnts == numInodes - 512 - 10) {
+        finish = true;
+      } else {
+        fmt::print("{} inodes left\n", cnts);
+      }
+    }
+
+    CO_ASSERT_INODE_CNTS(numInodes - 512 - 10);
+    CO_ASSERT_DIRENTRY_CNTS(numDirentries - 512 - 10);
+    CO_ASSERT_SESSION_CNTS(0);
+
+    auto queryAfterRemove = co_await queryTotalChunks(storage, *mgmtd);
+    CO_ASSERT_OK(queryAfterRemove);
+    CO_ASSERT_EQ(*queryAfterRemove, 0);
+
+    co_return;
+  }());
+}
+
+TYPED_TEST(TestGcManager, recursive) {
+  MockCluster::Config config;
+  config.mock_meta().gc().set_remove_chunks_batch_size(1);
+  config.mock_meta().gc().set_gc_directory_delay(3_s);
+  config.mock_meta().gc().set_gc_file_delay(3_s);
+  config.mock_meta().gc().set_scan_batch(128);
+  auto cluster = this->createMockCluster(config);
+  auto &meta = cluster.meta().getOperator();
+  folly::CPUThreadPoolExecutor exec(4);
+  std::vector<folly::SemiFuture<Void>> tasks;
+
+  InodeId dir, dir1, dir2, dir3, dir4;
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    dir = (co_await meta.mkdirs({SUPER_USER, "dir", p644, false}))->stat.id;
+    dir1 = (co_await meta.mkdirs({SUPER_USER, "dir/dir1", p644, false}))->stat.id;
+    dir2 = (co_await meta.mkdirs({SUPER_USER, "dir/dir2", p644, false}))->stat.id;
+    dir3 = (co_await meta.mkdirs({SUPER_USER, "dir/dir3", p644, false}))->stat.id;
+    dir4 = (co_await meta.mkdirs({SUPER_USER, "dir/dir4", p644, false}))->stat.id;
+  }());
+
+  auto worker = [&meta, dir, dir1, dir2, dir3, dir4](const size_t i) -> CoTask<void> {
+    for (int j = 0; j < 64; j++) {
+      auto fname = fmt::format("{}.{}", i, j);
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt(dir, fname), {}, O_RDONLY | O_EXCL, p644}));
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt(dir1, fname), {}, O_RDONLY | O_EXCL, p644}));
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt(dir2, fname), {}, O_RDONLY | O_EXCL, p644}));
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt(dir3, fname), {}, O_RDONLY | O_EXCL, p644}));
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt(dir4, fname), {}, O_RDONLY | O_EXCL, p644}));
+      CO_ASSERT_OK(co_await meta.hardLink({SUPER_USER, PathAt(dir, fname), PathAt(dir, fname + ".1"), AtFlags()}));
+    }
+    co_return;
+  };
+  for (size_t i = 0; i < 4; i++) {
+    auto task = folly::coro::co_invoke(worker, i).scheduleOn(&exec).start();
+    tasks.push_back(std::move(task));
+  }
+
+  for (auto &task : tasks) {
+    task.wait();
+  }
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    GET_INODE_CNTS(numInodes);
+    GET_DIRENTRY_CNTS(numEntries);
+
+    auto expectedInodes = numInodes - 5 - 4 * 5 * 64;
+    auto expectedEntries = numEntries - 5 - 4 * 6 * 64;
+
+    auto begin = UtcClock::now();
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "dir", AtFlags(), true}));
+
+    auto start = std::chrono::steady_clock::now();
+    bool finish = false;
+    while (!finish) {
+      CO_ASSERT_LE(std::chrono::steady_clock::now() - start, std::chrono::seconds(120));
+      co_await folly::coro::sleep(std::chrono::seconds(1));
+      GET_INODE_CNTS(cnts);
+      if (cnts <= expectedInodes) {
+        finish = true;
+      } else {
+        if (cnts < numInodes - 1) {
+          CO_ASSERT_GE(UtcClock::now(), begin + 6_s);
+        }
+        fmt::print("{} inodes left\n", cnts);
+      }
+    }
+
+    CO_ASSERT_INODE_CNTS(expectedInodes);
+    CO_ASSERT_DIRENTRY_CNTS(expectedEntries);
+  }());
+}
+
+TYPED_TEST(TestGcManager, delayThreshold) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(true);
+    config.mock_meta().gc().set_gc_file_delay(5_min);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &storage = cluster.meta().getStorageClient();
+
+    // create file, write then delete
+    auto session = MetaTestHelper::randomSession();
+    auto result = co_await meta.create({SUPER_USER, "file", session, O_RDWR, p644});
+    CO_ASSERT_OK(result);
+    co_await randomWrite(meta, storage, result->stat, folly::Random::rand64(100ULL << 30), 1ULL << 20);
+    CO_ASSERT_OK(co_await meta.close({SUPER_USER, result->stat.id, session, true, std::nullopt, std::nullopt}));
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "file", AtFlags(), false}));
+
+    co_await folly::coro::sleep(std::chrono::seconds(1));
+    CO_ASSERT_OK(co_await meta.stat({SUPER_USER, result->stat.id, AtFlags()}));
+
+    // only enable gc delay if free space > 95
+    config.mock_meta().gc().set_gc_delay_free_space_threshold(95);
+    // wait 1s, then inode is deleted
+    co_await folly::coro::sleep(std::chrono::seconds(1));
+    CO_ASSERT_INODE_NOT_EXISTS(result->stat.id);
+  }());
+}
+
+TYPED_TEST(TestGcManager, recurisvePerm) {
+  MockCluster::Config config;
+  config.mock_meta().set_recursive_remove_perm_check(0);
+  config.mock_meta().gc().set_enable(true);
+  config.mock_meta().gc().set_recursive_perm_check(true);
+
+  auto ua = flat::UserInfo(flat::Uid(1), flat::Gid(1));
+  auto ub = flat::UserInfo(flat::Uid(2), flat::Gid(2));
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "trash", p755, true}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "trash/gc-orphans", {}, 0, p644}));
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "share", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "share/ua", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ub, "share/ua/ub", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ub, "share/ua/ub/noperm-empty", p700, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ub, "share/ua/ub/noperm-notempty/subdir", p700, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "share/ua/protected-directory", p755, true}));
+    CO_ASSERT_OK(co_await meta.setAttr(
+        SetAttrReq::setIFlags(SUPER_USER, "share/ua/protected-directory", IFlags(FS_IMMUTABLE_FL))));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "share/ua/normal-dir", p755, true}));
+    CO_ASSERT_OK(co_await meta.create({ua, "share/ua/normal-dir/protected1", {}, 0, p644}));
+    CO_ASSERT_OK(co_await meta.create({ua, "share/ua/normal-dir/protected2", {}, 0, p644}));
+    CO_ASSERT_OK(co_await meta.setAttr(
+        SetAttrReq::setIFlags(SUPER_USER, "share/ua/normal-dir/protected1", IFlags(FS_IMMUTABLE_FL))));
+    CO_ASSERT_OK(co_await meta.setAttr(
+        SetAttrReq::setIFlags(SUPER_USER, "share/ua/normal-dir/protected2", IFlags(FS_IMMUTABLE_FL))));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "share/ua/data1/noperm/subdir", p755, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "share/ua/data2/noperm/subdir", p755, true}));
+    CO_ASSERT_OK(co_await meta.setAttr(
+        SetAttrReq::setPermission(SUPER_USER, "share/ua/data1/noperm", AtFlags(), {}, {}, flat::Permission(0))));
+    CO_ASSERT_OK(co_await meta.setAttr(
+        SetAttrReq::setPermission(SUPER_USER, "share/ua/data2/noperm", AtFlags(), {}, {}, flat::Permission(0))));
+
+    CO_ASSERT_OK(co_await meta.remove({ua, "share/ua", AtFlags(), true}));
+    co_await folly::coro::sleep(5_s);
+    CO_ASSERT_OK(co_await printTree(meta));
+  }());
+}
+
+TYPED_TEST(TestGcManager, DISABLED_BadFile) {
+  // generate too many logs, disable by default
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(true);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &storage = cluster.meta().getStorageClient();
+
+    // inject fault on chain 1
+    co_await dynamic_cast<storage::client::StorageClientInMem &>(storage).injectErrorOnChain(
+        flat::ChainId(100001),
+        makeError(StorageClientCode::kChecksumMismatch));
+
+    // create file on chain 1, then remove
+    for (size_t i = 0; i < 10000; i++) {
+      auto layout = Layout::newChainList(4096, {flat::ChainId(100001)});
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, fmt::format("{}.1", i), std::nullopt, O_RDONLY, p644, layout}));
+      CO_ASSERT_OK(co_await meta.remove({SUPER_USER, fmt::format("{}.1", i), AtFlags(), false}));
+    }
+    // create file on chain 2. then remove
+    for (size_t i = 0; i < 10000; i++) {
+      auto layout = Layout::newChainList(4096, {flat::ChainId(100002)});
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, fmt::format("{}.2", i), std::nullopt, O_RDONLY, p644, layout}));
+      CO_ASSERT_OK(co_await meta.remove({SUPER_USER, fmt::format("{}.2", i), AtFlags(), false}));
+    }
+
+    co_await folly::coro::sleep(std::chrono::seconds(5));
+    CO_ASSERT_INODE_CNTS(10010);
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/components/TestInodeIdAllocator.cc

@@ -0,0 +1,142 @@
+#include <chrono>
+#include <folly/Synchronized.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/CurrentExecutor.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <gtest/gtest-typed-test.h>
+#include <memory>
+#include <vector>
+
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "meta/components/InodeIdAllocator.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server::test {
+
+template <typename KV>
+class TestInodeIdAllocator : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestInodeIdAllocator, KVTypes);
+
+CoTask<void> doAllocate(std::shared_ptr<InodeIdAllocator> allocator,
+                        size_t times,
+                        folly::Synchronized<std::vector<InodeId>, std::mutex> &allocated,
+                        bool allowFailure = false) {
+  for (size_t i = 1; i <= times; i++) {
+    // since there is only one process, allocate shouldn't fail.
+    auto result = co_await allocator->allocate(std::chrono::seconds(10));
+    if (!allowFailure) {
+      CO_ASSERT_OK(result);
+    }
+    if (result.hasError()) {
+      CO_ASSERT_ERROR(result, MetaCode::kInodeIdAllocFailed);
+    } else {
+      allocated.lock()->push_back(result.value());
+      co_await folly::coro::co_reschedule_on_current_executor;
+    }
+  }
+  co_return;
+}
+
+TYPED_TEST(TestInodeIdAllocator, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto kv = this->kvEngine();
+    auto allocator = InodeIdAllocator::create(kv);
+
+    std::vector<InodeId> allocated;
+    for (size_t i = 0; i < 10000; i++) {
+      auto result = co_await allocator->allocate();
+      CO_ASSERT_OK(result);
+      allocated.push_back(result.value());
+    }
+
+    std::sort(allocated.begin(), allocated.end());
+    for (size_t i = 1; i < allocated.size(); i++) {
+      CO_ASSERT_GT(allocated.at(i), allocated.at(i - 1));
+    }
+    fmt::print("first {}, last {}\n", allocated.begin()->toHexString(), allocated.rbegin()->toHexString());
+  }());
+}
+
+TYPED_TEST(TestInodeIdAllocator, MultiThreads) {
+  folly::Synchronized<std::vector<InodeId>, std::mutex> allocated;
+  std::vector<folly::SemiFuture<folly::Unit>> futures;
+  folly::CPUThreadPoolExecutor exec(8);
+
+  auto kv = this->kvEngine();
+  auto allocator = InodeIdAllocator::create(kv);
+
+  for (size_t i = 0; i < 32; i++) {
+    // since there is only one process, allocate shouldn't fail.
+    futures.push_back(doAllocate(allocator, 10000, allocated, false).scheduleOn(&exec).start());
+  }
+  for (auto &f : futures) {
+    f.wait();
+  }
+
+  auto lock = allocated.lock();
+  std::sort(lock->begin(), lock->end());
+  fmt::print("succ {}, min {}, max {}\n", lock->size(), lock->begin()->toHexString(), lock->rbegin()->toHexString());
+  for (size_t i = 1; i < lock->size(); i++) {
+    ASSERT_GT(lock->at(i), lock->at(i - 1));
+  }
+}
+
+TYPED_TEST(TestInodeIdAllocator, MultiAllocator) {
+  folly::Synchronized<std::vector<InodeId>, std::mutex> allocated;
+  std::vector<folly::SemiFuture<folly::Unit>> futures;
+  folly::CPUThreadPoolExecutor exec(8);
+
+  auto kv = this->kvEngine();
+  std::vector<std::shared_ptr<InodeIdAllocator>> vec;
+  vec.reserve(32);
+  for (size_t i = 0; i < 32; i++) {
+    vec.push_back(InodeIdAllocator::create(kv));
+  }
+  for (size_t i = 0; i < 128; i++) {
+    auto &allocator = vec[i % vec.size()];
+    futures.push_back(doAllocate(allocator, 10000, allocated, true).scheduleOn(&exec).start());
+  }
+  for (auto &f : futures) {
+    f.wait();
+  }
+
+  auto lock = allocated.lock();
+  std::sort(lock->begin(), lock->end());
+  fmt::print("succ {}, min {}, max {}\n", lock->size(), lock->begin()->toHexString(), lock->rbegin()->toHexString());
+  for (size_t i = 1; i < lock->size(); i++) {
+    ASSERT_GT(lock->at(i), lock->at(i - 1));
+  }
+}
+
+TYPED_TEST(TestInodeIdAllocator, FaultInjection) {
+  folly::Synchronized<std::vector<InodeId>, std::mutex> allocated;
+  std::vector<folly::SemiFuture<folly::Unit>> futures;
+  folly::CPUThreadPoolExecutor exec(8);
+
+  auto kv = this->kvEngine();
+  auto allocator = InodeIdAllocator::create(kv);
+
+  for (size_t i = 0; i < 8; i++) {
+    FAULT_INJECTION_SET(10, -1);
+    futures.push_back(doAllocate(allocator, 10000, allocated, true).scheduleOn(&exec).start());
+  }
+  for (size_t i = 0; i < 8; i++) {
+    futures.push_back(doAllocate(allocator, 10000, allocated, false).scheduleOn(&exec).start());
+  }
+  for (auto &f : futures) {
+    f.wait();
+  }
+
+  auto lock = allocated.lock();
+  std::sort(lock->begin(), lock->end());
+  fmt::print("succ {}, min {}, max {}\n", lock->size(), lock->begin()->toHexString(), lock->rbegin()->toHexString());
+  for (size_t i = 1; i < lock->size(); i++) {
+    ASSERT_GT(lock->at(i), lock->at(i - 1));
+  }
+}
+
+}  // namespace hf3fs::meta::server::test

tests/meta/components/TestSessionManager.cc

@@ -0,0 +1,297 @@
+#include <algorithm>
+#include <chrono>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest-param-test.h>
+#include <limits>
+#include <map>
+#include <memory>
+#include <vector>
+
+#include "client/mgmtd/ICommonMgmtdClient.h"
+#include "client/storage/StorageClient.h"
+#include "common/app/ClientId.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Result.h"
+#include "common/utils/UtcTime.h"
+#include "common/utils/Uuid.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "gtest/gtest.h"
+#include "meta/components/GcManager.h"
+#include "meta/components/SessionManager.h"
+#include "meta/store/FileSession.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestSessionManager : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestSessionManager, KVTypes);
+
+TYPED_TEST(TestSessionManager, basic) {
+  // for (size_t i = 0; i < 10; i++) {
+  //   auto client = Uuid::random();
+  //   auto session = Uuid::random();
+  //   auto key = FileSession::packKey(client, session);
+  //   auto unpacked = FileSession::unpackByClientKey(key);
+  //   ASSERT_EQ(unpacked->first, client);
+  //   ASSERT_EQ(unpacked->second, session);
+  // }
+
+  for (size_t i = 0; i < 10; i++) {
+    auto inode = InodeId(folly::Random::rand64());
+    auto session = Uuid::random();
+    auto key = FileSession::packKey(inode, session);
+    auto unpacked = FileSession::unpackByInodeKey(key);
+    ASSERT_EQ(unpacked->first, inode);
+    ASSERT_EQ(unpacked->second, session);
+  }
+}
+
+TYPED_TEST(TestSessionManager, byInode) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto mgmtd = std::dynamic_pointer_cast<tests::FakeMgmtdClient>(cluster.mgmtdClient());
+
+    for (size_t i = 0; i < 10; i++) {
+      auto inode = InodeId(folly::Random::rand64());
+      size_t numSessions = folly::Random::rand32(5, 32);
+      for (size_t j = 0; j < numSessions; j++) {
+        READ_WRITE_TRANSACTION_OK({
+          auto session = FileSession::create(inode, ClientId::random(), Uuid::random());
+          CO_ASSERT_OK(co_await session.store(*txn));
+        });
+      }
+      READ_ONLY_TRANSACTION({
+        auto sessions = co_await FileSession::list(*txn, inode, false);
+        CO_ASSERT_OK(sessions);
+        CO_ASSERT_EQ(sessions->size(), numSessions);
+        auto hasSession = co_await FileSession::snapshotCheckExists(*txn, inode);
+        CO_ASSERT_OK(hasSession);
+        CO_ASSERT_TRUE(*hasSession);
+      });
+
+      READ_WRITE_TRANSACTION_OK({ CO_ASSERT_OK(co_await FileSession::removeAll(*txn, inode)); });
+      READ_WRITE_TRANSACTION_OK({
+        auto sessions = co_await FileSession::list(*txn, inode, false);
+        CO_ASSERT_OK(sessions);
+        CO_ASSERT_EQ(sessions->size(), 0);
+        auto hasSession = co_await FileSession::snapshotCheckExists(*txn, inode);
+        CO_ASSERT_OK(hasSession);
+        CO_ASSERT_FALSE(*hasSession);
+      });
+    }
+  }());
+}
+
+TYPED_TEST(TestSessionManager, scan) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto mgmtd = std::dynamic_pointer_cast<tests::FakeMgmtdClient>(cluster.mgmtdClient());
+
+    for (size_t i = 0; i < 100; i++) {
+      READ_WRITE_TRANSACTION_OK({
+        for (size_t i = 0; i < 500; i++) {
+          auto inode = InodeId(folly::Random::rand64(1 << 10, std::numeric_limits<uint64_t>::max()));
+          auto session = FileSession::create(inode, ClientId::random(), Uuid::random());
+          CO_ASSERT_OK(co_await session.store(*txn));
+          auto prune = FileSession::createPrune(session.clientId, session.sessionId);
+          CO_ASSERT_OK(co_await prune.store(*txn));
+        }
+      });
+    }
+
+    std::map<Uuid, FileSession> sessions;
+    for (size_t i = 0; i < FileSession::kShard; i++) {
+      std::vector<FileSession> vec;
+      bool more = true;
+      while (more) {
+        READ_ONLY_TRANSACTION({
+          auto sessions = co_await FileSession::scan(*txn, i, !vec.empty() ? std::optional(vec.back()) : std::nullopt);
+          CO_ASSERT_OK(sessions);
+          more = !sessions->empty();
+          vec.insert(vec.end(), sessions->begin(), sessions->end());
+        });
+      }
+      for (auto &s : vec) {
+        sessions.emplace(s.sessionId, s);
+      }
+    }
+    CO_ASSERT_EQ(sessions.size(), 100 * 500);
+
+    READ_ONLY_TRANSACTION({
+      auto prune = co_await FileSession::listPrune(*txn, sessions.size() * 2);
+      CO_ASSERT_OK(prune);
+      CO_ASSERT_EQ(prune->size(), sessions.size());
+      for (auto s : *prune) {
+        CO_ASSERT_TRUE(sessions.find(s.sessionId) != sessions.end());
+        sessions.erase(s.sessionId);
+      }
+    });
+    CO_ASSERT_TRUE(sessions.empty());
+  }());
+}
+
+// TYPED_TEST(TestSessionManager, byClient) {
+//   folly::coro::blockingWait([&]() -> CoTask<void> {
+//     auto cluster = this->createMockCluster();
+//     auto &sessionManager = cluster.meta().getSessionManager();
+//     auto mgmtd = std::dynamic_pointer_cast<tests::FakeMgmtdClient>(cluster.mgmtdClient());
+//     for (auto clientId : mgmtd->getActiveClients()) {
+//       size_t numSessions = folly::Random::rand32(5, 32);
+//       for (size_t j = 0; j < numSessions; j++) {
+//         READ_WRITE_TRANSACTION_OK({
+//           auto session = FileSession::create(InodeId(folly::Random::rand32()), ClientId(clientId), Uuid::random());
+//           CO_ASSERT_OK(co_await session.store(*txn));
+//         });
+//       }
+//
+//       READ_ONLY_TRANSACTION({
+//         auto sessions = co_await FileSession::list(*txn, clientId, true);
+//         CO_ASSERT_OK(sessions);
+//         CO_ASSERT_EQ(sessions->size(), numSessions);
+//       });
+//     }
+//
+//     std::vector<std::pair<Uuid, size_t>> deadClients;
+//     for (size_t i = 0; i < 10; i++) {
+//       auto clientId = Uuid::random();
+//       size_t numSessions = folly::Random::rand32(1, 32);
+//       deadClients.emplace_back(clientId, numSessions);
+//       for (size_t j = 0; j < numSessions; j++) {
+//         READ_WRITE_TRANSACTION_OK({
+//           auto session = FileSession::create(InodeId(folly::Random::rand32()), ClientId(clientId), Uuid::random());
+//           CO_ASSERT_OK(co_await session.store(*txn));
+//         });
+//       }
+//       READ_ONLY_TRANSACTION({
+//         auto sessions = co_await FileSession::list(*txn, clientId, false);
+//         CO_ASSERT_OK(sessions);
+//         CO_ASSERT_EQ(sessions->size(), numSessions);
+//       });
+//     }
+//     READ_ONLY_TRANSACTION({
+//       auto all = co_await sessionManager.listClients();
+//       CO_ASSERT_OK(all);
+//       CO_ASSERT_EQ(all->size(), deadClients.size() + mgmtd->getActiveClients().size());
+//       for (auto client : *all) {
+//         auto active = mgmtd->getActiveClients().contains(client.clientId.uuid);
+//         CO_ASSERT_EQ(active, client.active);
+//         CO_ASSERT_NE(client.sessions, 0);
+//       }
+//     });
+//   }());
+// }
+
+TYPED_TEST(TestSessionManager, prune) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().session_manager().set_enable(true);
+
+    auto cluster = this->createMockCluster(config);
+    auto mgmtd = std::dynamic_pointer_cast<tests::FakeMgmtdClient>(cluster.mgmtdClient());
+
+    auto now = UtcClock::now().toMicroseconds();
+    auto minutes = 60 * 1000000;
+
+    std::vector<Uuid> deadClients;
+    for (size_t i = 0; i < 100; i++) {
+      deadClients.push_back(Uuid::random());
+    }
+    std::vector<FileSession> active, steal, future;
+    for (size_t i = 0; i < folly::Random::rand32(1000); i++) {
+      auto client = mgmtd->getOneActiveClient();
+      READ_WRITE_TRANSACTION_OK({
+        auto session = FileSession::create(InodeId(folly::Random::rand32()),
+                                           client,
+                                           Uuid::random(),
+                                           UtcTime::fromMicroseconds(now - minutes));
+        active.push_back(session);
+        CO_ASSERT_OK(co_await session.store(*txn));
+
+        auto pruneSessionId = Uuid::random();
+        CO_ASSERT_OK(
+            co_await FileSession::create(InodeId(folly::Random::rand32()), client, pruneSessionId).store(*txn));
+        CO_ASSERT_OK(co_await FileSession::createPrune(client, pruneSessionId).store(*txn));
+        CO_ASSERT_OK(co_await FileSession::createPrune(ClientId(Uuid::random()), pruneSessionId).store(*txn));
+      });
+    }
+    for (size_t i = 0; i < folly::Random::rand32(1000); i++) {
+      READ_WRITE_TRANSACTION_OK({
+        auto session = FileSession::create(InodeId(folly::Random::rand32()),
+                                           ClientId(deadClients.at(folly::Random::rand32(deadClients.size()))),
+                                           Uuid::random(),
+                                           UtcTime::fromMicroseconds(now - minutes));
+        steal.push_back(session);
+        CO_ASSERT_OK(co_await session.store(*txn));
+      });
+    }
+    for (size_t i = 0; i < folly::Random::rand32(1000); i++) {
+      READ_WRITE_TRANSACTION_OK({
+        auto session = FileSession::create(InodeId(folly::Random::rand32()),
+                                           ClientId(deadClients.at(folly::Random::rand32(deadClients.size()))),
+                                           Uuid::random(),
+                                           UtcTime::fromMicroseconds(now + minutes));
+        future.push_back(session);
+        CO_ASSERT_OK(co_await session.store(*txn));
+      });
+    }
+
+    co_await folly::coro::sleep(std::chrono::seconds(1));
+    fmt::print("{} {}\n", active.size(), future.size());
+    CO_ASSERT_SESSION_CNTS(active.size() + future.size());
+
+    mgmtd->clearActiveClient();
+    co_await folly::coro::sleep(std::chrono::seconds(1));
+    CO_ASSERT_SESSION_CNTS(future.size());
+  }());
+}
+
+TYPED_TEST(TestSessionManager, syncOnPrune) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().set_sync_on_prune_session(true);
+    config.mock_meta().session_manager().set_enable(true);
+    config.mock_meta().session_manager().set_sync_on_prune_session(true);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &storage = cluster.meta().getStorageClient();
+    auto mgmtd = std::dynamic_pointer_cast<tests::FakeMgmtdClient>(cluster.mgmtdClient());
+
+    auto result = co_await meta.create({SUPER_USER, PathAt("file"), std::nullopt, O_RDONLY, p644});
+    CO_ASSERT_OK(result);
+    auto &inode = result->stat;
+    auto length = folly::Random::rand64(2 << 20, 4 << 20);
+    co_await randomWrite(meta, storage, inode, 0, length);
+
+    // create a dead session
+    READ_WRITE_TRANSACTION_OK({
+      auto session = FileSession::create(
+          inode.id,
+          ClientId(Uuid::random()),
+          Uuid::random(),
+          UtcTime::fromMicroseconds(UtcClock::now().toMicroseconds() - 60 * 60 * 1000000L /* 1h */));
+      CO_ASSERT_OK(co_await session.store(*txn));
+    });
+
+    // wait sometime, inode should be synced
+    CO_ASSERT_SESSION_CNTS(1);
+    co_await folly::coro::sleep(std::chrono::seconds(3));
+    CO_ASSERT_SESSION_CNTS(0);
+
+    auto stat = co_await meta.stat({SUPER_USER, PathAt("file"), AtFlags()});
+    CO_ASSERT_OK(stat);
+    CO_ASSERT_EQ(stat->stat.asFile().length, length);
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/components/TestUserToken.cc

@@ -0,0 +1,40 @@
+#include <folly/Random.h>
+
+#include "core/user/UserToken.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::meta::server::test {
+namespace {
+class UserTokenTest : public ::testing::Test {
+ protected:
+  void test(uint32_t uid, uint64_t randomv) {
+    auto token = core::encodeUserToken(uid, randomv);
+    XLOGF(DBG, "encode({}, {}) -> {}", uid, randomv, token);
+
+    auto decodeRes = core::decodeUserToken(token);
+    ASSERT_OK(decodeRes);
+
+    auto [duid, dts] = *decodeRes;
+    ASSERT_EQ(uid, duid);
+    ASSERT_EQ(randomv, dts);
+
+    auto decodeRes2 = core::decodeUidFromUserToken(token);
+    ASSERT_OK(decodeRes2);
+    ASSERT_EQ(uid, *decodeRes2);
+  }
+};
+
+TEST_F(UserTokenTest, test) {
+  for (uint32_t i = 0; i < 100; ++i) {
+    for (uint64_t j = 0; j < 100; ++j) {
+      test(i, j);
+    }
+  }
+  for (size_t i = 0; i < 10000; ++i) {
+    auto uid = folly::Random::rand32();
+    auto randomv = folly::Random::rand64();
+    test(uid, randomv);
+  }
+}
+}  // namespace
+}  // namespace hf3fs::meta::server::test

tests/meta/event/TestEventLogger.cc

@@ -0,0 +1,96 @@
+#include <array>
+#include <cerrno>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/FileUtil.h>
+#include <folly/Random.h>
+#include <folly/ScopeGuard.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/hash/Checksum.h>
+#include <folly/json.h>
+#include <folly/logging/LogConfigParser.h>
+#include <folly/logging/Logger.h>
+#include <folly/logging/LoggerDB.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <string_view>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <vector>
+
+#include "common/logging/FileHandlerFactory.h"
+#include "common/utils/MagicEnum.hpp"
+#include "common/utils/Uuid.h"
+#include "fbs/meta/Common.h"
+#include "meta/event/Event.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::meta::server {
+
+using flat::UserInfo;
+
+static void logRandomEvent(Event::Type type) {
+  auto parent = InodeId(folly::Random::rand64());
+  auto name = fmt::format("path-{}", Uuid::random());
+  auto inode = InodeId(folly::Random::rand64());
+  auto user = folly::Random::rand32();
+  Event(type).addField("parent", parent).addField("name", name).addField("inode", inode).addField("user", user).log();
+}
+
+static void checkEventLog(std::string_view path, size_t expected) {
+  auto fd = open(path.data(), O_RDONLY);
+  ASSERT_GE(fd, 0) << errno;
+
+  std::string line;
+  std::ifstream file(path.data());
+  size_t actual = 0;
+  while (std::getline(file, line)) {
+    ASSERT_NO_THROW(folly::parseJson(line)) << "Failed to parse " << line;
+    actual++;
+  }
+  ASSERT_EQ(actual, expected);
+}
+
+TEST(TestEventLogger, Basic) {
+  auto config = folly::LoggerDB::get().getConfig();
+  SCOPE_EXIT { folly::LoggerDB::get().resetConfig(config); };
+  std::string path = "test-event-logger.log";
+  unlink(path.c_str());
+  folly::LoggerDB::get().resetConfig(folly::parseLogConfig(R"JSON({
+    "categories": {
+      ".": { "level": "ERR", "handlers": ["stderr"] },
+      "eventlog": { "level": "INFO", "inherit": false, "propagate": "CRITICAL", "handlers": ["event"] },
+      "eventlog.CloseWrite": { "level": "FATAL", "inherit": false, "propagate": "CRITICAL", "handlers": ["event"] },
+    },
+    "handlers": {
+      "stderr": {
+        "type": "stream",
+        "options": { "stream": "stderr", "async": "false" }
+      },
+      "event": {
+        "type": "event", 
+        "options": { "path": "test-event-logger.log", "async": "true" }
+      }
+    }
+  })JSON"));
+  SCOPE_EXIT {
+    struct stat st;
+    stat(path.c_str(), &st);
+    fmt::print("log file {}, st_size {}\n", path, st.st_size);
+    unlink(path.c_str());
+  };
+
+  size_t numEvents = 1000;
+  for (size_t i = 0; i < numEvents; i++) {
+    logRandomEvent(Event::Type::Create);
+  }
+  // shouldn't log here
+  for (size_t i = 0; i < 10; i++) {
+    logRandomEvent(Event::Type::CloseWrite);
+  }
+
+  sleep(1);
+  checkEventLog(path, numEvents);
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/event/TestScan.cc

@@ -0,0 +1,273 @@
+#include <atomic>
+#include <chrono>
+#include <cstdlib>
+#include <double-conversion/utils.h>
+#include <fmt/compile.h>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/Synchronized.h>
+#include <folly/concurrency/ConcurrentHashMap.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+#include <folly/experimental/coro/CurrentExecutor.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Invoke.h>
+#include <folly/futures/Future.h>
+#include <folly/logging/xlog.h>
+#include <gflags/gflags.h>
+#include <gtest/gtest.h>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <variant>
+#include <vector>
+
+#include "common/kv/IKVEngine.h"
+#include "common/kv/mem/MemKVEngine.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Result.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/mgmtd/ChainRef.h"
+#include "fbs/mgmtd/MgmtdTypes.h"
+#include "meta/event/Scan.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "meta/store/Utils.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+using InodeMap = folly::ConcurrentHashMap<InodeId, Inode>;
+using DirEntryMap = folly::ConcurrentHashMap<InodeId, DirEntry>;
+
+template <typename KV>
+class TestScan : public MetaTestBase<KV> {
+ protected:
+  template <typename Map>
+  void create(Map &map) {
+    folly::CPUThreadPoolExecutor exec(8);
+    std::vector<folly::SemiFuture<Void>> tasks;
+    for (size_t i = 0; i < 8; i++) {
+      auto task = folly::coro::co_invoke([&]() -> CoTask<void> {
+        for (size_t i = 0; i < (1 << 10); i++) {
+          READ_WRITE_TRANSACTION_OK({
+            if constexpr (std::is_same_v<Map, InodeMap>) {
+              auto &inodes = map;
+              for (size_t j = 0; j < folly::Random::rand32(8, 12); j++) {
+                auto inode = MetaTestHelper::randomInode();
+                CO_ASSERT_OK(co_await inode.store(*txn));
+                CO_ASSERT_TRUE(inodes.insert(inode.id, inode).second);
+              }
+            } else {
+              auto &entries = map;
+              for (size_t j = 0; j < 10; j++) {
+                auto entry = MetaTestHelper::randomDirEntry();
+                CO_ASSERT_OK(co_await entry.store(*txn));
+                CO_ASSERT_TRUE(entries.insert(entry.id, entry).second);
+              }
+            }
+          });
+        }
+        co_return;
+      });
+      tasks.push_back(std::move(task).scheduleOn(&exec).start());
+    }
+    folly::coro::collectAllRange(std::move(tasks)).semi().wait();
+  }
+};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestScan, KVTypes);
+
+TYPED_TEST(TestScan, Inode) {
+  folly::ConcurrentHashMap<InodeId, Inode> allInodes;
+
+  // create some inodes
+  auto beginCreate = std::chrono::steady_clock::now();
+  this->create(allInodes);
+
+  auto beginScan = std::chrono::steady_clock::now();
+  MetaScan scan(MetaScan::Options(), this->kvEngine());
+
+  std::atomic_uint64_t scanned{0};
+  folly::ConcurrentHashMap<InodeId, Void> scannedInodes;
+  while (true) {
+    auto inodes = scan.getInodes();
+    if (inodes.empty()) {
+      break;
+    }
+    for (auto &inode : inodes) {
+      scanned.fetch_add(1);
+      ASSERT_EQ(inode, allInodes[inode.id]);
+      ASSERT_TRUE(scannedInodes.insert(inode.id, Void{}).second);
+    }
+  }
+
+  ASSERT_EQ(allInodes.size(), scanned.load());
+  ASSERT_EQ(scanned.load(), scannedInodes.size());
+
+  auto now = std::chrono::steady_clock::now();
+  fmt::print("create {}ms, scan {}ms, total {}\n",
+             std::chrono::duration_cast<std::chrono::milliseconds>(beginScan - beginCreate).count(),
+             std::chrono::duration_cast<std::chrono::milliseconds>(now - beginScan).count(),
+             scanned.load());
+}
+
+TYPED_TEST(TestScan, DirEntry) {
+  folly::ConcurrentHashMap<InodeId, DirEntry> allEntries;
+
+  // create some inodes
+  auto beginCreate = std::chrono::steady_clock::now();
+  this->create(allEntries);
+
+  auto beginScan = std::chrono::steady_clock::now();
+  MetaScan scan(MetaScan::Options(), this->kvEngine());
+
+  std::atomic_uint64_t scanned{0};
+  folly::ConcurrentHashMap<InodeId, DirEntry> scannedEntries;
+  while (true) {
+    auto entries = scan.getDirEntries();
+    if (entries.empty()) {
+      break;
+    }
+    for (auto &entry : entries) {
+      scanned.fetch_add(1);
+      ASSERT_EQ(entry, allEntries[entry.id]);
+      ASSERT_TRUE(scannedEntries.insert(entry.id, entry).second);
+    }
+  }
+
+  ASSERT_EQ(allEntries.size(), scanned.load());
+  ASSERT_EQ(scanned.load(), scannedEntries.size());
+
+  auto now = std::chrono::steady_clock::now();
+  fmt::print("create {}ms, scan {}ms, total {}\n",
+             std::chrono::duration_cast<std::chrono::milliseconds>(beginScan - beginCreate).count(),
+             std::chrono::duration_cast<std::chrono::milliseconds>(now - beginScan).count(),
+             scanned.load());
+}
+
+TYPED_TEST(TestScan, Exit) {
+  folly::ConcurrentHashMap<InodeId, DirEntry> allEntries;
+  this->create(allEntries);
+
+  // start scan but exit without consume all entries.
+  MetaScan scan(MetaScan::Options(), this->kvEngine());
+  auto entries = scan.getDirEntries();
+  ASSERT_FALSE(entries.empty());
+}
+
+DEFINE_string(fdb_cluster, "", "fdb cluster file path");
+DEFINE_uint32(scan_threads, 4, "scan threads");
+DEFINE_uint32(scan_coroutines, 8, "scan coroutines");
+DEFINE_bool(scan_all, false, "scan all inodes and direntries");
+DEFINE_bool(scan_check, false, "check parent exists");
+
+TEST(TestScanFDB, DISABLED_FDB) {
+  MetaScan::Options options;
+  options.fdb_cluster_file = FLAGS_fdb_cluster;
+  options.threads = FLAGS_scan_threads;
+  options.coroutines = FLAGS_scan_coroutines;
+  MetaScan scan(options);
+
+  std::set<InodeId> directories;
+  std::set<InodeId> missing;
+
+  auto begin = SteadyClock::now();
+  size_t totalInodes = 0;
+  while (true) {
+    auto inodes = scan.getInodes();
+    totalInodes += inodes.size();
+    if (inodes.empty() || !FLAGS_scan_all) {
+      break;
+    }
+    if (FLAGS_scan_check) {
+      for (auto &inode : inodes) {
+        if (inode.isDirectory()) {
+          directories.insert(inode.id);
+        }
+      }
+    }
+    XLOGF(DBG, "MetaScan get {} inodes", inodes.size());
+  }
+  XLOGF(INFO,
+        "MetaScan scan Inode finished, duration {}, get {} inodes\n",
+        std::chrono::duration_cast<std::chrono::milliseconds>(SteadyClock::now() - begin),
+        totalInodes);
+
+  begin = SteadyClock::now();
+  size_t totalEntries = 0;
+  while (true) {
+    auto entries = scan.getDirEntries();
+    totalEntries += entries.size();
+    if (entries.empty() || !FLAGS_scan_all) {
+      break;
+    }
+    if (FLAGS_scan_check) {
+      for (auto &entry : entries) {
+        if (!directories.contains(entry.parent)) {
+          missing.insert(entry.parent);
+        }
+      }
+    }
+    XLOGF(DBG, "MetaScan get {} entries", entries.size());
+  }
+  XLOGF(INFO,
+        "MetaScan scan DirEntry finished, duration {}, get {} entries",
+        std::chrono::duration_cast<std::chrono::milliseconds>(SteadyClock::now() - begin),
+        totalEntries);
+  XLOGF_IF(WARNING, !missing.empty(), "Missing parent: {}", fmt::join(missing.begin(), missing.end(), ", "));
+}
+
+DEFINE_int64(inode_id, 0, "inode id");
+
+TEST(TestScanFDB, DISABLED_PrintInode) {
+  MetaScan::Options options;
+  options.fdb_cluster_file = FLAGS_fdb_cluster;
+  options.threads = FLAGS_scan_threads;
+  options.coroutines = FLAGS_scan_coroutines;
+  MetaScan scan(options);
+  auto &kv = scan.kvEngine();
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto txn = kv.createReadonlyTransaction();
+    auto inodeId = InodeId(FLAGS_inode_id);
+    auto inode = co_await Inode::snapshotLoad(*txn, inodeId);
+    CO_ASSERT_OK(inode);
+    if (inode->has_value()) {
+      fmt::print("inode: {}\n", **inode);
+    } else {
+      fmt::print("inode {} doesn't exist!", inodeId);
+    }
+    fmt::print("============\n");
+
+    std::string prev;
+    while (true) {
+      auto txn = kv.createReadonlyTransaction();
+      auto entries = co_await DirEntryList::snapshotLoad(*txn, inodeId, prev, -1);
+      CO_ASSERT_OK(entries);
+      for (auto entry : entries->entries) {
+        auto inode = co_await Inode::snapshotLoad(*kv.createReadWriteTransaction(), entry.id);
+        CO_ASSERT_OK(inode);
+        fmt::print("entry: {} inode: {}\n", entry, inode->has_value() ? fmt::format("{}", **inode) : "");
+        prev = entry.name;
+      }
+      if (!entries->more) break;
+    }
+    fmt::print("============\n");
+
+    if (inode->has_value() && inode->value().isDirectory()) {
+      auto txn = kv.createReadonlyTransaction();
+      auto entry = co_await inode->value().snapshotLoadDirEntry(*txn);
+      CO_ASSERT_OK(entry);
+      fmt::print("entry points to {}: {}\n", inodeId, *entry);
+    }
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/store/TestAuth.cc

@@ -0,0 +1,107 @@
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <fcntl.h>
+#include <folly/Random.h>
+#include <folly/Synchronized.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/experimental/coro/Task.h>
+#include <folly/futures/Barrier.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <vector>
+
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Result.h"
+#include "common/utils/StatusCode.h"
+#include "common/utils/UtcTime.h"
+#include "core/user/UserStore.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/storage/Common.h"
+#include "gtest/gtest.h"
+#include "meta/components/SessionManager.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestAuth : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestAuth, KVTypes);
+
+TYPED_TEST(TestAuth, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().set_authenticate(true);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    flat::Uid rootId(0), aliceId(1), bobId(2), charlieId(3);
+    UserAttr root, alic, bob, charlie;
+    READ_WRITE_TRANSACTION_OK({
+      core::UserStore userStore;
+      root = (co_await userStore.addUser(*txn, rootId, "root", {}, true)).value();
+      alic = (co_await userStore.addUser(*txn, aliceId, "alic", {flat::Gid(2), flat::Gid(1001)}, true)).value();
+      bob = (co_await userStore.addUser(*txn, bobId, "bob", {flat::Gid(1001)}, false)).value();
+      charlie = (co_await userStore.addUser(*txn, charlieId, "charlie", {}, false)).value();
+    });
+
+    std::vector<std::pair<UserInfo, UserInfo>> succ{
+        // root & sudoer can auth with any uid that exists
+        {{rootId, root.gid, root.token}, {rootId, root.gid, root.groups, root.token}},
+        {{aliceId, alic.gid, root.token}, {aliceId, alic.gid, alic.groups, root.token}},
+        {{bobId, bob.gid, root.token}, {bobId, bob.gid, bob.groups, root.token}},
+        {{bobId, charlie.gid, root.token}, {bobId, charlie.gid, bob.groups, root.token}},
+        {{rootId, root.gid, alic.token}, {rootId, root.gid, root.groups, alic.token}},
+        {{aliceId, alic.gid, alic.token}, {aliceId, alic.gid, alic.groups, alic.token}},
+        {{bobId, bob.gid, alic.token}, {bobId, bob.gid, bob.groups, alic.token}},
+        {{bobId, charlie.gid, alic.token}, {bobId, charlie.gid, bob.groups, alic.token}},
+        // other user can only auth with self uid and gid
+        {{bobId, bob.gid, bob.token}, {bobId, bob.gid, bob.groups, bob.token}},
+        {{charlieId, charlie.gid, charlie.token}, {charlieId, charlie.gid, charlie.groups, charlie.token}},
+    };
+    for (auto [req, expected] : succ) {
+      auto result = co_await meta.authenticate(AuthReq(req));
+      CO_ASSERT_OK(result);
+      const auto &actual = result->user;
+      CO_ASSERT_EQ(actual, expected) << fmt::format("{} vs {}; ", actual, expected)
+                                     << fmt::format("{} vs {}",
+                                                    fmt::join(actual.groups.begin(), actual.groups.end(), ","),
+                                                    fmt::join(expected.groups.begin(), expected.groups.end(), ","));
+    }
+
+    std::vector<UserInfo> fail{// don't allow unknown uid
+                               {flat::Uid(999), flat::Gid(0), root.token},
+                               {flat::Uid(1024), flat::Gid(0), root.token},
+                               // other user can only auth with self uid and gid
+                               {bobId, charlie.gid, bob.token},
+                               {charlieId, bob.gid, bob.token},
+                               {charlieId, bob.gid, charlie.token},
+                               {bobId, charlie.gid, charlie.token},
+                               // invalid token
+                               {bobId, bob.gid, ""}};
+    for (const auto &req : fail) {
+      fmt::print("{}\n", req);
+      auto result = co_await meta.authenticate(AuthReq(req));
+      CO_ASSERT_ERROR(result, StatusCode::kAuthenticationFail);
+    }
+  }());
+}
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/TestDirEntry.cc

@@ -0,0 +1,139 @@
+#include <cstdlib>
+#include <fmt/core.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "common/kv/IKVEngine.h"
+#include "common/kv/mem/MemKVEngine.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/String.h"
+#include "common/utils/UtcTime.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+using namespace hf3fs::kv;
+
+class TestDirEntry : public ::testing::Test {
+ protected:
+  MemKVEngine engine_;
+};
+
+const Acl acl(Uid(0), Gid(0), Permission(0777));
+
+TEST_F(TestDirEntry, LoadStore) {
+  srand(time(nullptr));
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    static constexpr size_t kDirEntries = 1000;
+    std::vector<DirEntry> entries;
+    for (size_t i = 0; i < kDirEntries; i++) {
+      auto entry = MetaTestHelper::randomDirEntry();
+      auto storeTxn = engine_.createReadWriteTransaction();
+      auto storeResult = co_await entry.store(*storeTxn);
+      CO_ASSERT_TRUE(storeResult.hasValue());
+      auto commitResult = co_await storeTxn->commit();
+      CO_ASSERT_OK(commitResult);
+
+      entries.push_back(std::move(entry));
+    }
+
+    auto loadTxn = engine_.createReadonlyTransaction();
+    for (auto &entry : entries) {
+      auto loadResult =
+          (co_await DirEntry::snapshotLoad(*loadTxn, entry.parent, entry.name)).then(checkMetaFound<DirEntry>);
+      CO_ASSERT_OK(loadResult);
+      CO_ASSERT_EQ(entry, *loadResult);
+    }
+
+    for (auto &entry : entries) {
+      auto removeTxn = engine_.createReadWriteTransaction();
+      auto removeResult = co_await entry.remove(*removeTxn);
+      CO_ASSERT_TRUE(removeResult.hasValue());
+      auto commitResult = co_await removeTxn->commit();
+      CO_ASSERT_TRUE(commitResult.hasValue());
+    }
+
+    for (auto &entry : entries) {
+      DirEntry other(entry.parent, entry.name);
+      auto loadTxn = engine_.createReadonlyTransaction();
+      auto loadResult =
+          (co_await DirEntry::snapshotLoad(*loadTxn, entry.parent, entry.name)).then(checkMetaFound<DirEntry>);
+      CO_ASSERT_ERROR(loadResult, MetaCode::kNotFound);
+    }
+  }());
+}
+
+TEST_F(TestDirEntry, List) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    std::vector<DirEntry> entries;
+    InodeId parent = MetaTestHelper::randomInodeId();
+    for (size_t i = 0; i < 100; i++) {
+      String name = fmt::format("file-{}", i);
+      InodeId id = MetaTestHelper::randomInodeId();
+      DirEntry entry = DirEntry::newFile(parent, name, id);
+
+      auto storeTxn = engine_.createReadWriteTransaction();
+      auto storeResult = co_await entry.store(*storeTxn);
+      CO_ASSERT_TRUE(storeResult.hasValue());
+
+      auto commitResult = co_await storeTxn->commit();
+      CO_ASSERT_OK(commitResult);
+    }
+
+    auto loadTxn = engine_.createReadonlyTransaction();
+    auto listResult = co_await DirEntryList::snapshotLoad(*loadTxn, parent, {}, 10, false);
+    CO_ASSERT_TRUE(listResult.hasValue());
+    auto list = listResult.value();
+    CO_ASSERT_EQ(list.entries.size(), 10);
+    CO_ASSERT_TRUE(list.inodes.empty());
+    CO_ASSERT_TRUE(list.more);
+
+    // load last entries
+    listResult = co_await DirEntryList::snapshotLoad(*loadTxn, parent, list.entries.at(9).name, 90, false);
+    CO_ASSERT_TRUE(listResult.hasValue());
+    list = listResult.value();
+    CO_ASSERT_EQ(list.entries.size(), 90);
+    CO_ASSERT_TRUE(list.inodes.empty());
+    CO_ASSERT_FALSE(list.more);
+  }());
+}
+
+TEST_F(TestDirEntry, Conflict) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    for (int i = 0; i < 1000; i++) {
+      // txn1 update entry, txn2 include entry into it's read conflict set (by load or addReadConflict) and set a random
+      // key, txn1 commit first, txn2 should fail
+      DirEntry entry(MetaTestHelper::randomInodeId(), std::to_string(i));
+      entry.id = MetaTestHelper::randomInodeId();
+      auto txn1 = engine_.createReadWriteTransaction();
+      auto txn2 = engine_.createReadWriteTransaction();
+
+      auto store = co_await entry.store(*txn1);
+      CO_ASSERT_OK(store);
+
+      if (rand() % 2) {
+        CO_ASSERT_OK(co_await DirEntry::load(*txn2, entry.parent, entry.name));
+      } else {
+        auto addConflict = co_await entry.addIntoReadConflict(*txn2);
+        CO_ASSERT_OK(addConflict);
+      }
+      auto set = co_await txn2->set(std::to_string(rand()), std::to_string(rand()));
+      CO_ASSERT_OK(set);
+
+      auto txn1Result = co_await txn1->commit();
+      CO_ASSERT_OK(txn1Result);
+      auto txn2Result = co_await txn2->commit();
+      CO_ASSERT_TRUE(txn2Result.hasError());
+      CO_ASSERT_EQ(txn2Result.error().code(), TransactionCode::kConflict);
+    }
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/store/TestInode.cc

@@ -0,0 +1,244 @@
+#include <cstdlib>
+#include <double-conversion/utils.h>
+#include <fmt/compile.h>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <gtest/gtest.h>
+#include <set>
+#include <string>
+#include <variant>
+#include <vector>
+
+#include "common/kv/IKVEngine.h"
+#include "common/kv/mem/MemKVEngine.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/mgmtd/ChainRef.h"
+#include "fbs/mgmtd/MgmtdTypes.h"
+#include "meta/store/Inode.h"
+#include "meta/store/Utils.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+using flat::ChainId;
+using flat::ChainRef;
+using flat::ChainTableId;
+using flat::ChainTableVersion;
+
+TEST(TestInodeId, Special) {
+  auto root = InodeId::root();
+  auto gcRoot = InodeId::gcRoot();
+  ASSERT_NE(root, gcRoot);
+  ASSERT_EQ(root.u64(), 0);
+  ASSERT_EQ(gcRoot.u64(), 1);
+  ASSERT_EQ(root, InodeId::root());
+  ASSERT_EQ(gcRoot, InodeId::gcRoot());
+
+  ASSERT_EQ(fmt::format("{}", root), "0x0000000000000000");
+  ASSERT_EQ(fmt::format("{}", gcRoot), "0x0000000000000001");
+  ASSERT_EQ(root.toHexString(), "0x0000000000000000");
+  ASSERT_EQ(gcRoot.toHexString(), "0x0000000000000001");
+}
+
+template <typename KV>
+class TestInode : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestInode, KVTypes);
+
+TYPED_TEST(TestInode, File) {
+  Inode inode = Inode::newFile(MetaTestHelper::randomInodeId(),
+                               Acl(Uid(0), Gid(0), Permission(0644)),
+                               Layout(),
+                               UtcClock::now().castGranularity(1_s));
+
+  ASSERT_TRUE(inode.isFile());
+  ASSERT_FALSE(inode.isDirectory());
+  ASSERT_FALSE(inode.isSymlink());
+  ASSERT_EQ(inode.getType(), InodeType::File);
+  ASSERT_NO_THROW(inode.asFile());
+}
+
+TYPED_TEST(TestInode, FileLayout) {
+  auto layout = Layout::newEmpty(ChainTableId(1), 1000, 4);
+  ASSERT_TRUE(std::holds_alternative<Layout::Empty>(layout.chains));
+  ASSERT_FALSE(layout.valid(true));
+  layout = Layout::newEmpty(ChainTableId(1), 1024, 4);
+  ASSERT_TRUE(layout.valid(true));
+  ASSERT_FALSE(layout.valid(false));
+  ASSERT_EQ(layout.getChainIndexList().size(), 0);
+
+  layout.chains = Layout::ChainList{{1}};
+  ASSERT_FALSE(layout.valid(false));
+
+  layout.chains = Layout::ChainList{{1, 2, 3, 4}};
+  ASSERT_FALSE(layout.valid(false)) << fmt::format("{}", layout);
+  layout.tableVersion = ChainTableVersion(1);
+  ASSERT_TRUE(layout.valid(false)) << fmt::format("{}", layout);
+  ASSERT_TRUE(std::holds_alternative<Layout::ChainList>(layout.chains));
+  ASSERT_EQ(layout.getChainIndexList().size(), layout.stripeSize);
+
+  layout.chains = Layout::Empty();
+  layout.chains = Layout::ChainRange(1, Layout::ChainRange::STD_SHUFFLE_MT19937, 0);
+  ASSERT_TRUE(layout.valid(false));
+  ASSERT_TRUE(std::holds_alternative<Layout::ChainRange>(layout.chains));
+  ASSERT_EQ(layout.getChainIndexList().size(), layout.stripeSize);
+
+  std::set<uint32_t> set;
+  for (auto chain : layout.getChainIndexList()) {
+    ASSERT_FALSE(set.contains(chain));
+    set.emplace(chain);
+  }
+}
+
+TYPED_TEST(TestInode, ChunkId) {
+  Inode inode = Inode::newFile(MetaTestHelper::randomInodeId(),
+                               Acl(Uid(0), Gid(0), Permission(0644)),
+                               Layout(),
+                               UtcClock::now().castGranularity(1_s));
+  ASSERT_FALSE(inode.asFile().getChunkId(inode.id, 0));
+
+  inode.asFile().layout = Layout::newEmpty(ChainTableId(1), 4096, 1);
+  ASSERT_TRUE(inode.asFile().getChunkId(inode.id, 0));
+  ASSERT_EQ(*inode.asFile().getChunkId(inode.id, 100), ChunkId(inode.id, 0, 0));
+  ASSERT_EQ(*inode.asFile().getChunkId(inode.id, 9999), ChunkId(inode.id, 0, 2));
+  ASSERT_EQ(inode.asFile().getChunkId(inode.id, 1ul << 63).error().code(), MetaCode::kFileTooLarge);
+
+  for (int i = 0; i < 10; i++) {
+    auto inode = meta::InodeId(folly::Random::rand64());
+    auto chunk = folly::Random::rand32();
+    ChunkId chunkId(inode, 0, chunk);
+    ASSERT_EQ(chunkId.chunk(), chunk);
+    ASSERT_EQ(chunkId.inode(), inode);
+    auto packed = chunkId.pack();
+    ASSERT_EQ(chunkId, ChunkId::unpack(packed));
+  }
+}
+
+TYPED_TEST(TestInode, SerializedSize) {
+  auto layout = Layout::newEmpty(ChainTableId(1), 4096, 1);
+  layout.chains = Layout::ChainRange(5, Layout::ChainRange::NO_SHUFFLE, 0);
+  auto file = Inode::newFile(MetaTestHelper::randomInodeId(),
+                             Acl(Uid(0), Gid(0), Permission(0644)),
+                             layout,
+                             UtcClock::now().castGranularity(1_s));
+  file.asFile().length = 100_MB;
+  fmt::print("File with range layout, serialized {} bytes, data {} type {}, asFile {}.\n",
+             serde::serialize(file).size(),
+             serde::serialize(file.data()).size(),
+             serde::serialize(file.data().type).size(),
+             serde::serialize(file.data().asFile()).size());
+
+  auto directory = Inode::newDirectory(MetaTestHelper::randomInodeId(),
+                                       MetaTestHelper::randomInodeId(),
+                                       "directory-name",
+                                       Acl(Uid(0), Gid(0), Permission(0644)),
+                                       layout,
+                                       UtcClock::now().castGranularity(1_s));
+  fmt::print("Directory with layout, serialized {} {} bytes.\n",
+             serde::serialize(directory.data()).size(),
+             serde::serialize(directory).size());
+}
+
+TYPED_TEST(TestInode, Directory) {
+  Inode inode = Inode::newDirectory(MetaTestHelper::randomInodeId(),
+                                    MetaTestHelper::randomInodeId(),
+                                    "directory",
+                                    Acl(Uid(0), Gid(0), Permission(0644)),
+                                    Layout::newEmpty(ChainTableId(1), 512 << 10, 64),
+                                    UtcClock::now().castGranularity(1_s));
+
+  ASSERT_FALSE(inode.isFile());
+  ASSERT_TRUE(inode.isDirectory());
+  ASSERT_FALSE(inode.isSymlink());
+  ASSERT_EQ(inode.getType(), InodeType::Directory);
+  ASSERT_NO_THROW(inode.asDirectory());
+}
+
+TYPED_TEST(TestInode, Symlink) {
+  Inode inode = Inode::newSymlink(MetaTestHelper::randomInodeId(),
+                                  std::to_string(folly::Random::rand32()),
+                                  Uid(0),
+                                  Gid(0),
+                                  UtcClock::now().castGranularity(1_s));
+
+  ASSERT_FALSE(inode.isFile());
+  ASSERT_FALSE(inode.isDirectory());
+  ASSERT_TRUE(inode.isSymlink());
+  ASSERT_EQ(inode.getType(), InodeType::Symlink);
+  ASSERT_NO_THROW(inode.asSymlink());
+}
+
+TYPED_TEST(TestInode, LoadStore) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    static constexpr size_t kInodes = 1000;
+    std::vector<Inode> inodes;
+    for (size_t i = 0; i < kInodes; i++) {
+      auto inode = MetaTestHelper::randomInode();
+      READ_WRITE_TRANSACTION_OK({
+        auto storeResult = co_await inode.store(*txn);
+        CO_ASSERT_OK(storeResult);
+      });
+
+      inodes.push_back(inode);
+    }
+
+    for (auto &inode : inodes) {
+      READ_ONLY_TRANSACTION({
+        auto loadResult = (co_await Inode::snapshotLoad(*txn, inode.id)).then(checkMetaFound<Inode>);
+        CO_ASSERT_OK(loadResult);
+        auto other = std::move(loadResult.value());
+        CO_ASSERT_EQ(inode, other) << inode.isFile();
+        if (inode.isFile()) {
+          auto cnts = inode.asFile().layout.getChainIndexList().size();
+          CO_ASSERT_EQ(cnts, other.asFile().layout.getChainIndexList().size());
+          for (size_t i = 0; i < cnts; i++) {
+            CO_ASSERT_EQ(inode.asFile().layout.getChainIndexList()[i], other.asFile().layout.getChainIndexList()[i]);
+          }
+        }
+      });
+    }
+
+    for (auto &inode : inodes) {
+      READ_WRITE_TRANSACTION_OK({
+        auto removeResult = co_await inode.remove(*txn);
+        CO_ASSERT_OK(removeResult);
+      });
+    }
+
+    for (auto &inode : inodes) {
+      READ_ONLY_TRANSACTION({
+        auto loadResult = (co_await Inode::snapshotLoad(*txn, inode.id)).then(checkMetaFound<Inode>);
+        CO_ASSERT_ERROR(loadResult, MetaCode::kNotFound);
+      });
+    }
+  }());
+}
+
+TYPED_TEST(TestInode, Conflict) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    for (int i = 0; i < 1000; i++) {
+      // txn1 update inode, txn2 include inode into it's read conflict set (by load or addReadConflict) and set a random
+      // key, txn1 commit first, txn2 should fail
+      Inode inode = MetaTestHelper::randomInode();
+      CO_ASSERT_CONFLICT([&](auto &txn) -> CoTask<void> { CO_ASSERT_OK(co_await inode.store(txn)); },
+                         [&](auto &txn) -> CoTask<void> {
+                           if (folly::Random::rand32() % 2) {
+                             CO_ASSERT_OK(co_await Inode::load(txn, inode.id));
+                           } else {
+                             // use snapshotLoad to get a read version
+                             CO_ASSERT_OK(co_await Inode::snapshotLoad(txn, inode.id));
+                             CO_ASSERT_OK(co_await Inode(inode.id).addIntoReadConflict(txn));
+                           }
+                           auto set = co_await txn.set(std::to_string(folly::Random::rand32()),
+                                                       std::to_string(folly::Random::rand32()));
+                           CO_ASSERT_OK(set);
+                         });
+    }
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestBatch.cc

@@ -0,0 +1,184 @@
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/Synchronized.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/experimental/coro/Task.h>
+#include <folly/futures/Barrier.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <sys/stat.h>
+#include <tuple>
+#include <vector>
+
+#include "client/storage/StorageClient.h"
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/Result.h"
+#include "common/utils/StatusCode.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/core/user/User.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/meta/Service.h"
+#include "fbs/storage/Common.h"
+#include "fdb/FDBRetryStrategy.h"
+#include "gtest/gtest.h"
+#include "meta/components/SessionManager.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/ops/BatchOperation.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestBatchOp : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestBatchOp, KVTypes);
+
+TYPED_TEST(TestBatchOp, basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.set_num_meta(1);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+    auto &engine = *this->kvEngine();
+
+    // create a inodeId to test
+    auto create = co_await meta.create({SUPER_USER, PathAt("test"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(create);
+    auto inodeId = create->stat.id;
+
+    auto u1 = flat::UserInfo(flat::Uid(1), flat::Gid(1), "");
+    auto u2 = flat::UserInfo(flat::Uid(2), flat::Gid(2), "");
+
+    auto waiter1 = BatchedOp::Waiter<SetAttrReq, SetAttrRsp>(
+        SetAttrReq::setPermission(SUPER_USER, PathAt(inodeId), AtFlags(), flat::Uid(1), std::nullopt, std::nullopt));
+    auto waiter2 = BatchedOp::Waiter<SetAttrReq, SetAttrRsp>(
+        SetAttrReq::setPermission(u1, PathAt(inodeId), AtFlags(), std::nullopt, flat::Gid(1), std::nullopt));
+    auto waiter3 = BatchedOp::Waiter<SetAttrReq, SetAttrRsp>(
+        SetAttrReq::setPermission(u2, PathAt(inodeId), AtFlags(), std::nullopt, flat::Gid(2), std::nullopt));
+    auto waiter4 = BatchedOp::Waiter<SyncReq, SyncRsp>(
+        SyncReq(SUPER_USER, inodeId, true, SETATTR_TIME_NOW, SETATTR_TIME_NOW, false, meta::VersionedLength{1000, 0}));
+    auto waiter5 = BatchedOp::Waiter<CloseReq, CloseRsp>(
+        CloseReq(SUPER_USER, inodeId, MetaTestHelper::randomSession(), true, std::nullopt, std::nullopt));
+    auto waiter6 = BatchedOp::Waiter<CloseReq, CloseRsp>(
+        CloseReq(SUPER_USER, inodeId, std::nullopt, true, std::nullopt, std::nullopt));
+
+    {
+      FAULT_INJECTION_SET(10, 3);
+      auto batch = std::make_unique<BatchedOp>(store, inodeId);
+      batch->add(waiter1);
+      batch->add(waiter2);
+      batch->add(waiter3);
+      batch->add(waiter4);
+      batch->retry(Status(StatusCode::kInvalidArg));
+
+      auto txn = engine.createReadWriteTransaction();
+      auto driver = OperationDriver(*batch, Void{});
+      auto result = co_await driver.run(std::move(txn), {}, false, false);
+      CO_ASSERT_OK(result);
+      CO_ASSERT_EQ(result->asFile().getVersionedLength(), (meta::VersionedLength{1000, 0}));
+      CO_ASSERT_EQ(result->acl.uid, flat::Uid(1));
+      CO_ASSERT_EQ(result->acl.gid, flat::Gid(1));
+
+      CO_ASSERT_OK(waiter1.getResult());
+      CO_ASSERT_OK(waiter2.getResult());
+      CO_ASSERT_ERROR(waiter3.getResult(), MetaCode::kNoPermission);
+      CO_ASSERT_OK(waiter4.getResult());
+    };
+
+    {
+      FAULT_INJECTION_SET(10, 3);
+      auto batch = std::make_unique<BatchedOp>(store, inodeId);
+      batch->add(waiter4);
+      batch->add(waiter5);
+      batch->add(waiter6);
+      batch->retry(Status(StatusCode::kInvalidArg));
+
+      auto txn = engine.createReadWriteTransaction();
+      auto driver = OperationDriver(*batch, Void{});
+      auto result = co_await driver.run(std::move(txn), {}, false, false);
+      if (!result.hasError()) {
+        CO_ASSERT_EQ(result->asFile().getVersionedLength(), (meta::VersionedLength{0, 1}));
+        CO_ASSERT_OK(waiter4.getResult());
+        CO_ASSERT_OK(waiter5.getResult());
+        CO_ASSERT_ERROR(waiter6.getResult(), StatusCode::kInvalidArg);
+      }
+    };
+  }());
+}
+
+TYPED_TEST(TestBatchOp, batch) {
+  auto cluster = this->createMockCluster();
+  auto &meta = cluster.meta().getOperator();
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto req1 = SyncReq(SUPER_USER, InodeId(1), true, {}, {});
+    auto req2 = SyncReq(SUPER_USER, InodeId(2), true, {}, {});
+
+    BatchedOp::Waiter<SyncReq, SyncRsp> waiter1(req1), waiter2(req1), waiter3(req1), waiter4(req2), waiter5(req1);
+    auto batch1 = meta.addBatchReq(InodeId(1), waiter1);
+    CO_ASSERT_NE(batch1.get(), nullptr);
+    auto batch2 = meta.addBatchReq(InodeId(1), waiter2);
+    CO_ASSERT_NE(batch2.get(), nullptr);
+    auto batch3 = meta.addBatchReq(InodeId(1), waiter3);
+    CO_ASSERT_EQ(batch3.get(), nullptr);
+    auto batch4 = meta.addBatchReq(InodeId(2), waiter4);
+    CO_ASSERT_NE(batch4.get(), nullptr);
+    CO_ASSERT_NE(batch1.get(), batch4.get());
+
+    // batch 1 can run, but batch 2 can't
+    CO_ASSERT_TRUE(waiter1.baton.ready());
+    CO_ASSERT_FALSE(waiter2.baton.ready());
+    CO_ASSERT_OK(co_await meta.runBatch(InodeId(1), std::move(batch1)));
+    CO_ASSERT_TRUE(waiter2.baton.ready());
+    CO_ASSERT_FALSE(waiter3.baton.ready());
+    CO_ASSERT_OK(co_await meta.runBatch(InodeId(1), std::move(batch2)));
+    CO_ASSERT_TRUE(waiter3.baton.ready());
+
+    auto batch5 = meta.addBatchReq(InodeId(1), waiter5);
+    CO_ASSERT_NE(batch5.get(), nullptr);
+    CO_ASSERT_TRUE(waiter5.baton.ready());
+
+    std::unique_ptr<BatchedOp> batch6;
+    std::vector<std::unique_ptr<BatchedOp::Waiter<SyncReq, SyncRsp>>> waiters;
+    for (size_t i = 0; i < 5000; i++) {
+      auto waiter = std::make_unique<BatchedOp::Waiter<SyncReq, SyncRsp>>(req1);
+      auto batch = meta.addBatchReq(InodeId(1), *waiter);
+      if (!batch6) {
+        CO_ASSERT_TRUE(batch);
+        batch6 = std::move(batch);
+      }
+      if (i >= cluster.config().mock_meta().max_batch_operations()) {
+        CO_ASSERT_TRUE(waiter->baton.ready());
+        CO_ASSERT_EQ(waiter->getResult().error().code(), MetaCode::kBusy);
+      } else {
+        CO_ASSERT_FALSE(waiter->baton.ready()) << fmt::format("{}", i);
+      }
+      waiters.push_back(std::move(waiter));
+    }
+
+    // reject timeout operation
+    CO_ASSERT_ERROR(co_await meta.runBatch(InodeId(1), std::move(batch5), SteadyClock::now()),
+                    MetaCode::kOperationTimeout);
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestClose.cc

@@ -0,0 +1,246 @@
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <fcntl.h>
+#include <folly/Random.h>
+#include <folly/Synchronized.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Collect.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/experimental/coro/Task.h>
+#include <folly/futures/Barrier.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <vector>
+
+#include "client/storage/StorageClient.h"
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/Result.h"
+#include "common/utils/StatusCode.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/storage/Common.h"
+#include "gtest/gtest.h"
+#include "meta/components/SessionManager.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestClose : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestClose, KVTypes);
+
+TYPED_TEST(TestClose, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    // close directory
+    CO_ASSERT_ERROR(
+        co_await meta.close(
+            {SUPER_USER, InodeId::root(), MetaTestHelper::randomSession(), true, std::nullopt, std::nullopt}),
+        MetaCode::kNotFile);
+
+    auto create = co_await meta.create({SUPER_USER, PathAt("test-file"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(create);
+    auto inode = create->stat;
+    CO_ASSERT_EQ(inode.asFile().length, 0);
+
+    // file has no session
+    READ_WRITE_TRANSACTION_NO_COMMIT({
+      auto result = co_await FileSession::checkExists(*txn, inode.id);
+      CO_ASSERT_OK(result);
+      CO_ASSERT_FALSE(*result);
+    });
+
+    // open file with read
+    CO_ASSERT_OK(co_await meta.open({SUPER_USER, PathAt("test-file"), {}, O_RDONLY}));
+    // file has no session
+    READ_WRITE_TRANSACTION_NO_COMMIT({
+      auto result = co_await FileSession::checkExists(*txn, inode.id);
+      CO_ASSERT_OK(result);
+      CO_ASSERT_FALSE(*result);
+    });
+
+    // open file with write
+    auto session = MetaTestHelper::randomSession();
+    CO_ASSERT_OK(co_await meta.open({SUPER_USER, PathAt("test-file"), session, O_WRONLY}));
+
+    // file has session & client has session
+    READ_WRITE_TRANSACTION_NO_COMMIT({
+      auto result = co_await FileSession::checkExists(*txn, inode.id);
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(*result);
+      auto inodeSessions = co_await FileSession::list(*txn, inode.id, false);
+      CO_ASSERT_TRUE(inodeSessions.hasValue() && !inodeSessions->empty());
+      // auto clientSessions = co_await FileSession::list(*txn, session.client.uuid, false);
+      // CO_ASSERT_TRUE(clientSessions.hasValue() && !clientSessions->empty());
+    });
+
+    // close file, should set session if update length is enabled
+    CO_ASSERT_OK(co_await meta.close({SUPER_USER, inode.id, {}, false, UtcClock::now(), {}}));
+    CO_ASSERT_ERROR(co_await meta.close({SUPER_USER, inode.id, {}, true, UtcClock::now(), {}}),
+                    StatusCode::kInvalidArg);
+
+    CO_ASSERT_OK(co_await meta.close({SUPER_USER, inode.id, session, true, {}, {}}));
+
+    // file has no session, client has no session
+    READ_WRITE_TRANSACTION_NO_COMMIT({
+      auto result = co_await FileSession::checkExists(*txn, inode.id);
+      CO_ASSERT_OK(result);
+      CO_ASSERT_FALSE(*result);
+      auto inodeSessions = co_await FileSession::list(*txn, inode.id, false);
+      CO_ASSERT_TRUE(inodeSessions.hasValue() && inodeSessions->empty());
+      // todo: maybe should check session count
+      // or: just dump all sessions
+      // auto clientSessions = co_await FileSession::list(*txn, session.client.uuid, false);
+      // CO_ASSERT_TRUE(clientSessions.hasValue() && clientSessions->empty());
+    });
+  }());
+}
+
+// TYPED_TEST(TestClose, CheckHole) {
+//   folly::coro::blockingWait([&]() -> CoTask<void> {
+//     MockCluster::Config cfg;
+//     cfg.mock_meta().set_check_file_hole(true);
+//     auto cluster = this->createMockCluster(cfg);
+//     auto &meta = cluster.meta().getOperator();
+//     auto &storage = cluster.meta().getStorageClient();
+
+//     auto create = co_await meta.create({SUPER_USER, PathAt("test-file"), {}, O_EXCL, p644});
+//     CO_ASSERT_OK(create);
+//     Inode inode(create->stat);
+//     CO_ASSERT_EQ(inode.asFile().length, 0);
+
+//     auto session1 = MetaTestHelper::randomSession();
+//     auto session2 = MetaTestHelper::randomSession();
+//     CO_ASSERT_OK(co_await meta.open({SUPER_USER, PathAt("test-file"), session1, O_WRONLY}));
+//     CO_ASSERT_OK(co_await meta.open({SUPER_USER, PathAt("test-file"), session2, O_WRONLY}));
+
+//     // do some write to generate hole
+//     co_await randomWrite(meta, storage, inode, 1 << 20, 1 << 20);
+//     // close session1, return ok
+//     CO_ASSERT_OK(co_await meta.close({SUPER_USER, inode.id, session1, true, {}, {}}));
+
+//     // close session2, return has hole
+//     CO_ASSERT_ERROR(co_await meta.close({SUPER_USER, inode.id, session2, true, {}, {}}), MetaCode::kFileHasHole);
+
+//     // don't allow open O_RDONLY
+//     CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, PathAt("test-file"), {}, O_RDONLY}), MetaCode::kFileHasHole);
+
+//     // open read write
+//     auto session3 = MetaTestHelper::randomSession();
+//     CO_ASSERT_OK(co_await meta.open({SUPER_USER, PathAt("test-file"), session3, O_WRONLY}));
+
+//     // write fix hole
+//     co_await randomWrite(meta, storage, inode, 0, 3 << 20);
+//     // close should ok
+//     CO_ASSERT_OK(co_await meta.close({SUPER_USER, inode.id, session3, true, {}, {}}));
+
+//     // open and check length
+//     auto oresult = co_await meta.open({SUPER_USER, PathAt("test-file"), {}, O_RDONLY});
+//     CO_ASSERT_OK(oresult);
+//     CO_ASSERT_EQ(oresult->stat.asFile().length, 3 << 20);
+//   }());
+// }
+
+TYPED_TEST(TestClose, Concurrent) {
+  folly::CPUThreadPoolExecutor exec(8);
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config cfg;
+    cfg.mock_meta().set_check_file_hole(true);
+    auto cluster = this->createMockCluster(cfg);
+    auto &meta = cluster.meta().getOperator();
+    auto &storage = cluster.meta().getStorageClient();
+
+    auto worker = [&](Path path,
+                      size_t offset,
+                      size_t length,
+                      std::atomic_int *failed,
+                      folly::futures::Barrier *beginBarr,
+                      folly::futures::Barrier *endBarr) -> CoTask<void> {
+      // open, should succ
+      if (!beginBarr && folly::Random::oneIn(4)) {
+        co_await folly::coro::sleep(std::chrono::milliseconds(folly::Random::rand32(300)));
+      }
+      auto session = MetaTestHelper::randomSession();
+      auto oresult = co_await meta.create({SUPER_USER, path, session, O_RDWR, p644});
+      CO_ASSERT_OK(oresult);
+      Inode inode = oresult->stat;
+      if (beginBarr) co_await beginBarr->wait();
+
+      co_await randomWrite(meta, storage, inode, offset, length);
+
+      if (endBarr) co_await endBarr->wait();
+      auto cresult = co_await meta.close({SUPER_USER, inode.id, session, true, {}, UtcClock::now()});
+      if (cresult.hasError()) {
+        CO_ASSERT_ERROR(cresult, MetaCode::kFileHasHole);
+        if (failed) (*failed)++;
+      }
+      co_return;
+    };
+
+    std::vector<std::tuple<bool, bool, bool>> tests;
+    for (auto hole : {false, true}) {
+      for (auto beginBarr : {false, true})
+        for (auto endBarr : {false, true}) tests.push_back({beginBarr, endBarr, hole});
+    }
+
+    for (auto [hole, beginBarr, endBarr] : tests) {
+      XLOGF(ERR, "test: beginBarr {}, endBarr {}, hole {}", beginBarr, endBarr, hole);
+      Path path = fmt::format("file-{}", folly::Random::rand32());
+      folly::futures::Barrier barr1(16), barr2(16);
+      std::vector<folly::SemiFuture<folly::Unit>> futures;
+      std::atomic_int failed(0);
+      size_t offset = hole ? folly::Random::rand32(1 << 20, 8 << 20) : 0;
+      for (size_t i = 0; i < 16; i++) {
+        size_t length = folly::Random::rand32(128 << 10, 2 << 20);
+        auto task = folly::coro::co_invoke(worker,
+                                           path,
+                                           offset,
+                                           length,
+                                           &failed,
+                                           beginBarr ? &barr1 : nullptr,
+                                           endBarr ? &barr2 : nullptr)
+                        .scheduleOn(&exec)
+                        .start();
+        futures.push_back(std::move(task));
+        offset += length;
+      }
+      for (auto &f : futures) {
+        f.wait();
+      }
+
+      // todo: support check hole in future
+      // if (!hole) {
+      //   if (beginBarr) {
+      //     CO_ASSERT_EQ(failed, 0);
+      //   }
+      //   CO_ASSERT_OK(co_await meta.open({SUPER_USER, path, {}, O_RDONLY}));
+      // } else {
+      //   XLOGF(ERR, "{} close found hole", failed.load());
+      //   CO_ASSERT_NE(failed, 0);
+      //   CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, path, {}, O_RDONLY}), MetaCode::kFileHasHole);
+      // }
+    }
+
+    co_return;
+  }());
+}
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestCreate.cc

@@ -0,0 +1,409 @@
+#include <array>
+#include <fcntl.h>
+#include <folly/Random.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Task.h>
+#include <gtest/gtest.h>
+#include <memory>
+#include <optional>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "common/kv/ITransaction.h"
+#include "common/kv/mem/MemKV.h"
+#include "common/kv/mem/MemKVEngine.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/Path.h"
+#include "common/utils/RandomUtils.h"
+#include "common/utils/Result.h"
+#include "common/utils/StatusCode.h"
+#include "fbs/core/user/User.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/meta/Service.h"
+#include "fdb/FDB.h"
+#include "fmt/core.h"
+#include "gtest/gtest.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/ops/BatchOperation.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestCreate : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestCreate, KVTypes);
+
+TYPED_TEST(TestCreate, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config cfg;
+    cfg.mock_meta().set_check_file_hole(true);
+    auto cluster = this->createMockCluster(cfg);
+    auto &meta = cluster.meta().getOperator();
+
+    // create "file", should success
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p644}));
+
+    // create "file" again should success
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p644}));
+
+    // create file and open write without session
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, "file2", {}, O_RDWR, p644}), StatusCode::kInvalidArg);
+
+    // create "file" again with or without O_EXCL
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, "file", {}, O_EXCL, p644}), MetaCode::kExists);
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p644}));
+
+    // other have no create permission, but can open files that already exists.
+    flat::UserInfo otherUser(Uid(1), Gid(1), String());
+    CO_ASSERT_OK(co_await meta.create({otherUser, "file", {}, O_RDONLY, p644}));
+    CO_ASSERT_ERROR(co_await meta.create({otherUser, "file2", {}, O_RDONLY, p644}), MetaCode::kNoPermission);
+
+    // create under a directory that not exist
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, "directory/file", {}, O_RDONLY, p644}), MetaCode::kNotFound);
+    CO_ASSERT_ERROR(
+        co_await meta.create({SUPER_USER, PathAt(MetaTestHelper::randomInodeId(), "file"), {}, O_RDONLY, p644}),
+        MetaCode::kNotFound);
+
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, "/", {}, O_RDONLY, p644}), StatusCode::kInvalidArg);
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, "a/", {}, O_RDONLY, p644}), StatusCode::kInvalidArg);
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, "/a/b/.", {}, O_RDONLY, p644}), StatusCode::kInvalidArg);
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, "/a/..", {}, O_RDONLY, p644}), StatusCode::kInvalidArg);
+
+    // create under directory
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "directory/sub-directory", p755, true}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "directory/file", {}, O_EXCL, p644}));
+    auto oresult = co_await meta.open({SUPER_USER, "directory", {}, O_DIRECTORY});
+    CO_ASSERT_OK(oresult);
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt(oresult->stat.id, "file2"), {}, O_EXCL, p644}));
+
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, "directory/sub-directory", {}, O_RDONLY, p644}),
+                    MetaCode::kIsDirectory);
+
+    READ_ONLY_TRANSACTION({
+      auto result = co_await DirEntryList::snapshotLoad(*txn, InodeId::root(), "", 4096);
+      CO_ASSERT_OK(result);
+      CO_ASSERT_EQ(result->entries.size(), 2);  // file, directory
+      for (size_t i = 0; i < result->entries.size(); i++) {
+        fmt::print("{}/{} -> {}\n", result->entries.at(i).parent, result->entries.at(i).name, result->entries.at(i).id);
+      }
+    });
+  }());
+}
+
+TYPED_TEST(TestCreate, TestSetGid) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    auto shareGroup = Gid(12345);
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "shared", Permission(S_ISGID | 0777), false}));
+    CO_ASSERT_OK(co_await meta.setAttr(
+        SetAttrReq::setPermission(SUPER_USER, "shared", AtFlags(), std::nullopt, shareGroup, std::nullopt)));
+    auto dir = co_await meta.mkdirs({SUPER_USER, "shared/subdir1/subdir2", Permission(0070), true});
+    CO_ASSERT_OK(dir);
+    CO_ASSERT_TRUE(dir->stat.acl.perm & S_ISGID) << fmt::format("{:o}", dir->stat.acl.uid);
+
+    auto file = co_await meta.create({SUPER_USER, "shared/subdir1/subdir2/file", {}, O_RDONLY, Permission(0640)});
+    CO_ASSERT_OK(file);
+    CO_ASSERT_EQ(file->stat.acl.gid, shareGroup);
+    CO_ASSERT_FALSE(file->stat.acl.perm & S_ISGID);
+
+    CO_ASSERT_OK(
+        co_await meta.open({flat::UserInfo{Uid(1), shareGroup, ""}, "shared/subdir1/subdir2/file", {}, O_RDONLY}));
+  }());
+}
+
+TYPED_TEST(TestCreate, ConflictSet) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.set_num_meta(1);
+    auto cluster = this->createMockCluster(config);
+    auto &store = cluster.meta().getStore();
+    auto &meta = cluster.meta().getOperator();
+
+    {
+      auto path = "create-new-file";
+      auto session = MetaTestHelper::randomSession();
+      InodeId inodeId;
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = CreateReq(SUPER_USER, Path(path), session, O_RDWR, p644);
+            auto createResult = co_await BatchedOp::create(store, txn, req);
+            CO_ASSERT_OK(createResult);
+            inodeId = createResult->stat.id;
+          },
+          (std::vector<String>{
+              MetaTestHelper::getInodeKey(InodeId::root()),           // parent Inode
+              MetaTestHelper::getDirEntryKey(InodeId::root(), path),  // dir entry
+              MetaTestHelper::getInodeKey(inodeId),
+              std::string(kv::kMetadataVersionKey),
+          }),
+          (std::vector<String>{
+              MetaTestHelper::getDirEntryKey(InodeId::root(), path),  // dir entry
+              MetaTestHelper::getInodeKey(inodeId),
+              MetaTestHelper::getInodeSessionKey(FileSession::create(inodeId, session)),
+              // MetaTestHelper::getClientSessionKey(FileSession::create(inodeId, session)),
+              // todo: if per directory chain allocation enabled, should contains parent
+              // MetaTestHelper::getInodeKey(InodeId::root())
+          }),
+          true);
+    }
+
+    {
+      auto path = "create-trunc-exists-file";
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      auto session = MetaTestHelper::randomSession();
+      InodeId inodeId;
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = CreateReq(SUPER_USER, Path(path), session, O_TRUNC | O_WRONLY, p644);
+            auto createResult = co_await BatchedOp::create(store, txn, req);
+            CO_ASSERT_OK(createResult);
+            inodeId = createResult->stat.id;
+          },
+          (std::vector<String>{std::string(kv::kMetadataVersionKey)}),
+          (std::vector<String>{
+              MetaTestHelper::getInodeSessionKey(FileSession::create(inodeId, session)),
+              // MetaTestHelper::getClientSessionKey(FileSession::create(inodeId, session))
+          }),
+          true);
+    }
+
+    // don't allow replace exists file on create
+    // {
+    //   auto path = "create-trunc-replace-exists-file";
+    //   auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+    //   CO_ASSERT_OK(result);
+    //   auto oldInodeId = result->stat.id;
+    //   CO_ASSERT_OK(co_await meta.sync(
+    //       {SUPER_USER, oldInodeId, true, std::nullopt, std::nullopt, false, VersionedLength{2ULL << 30, 0}}));
+    //   auto session = MetaTestHelper::randomSession();
+    //   InodeId newInodeId;
+    //   CHECK_CONFLICT_SET(
+    //       [&](auto &txn) -> CoTask<void> {
+    //         auto req = CreateReq(SUPER_USER, Path(path), session, O_TRUNC | O_WRONLY, p644);
+    //         auto createResult = co_await BatchedOp::create(store,  txn, req);
+    //         CO_ASSERT_OK(createResult);
+    //         newInodeId = createResult->stat.id;
+    //         CO_ASSERT_NE(oldInodeId, newInodeId);
+    //       },
+    //       (std::vector<String>{MetaTestHelper::getDirEntryKey(InodeId::root(), path),
+    //                            MetaTestHelper::getInodeKey(oldInodeId)}),
+    //       (std::vector<String>{MetaTestHelper::getDirEntryKey(InodeId::root(), path),
+    //                            MetaTestHelper::getInodeKey(newInodeId),
+    //                            MetaTestHelper::getInodeSessionKey(FileSession::create(newInodeId, session)),
+    //                            MetaTestHelper::getClientSessionKey(FileSession::create(newInodeId, session))}),
+    //       false);
+    // }
+
+    {
+      auto path = "create-open-exists-file";
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY | O_CREAT, p644});
+      CO_ASSERT_OK(result);
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = CreateReq(SUPER_USER, Path(path), {}, O_RDONLY | O_CREAT, p644);
+            CO_ASSERT_OK(co_await BatchedOp::create(store, txn, req));
+          },
+          (std::vector<String>{std::string(kv::kMetadataVersionKey)}),
+          (std::vector<String>{}),
+          true);
+    }
+  }());
+}
+
+TYPED_TEST(TestCreate, Concurrent) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.set_num_meta(1);
+    auto cluster = this->createMockCluster(config);
+    auto &store = cluster.meta().getStore();
+
+    // concurrent create on same path should conflict
+    for (int i = 0; i < 100; i++) {
+      auto path = std::to_string(i) + ".txt";
+      auto createTask = [&](auto &txn) -> CoTask<void> {
+        auto req = CreateReq(SUPER_USER, Path(path), {}, O_RDONLY, p644);
+        auto createResult = co_await BatchedOp::create(store, txn, req);
+        CO_ASSERT_OK(createResult);
+      };
+      CO_ASSERT_CONFLICT(createTask, createTask);
+    }
+
+    // concurrent create a different path is conflict because per directory chain allocation counter.
+    // concurrent create on different path shouldn't conflict
+    // for (int i = 0; i < 100; i++) {
+    //   auto create1 = [&](auto &txn) -> CoTask<void> {
+    //     auto req = CreateReq(SUPER_USER, Path(std::to_string(i) + ".1"), {}, O_RDONLY, p644);
+    //     auto createResult = co_await BatchedOp::create(store, txn, req);
+    //     CO_ASSERT_OK(createResult);
+    //   };
+    //   auto create2 = [&](auto &txn) -> CoTask<void> {
+    //     auto req = CreateReq(SUPER_USER, Path(std::to_string(i) + ".2"), {}, O_RDONLY, p644);
+    //     auto createResult = co_await BatchedOp::create(store, txn, req);
+    //     CO_ASSERT_OK(createResult);
+    //   };
+    //   CO_ASSERT_NO_CONFLICT(create1, create2);
+    // }
+  }());
+}
+
+TYPED_TEST(TestCreate, FaultInjection) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    auto result = co_await meta.mkdirs({SUPER_USER, "dirA/dirB/dirC", p755, true});
+    CO_ASSERT_OK(result);
+    std::vector<Path> parents =
+        {"/", "/dirA", "/dirA/dirB", "/dirA/dirB/dirC", "/dirA/dirB/../", "/dirA/dirB/../dirB", "not_exists"};
+
+    for (auto i = 0; i < 100; i++) {
+      FAULT_INJECTION_SET(10, 3);  // 10%, 3 faults.
+      auto parent = parents[folly::Random::rand32(parents.size())];
+      auto path = parent;
+      path.append(std::to_string(i));
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+      if (parent == "not_exists") {
+        CO_ASSERT_ERROR(result, MetaCode::kNotFound);
+      } else {
+        CO_ASSERT_OK(result);
+      }
+    }
+  }());
+}
+
+TYPED_TEST(TestCreate, OTrunc) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config cfg;
+    cfg.mock_meta().set_check_file_hole(true);
+    auto cluster = this->createMockCluster(cfg);
+    auto &meta = cluster.meta().getOperator();
+
+    for (size_t i = 0; i < 10; i++) {
+      auto fname = fmt::format("truncate-inplace-{}", i);
+      auto create = co_await meta.create({SUPER_USER, fname, {}, O_EXCL, p644});
+      CO_ASSERT_OK(create);
+      CO_ASSERT_OK(co_await meta.sync(
+          {SUPER_USER, create->stat.id, true, std::nullopt, std::nullopt, false, VersionedLength{1_MB, 0}}));
+
+      auto createAgain = co_await meta.create({SUPER_USER, fname, MetaTestHelper::randomSession(), O_RDWR, p644});
+      CO_ASSERT_OK(createAgain);
+      CO_ASSERT_FALSE(createAgain->needTruncate);
+
+      auto session = MetaTestHelper::randomSession();
+      auto open = co_await meta.create({SUPER_USER, fname, session, O_TRUNC | O_RDWR, p644});
+      CO_ASSERT_OK(open);
+      CO_ASSERT_EQ(open->stat.id, create->stat.id);
+      CO_ASSERT_TRUE(open->needTruncate);
+      CO_ASSERT_NE(open->stat.asFile().length, 0);
+    }
+  }());
+}
+
+TYPED_TEST(TestCreate, EventLog) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    for (size_t i = 0; i < 10; i++) {
+      FAULT_INJECTION_SET(10, 2);
+      auto create = co_await meta.create({SUPER_USER, fmt::format("{}", i), {}, O_EXCL, p644});
+      CO_ASSERT_OK(create);
+    }
+
+    for (size_t i = 0; i < 10; i++) {
+      // create again, shouldn't have log.
+      FAULT_INJECTION_SET(10, 2);
+      auto create = co_await meta.create({SUPER_USER, fmt::format("{}", i), {}, O_RDONLY, p644});
+      CO_ASSERT_OK(create);
+    }
+
+    // todo: how to verify log counts here?
+  }());
+}
+
+TYPED_TEST(TestCreate, batch) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.set_num_meta(1);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+    auto &engine = *this->kvEngine();
+
+    for (size_t i = 0; i < 10; i++) {
+      auto dir1 = (co_await meta.mkdirs({SUPER_USER, fmt::format("{}.1", i), p755, false}))->stat.id;
+      auto dir2 = (co_await meta.mkdirs({SUPER_USER, fmt::format("{}.2", i), p755, false}))->stat.id;
+      std::vector<std::pair<CreateReq, Result<CreateRsp>>> reqs;
+      auto cnts = folly::Random::rand32(200);
+      for (auto i = 0u; i < cnts; i++) {
+        auto path = folly::Random::rand32(10);
+        auto user = folly::Random::oneIn(2) ? SUPER_USER : flat::UserInfo{Uid(1), Gid(1)};
+        auto perm = folly::Random::rand32(0777);
+        auto flags = folly::Random::rand32(O_RDWR + 1);
+        for (auto flag : {O_CREAT, O_TRUNC, O_EXCL, O_DIRECTORY}) {
+          if (folly::Random::oneIn(2)) {
+            flags |= flag;
+          }
+        }
+        FAULT_INJECTION_SET(5, 3);
+        CreateReq req{user,
+                      PathAt(dir1, folly::to<std::string>(path)),
+                      flags != O_RDONLY ? MetaTestHelper::randomSession() : std::optional<SessionInfo>(),
+                      OpenFlags(flags),
+                      Permission(perm)};
+        auto rsp = co_await meta.create(req);
+        reqs.push_back({req, rsp});
+      }
+
+      std::vector<std::unique_ptr<BatchedOp::Waiter<CreateReq, CreateRsp>>> waiters;
+      auto batch = std::make_unique<BatchedOp>(store, dir2);
+      for (auto &[req, rsp] : reqs) {
+        auto req1 = req;
+        req1.path.parent = dir2;
+        auto waiter = std::make_unique<BatchedOp::Waiter<CreateReq, CreateRsp>>(req1);
+        batch->add(*waiter);
+        waiters.emplace_back(std::move(waiter));
+      }
+
+      // Note: test can't pass with fault injection enabled, result may change if inject MaybeCommtted error.
+      // FAULT_INJECTION_SET(5, 3);
+      auto txn = engine.createReadWriteTransaction();
+      auto driver = OperationDriver(*batch, Void{});
+      CO_ASSERT_OK(co_await driver.run(std::move(txn), {}, false, false));
+
+      for (auto i = 0u; i < reqs.size(); i++) {
+        auto req = reqs[i].first;
+        auto expected = reqs[i].second;
+        auto result = waiters[i]->getResult();
+        if (expected.hasError()) {
+          CO_ASSERT_TRUE(result.hasError())
+              << fmt::format("req {} expected {}, result {}", req, expected.error(), result.value());
+          CO_ASSERT_EQ(result.error().code(), expected.error().code());
+          CO_ASSERT_NE(result.error().code(), MetaCode::kFoundBug);
+        } else {
+          CO_ASSERT_FALSE(result.hasError())
+              << fmt::format("req {} expected {}, result {}", req, expected.value(), result.error());
+          CO_ASSERT_EQ(expected->stat.acl, result->stat.acl);
+          CO_ASSERT_EQ(expected->needTruncate, result->needTruncate);
+          CO_ASSERT_FALSE((expected->needTruncate && !req.flags.contains(O_TRUNC)));
+        }
+      }
+    }
+  }());
+}
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestHardLink.cc

@@ -0,0 +1,106 @@
+#include <array>
+#include <cstdint>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Task.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <vector>
+
+#include "common/kv/ITransaction.h"
+#include "common/kv/mem/MemKV.h"
+#include "common/kv/mem/MemKVEngine.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/Path.h"
+#include "common/utils/Result.h"
+#include "common/utils/StatusCode.h"
+#include "common/utils/String.h"
+#include "fdb/FDB.h"
+#include "gtest/gtest.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+template <typename KV>
+class TestHardLink : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestHardLink, KVTypes);
+
+TYPED_TEST(TestHardLink, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    FAULT_INJECTION_SET(10, 5);
+
+    auto createResult = co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p644});
+    CO_ASSERT_OK(createResult);
+    auto &file = createResult->stat;
+
+    auto mkdirResult = co_await meta.mkdirs({SUPER_USER, "directory", p755, true});
+    CO_ASSERT_OK(mkdirResult);
+    auto &dir = mkdirResult->stat;
+
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, PathAt("symlink-file"), "file"}));
+    auto symlinkFileResult = co_await meta.stat({SUPER_USER, PathAt("symlink-file"), AtFlags(AT_SYMLINK_NOFOLLOW)});
+    CO_ASSERT_OK(symlinkFileResult);
+    auto &symlinkFile = symlinkFileResult->stat;
+
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, PathAt("symlink-dir"), "directory"}));
+    auto symlinkDirResult = co_await meta.stat({SUPER_USER, PathAt("symlink-dir"), AtFlags(AT_SYMLINK_NOFOLLOW)});
+    CO_ASSERT_OK(symlinkDirResult);
+    auto &symlinkDir = symlinkDirResult->stat;
+
+    using TestArg = std::tuple<std::string /* oldpath*/,
+                               std::string /* newpath*/,
+                               bool /* follow */,
+                               std::optional<Status> /* error */,
+                               InodeId /* target */,
+                               uint16_t /* nlink*/>;
+    for (auto [oldPath, newPath, follow, error, targetId, nlink] :
+         {TestArg("file", "file-hardlink1", false, {}, file.id, 2),
+          TestArg("file", "file-hardlink2", false, {}, file.id, 3),
+          TestArg("directory", "directory-hardlink", false, MetaCode::kIsDirectory, dir.id, 1),
+          TestArg("symlink-file", "symlink-file-hardlink-follow", true, {}, file.id, 4),
+          TestArg("symlink-file", "symlink-file-hardlink-no-follow", false, {}, symlinkFile.id, 2),
+          TestArg("symlink-dir", "symlink-dir-hardlink-follow", true, MetaCode::kIsDirectory, dir.id, 1),
+          TestArg("symlink-dir", "symlink-dir-hardlink-no-follow", false, {}, symlinkDir.id, 2)}) {
+      fmt::print("hardlink {} -> {}, follow {}\n", oldPath, newPath, follow);
+      auto result = co_await meta.hardLink(
+          {SUPER_USER, oldPath, newPath, follow ? AtFlags(AT_SYMLINK_FOLLOW) : AtFlags(AT_SYMLINK_NOFOLLOW)});
+      if (error) {
+        CO_ASSERT_ERROR(result, error->code());
+        continue;
+      }
+      CO_ASSERT_OK(result);
+      auto &inode = result->stat;
+      CO_ASSERT_EQ(inode.id, targetId);
+      CO_ASSERT_EQ(inode.nlink, nlink);
+    }
+
+    FAULT_INJECTION_SET(0, 0);
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "file", AtFlags(), false}));
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "file-hardlink1", AtFlags(), false}));
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "symlink-file-hardlink-follow", AtFlags(), false}));
+    auto result = co_await meta.stat({SUPER_USER, "file-hardlink2", AtFlags(AT_SYMLINK_NOFOLLOW)});
+    CO_ASSERT_OK(result);
+    auto &stat = result->stat;
+    CO_ASSERT_EQ(stat.nlink, 1);
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "file-hardlink2", AtFlags(), false}));
+    CO_ASSERT_ERROR(co_await meta.stat({SUPER_USER, "file-hardlink2", AtFlags(AT_SYMLINK_NOFOLLOW)}),
+                    MetaCode::kNotFound);
+  }());
+}
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestIFlags.cc

@@ -0,0 +1,130 @@
+#include <chrono>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/experimental/coro/Task.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <linux/fs.h>
+#include <optional>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Duration.h"
+#include "common/utils/String.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/core/user/User.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/meta/Service.h"
+#include "gtest/gtest.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/Utils.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestIFlags : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestIFlags, KVTypes);
+
+TYPED_TEST(TestIFlags, FS_HUGE_FILE_FL) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(true);
+    config.mock_meta().gc().set_gc_directory_delay(0_s);
+    config.mock_meta().gc().set_gc_file_delay(0_s);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    auto ua = flat::UserInfo{flat::Uid(1), flat::Gid(1), String()};
+    auto ub = flat::UserInfo{flat::Uid(2), flat::Gid(2), String()};
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir", p777, true}));
+    CO_ASSERT_OK(co_await meta.create({ua, "dir/file_ua", {}, 0, p644}));
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(ua, "dir/file_ua", IFlags(FS_HUGE_FILE_FL))));
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(SUPER_USER, "dir/file_ua", IFlags())));
+    CO_ASSERT_ERROR(co_await meta.setAttr(SetAttrReq::setIFlags(ub, "dir/file_ua", IFlags(FS_HUGE_FILE_FL))),
+                    MetaCode::kNoPermission);
+  }());
+}
+
+TYPED_TEST(TestIFlags, FS_IMMUTABLE_FL) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(true);
+    config.mock_meta().gc().set_gc_directory_delay(0_s);
+    config.mock_meta().gc().set_gc_file_delay(0_s);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    auto ua = flat::UserInfo{flat::Uid(1), flat::Gid(1), String()};
+    auto ub = flat::UserInfo{flat::Uid(2), flat::Gid(2), String()};
+
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "file", {}, 0, p644}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "protected-file", {}, 0, p644}));
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir/protected-dir", p777, true}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "dir/file", {}, 0, p644}));
+    CO_ASSERT_OK(co_await meta.create({ua, "dir/file_ua", {}, 0, p644}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "dir/protected-file", {}, 0, p644}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "dir/protected-dir/file", {}, 0, p644}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "dir/protected-dir/protected-file", {}, 0, p644}));
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir/normal-dir", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir/normal-dir2", p777, true}));
+    CO_ASSERT_OK(co_await meta.hardLink(
+        {SUPER_USER, "dir/protected-file", "dir/normal-dir/protected-file", AtFlags(AT_SYMLINK_FOLLOW)}));
+    CO_ASSERT_OK(co_await meta.hardLink(
+        {SUPER_USER, "dir/protected-file", "dir/normal-dir2/protected-file", AtFlags(AT_SYMLINK_FOLLOW)}));
+
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(ua, "dir/file_ua", IFlags())));
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(ub, "dir/file_ua", IFlags())));
+    config.mock_meta().set_allow_owner_change_immutable(false);
+    CO_ASSERT_ERROR(co_await meta.setAttr(SetAttrReq::setIFlags(ua, "dir/file_ua", IFlags(FS_IMMUTABLE_FL))),
+                    MetaCode::kNoPermission);
+    config.mock_meta().set_allow_owner_change_immutable(true);
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(ua, "dir/file_ua", IFlags(FS_IMMUTABLE_FL))));
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(ub, "dir/file_ua", IFlags(FS_IMMUTABLE_FL))));
+    CO_ASSERT_ERROR(co_await meta.setAttr(SetAttrReq::setIFlags(ub, "dir/file_ua", IFlags())), MetaCode::kNoPermission);
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(ua, "dir/file_ua", IFlags())));
+
+    std::vector<InodeId> inodes;
+    for (auto &path : std::vector<std::string>{"protected-file",
+                                               "dir/protected-dir",
+                                               "dir/protected-file",
+                                               "dir/protected-dir/protected-file"}) {
+      auto res = co_await meta.setAttr(SetAttrReq::setIFlags(SUPER_USER, path, IFlags(FS_IMMUTABLE_FL)));
+      CO_ASSERT_OK(res);
+      inodes.push_back(res->stat.id);
+      CO_ASSERT_ERROR(co_await meta.remove({SUPER_USER, path, AtFlags(0), true}), MetaCode::kNoPermission);
+    }
+    auto res = co_await meta.stat({SUPER_USER, "dir/protected-dir/file", AtFlags()});
+    CO_ASSERT_OK(res);
+    inodes.push_back(res->stat.id);
+
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "file", AtFlags(0), true}));
+    CO_ASSERT_ERROR(co_await meta.remove({SUPER_USER, "dir", AtFlags(0), true}), MetaCode::kNoPermission);
+    config.mock_meta().set_recursive_remove_perm_check(0);
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "dir", AtFlags(0), true}));
+    co_await folly::coro::sleep(std::chrono::seconds(3));
+
+    for (auto &inodeId : inodes) {
+      CO_ASSERT_INODE_EXISTS(inodeId);
+    }
+
+    CO_ASSERT_OK(co_await printTree(meta));
+    co_return;
+  }());
+}
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestList.cc

@@ -0,0 +1,81 @@
+#include <algorithm>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <gtest/gtest.h>
+#include <vector>
+
+#include "common/utils/FaultInjection.h"
+#include "common/utils/StatusCode.h"
+#include "meta/store/MetaStore.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestList : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestList, KVTypes);
+
+TYPED_TEST(TestList, FaultInjection) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    std::vector<std::string> paths;
+    size_t numFiles = folly::Random::rand32(0, 4096);
+    for (size_t i = 0; i < numFiles; i++) {
+      std::string path = std::to_string(i);
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      paths.push_back(path);
+    }
+    std::sort(paths.begin(), paths.end());
+
+    for (size_t i = 0; i < 10; i++) {
+      FAULT_INJECTION_SET(10, 3);  // 10%, 3 faults
+      std::vector<DirEntry> entries;
+      bool hasMore = true;
+      const bool needStatus = folly::Random::oneIn(2);
+      String prev;
+
+      // do load
+      while (hasMore) {
+        auto limit = folly::Random::rand32(0, 512);
+        auto req = i % 2 == 0 ? ListReq(SUPER_USER, Path("/"), prev, (int)limit, needStatus)
+                              : ListReq(SUPER_USER, InodeId::root(), prev, (int)limit, needStatus);
+        auto list = co_await meta.list(req);
+        CO_ASSERT_OK(list);
+        if (needStatus) {
+          CO_ASSERT_TRUE(list->inodes.size() == list->entries.size());
+        }
+        for (size_t i = 0; i < list->entries.size(); i++) {
+          if (needStatus) {
+            auto entry = list->entries.at(i);
+            auto inode = list->inodes.at(i);
+            entries.push_back(entry);
+            CO_ASSERT_EQ(entry.id, inode.id);
+          } else {
+            entries.push_back(list->entries.at(i));
+          }
+        }
+
+        hasMore = list->more;
+        if (list->entries.size() == 0) {
+          CO_ASSERT_FALSE(hasMore);
+        }
+        if (hasMore) {
+          prev = list->entries.at(list->entries.size() - 1).name;
+        }
+      }
+
+      // check
+      CO_ASSERT_EQ(entries.size(), numFiles);
+      for (size_t i = 0; i < numFiles; i++) {
+        auto entry = entries[i];
+        CO_ASSERT_EQ(entry.name, paths[i]);
+      }
+    }
+  }());
+}
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestMkdirs.cc

@@ -0,0 +1,152 @@
+#include <fcntl.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Task.h>
+#include <gtest/gtest.h>
+#include <linux/fs.h>
+
+#include "common/utils/FaultInjection.h"
+#include "common/utils/StatusCode.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Service.h"
+#include "meta/store/MetaStore.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestMkdirs : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestMkdirs, KVTypes);
+
+TYPED_TEST(TestMkdirs, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "a", p755, false}));
+    CO_ASSERT_ERROR(co_await meta.mkdirs({SUPER_USER, "b/c", p755, false}), MetaCode::kNotFound);
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "b/c", p755, true}));
+
+    for (auto path : {"a", "b", "b/c"}) {
+      auto result = co_await meta.open({SUPER_USER, path, {}, O_DIRECTORY});
+      CO_ASSERT_OK(result);
+    }
+
+    auto result = co_await meta.mkdirs({SUPER_USER, "b/c", p755, true});
+    CO_ASSERT_ERROR(result, MetaCode::kExists);
+    auto result2 = co_await meta.mkdirs({SUPER_USER, "/", p700, true});
+    CO_ASSERT_ERROR(result2, MetaCode::kExists);
+
+    flat::UserInfo otherUser(Uid(1), Gid(1), String());
+    CO_ASSERT_ERROR(co_await meta.mkdirs({otherUser, "d/e", p755, true}), MetaCode::kNoPermission);
+
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(SUPER_USER, "/", IFlags(FS_CHAIN_ALLOCATION_FL))));
+    auto result3 = co_await meta.stat({SUPER_USER, "/", AtFlags()});
+    CO_ASSERT_OK(result3);
+    CO_ASSERT_TRUE(result3->stat.acl.iflags & FS_CHAIN_ALLOCATION_FL);
+
+    auto result4 = co_await meta.mkdirs({SUPER_USER, "f/g", p755, true});
+    CO_ASSERT_OK(result4);
+    CO_ASSERT_TRUE(result4->stat.acl.iflags & FS_CHAIN_ALLOCATION_FL);
+    CO_ASSERT_EQ(result4->stat.asDirectory().chainAllocCounter, -1);
+
+    auto dir = result4->stat.id;
+    auto result5 = co_await meta.create(CreateReq(SUPER_USER, PathAt(dir, "a"), std::nullopt, OpenFlags(), p755));
+    CO_ASSERT_OK(result5);
+    CO_ASSERT_EQ(result5->stat.id.useNewChunkEngine(), cluster.config().mock_meta().enable_new_chunk_engine());
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setIFlags(SUPER_USER, dir, IFlags(FS_NEW_CHUNK_ENGINE))));
+    auto result6 = co_await meta.create(CreateReq(SUPER_USER, PathAt(dir, "b"), std::nullopt, OpenFlags(), p755));
+    CO_ASSERT_OK(result6);
+    CO_ASSERT_TRUE(result6->stat.id.useNewChunkEngine());
+  }());
+}
+
+TYPED_TEST(TestMkdirs, ConflictSet) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &store = cluster.meta().getStore();
+
+    for (int i = 0; i < 100; i++) {
+      std::string path = std::to_string(i) + ".txt";
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = MkdirsReq(SUPER_USER, Path(path), p755, false);
+            auto createResult = co_await store.mkdirs(req)->run(txn);
+            CO_ASSERT_OK(createResult);
+          },
+          (std::vector<String>{
+              MetaTestHelper::getInodeKey(InodeId::root()),           // parent Inode
+              MetaTestHelper::getDirEntryKey(InodeId::root(), path),  // dir entry
+          }),
+          (std::vector<String>{
+              MetaTestHelper::getDirEntryKey(InodeId::root(), path),  // dir entry
+          }),
+          false);
+    }
+  }());
+}
+
+TYPED_TEST(TestMkdirs, Concurrent) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &store = cluster.meta().getStore();
+
+    // concurrent mkdir on same path should conflict
+    for (int i = 0; i < 100; i++) {
+      std::string path = std::to_string(i) + ".dir";
+      auto mkdir = [&](auto &txn) -> CoTask<void> {
+        auto req = MkdirsReq(SUPER_USER, Path(path), p755, false);
+        auto mkdirResult = co_await store.mkdirs(req)->run(txn);
+        CO_ASSERT_OK(mkdirResult);
+      };
+      CO_ASSERT_CONFLICT(mkdir, mkdir);
+    }
+
+    // concurrent mkdir on different path shouldn't conflict
+    for (int i = 0; i < 100; i++) {
+      auto mkdir1 = [&](auto &txn) -> CoTask<void> {
+        auto req = MkdirsReq(SUPER_USER, Path(std::to_string(i) + ".1"), p755, false);
+        auto mkdirResult = co_await store.mkdirs(req)->run(txn);
+        CO_ASSERT_OK(mkdirResult);
+      };
+      auto mkdir2 = [&](auto &txn) -> CoTask<void> {
+        auto req = MkdirsReq(SUPER_USER, Path(std::to_string(i) + ".2"), p755, false);
+        auto mkdirResult = co_await store.mkdirs(req)->run(txn);
+        CO_ASSERT_OK(mkdirResult);
+      };
+      CO_ASSERT_NO_CONFLICT(mkdir1, mkdir2);
+    }
+  }());
+}
+
+TYPED_TEST(TestMkdirs, FaultInjection) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    auto result = co_await meta.mkdirs({SUPER_USER, "dirA/dirB/dirC", p755, true});
+    CO_ASSERT_OK(result);
+    std::vector<Path> parents =
+        {"/", "/dirA", "/dirA/dirB", "/dirA/dirB/dirC", "/dirA/dirB/../", "/dirA/dirB/../dirB", "not_exists"};
+
+    for (auto i = 0; i < 100; i++) {
+      FAULT_INJECTION_SET(5, 3);  // 5%, 3 faults.
+      auto parent = parents[folly::Random::rand32(parents.size())];
+      auto path = parent;
+      path.append(std::to_string(i));
+      auto result = co_await meta.mkdirs({SUPER_USER, path, p755, false});
+      if (parent == "not_exists") {
+        CO_ASSERT_ERROR(result, MetaCode::kNotFound);
+      } else {
+        CO_ASSERT_OK(result);
+      }
+    }
+  }());
+}
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestOpen.cc

@@ -0,0 +1,343 @@
+#include <algorithm>
+#include <fcntl.h>
+#include <folly/Random.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/experimental/coro/Task.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <random>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "common/utils/FaultInjection.h"
+#include "common/utils/StatusCode.h"
+#include "fmt/core.h"
+#include "gtest/gtest.h"
+#include "meta/store/MetaStore.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestOpen : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestOpen, KVTypes);
+
+TYPED_TEST(TestOpen, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    // create file, directory and symlink
+    CO_ASSERT_OK(
+        co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p644, std::nullopt, true /* dynamic stripe */}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "symlink", "file"}));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir/dir2", p700, true}));
+
+    // open file and symlink should success, and both point to file
+    auto result1 = co_await meta.open({SUPER_USER, "file", {}, O_RDONLY});
+    auto result2 = co_await meta.open({SUPER_USER, "symlink", {}, O_RDONLY});
+    CO_ASSERT_OK(result1);
+    CO_ASSERT_OK(result2);
+    CO_ASSERT_EQ(result1->stat, result2->stat);
+
+    // open with InodeId
+    auto result3 = co_await meta.open({SUPER_USER, result1->stat.id, {}, O_RDONLY});
+    CO_ASSERT_OK(result3);
+    CO_ASSERT_EQ(result1->stat, result3->stat);
+    CO_ASSERT_NE(result3->stat.asFile().dynStripe, 0);
+
+    auto rw1 = co_await meta.open(
+        {SUPER_USER, result1->stat.id, MetaTestHelper::randomSession(), O_RDWR, true /* dynamic stripe */});
+    CO_ASSERT_OK(rw1);
+    CO_ASSERT_NE(rw1->stat.asFile().dynStripe, 0);
+
+    auto rw2 = co_await meta.open(
+        {SUPER_USER, result1->stat.id, MetaTestHelper::randomSession(), O_RDWR, false /* dynamic stripe */});
+    CO_ASSERT_OK(rw2);
+    CO_ASSERT_EQ(rw2->stat.asFile().dynStripe, 0);
+
+    // open a not exists file, should return MetaCode::kNotFound
+    CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, "not-exist", {}, O_RDONLY}), MetaCode::kNotFound);
+    CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, "not-exist/not-exist", {}, O_RDONLY}), MetaCode::kNotFound);
+
+    flat::UserInfo otherUser(Uid(1), Gid(1), String());
+    auto session = MetaTestHelper::randomSession();
+    // other user should can only open file in read only mode
+    CO_ASSERT_OK(co_await meta.open({otherUser, "file", {}, O_RDONLY}));
+    CO_ASSERT_ERROR(co_await meta.open({otherUser, "file", session, O_RDWR}), MetaCode::kNoPermission);
+
+    // open not exists file under directory, su should get kNotFound, other user should get kNoPermission
+    CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, "dir/file", {}, O_RDONLY}), MetaCode::kNotFound);
+    CO_ASSERT_ERROR(co_await meta.open({otherUser, "dir/file", {}, O_RDONLY}), MetaCode::kNoPermission);
+
+    // open file with O_DIRECTORY should get kNotDirectory, write open directory should get kIsDirectory
+    CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, "file", {}, O_RDONLY | O_DIRECTORY}), MetaCode::kNotDirectory);
+    CO_ASSERT_OK(co_await meta.open({SUPER_USER, "dir/dir2", {}, O_RDONLY | O_DIRECTORY}));
+
+    CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, "dir/dir2", session, O_WRONLY}), MetaCode::kIsDirectory);
+
+    // open without session
+    CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, "dir/dir2", {}, O_WRONLY}), StatusCode::kInvalidArg);
+    CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, "dir/dir2", {}, O_RDWR}), StatusCode::kInvalidArg);
+
+    config.mock_meta().set_readonly(true);
+    CO_ASSERT_OK(co_await meta.open({otherUser, "file", {}, O_RDONLY}));
+    CO_ASSERT_ERROR(co_await meta.open({SUPER_USER, "file", session, O_WRONLY}), StatusCode::kReadOnlyMode);
+  }());
+}
+
+TYPED_TEST(TestOpen, Concurrent) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+
+    auto path = "open-file-with-write";
+    auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+    CO_ASSERT_OK(result);
+    InodeId inodeId = result->stat.id;
+
+    fmt::print("{}\n", result->stat);
+
+    auto session = MetaTestHelper::randomSession();
+    CHECK_CONFLICT_SET(
+        [&](auto &txn) -> CoTask<void> {
+          auto req = OpenReq(SUPER_USER, Path(path), session, O_RDWR);
+          CO_ASSERT_OK(co_await store.open(req)->run(txn));
+        },
+        (std::vector<String>{}),
+        (std::vector<String>{
+            MetaTestHelper::getInodeSessionKey(FileSession::create(inodeId, session)),
+            // MetaTestHelper::getClientSessionKey(FileSession::create(inodeId, session)),
+        }),
+        true);
+  }());
+}
+
+TYPED_TEST(TestOpen, FaultInjection) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    std::vector<std::string> paths;
+    for (auto i = 0; i < 10; i++) {
+      std::string path = std::to_string(i);
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      paths.push_back(path);
+    }
+
+    paths.push_back("not_exists");
+    for (auto i = 0; i < 100; i++) {
+      FAULT_INJECTION_SET(10, 3);  // 10%, 3 faults
+      auto path = paths[folly::Random::rand32(paths.size())];
+      auto result = co_await meta.open({SUPER_USER, path, {}, O_RDONLY});
+      if (path == "not_exists") {
+        CO_ASSERT_ERROR(result, MetaCode::kNotFound);
+      } else {
+        CO_ASSERT_OK(result);
+      }
+    }
+  }());
+}
+
+// TYPED_TEST(TestOpen, PruneSession) {
+//   folly::coro::blockingWait([&]() -> CoTask<void> {
+//     MockCluster::Config cfg;
+//     cfg.mock_meta().set_check_file_hole(true);
+//     auto cluster = this->createMockCluster(cfg);
+//     auto &meta = cluster.meta().getOperator();
+//     auto &storage = cluster.meta().getStorageClient();
+
+//     std::vector<std::string> path;
+//     Uuid clientId = Uuid::random();
+//     std::vector<Uuid> sessions;
+//     std::vector<std::pair<InodeId, uint64_t>> inodes;
+//     for (auto i = 0; i < 20; i++) {
+//       std::string path = std::to_string(i);
+//       auto sessionId = Uuid::random();
+//       auto result = co_await meta.create({SUPER_USER, path, SessionInfo(ClientId(clientId), sessionId), O_RDWR,
+//       p644}); CO_ASSERT_OK(result);
+
+//       // write to make a hole
+//       auto offset = folly::Random::rand32(1 << 20, 4 << 20);
+//       auto length = folly::Random::rand32(2 << 20) + 1;
+//       co_await randomWrite(meta, storage, result->stat, offset, length);
+//       sessions.push_back(sessionId);
+//       inodes.push_back({result->stat.id, offset + length});
+//     }
+//     for (auto i = 0; i < 100; i++) {
+//       sessions.push_back(Uuid::random());
+//     }
+//     std::shuffle(sessions.begin(), sessions.end(), std::mt19937());
+//     CO_ASSERT_OK(co_await meta.pruneSession({ClientId(clientId), sessions}));
+
+//     co_await folly::coro::sleep(std::chrono::seconds(1));
+//     for (auto [inodeId, length] : inodes) {
+//       auto result = co_await meta.stat({SUPER_USER, inodeId, AtFlags(AT_SYMLINK_FOLLOW)});
+//       CO_ASSERT_OK(result);
+//       auto stat = result->stat;
+//       CO_ASSERT_EQ(stat.asFile().length, length);
+//       CO_ASSERT_TRUE(stat.asFile().hasHole());
+//     }
+//   }());
+// }
+
+TYPED_TEST(TestOpen, OTrunc) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config cfg;
+    cfg.mock_meta().set_check_file_hole(true);
+    auto cluster = this->createMockCluster(cfg);
+    auto &meta = cluster.meta().getOperator();
+    auto &storage = cluster.meta().getStorageClient();
+
+    for (size_t i = 0; i < 10; i++) {
+      auto fname = fmt::format("truncate-inplace-{}", i);
+      auto create = co_await meta.create({SUPER_USER, fname, {}, O_EXCL, p644});
+      CO_ASSERT_OK(create);
+      co_await truncate(meta, storage, create->stat, 1_MB);
+
+      FAULT_INJECTION_SET(20, 3);
+      auto session = MetaTestHelper::randomSession();
+      auto open = co_await meta.open({SUPER_USER, fname, session, O_TRUNC | O_RDWR});
+      CO_ASSERT_EQ(open->stat.id, create->stat.id);
+      CO_ASSERT_TRUE(open->needTruncate);
+      CO_ASSERT_NE(open->stat.asFile().length, 0);
+    }
+
+    for (size_t i = 0; i < 10; i++) {
+      auto fname = fmt::format("truncate-by-inode-{}", i);
+      auto create = co_await meta.create({SUPER_USER, fname, {}, O_EXCL, p644});
+      CO_ASSERT_OK(create);
+      co_await truncate(meta, storage, create->stat, 2_GB);
+
+      FAULT_INJECTION_SET(20, 3);
+      auto session = MetaTestHelper::randomSession();
+      auto open = co_await meta.open({SUPER_USER, create->stat.id, session, O_TRUNC | O_RDWR});
+      CO_ASSERT_EQ(open->stat.id, create->stat.id);
+      CO_ASSERT_TRUE(open->needTruncate);
+      CO_ASSERT_NE(open->stat.asFile().length, 0);
+    }
+
+    for (size_t i = 0; i < 10; i++) {
+      auto fname = fmt::format("truncate-replace-{}", i);
+      auto create = co_await meta.create({SUPER_USER, fname, {}, O_EXCL, p644});
+      CO_ASSERT_OK(create);
+      CO_ASSERT_EQ(create->stat.asFile().dynStripe, 0);
+      co_await truncate(meta, storage, create->stat, 2_GB);
+
+      auto session1 = MetaTestHelper::randomSession();
+      auto session2 = MetaTestHelper::randomSession();
+      auto session3 = MetaTestHelper::randomSession();
+      auto open1 = co_await meta.open({SUPER_USER, fname, session1, O_TRUNC | O_RDWR, true /* dynamic stripe */});
+      CO_ASSERT_NE(open1->stat.id, create->stat.id);
+      CO_ASSERT_NE(open1->stat.asFile().dynStripe, 0);
+
+      auto open2 = co_await meta.open({SUPER_USER, fname, session2, O_TRUNC | O_RDWR});
+      CO_ASSERT_EQ(open1->stat.id, open2->stat.id);
+
+      auto result = co_await meta.stat({SUPER_USER, open1->stat.id, AtFlags(AT_SYMLINK_FOLLOW)});
+      CO_ASSERT_OK(result);
+      CO_ASSERT_EQ(result->stat.asFile().length, 0);
+
+      co_await truncate(meta, storage, open1->stat, 2_GB);
+
+      CO_ASSERT_OK(co_await meta.close({SUPER_USER, open1->stat.id, session1, true, {}, {}}));
+      CO_ASSERT_OK(co_await meta.close({SUPER_USER, open1->stat.id, session2, true, {}, {}}));
+
+      auto open3 = co_await meta.open({SUPER_USER, fname, session3, O_TRUNC | O_RDWR});
+      CO_ASSERT_NE(open3->stat.id, open1->stat.id);
+    }
+  }());
+}
+
+TYPED_TEST(TestOpen, ConflictSet) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &store = cluster.meta().getStore();
+    auto &meta = cluster.meta().getOperator();
+    auto &storage = cluster.meta().getStorageClient();
+
+    {
+      auto path = "open-trunc-exists-file";
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      auto session = MetaTestHelper::randomSession();
+      InodeId inodeId = result->stat.id;
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = OpenReq(SUPER_USER, Path(path), session, O_TRUNC | O_WRONLY);
+            auto openResult = co_await store.open(req)->run(txn);
+            CO_ASSERT_OK(openResult);
+            CO_ASSERT_EQ(inodeId, openResult->stat.id);
+            inodeId = openResult->stat.id;
+          },
+          (std::vector<String>{}),
+          (std::vector<String>{
+              MetaTestHelper::getInodeSessionKey(FileSession::create(inodeId, session)),
+              // MetaTestHelper::getClientSessionKey(FileSession::create(inodeId, session))
+          }),
+          true);
+    }
+
+    {
+      auto path = "open-trunc-file-by-inode";
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      auto inodeId = result->stat.id;
+      co_await truncate(meta, storage, result->stat, 2_GB);
+      auto session = MetaTestHelper::randomSession();
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = OpenReq(SUPER_USER, inodeId, session, O_TRUNC | O_WRONLY);
+            auto openResult = co_await store.open(req)->run(txn);
+            CO_ASSERT_OK(openResult);
+            CO_ASSERT_EQ(openResult->stat.id, inodeId);
+          },
+          (std::vector<String>{}),
+          (std::vector<String>{
+              MetaTestHelper::getInodeSessionKey(FileSession::create(inodeId, session)),
+              // MetaTestHelper::getClientSessionKey(FileSession::create(inodeId, session))
+          }),
+          false);
+    }
+
+    {
+      auto path = "open-trunc-replace-exists-file";
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      auto oldInodeId = result->stat.id;
+      co_await truncate(meta, storage, result->stat, 2_GB);
+      auto session = MetaTestHelper::randomSession();
+      InodeId newInodeId;
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = OpenReq(SUPER_USER, Path(path), session, O_TRUNC | O_WRONLY);
+            auto openResult = co_await store.open(req)->run(txn);
+            CO_ASSERT_OK(openResult);
+            newInodeId = openResult->stat.id;
+            CO_ASSERT_NE(oldInodeId, newInodeId);
+          },
+          (std::vector<String>{MetaTestHelper::getDirEntryKey(InodeId::root(), path),
+                               MetaTestHelper::getInodeKey(oldInodeId)}),
+          (std::vector<String>{
+              MetaTestHelper::getDirEntryKey(InodeId::root(), path),
+              MetaTestHelper::getInodeKey(newInodeId),
+              MetaTestHelper::getInodeSessionKey(FileSession::create(newInodeId, session)),
+              // MetaTestHelper::getClientSessionKey(FileSession::create(newInodeId, session))
+          }),
+          false);
+    }
+  }());
+}
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestRealPath.cc

@@ -0,0 +1,97 @@
+#include <fcntl.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Task.h>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#include "common/utils/Coroutine.h"
+#include "common/utils/Path.h"
+#include "common/utils/StatusCode.h"
+#include "fbs/meta/Common.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/PathResolve.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestRealPath : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestRealPath, KVTypes);
+
+TYPED_TEST(TestRealPath, basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, InodeId::root(), true}))->path, "/");
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "/a/b/c", p777, true}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "/a/b/c/file", {}, O_RDONLY, p777}));
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "/a", true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "/a/", true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "/a/.", true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "/a/..", true}))->path, "/");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "/a/../../..", true}))->path, "/");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "/a/b", true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "/a/b/c", true}))->path, "/a/b/c");
+
+    auto b = (co_await meta.stat({SUPER_USER, "/a/b", AtFlags()}))->stat.id;
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(b, "."), false}))->path, ".");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(b, "."), true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(b, "c"), true}))->path, "/a/b/c");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(b, "c/.."), false}))->path, ".");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(b, "c/.."), true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(b, ".."), true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(b, "c/file"), false}))->path, "c/file");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(b, "c/file"), true}))->path, "/a/b/c/file");
+
+    auto file = (co_await meta.stat({SUPER_USER, "/a/b/c/file", AtFlags()}))->stat.id;
+    CO_ASSERT_ERROR((co_await meta.getRealPath({SUPER_USER, file, true})), MetaCode::kNotDirectory);
+
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/symlink-1", "/a/b"}));
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "symlink-1/.", false}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "symlink-1/.", true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "symlink-1/c", true}))->path, "/a/b/c");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "symlink-1/c/..", false}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "symlink-1/c/..", true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "symlink-1/..", true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "symlink-1/c/file", false}))->path, "/a/b/c/file");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "symlink-1/c/file", true}))->path, "/a/b/c/file");
+
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "a/symlink-2", "b"}));
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/.", false}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/..", false}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/.", true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/..", true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/c", true}))->path, "/a/b/c");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/c/..", false}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/c/..", true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/..", true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/c/file", false}))->path, "/a/b/c/file");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, "a/symlink-2/c/file", true}))->path, "/a/b/c/file");
+
+    auto a = (co_await meta.stat({SUPER_USER, "/a", AtFlags()}))->stat.id;
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/."), false}))->path, "b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/.."), false}))->path, ".");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/."), true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/c"), true}))->path, "/a/b/c");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/.."), true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/c/.."), false}))->path, "b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/c/.."), true}))->path, "/a/b");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/.."), true}))->path, "/a");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/c/file"), false}))->path, "b/c/file");
+    CO_ASSERT_EQ((co_await meta.getRealPath({SUPER_USER, PathAt(a, "symlink-2/c/file"), true}))->path, "/a/b/c/file");
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestRemove.cc

@@ -0,0 +1,397 @@
+#include <chrono>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/Random.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Sleep.h>
+#include <folly/experimental/coro/Task.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <string>
+#include <thread>
+
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Duration.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/core/user/User.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Service.h"
+#include "gtest/gtest.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/Utils.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestRemove : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestRemove, KVTypes);
+
+TYPED_TEST(TestRemove, GC) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(false);
+    config.mock_meta().gc().set_gc_directory_delay(0_s);
+    config.mock_meta().gc().set_gc_file_delay(0_s);
+    config.mock_meta().gc().set_scan_interval(10_ms);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &gcManager = cluster.meta().getGcManager();
+
+    int fileCnt = 512;
+
+    // create a directory, create some files, subdirectories, symlinks under it, then remove it recursively
+    auto mkdirResult = co_await meta.mkdirs({SUPER_USER, "directory", p755, false});
+    CO_ASSERT_OK(mkdirResult);
+    auto &directory = mkdirResult->stat;
+    for (int i = 0; i < fileCnt; i++) {
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt(directory.id, std::to_string(i) + ".file"), {}, 0, p644}));
+      CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, PathAt(directory.id, std::to_string(i) + ".dir"), p755, false}));
+      CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, PathAt(directory.id, std::to_string(i) + ".symlink"), "target"}));
+    }
+
+    GET_INODE_CNTS(inodes);
+    GET_DIRENTRY_CNTS(entries);
+    fmt::print("inodes {} dirEntries {}\n", inodes, entries);
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "directory", AtFlags(), true}));
+
+    GET_INODE_CNTS(numInodes);
+
+    config.mock_meta().gc().set_enable(true);
+    std::this_thread::sleep_for(std::chrono::seconds(4));
+
+    READ_WRITE_TRANSACTION_NO_COMMIT({
+      for (auto gcDir : gcManager.currGcDirectories()) {
+        auto empty = co_await DirEntryList::checkEmpty(*txn, gcDir->dirId());
+        CO_ASSERT_OK(empty);
+        CO_ASSERT_TRUE(empty.value());
+      }
+
+      // all inodes except root, gcRoot, gcDirectory should be removed
+      CO_ASSERT_INODE_CNTS(numInodes - 1 - fileCnt * 3);
+    });
+  }());
+}
+
+TYPED_TEST(TestRemove, Remove) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(false);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    FAULT_INJECTION_SET(10, 3);
+
+    CO_ASSERT_ERROR(co_await meta.remove({SUPER_USER, "not-exists", AtFlags(), false}), MetaCode::kNotFound);
+    CO_ASSERT_ERROR(co_await meta.remove({SUPER_USER, "/", AtFlags(), false}), StatusCode::kInvalidArg);
+    CO_ASSERT_ERROR(co_await meta.remove({SUPER_USER, "not-exists/", AtFlags(), false}), StatusCode::kInvalidArg);
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "a/b", p755, true}));
+
+    // remove a not empty directory
+    CO_ASSERT_ERROR(co_await meta.remove({SUPER_USER, "a", AtFlags(), false}), MetaCode::kNotEmpty);
+    // remove a not empty directory recursively
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "a", AtFlags(), true}));
+    // stat removed directory
+    CO_ASSERT_ERROR(co_await meta.stat({SUPER_USER, "a", AtFlags(AT_SYMLINK_NOFOLLOW)}), MetaCode::kNotFound);
+
+    auto result = co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p644});
+    CO_ASSERT_OK(result);
+    auto &inode = result->stat;
+    CO_ASSERT_OK(co_await meta.hardLink({SUPER_USER, "file", "file-hardlink", AtFlags(AT_SYMLINK_NOFOLLOW)}));
+    auto statResult = co_await meta.stat({SUPER_USER, "file-hardlink", AtFlags(AT_SYMLINK_NOFOLLOW)});
+    CO_ASSERT_OK(statResult);
+    CO_ASSERT_EQ(inode.nlink, 1);
+    CO_ASSERT_EQ(statResult->stat.nlink, 2);
+    CO_ASSERT_EQ(inode.id, statResult->stat.id);
+
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "file", AtFlags(), false}));
+    statResult = co_await meta.stat({SUPER_USER, "file-hardlink", AtFlags(AT_SYMLINK_NOFOLLOW)});
+    CO_ASSERT_OK(statResult);
+    CO_ASSERT_EQ(statResult->stat.nlink, 1);
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "file-hardlink", AtFlags(), false}));
+    CO_ASSERT_ERROR(co_await meta.stat({SUPER_USER, "file-hardlink", AtFlags(AT_SYMLINK_NOFOLLOW)}),
+                    MetaCode::kNotFound);
+  }());
+}
+
+TYPED_TEST(TestRemove, RemoveRecursivePerm) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto ua = flat::UserInfo(flat::Uid(1), flat::Gid(1));
+    auto ub = flat::UserInfo(flat::Uid(2), flat::Gid(2));
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "shared", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "shared/ua/subdir", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "shared/ua2/subdir", flat::Permission(0222), true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "shared/ua_ub/subdir", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ub, "shared/ua_ub/subdir/dir", p700, false}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "shared/ua_ub2/subdir", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ub, "shared/ua_ub2/subdir/dir", flat::Permission(0777 & S_ISVTX), false}));
+
+    FAULT_INJECTION_SET(10, 3);
+    CO_ASSERT_ERROR(co_await meta.remove({ua, "shared/ua2", AtFlags(), true}), MetaCode::kNoPermission);
+    CO_ASSERT_ERROR(co_await meta.remove({ub, "shared/ua", AtFlags(), true}), MetaCode::kNoPermission);
+    CO_ASSERT_OK(co_await meta.remove({ua, "shared/ua", AtFlags(), true}));
+    CO_ASSERT_ERROR(co_await meta.remove({ua, "shared/ua_ub", AtFlags(), true}), MetaCode::kNoPermission);
+    CO_ASSERT_ERROR(co_await meta.remove({ua, "shared/ua_ub2", AtFlags(), true}), MetaCode::kNoPermission);
+  }());
+}
+
+TYPED_TEST(TestRemove, RemoveRecursive) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(false);
+    config.mock_meta().gc().set_gc_directory_delay(0_s);
+    config.mock_meta().gc().set_gc_file_delay(0_s);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    // remove a non-empty directory recursively should remove subdirectory too.
+    auto mkdirResult = co_await meta.mkdirs({SUPER_USER, "dir", p755, true});
+    CO_ASSERT_OK(mkdirResult);
+    auto dirId = mkdirResult->stat.id;
+    // create a sub directory
+    auto subDirResult = co_await meta.mkdirs({SUPER_USER, "dir/subdir", p755, true});
+    CO_ASSERT_OK(subDirResult);
+    auto subDirId = subDirResult->stat.id;
+    // create a file under directory
+    auto createResult = co_await meta.create({SUPER_USER, PathAt(subDirId, "file"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(createResult);
+    auto fileId = createResult->stat.id;
+
+    for (auto path : {"dir/.", "dir/..", "dir/subdir", "dir/subdir/file"}) {
+      fmt::print("path {}\n", path);
+      CO_ASSERT_OK(co_await meta.stat({SUPER_USER, PathAt(path), AtFlags(AT_SYMLINK_FOLLOW)}));
+    }
+
+    // remove directory
+    auto removeResult = co_await meta.remove({SUPER_USER, "dir", AtFlags(), true});
+    CO_ASSERT_OK(removeResult);
+    READ_ONLY_TRANSACTION({
+      // all inodes present
+      for (auto inodeId : {dirId, subDirId, fileId}) {
+        CO_ASSERT_OK((co_await Inode::snapshotLoad(*txn, inodeId)).then(checkMetaFound<Inode>));
+      }
+      // dir entry should present
+      for (auto [parent, name] : {std::pair(dirId, "subdir"), std::pair(subDirId, "file")}) {
+        fmt::print("parent {}, name {}\n", parent, name);
+        CO_ASSERT_OK((co_await DirEntry::snapshotLoad(*txn, parent, name)).then(checkMetaFound<DirEntry>));
+      }
+    });
+
+    // lookup at deleted directory
+    for (auto [parent, name] : {std::pair{dirId, "subdir"}, {subDirId, "file"}}) {
+      fmt::print("parent {}, name {}\n", parent, name);
+      CO_ASSERT_OK(co_await meta.stat({SUPER_USER, PathAt(parent, name), AtFlags(AT_SYMLINK_FOLLOW)}));
+    }
+    for (auto path : {"dir/.", "dir/..", "dir/subdir", "dir/subdir/file"}) {
+      fmt::print("path {}\n", path);
+      CO_ASSERT_ERROR(co_await meta.stat({SUPER_USER, PathAt(path), AtFlags(AT_SYMLINK_FOLLOW)}), MetaCode::kNotFound);
+    }
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, PathAt(dirId, "file2"), {}, O_EXCL, p644}), MetaCode::kNotFound);
+    auto file2 = co_await meta.create({SUPER_USER, PathAt(subDirId, "file2"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(file2);
+
+    config.mock_meta().gc().set_enable(true);
+    std::this_thread::sleep_for(std::chrono::seconds(2));
+
+    READ_ONLY_TRANSACTION({
+      // all inodes removed
+      for (auto inodeId : {dirId, subDirId, fileId, file2->stat.id}) {
+        CO_ASSERT_ERROR((co_await Inode::snapshotLoad(*txn, inodeId)).then(checkMetaFound<Inode>), MetaCode::kNotFound);
+      }
+    });
+    // after GC, directory has deleted, create under directory should get kNotFound
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, PathAt(dirId, "another-file"), {}, O_EXCL, p644}),
+                    MetaCode::kNotFound);
+    CO_ASSERT_ERROR(co_await meta.create({SUPER_USER, PathAt(subDirId, "another-file"), {}, O_EXCL, p644}),
+                    MetaCode::kNotFound);
+    // lookup at deleted directory should get kNotFound
+    for (auto pair : {std::pair{dirId, "."}, {dirId, ".."}, {dirId, "subdir"}, {subDirId, "file"}}) {
+      CO_ASSERT_ERROR(co_await meta.stat({SUPER_USER, PathAt(pair.first, pair.second), AtFlags(AT_SYMLINK_FOLLOW)}),
+                      MetaCode::kNotFound);
+    }
+  }());
+}
+
+TYPED_TEST(TestRemove, RemoveDirectoryByInode) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(false);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    FAULT_INJECTION_SET(10, 3);
+
+    // remove a non-empty directory recursively should remove subdirectory too.
+    auto mkdirResult = co_await meta.mkdirs({SUPER_USER, "dir", p755, true});
+    CO_ASSERT_OK(mkdirResult);
+    auto dirId = mkdirResult->stat.id;
+    // create a sub directory
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir/subdir", p755, true}));
+    auto createResult = co_await meta.create({SUPER_USER, PathAt(dirId, "file"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(createResult);
+    auto fileId = createResult->stat.id;
+
+    CO_ASSERT_ERROR(co_await meta.remove({SUPER_USER, fileId, AtFlags(), false}), MetaCode::kNotDirectory);
+    // remove directory
+    CO_ASSERT_ERROR(co_await meta.remove({SUPER_USER, dirId, AtFlags(), false}), MetaCode::kNotEmpty);
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, dirId, AtFlags(), true}));
+  }());
+}
+
+TYPED_TEST(TestRemove, Symlink) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(false);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt("file"), {}, O_EXCL, p644}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, PathAt("symlink1"), "file"}));
+    CO_ASSERT_OK(co_await meta.stat({SUPER_USER, PathAt("symlink1"), AtFlags(AT_SYMLINK_FOLLOW)}));
+    CO_ASSERT_OK(
+        co_await meta.hardLink({SUPER_USER, PathAt("symlink1"), PathAt("symlink2"), AtFlags(AT_SYMLINK_NOFOLLOW)}));
+    CO_ASSERT_OK(co_await meta.stat({SUPER_USER, PathAt("symlink2"), AtFlags(AT_SYMLINK_FOLLOW)}));
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, PathAt("symlink1"), AtFlags(AT_SYMLINK_NOFOLLOW), false}));
+    CO_ASSERT_OK(co_await meta.stat({SUPER_USER, PathAt("symlink2"), AtFlags(AT_SYMLINK_FOLLOW)}));
+  }());
+}
+
+TYPED_TEST(TestRemove, ConflictSet) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(false);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+
+    // remove file
+    for (int i = 0; i < 100; i++) {
+      std::string path = std::to_string(i) + ".file";
+      auto result = co_await meta.create({SUPER_USER, path, {}, 0, p644});
+      CO_ASSERT_OK(result);
+      auto inodeId = result->stat.id;
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = RemoveReq(SUPER_USER, Path(path), AtFlags(0), false);
+            auto removeResult = co_await store.remove(req)->run(txn);
+            CO_ASSERT_OK(removeResult);
+          },
+          (std::vector<String>{
+              MetaTestHelper::getDirEntryKey(InodeId::root(), path),
+              MetaTestHelper::getInodeKey(inodeId),
+          }),
+          (std::vector<String>{
+              MetaTestHelper::getInodeKey(inodeId),
+          }),
+          false);
+    }
+
+    // remove directory
+    for (int i = 0; i < 1; i++) {
+      std::string path = std::to_string(i) + ".directory";
+      auto result = co_await meta.mkdirs({SUPER_USER, path, p755, false});
+      CO_ASSERT_OK(result);
+      auto inodeId = result->stat.id;
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = RemoveReq(SUPER_USER, Path(path), AtFlags(0), false);
+            auto removeResult = co_await store.remove(req)->run(txn);
+            CO_ASSERT_OK(removeResult);
+          },
+          (std::vector<String>{
+              MetaTestHelper::getDirEntryKey(InodeId::root(), path),
+          }),
+          (std::vector<String>{
+              MetaTestHelper::getInodeKey(inodeId),  // src Inode
+          }),
+          false);
+    }
+  }());
+}
+
+TYPED_TEST(TestRemove, Idempotent) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(false);
+    config.mock_meta().set_idempotent_record_expire(5_s);
+    config.mock_meta().set_idempotent_record_clean(1_s);
+    config.set_num_meta(1);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    auto remove1 = RemoveReq({SUPER_USER, PathAt("file"), AtFlags(AT_SYMLINK_NOFOLLOW), false});
+    auto remove2 = RemoveReq({SUPER_USER, PathAt("file"), AtFlags(AT_SYMLINK_NOFOLLOW), false});
+    CO_ASSERT_ERROR(co_await meta.remove(remove1), MetaCode::kNotFound);
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt("file"), {}, O_EXCL, p644}));
+    CO_ASSERT_ERROR(co_await meta.remove(remove1), MetaCode::kNotFound);
+    CO_ASSERT_OK(co_await meta.remove(remove2));
+    CO_ASSERT_OK(co_await meta.remove(remove2));
+    CO_ASSERT_ERROR(co_await meta.remove(remove1), MetaCode::kNotFound);
+    co_await folly::coro::sleep(2_s);
+    CO_ASSERT_OK(co_await meta.remove(remove2));
+    co_await folly::coro::sleep(5_s);
+    CO_ASSERT_ERROR(co_await meta.remove(remove2), MetaCode::kNotFound);
+    // remove check inode id
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, PathAt("file"), {}, O_EXCL, p644}));
+    CO_ASSERT_ERROR(
+        co_await meta.remove(
+            {SUPER_USER, PathAt("file"), AtFlags(AT_SYMLINK_NOFOLLOW), false, false, InodeId(folly::Random::rand64())}),
+        MetaCode::kNotFound);
+    co_return;
+  }());
+}
+
+TYPED_TEST(TestRemove, ConcurrentCreate) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(false);
+    config.set_num_meta(1);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+
+    // rmdir and concurrent create should conflict
+    for (int i = 0; i < 100; i++) {
+      auto dirPath = std::to_string(i) + ".directory";
+      auto childPath = "child";
+      auto mkdir = co_await meta.mkdirs({SUPER_USER, dirPath, p755, true});
+      CO_ASSERT_OK(mkdir);
+      auto parentId = mkdir->stat.id;
+      auto remove = [&](auto &txn) -> CoTask<void> {
+        auto req = RemoveReq(SUPER_USER, Path(dirPath), AtFlags(0), false);
+        auto removeResult = co_await store.remove(req)->run(txn);
+        CO_ASSERT_OK(removeResult);
+      };
+      auto create = [&](auto &txn) -> CoTask<void> {
+        if (i % 2 == 0) {
+          auto req = MkdirsReq(SUPER_USER, PathAt(parentId, childPath), p755, false);
+          auto mkdirResult = co_await store.mkdirs(req)->run(txn);
+          CO_ASSERT_OK(mkdirResult);
+        } else {
+          auto req = CreateReq(SUPER_USER, PathAt(parentId, childPath), {}, O_RDONLY, p644);
+          auto createResult = co_await BatchedOp::create(store, txn, req);
+          CO_ASSERT_OK(createResult);
+        }
+      };
+      if (i % 2 == 0) {
+        CO_ASSERT_CONFLICT(remove, create);
+      } else {
+        CO_ASSERT_CONFLICT(create, remove);
+      }
+    }
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestRename.cc

@@ -0,0 +1,332 @@
+#include <cstdlib>
+#include <fcntl.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Task.h>
+#include <functional>
+#include <gtest/gtest.h>
+#include <optional>
+#include <string>
+
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "fbs/core/user/User.h"
+#include "fbs/meta/Common.h"
+#include "gtest/gtest.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/ops/BatchOperation.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestRename : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestRename, KVTypes);
+
+TYPED_TEST(TestRename, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    FAULT_INJECTION_SET(10, 5);
+
+    // create a, rename a -> b, stat a -> InodeId::root(), stat b -> Inode
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "a", {}, O_RDONLY, p644}));
+    CO_ASSERT_OK(co_await meta.rename({SUPER_USER, "a", "b"}));
+    auto a = co_await meta.stat({SUPER_USER, "a", AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_ERROR(a, MetaCode::kNotFound);
+    auto b = co_await meta.stat({SUPER_USER, "b", AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_OK(b);
+
+    // create file c, d, rename c -> d, c will replace d
+    InodeId cId;
+    auto c = co_await meta.create({SUPER_USER, "c", {}, O_RDONLY, p644});
+    auto d = co_await meta.create({SUPER_USER, "d", {}, O_RDONLY, p644});
+    CO_ASSERT_OK(c);
+    CO_ASSERT_OK(d);
+    cId = c->stat.id;
+
+    // check rename return inode
+    auto renameResult = co_await meta.rename({SUPER_USER, "c", "d"});
+    CO_ASSERT_OK(renameResult);
+    CO_ASSERT_TRUE(renameResult->stat.has_value());
+    CO_ASSERT_EQ(renameResult->stat->id, cId);
+    auto statResult = co_await meta.stat({SUPER_USER, "d", AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_OK(statResult);
+    CO_ASSERT_EQ(statResult->stat, renameResult->stat);
+
+    auto c1 = co_await meta.stat({SUPER_USER, "c", AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_ERROR(c1, MetaCode::kNotFound);
+    auto d1 = co_await meta.stat({SUPER_USER, "d", AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_OK(d1);
+    CO_ASSERT_EQ(d1->stat.id, cId);
+
+    // create directory e, f/g, rename e -> f, should return not empty
+    auto f = co_await meta.mkdirs({SUPER_USER, "e", p755, false});
+    auto g = co_await meta.mkdirs({SUPER_USER, "f/g", p755, true});
+    CO_ASSERT_OK(f);
+    CO_ASSERT_OK(g);
+
+    CO_ASSERT_ERROR(co_await meta.rename({SUPER_USER, "e", "f"}), MetaCode::kNotEmpty);
+
+    // remove f/g
+    FAULT_INJECTION_SET(0, 0);
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "f/g", AtFlags(), false}));
+
+    // rename e -> f again
+    FAULT_INJECTION_SET(10, 5);
+    CO_ASSERT_OK(co_await meta.rename({SUPER_USER, "e", "f"}));
+  }());
+}
+
+TYPED_TEST(TestRename, Directory) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir1", p777, false}));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir2", p777, false}));
+    auto stat1 = (co_await meta.stat({SUPER_USER, "dir1", AtFlags()}))->stat;
+    CO_ASSERT_EQ(stat1.asDirectory().name, "dir1");
+    CO_ASSERT_OK(co_await meta.rename({SUPER_USER, "dir1", "dir2"}));
+    auto stat2 = (co_await meta.stat({SUPER_USER, "dir2", AtFlags()}))->stat;
+    CO_ASSERT_EQ(stat2.asDirectory().name, "dir2");
+    CO_ASSERT_EQ(stat2.id, stat1.id);
+  }());
+}
+
+TYPED_TEST(TestRename, Trash) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(true);
+    config.mock_meta().gc().set_gc_directory_delay(0_s);
+    config.mock_meta().gc().set_gc_file_delay(0_s);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "trash", p777, false}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "dir/a", {}, O_RDONLY, p644}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "dir/b", {}, O_RDONLY, p644}));
+    CO_ASSERT_OK(co_await meta.rename({SUPER_USER, "dir/a", "trash/trash_name", true}));
+    CO_ASSERT_ERROR(co_await meta.rename({SUPER_USER, "dir/b", "trash/trash_name", true}), MetaCode::kExists);
+  }());
+}
+
+TYPED_TEST(TestRename, GC) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.mock_meta().gc().set_enable(true);
+    config.mock_meta().gc().set_gc_directory_delay(0_s);
+    config.mock_meta().gc().set_gc_file_delay(0_s);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+
+    // rename directory to trash, should be owner and rwx permission
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "trash", p777, true}));
+    auto ua = flat::UserInfo(flat::Uid(1), flat::Gid(1));
+    auto ub = flat::UserInfo(flat::Uid(2), flat::Gid(2));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "ua", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "ua/data-500", flat::Permission(0500), true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "ua/data-777", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "trash/ua", p777, true}));
+    CO_ASSERT_ERROR(co_await meta.rename({ua, "ua/data-777", "trash/data-777"}), MetaCode::kNoPermission);
+    CO_ASSERT_OK(co_await meta.rename({ua, "ua/data-777", "trash/data-777", true}));
+    CO_ASSERT_ERROR(co_await meta.rename({ua, "ua/data-500", "trash/data-500", true}), MetaCode::kNoPermission);
+    CO_ASSERT_OK(co_await meta.rename({ua, "trash/ua", "trash/ua-dest"}));
+
+    CO_ASSERT_OK(co_await meta.mkdirs({ua, "ua/dir_has_root", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "ua/dir_has_root/root", p700, true}));
+    CO_ASSERT_ERROR(co_await meta.rename({ua, "ua/dir_has_root", "trash/dir_has_root", true}), MetaCode::kNoPermission);
+
+    // rename directory to another directory which is under GC, should fail
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir1/subdir", p777, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "dir2", p777, false}));
+    auto stat = (co_await meta.stat({SUPER_USER, "dir1", AtFlags()}))->stat;
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, "dir1", AtFlags(), true}));
+    CO_ASSERT_ERROR(co_await meta.rename({SUPER_USER, "dir2", PathAt(stat.id, "dir")}), MetaCode::kNotFound);
+    CO_ASSERT_ERROR(co_await meta.rename({SUPER_USER, "dir2", PathAt(InodeId::gcRoot(), "orphan")}),
+                    MetaCode::kNoPermission);
+
+    // rename replace a file, the file should be removed by GC.
+    InodeId oldDstId;
+    // create src and dst
+    auto createSrc = co_await meta.create({SUPER_USER, "a", {}, O_RDONLY, p644});
+    auto createDst = co_await meta.create({SUPER_USER, "b", {}, O_RDONLY, p644});
+    CO_ASSERT_OK(createSrc);
+    CO_ASSERT_OK(createDst);
+    oldDstId = createDst->stat.id;
+
+    // do rename to replace old dst
+    CO_ASSERT_OK(co_await meta.rename({SUPER_USER, "a", "b"}));
+    // wait GC tasks
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    // GC should remove old dst
+    CO_ASSERT_INODE_NOT_EXISTS(oldDstId);
+  }());
+}
+
+TYPED_TEST(TestRename, ConflictSet) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+
+    for (int i = 0; i < 100; i++) {
+      std::string src = std::to_string(i) + ".src";
+      std::string dst = std::to_string(i) + ".dst";
+      InodeId srcId;
+      // create src first
+      auto result = co_await meta.mkdirs({SUPER_USER, src, p755, false});
+      CO_ASSERT_OK(result);
+      srcId = result->stat.id;
+
+      CHECK_CONFLICT_SET(
+          [&](auto &txn) -> CoTask<void> {
+            auto req = RenameReq(SUPER_USER, Path(src), Path(dst));
+            auto removeResult = co_await store.rename(req)->run(txn);
+            CO_ASSERT_OK(removeResult);
+          },
+          (std::vector<String>{
+              MetaTestHelper::getDirEntryKey(InodeId::root(), src),  // src dirEntry
+              MetaTestHelper::getDirEntryKey(InodeId::root(), dst),  // dst dirEntry
+              MetaTestHelper::getInodeKey(InodeId::root()),          // dst parent Inode
+              MetaTestHelper::getInodeKey(srcId),                    // src Inode
+          }),
+          (std::vector<String>{
+              MetaTestHelper::getDirEntryKey(InodeId::root(), src),  // src dirEntry
+              MetaTestHelper::getDirEntryKey(InodeId::root(), dst),  // dst dirEntry
+              MetaTestHelper::getInodeKey(srcId),                    // src Inode
+          }),
+          false);
+    }
+  }());
+}
+
+TYPED_TEST(TestRename, Concurrent) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.set_num_meta(1);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+
+    auto rename = [&](Path src, Path dst) -> std::function<CoTask<void>(IReadWriteTransaction &)> {
+      return [=, &store](auto &txn) -> CoTask<void> {  // rename src to dst
+        auto req = RenameReq(SUPER_USER, Path(src), Path(dst));
+        auto renameResult = co_await store.rename(req)->run(txn);
+        CO_ASSERT_OK(renameResult);
+      };
+    };
+    auto create =
+        [&](InodeId parent, Path path, bool directory) -> std::function<CoTask<void>(IReadWriteTransaction &)> {
+      EXPECT_FALSE(path.has_parent_path());
+      return [=, &store](auto &txn) -> CoTask<void> {  // create or mkdir at dst
+        if (directory) {
+          auto req = MkdirsReq(SUPER_USER, PathAt(parent, path), p755, true);
+          auto mkdirResult = co_await store.mkdirs(req)->run(txn);
+          CO_ASSERT_OK(mkdirResult);
+        } else {
+          auto req = CreateReq(SUPER_USER, PathAt(parent, path), {}, O_RDONLY, p644);
+          CO_ASSERT_OK(co_await BatchedOp::create(store, txn, req));
+        }
+      };
+    };
+
+    // rename src -> dst and concurrent create dst should conflict
+    for (int i = 0; i < 100; i++) {
+      std::string src = std::to_string(i) + ".src-1";
+      std::string dst = std::to_string(i) + ".dst-1";
+      // create src first
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, src, {}, O_RDONLY, p644}));
+      if (i % 2 == 0) {
+        CO_ASSERT_CONFLICT(rename(src, dst), create(InodeId::root(), dst, i % 2 == 0));
+      } else {
+        CO_ASSERT_CONFLICT(create(InodeId::root(), dst, i % 2 == 0), rename(src, dst));
+      }
+    }
+
+    // rename to same destination should conflict
+    for (int i = 0; i < 100; i++) {
+      std::string src1 = std::to_string(i) + ".src-2.1";
+      std::string src2 = std::to_string(i) + ".src-2.2";
+      std::string dst = std::to_string(i) + ".dst-2";
+      for (auto path : {src1, src2}) {
+        CO_ASSERT_OK(co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644}));
+      }
+      auto rename1 = rename(src1, dst);  // rename src1 to dst
+      auto rename2 = rename(src2, dst);  // rename src2 to dst
+      if (i % 2 == 0) {
+        CO_ASSERT_CONFLICT(rename1, rename2);
+      } else {
+        CO_ASSERT_CONFLICT(rename2, rename1);
+      }
+    }
+
+    // rename src to different dst should be conflict
+    for (int i = 0; i < 100; i++) {
+      std::string src = std::to_string(i) + ".src-3";
+      std::string dst1 = std::to_string(i) + ".dst-3.1";
+      std::string dst2 = std::to_string(i) + ".dst-3.2";
+      // create src1, src2 first
+      CO_ASSERT_OK(co_await meta.create({SUPER_USER, Path(src), {}, O_RDONLY, p644}));
+      auto rename1 = rename(src, dst1);  // rename src to dst1
+      auto rename2 = rename(src, dst2);  // rename src to dst2
+      if (i % 2 == 0) {
+        CO_ASSERT_CONFLICT(rename1, rename2);
+      } else {
+        CO_ASSERT_CONFLICT(rename2, rename1);
+      }
+    }
+
+    // rename different srcs to different dsts shouldn't be conflict
+    for (int i = 0; i < 100; i++) {
+      std::string src1 = std::to_string(i) + ".src-4.1";
+      std::string src2 = std::to_string(i) + ".src-4.2";
+      std::string dst1 = std::to_string(i) + ".dst-4.1";
+      std::string dst2 = std::to_string(i) + ".dst-4.2";
+      for (auto path : {src1, src2}) {
+        CO_ASSERT_OK(co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644}));
+      }
+      auto rename1 = rename(src1, dst1);  // rename src1 to dst1
+      auto rename2 = rename(src2, dst2);  // rename src2 to dst2
+      CO_ASSERT_NO_CONFLICT(rename1, rename2);
+    }
+
+    // rename replace a empty directory and create under directory should be conflict.
+    for (int i = 0; i < 100; i++) {
+      std::string src = std::to_string(i) + ".src-5";
+      std::string dst = std::to_string(i) + ".dst-5";
+      InodeId oldDstId;
+      // create src and dst first
+      auto result = co_await meta.mkdirs({SUPER_USER, src, p755, false});
+      CO_ASSERT_OK(result);
+      result = co_await meta.mkdirs({SUPER_USER, dst, p755, false});
+      CO_ASSERT_OK(result);
+      oldDstId = result->stat.id;
+      if (i % 2 == 0) {
+        CO_ASSERT_CONFLICT(rename(src, dst), create(oldDstId, "child", (i % 2 == 0)));
+      } else {
+        CO_ASSERT_CONFLICT(create(oldDstId, "child", (i % 2 == 0)), rename(src, dst));
+      }
+    }
+
+    // mkdir /a /b/d, rename /a -> /b/d/e and rename /b/d -> /a/c should conflict
+    // mkdir /a, /b/d/e
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "a", p755, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "b/d", p755, true}));
+    if (folly::Random::oneIn(2)) {
+      CO_ASSERT_CONFLICT(rename("/a", "/b/d/e"), rename("/b/d", "/a/c"));
+    } else {
+      CO_ASSERT_CONFLICT(rename("/b/d", "/a/c"), rename("/a", "/b/d/e"));
+    }
+  }());
+}
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestResolve.cc

@@ -0,0 +1,344 @@
+#include <bits/types/FILE.h>
+#include <fcntl.h>
+#include <fmt/core.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Task.h>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <iterator>
+#include <variant>
+#include <vector>
+
+#include "common/utils/Coroutine.h"
+#include "common/utils/StatusCode.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/meta/Common.h"
+#include "meta/components/AclCache.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/PathResolve.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestResolve : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestResolve, KVTypes);
+
+TYPED_TEST(TestResolve, ResolveComponent) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    AclCache aclCache(1 << 20);
+
+    READ_ONLY_TRANSACTION({
+      // when resolve with inexistent dentry, do not return error
+      DirEntry parentEntry = DirEntry::newDirectory(MetaTestHelper::randomInodeId(),
+                                                    "not-exists-dir",
+                                                    MetaTestHelper::randomInodeId(),
+                                                    rootp777);
+      auto resolveResult = co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathComponent(parentEntry, "not-exists");
+      CO_ASSERT_OK(resolveResult);
+
+      // parentId other than rootId will trigger inode load
+      auto parentId = MetaTestHelper::randomInodeId();
+      resolveResult = co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathComponent(parentId, "not-exists");
+      CO_ASSERT_ERROR(resolveResult, MetaCode::kNotFound);
+    });
+
+    {  // if parent is not directory, should return kNotDirectory
+      Inode parentInode = Inode::newFile(MetaTestHelper::randomInodeId(),
+                                         rootp644,
+                                         MetaTestHelper::randomLayout(),
+                                         UtcClock::now().castGranularity(1_s));
+      READ_WRITE_TRANSACTION_OK({
+        auto storeInodeResult = co_await parentInode.store(*txn);
+        CO_ASSERT_OK(storeInodeResult);
+      });
+      READ_ONLY_TRANSACTION({
+        auto resolveResult =
+            co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathComponent(parentInode.id, "not-exists");
+        CO_ASSERT_ERROR(resolveResult, MetaCode::kNotDirectory);
+      });
+    }
+
+    {
+      // lookup not exist path, if user doesn't have permission, should return kNoPermission, otherwise return
+      // kNotFound.
+      Inode parentInode = Inode::newDirectory(MetaTestHelper::randomInodeId(),
+                                              InodeId::root(),
+                                              "parent",
+                                              rootp700,
+                                              Layout(),
+                                              UtcClock::now().castGranularity(1_s));
+      DirEntry parentEntry = DirEntry::newDirectory(InodeId::root(), "dir-name", parentInode.id, rootp700);
+      CO_ASSERT_TRUE(parentInode.isDirectory());
+      READ_WRITE_TRANSACTION_OK({
+        // create a directory
+        auto storeInodeResult = co_await parentInode.store(*txn);
+        CO_ASSERT_OK(storeInodeResult);
+      });
+      READ_ONLY_TRANSACTION({
+        // check permission
+        flat::UserInfo otherUser(Uid(1), Gid(1), String());
+        auto resolveResult =
+            co_await PathResolveOp(*txn, aclCache, otherUser).pathComponent(parentInode.id, "not-exists");
+        CO_ASSERT_ERROR(resolveResult, MetaCode::kNoPermission);
+
+        resolveResult = co_await PathResolveOp(*txn, aclCache, otherUser).pathComponent(parentEntry, "not-exists");
+        CO_ASSERT_ERROR(resolveResult, MetaCode::kNoPermission);
+
+        // should return parent info if parent exists but dirEntry not exists
+        // pass in parentId other than rootId will trigger inode load
+        resolveResult = co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathComponent(parentInode.id, "not-exists");
+        CO_ASSERT_OK(resolveResult);
+        CO_ASSERT_EQ(resolveResult->getParentId(), parentInode.id);
+        CO_ASSERT_EQ(resolveResult->getParentAcl(), parentInode.acl);
+
+        resolveResult = co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathComponent(parentEntry, "not-exists");
+        CO_ASSERT_OK(resolveResult);
+        CO_ASSERT_TRUE(!std::holds_alternative<Inode>(resolveResult->parent));
+        CO_ASSERT_EQ(resolveResult->getParentId(), parentInode.id);
+      });
+    }
+
+    {
+      // create a directory and a entry
+      Inode parentInode = Inode::newDirectory(MetaTestHelper::randomInodeId(),
+                                              InodeId::root(),
+                                              "parent",
+                                              rootp700,
+                                              Layout(),
+                                              UtcClock::now().castGranularity(1_s));
+      DirEntry childEntry = DirEntry::newFile(parentInode.id, "file", MetaTestHelper::randomInodeId());
+      CO_ASSERT_TRUE(parentInode.isDirectory());
+      READ_WRITE_TRANSACTION_OK({
+        auto storeInodeResult = co_await parentInode.store(*txn);
+        auto storeEntryResult = co_await childEntry.store(*txn);
+        CO_ASSERT_OK(storeInodeResult);
+        CO_ASSERT_OK(storeEntryResult);
+      });
+      READ_ONLY_TRANSACTION({
+        auto resolveResult = co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathComponent(parentInode.id, "file");
+        CO_ASSERT_OK(resolveResult);
+        CO_ASSERT_TRUE(std::holds_alternative<Inode>(resolveResult->parent));
+        CO_ASSERT_EQ(std::get<Inode>(resolveResult->parent), parentInode);
+        CO_ASSERT_TRUE(resolveResult->dirEntry.has_value());
+        CO_ASSERT_EQ(resolveResult->dirEntry.value(), childEntry);
+      });
+    }
+  }());
+}
+
+TYPED_TEST(TestResolve, pathRange) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    AclCache aclCache(1 << 20);
+    READ_ONLY_TRANSACTION({
+      // if parent doesn't exists, do not return error
+      // if path is empty, return kNotFound
+      auto parentId = MetaTestHelper::randomInodeId();
+      auto path = PathAt(parentId, "/a/b/c");
+      auto result1 = co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathRange(path);
+      CO_ASSERT_OK(result1);
+
+      path = PathAt(parentId, "a/b/c");
+      auto result2 = co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathRange(path);
+      CO_ASSERT_ERROR(result2, MetaCode::kNotFound);
+    });
+
+    // create some directory file and link
+    auto ts = UtcClock::now().castGranularity(1_s);
+    auto root = Inode::newDirectory(InodeId::root(), InodeId::root(), "/", rootp755, Layout(), ts);
+    auto a =
+        Inode::newDirectory(MetaTestHelper::randomInodeId(), InodeId::root(), "a", rootp700, Layout(), ts);  // dir: /a
+    auto aEntry = DirEntry::newDirectory(InodeId::root(), "a", a.id, rootp700);
+    auto b = Inode::newDirectory(MetaTestHelper::randomInodeId(), a.id, "b", rootp700, Layout(), ts);  // dir: /a/b
+    auto bEntry = DirEntry::newDirectory(a.id, "b", b.id, rootp700);
+    auto c = Inode::newDirectory(MetaTestHelper::randomInodeId(), b.id, "c", rootp700, Layout(), ts);  // dir: /a/b/c
+    auto cEntry = DirEntry::newDirectory(b.id, "c", c.id, rootp700);
+    auto d = Inode::newFile(MetaTestHelper::randomInodeId(),
+                            rootp644,
+                            MetaTestHelper::randomLayout(),
+                            ts);  // file: /a/b/c/d
+    auto dEntry = DirEntry::newFile(c.id, "d", d.id);
+    auto e = Inode::newSymlink(MetaTestHelper::randomInodeId(),
+                               "../c",
+                               rootUid,
+                               rootGid,
+                               ts);  // symlink: /a/b/c/e -> '../c'
+    auto eEntry = DirEntry::newSymlink(c.id, "e", e.id);
+    auto f = Inode::newSymlink(MetaTestHelper::randomInodeId(), "d", rootUid, rootGid, ts);  // symlink: /a/b/c/f -> 'd'
+    auto fEntry = DirEntry::newSymlink(c.id, "f", f.id);
+    auto g = Inode::newSymlink(MetaTestHelper::randomInodeId(),
+                               "g",
+                               rootUid,
+                               rootGid,
+                               ts);  // symlink: /a/b/c/g -> 'g' (loop)
+    auto gEntry = DirEntry::newSymlink(c.id, "g", g.id);
+    std::vector<Inode> inodes = {root, a, b, c, d, e, f, g};
+    std::vector<DirEntry> entries = {aEntry, bEntry, cEntry, dEntry, eEntry, fEntry, gEntry};
+    READ_WRITE_TRANSACTION_OK({
+      for (auto &inode : inodes) {
+        CO_ASSERT_OK(co_await inode.store(*txn));
+      }
+      for (auto &entry : entries) {
+        CO_ASSERT_OK(co_await entry.store(*txn));
+      }
+    });
+
+    READ_ONLY_TRANSACTION({
+      // resolve: /a/b/c/d -> d
+      auto path = PathAt("/a/b/c/d");
+      Path trace;
+      auto result = co_await PathResolveOp(*txn, aclCache, SUPER_USER, &trace).pathRange(path);
+      std::cout << "resolve " << *path.path << " -> " << trace << " " << trace.normalize() << std::endl;
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(result->missing.empty()) << result->missing;
+      CO_ASSERT_EQ(result->getParentId(), c.id);
+      CO_ASSERT_EQ(result->dirEntry.value(), dEntry);
+    });
+
+    READ_ONLY_TRANSACTION({
+      // resolve: /a/b/c/e -> e
+      auto path = Path("/a/b/c/e");
+      Path trace;
+      auto result = co_await PathResolveOp(*txn, aclCache, SUPER_USER, &trace).pathRange(path);
+      std::cout << "resolve " << path << " -> " << trace << " " << trace.normalize() << std::endl;
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(result->missing.empty()) << result->missing;
+      CO_ASSERT_EQ(result->getParentId(), c.id);
+      CO_ASSERT_EQ(result->dirEntry.value(), eEntry);
+    });
+
+    READ_ONLY_TRANSACTION({
+      // resolve: /a/b/c/e/d -> should get d
+      auto path = PathAt("/a/b/c/e/d");
+      Path trace;
+      auto result = co_await PathResolveOp(*txn, aclCache, SUPER_USER, &trace).pathRange(path);
+      CO_ASSERT_OK(result);
+      std::cout << "resolve " << *path.path << " -> " << trace << " " << trace.normalize() << std::endl;
+      CO_ASSERT_TRUE(result->missing.empty()) << result->missing;
+      CO_ASSERT_EQ(result->getParentId(), c.id);
+      CO_ASSERT_EQ(result->dirEntry.value(), dEntry);
+    });
+
+    READ_ONLY_TRANSACTION({
+      // resolve: /a/b/c/f/xyz -> /a/b/c/f is symlink point to file, should get kNotDirectory
+      auto path = PathAt("/a/b/c/f/xyz");
+      Path trace;
+      auto result = co_await PathResolveOp(*txn, aclCache, SUPER_USER, &trace).pathRange(path);
+      CO_ASSERT_ERROR(result, MetaCode::kNotDirectory);
+    });
+
+    READ_ONLY_TRANSACTION({
+      // resolve: /a/b/c/g/xyz, /a/b/c/g is a symlink that cause loop, should get kTooManySymlinks
+      auto path = PathAt("a/b/c/g/xyz");  // should still work
+      Path trace;
+      auto result = co_await PathResolveOp(*txn, aclCache, SUPER_USER, &trace).pathRange(path);
+      CO_ASSERT_ERROR(result, MetaCode::kTooManySymlinks);
+    });
+
+    READ_ONLY_TRANSACTION({
+      // resolve /a/x/y/z -> stop at x
+      auto path = PathAt("/a/x/y/z");
+      auto result = co_await PathResolveOp(*txn, aclCache, SUPER_USER).pathRange(path);
+      CO_ASSERT_OK(result);
+      CO_ASSERT_EQ(result->missing, "x/y/z");
+    });
+  }());
+}
+
+TYPED_TEST(TestResolve, path) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    AclCache aclCache(1 << 20);
+
+    // some path /a/b/c here
+    auto ts = UtcClock::now().castGranularity(1_s);
+    auto root = Inode::newDirectory(InodeId::root(), InodeId::root(), "/", rootp755, Layout(), ts);
+    auto a =
+        Inode::newDirectory(MetaTestHelper::randomInodeId(), InodeId::root(), "a", rootp700, Layout(), ts);  // dir: /a
+    auto aEntry = DirEntry::newDirectory(InodeId::root(), "a", a.id, rootp700);
+    auto b = Inode::newDirectory(MetaTestHelper::randomInodeId(), a.id, "b", rootp700, Layout(), ts);  // dir: /a/b
+    auto bEntry = DirEntry::newDirectory(a.id, "b", b.id, rootp700);
+    auto c = Inode::newDirectory(MetaTestHelper::randomInodeId(), b.id, "c", rootp700, Layout(), ts);  // dir: /a/b/c
+    auto cEntry = DirEntry::newDirectory(b.id, "c", c.id, rootp700);
+    std::vector<Inode> inodes = {root, a, b, c};
+    std::vector<DirEntry> entries = {aEntry, bEntry, cEntry};
+    READ_WRITE_TRANSACTION_OK({
+      for (auto &inode : inodes) {
+        auto result = co_await inode.store(*txn);
+        CO_ASSERT_OK(result);
+      }
+      for (auto &entry : entries) {
+        auto result = co_await entry.store(*txn);
+        CO_ASSERT_OK(result);
+      }
+    });
+
+    // todo: should add more test for follow symlink
+
+    READ_ONLY_TRANSACTION({
+      // resolve /a/b/c
+      auto result =
+          co_await PathResolveOp(*txn, aclCache, SUPER_USER).path(Path("/a/b/c"), AtFlags(AT_SYMLINK_NOFOLLOW));
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(!std::holds_alternative<Inode>(result->parent));
+      CO_ASSERT_EQ(result->getParentId(), b.id);
+      CO_ASSERT_TRUE(result->dirEntry.has_value());
+      CO_ASSERT_EQ(result->dirEntry.value(), cEntry);
+
+      // resolve /a/b/../b/c
+      result =
+          co_await PathResolveOp(*txn, aclCache, SUPER_USER).path(Path("/a/b/../b/c"), AtFlags(AT_SYMLINK_NOFOLLOW));
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(!std::holds_alternative<Inode>(result->parent));
+      CO_ASSERT_EQ(result->getParentId(), b.id);
+      CO_ASSERT_TRUE(result->dirEntry.has_value());
+      CO_ASSERT_EQ(result->dirEntry.value(), cEntry);
+
+      // resolve /a/b/c/../c
+      result =
+          co_await PathResolveOp(*txn, aclCache, SUPER_USER).path(Path("/a/b/../b/c"), AtFlags(AT_SYMLINK_NOFOLLOW));
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(!std::holds_alternative<Inode>(result->parent));
+      CO_ASSERT_EQ(result->getParentId(), b.id);
+      CO_ASSERT_TRUE(result->dirEntry.has_value());
+      CO_ASSERT_EQ(result->dirEntry.value(), cEntry);
+    });
+
+    READ_ONLY_TRANSACTION({
+      // resolve /a/d -> parent found but dirEntry not found
+      auto result = co_await PathResolveOp(*txn, aclCache, SUPER_USER).path(Path("/a/d"), AtFlags(AT_SYMLINK_NOFOLLOW));
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(!std::holds_alternative<Inode>(result->parent));
+      CO_ASSERT_EQ(result->getParentId(), a.id);
+      CO_ASSERT_FALSE(result->dirEntry.has_value());
+
+      // resolve /a/../a/d
+      result = co_await PathResolveOp(*txn, aclCache, SUPER_USER).path(Path("/a/d"), AtFlags(AT_SYMLINK_NOFOLLOW));
+      CO_ASSERT_OK(result);
+      CO_ASSERT_TRUE(!std::holds_alternative<Inode>(result->parent));
+      CO_ASSERT_EQ(result->getParentId(), a.id);
+      CO_ASSERT_FALSE(result->dirEntry.has_value());
+    });
+
+    READ_ONLY_TRANSACTION({
+      // resolve /a/d/e -> kNotFound
+      auto result =
+          co_await PathResolveOp(*txn, aclCache, SUPER_USER).path(Path("/a/d/e"), AtFlags(AT_SYMLINK_NOFOLLOW));
+      CO_ASSERT_ERROR(result, MetaCode::kNotFound);
+
+      // resolve /a/d/../d/e
+      result = co_await PathResolveOp(*txn, aclCache, SUPER_USER).path(Path("/a/d/e"), AtFlags(AT_SYMLINK_NOFOLLOW));
+      CO_ASSERT_ERROR(result, MetaCode::kNotFound);
+    });
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestSetPermission.cc

@@ -0,0 +1,145 @@
+#include <algorithm>
+#include <folly/Random.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/futures/ManualTimekeeper.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <vector>
+
+#include "common/utils/FaultInjection.h"
+#include "common/utils/StatusCode.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/meta/Service.h"
+#include "gtest/gtest.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/MetaStore.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestSetPermission : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestSetPermission, KVTypes);
+
+TYPED_TEST(TestSetPermission, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p777}));
+    CO_ASSERT_OK(co_await meta.stat({SUPER_USER, "file", AtFlags(AT_SYMLINK_FOLLOW)}));
+
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, "file", AtFlags(), {}, {}, p700)));
+
+    flat::UserInfo ua(Uid(1), Gid(1), String());
+    CO_ASSERT_ERROR(co_await meta.setAttr(SetAttrReq::setPermission(ua, "file", AtFlags(), {}, {}, p755)),
+                    MetaCode::kNoPermission);
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, "file", AtFlags(), ua.uid, ua.gid, {})));
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setPermission(ua, "file", AtFlags(), {}, {}, p700)));
+    CO_ASSERT_ERROR(co_await meta.setAttr(SetAttrReq::setPermission(ua, "file", AtFlags(), ua.uid, rootGid, {})),
+                    MetaCode::kNoPermission);
+  }());
+}
+
+TYPED_TEST(TestSetPermission, ByInodeId) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    auto file = (co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p777}))->stat;
+    auto directory = (co_await meta.mkdirs({SUPER_USER, "directory", p777, false}))->stat;
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, file.id, AtFlags(), {}, {}, p700)));
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, directory.id, AtFlags(), {}, {}, p700)));
+
+    auto fileStat = (co_await meta.stat({SUPER_USER, "file", AtFlags()}))->stat;
+    CO_ASSERT_EQ(fileStat.acl.perm, p700);
+    auto dirStat = (co_await meta.stat({SUPER_USER, "directory", AtFlags()}))->stat;
+    CO_ASSERT_EQ(dirStat.acl.perm, p700);
+    auto dirStat2 = (co_await meta.stat({SUPER_USER, "directory", AtFlags()}))->stat;
+    CO_ASSERT_EQ(dirStat2.acl.perm, p700);
+
+    READ_WRITE_TRANSACTION_OK({
+      directory.asDirectory().name = "";
+      CO_ASSERT_OK(co_await meta::server::Inode(directory).store(*txn));
+    });
+
+    CO_ASSERT_OK(co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, directory.id, AtFlags(), {}, {}, p755)));
+    auto dirStat3 = (co_await meta.stat({SUPER_USER, "directory", AtFlags()}))->stat;
+    CO_ASSERT_EQ(dirStat3.acl.perm, p755);
+    CO_ASSERT_EQ(dirStat3.asDirectory().name, "directory");
+
+    CO_ASSERT_OK(co_await meta.setAttr(
+        SetAttrReq::setPermission(SUPER_USER, PathAt(directory.id, "."), AtFlags(), {}, {}, p700)));
+    CO_ASSERT_EQ((co_await meta.stat({SUPER_USER, "directory", AtFlags()}))->stat.acl.perm, p700);
+    READ_ONLY_TRANSACTION({
+      CO_ASSERT_EQ((co_await DirEntry::snapshotLoad(*txn, InodeId::root(), "directory")).value()->dirAcl->perm, p700);
+      CO_ASSERT_EQ((co_await DirEntry::snapshotLoad(*txn, directory.id, ".")).value(), std::nullopt);
+    });
+
+    CO_ASSERT_OK(co_await meta.remove({SUPER_USER, PathAt(InodeId::root(), "directory"), AtFlags(), false}));
+    CO_ASSERT_ERROR(co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, directory.id, AtFlags(), {}, {}, p700)),
+                    MetaCode::kNotFound);
+    CO_ASSERT_ERROR(co_await meta.setAttr(
+                        SetAttrReq::setPermission(SUPER_USER, PathAt(directory.id, "."), AtFlags(), {}, {}, p700)),
+                    MetaCode::kNotFound);
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "directory", p777, false}));
+    CO_ASSERT_ERROR(co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, directory.id, AtFlags(), {}, {}, p700)),
+                    MetaCode::kNotFound);
+
+    CO_ASSERT_ERROR(
+        co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, InodeId::root(), AtFlags(), {}, {}, p700)),
+        MetaCode::kNoPermission);
+  }());
+}
+
+TYPED_TEST(TestSetPermission, ConflictSet) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+    // create file, directory
+    auto createResult = co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p777});
+    CO_ASSERT_OK(createResult);
+    auto fileId = createResult->stat.id;
+
+    auto mkdirResult = co_await meta.mkdirs({SUPER_USER, "directory", p777, false});
+    CO_ASSERT_OK(mkdirResult);
+    auto dirId = mkdirResult->stat.id;
+
+    CHECK_CONFLICT_SET(
+        [&](auto &txn) -> CoTask<void> {
+          auto req = SetAttrReq::setPermission(SUPER_USER, "directory", AtFlags(0), {}, {}, p700);
+          auto result = co_await store.setAttr(req)->run(txn);
+          CO_ASSERT_OK(result);
+        },
+        (std::vector<String>{
+            MetaTestHelper::getDirEntryKey(InodeId::root(), "directory"),  // dir entry
+            MetaTestHelper::getInodeKey(dirId),                            // inode
+        }),
+        (std::vector<String>{
+            MetaTestHelper::getDirEntryKey(InodeId::root(), "directory"),  // dir entry
+            MetaTestHelper::getInodeKey(dirId),                            // inode
+        }),
+        true);
+
+    CHECK_CONFLICT_SET(
+        [&](auto &txn) -> CoTask<void> {
+          auto req = SetAttrReq::setPermission(SUPER_USER, "file", AtFlags(0), {}, {}, p700);
+          auto result = co_await store.setAttr(req)->run(txn);
+          CO_ASSERT_OK(result);
+        },
+        (std::vector<String>{
+            MetaTestHelper::getInodeKey(fileId),  // inode
+        }),
+        (std::vector<String>{
+            MetaTestHelper::getInodeKey(fileId),  // inode
+        }),
+        true);
+  }());
+}
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestStat.cc

@@ -0,0 +1,100 @@
+#include <fcntl.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/Task.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <string>
+
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/Result.h"
+#include "common/utils/StatusCode.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestStat : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestStat, KVTypes);
+
+TYPED_TEST(TestStat, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    // create a
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "a/b/c", p755, true}));
+
+    auto result = co_await meta.stat({SUPER_USER, "a", AtFlags(AT_SYMLINK_NOFOLLOW)});
+    CO_ASSERT_OK(result);
+    auto a = result->stat;
+    CO_ASSERT_ERROR(co_await meta.stat({SUPER_USER, "b", AtFlags(AT_SYMLINK_NOFOLLOW)}), MetaCode::kNotFound);
+    CO_ASSERT_OK(co_await meta.stat({SUPER_USER, PathAt(a.id, "b"), AtFlags(AT_SYMLINK_NOFOLLOW)}));
+
+    result = co_await meta.stat({SUPER_USER, a.id, AtFlags(AT_SYMLINK_NOFOLLOW)});
+    CO_ASSERT_OK(result);
+    CO_ASSERT_EQ(result->stat, std::optional(a));
+
+    // create file
+    auto cresult = co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p755});
+    CO_ASSERT_OK(cresult);
+    auto file = cresult->stat;
+
+    CO_ASSERT_ERROR(co_await meta.stat({SUPER_USER, PathAt(file.id, "."), AtFlags(AT_SYMLINK_NOFOLLOW)}),
+                    MetaCode::kNotDirectory);
+
+    result = co_await meta.stat({SUPER_USER, file.id, AtFlags(AT_SYMLINK_NOFOLLOW)});
+    CO_ASSERT_OK(result);
+    CO_ASSERT_EQ(result->stat, std::optional(file));
+  }());
+}
+
+TYPED_TEST(TestStat, FollowSymlink) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    // create a/b/c
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "a/b/c", p755, true}));
+    // symlink a/d -> a/b
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "a/d", "/a/b"}));
+    // a/d is symlink but not the last symlink
+    CO_ASSERT_OK(co_await meta.stat({SUPER_USER, "a/d/c", AtFlags(AT_SYMLINK_NOFOLLOW)}));
+  }());
+}
+
+TYPED_TEST(TestStat, FaultInjection) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    std::vector<std::string> paths;
+    for (auto i = 0; i < 10; i++) {
+      std::string path = std::to_string(i);
+      auto result = co_await meta.create({SUPER_USER, path, {}, O_RDONLY, p644});
+      CO_ASSERT_OK(result);
+      paths.push_back(path);
+    }
+
+    paths.push_back("not_exists");
+    for (auto i = 0; i < 100; i++) {
+      FAULT_INJECTION_SET(10, 3);  // 10%, 3 faults
+      auto path = paths[folly::Random::rand32(paths.size())];
+      auto result = co_await meta.stat({SUPER_USER, path, AtFlags(AT_SYMLINK_FOLLOW)});
+      if (path == "not_exists") {
+        CO_ASSERT_ERROR(result, MetaCode::kNotFound);
+      } else {
+        CO_ASSERT_OK(result);
+      }
+    }
+  }());
+}
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestSymlink.cc

@@ -0,0 +1,98 @@
+#include <fcntl.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Task.h>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#include "common/utils/Coroutine.h"
+#include "common/utils/Path.h"
+#include "common/utils/StatusCode.h"
+#include "meta/store/DirEntry.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/PathResolve.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+
+template <typename KV>
+class TestSymlink : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestSymlink, KVTypes);
+
+TYPED_TEST(TestSymlink, RealPath) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    FAULT_INJECTION_SET(10, 5);
+
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "/a/b", p755, true}));
+    CO_ASSERT_OK(co_await meta.mkdirs({SUPER_USER, "/a/c", p755, true}));
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p644}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "symlink1", "file"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "symlink2", "/file"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/symlink1", "../file"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/symlink2", "/file"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/symlink3", "../../file"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/b/symlink1", "../../file"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/b/symlink2", "../.././file"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/b/symlink3", "../c/./../../../file"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/b/symlink4", "symlink3"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/c/symlink1", "/a/b/symlink4"}));
+
+    for (auto path : {"/symlink1",
+                      "symlink1",
+                      "symlink2",
+                      "/a/symlink1",
+                      "/a/symlink2",
+                      "/a/symlink3",
+                      "/a/b/symlink1",
+                      "/a/b/symlink2",
+                      "/a/b/symlink3",
+                      "/a/c/symlink1"}) {
+      auto result = co_await meta.getRealPath({SUPER_USER, path, false});
+      CO_ASSERT_OK(result);
+      CO_ASSERT_EQ(result->path, "/file") << path << " -> " << result->path;
+    }
+
+    auto b = co_await meta.open({SUPER_USER, "/a/b", {}, O_RDONLY | O_DIRECTORY});
+    CO_ASSERT_OK(b);
+    for (auto path : {"symlink1",
+                      "symlink2",
+                      "./symlink3",
+                      "../b/symlink3",
+                      "../symlink1",
+                      "../symlink2",
+                      "../symlink3",
+                      "../../a/symlink3",
+                      "../../../a/b/symlink3"}) {
+      auto result = co_await meta.getRealPath({SUPER_USER, PathAt(b->stat.id, path), false});
+      CO_ASSERT_OK(result);
+      CO_ASSERT_EQ(result->path, "/file") << path << " -> " << result->path;
+    }
+
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/c/symlink2", "../b/../b"}));
+    CO_ASSERT_OK(co_await meta.symlink({SUPER_USER, "/a/c/symlink3", "../b/../.."}));
+    auto c = co_await meta.open({SUPER_USER, "/a/c", {}, O_RDONLY | O_DIRECTORY});
+    CO_ASSERT_OK(c);
+
+    for (auto [path, rpath] : {std::make_pair("symlink2", "../b"),
+                               std::make_pair("symlink3", "../.."),
+                               {"symlink2/symlink1", "/file"},
+                               {"symlink2/../b", "../b"}}) {
+      auto result = co_await meta.getRealPath({SUPER_USER, PathAt(c->stat.id, path), false});
+      CO_ASSERT_OK(result);
+      CO_ASSERT_EQ(result->path, rpath) << path << " -> " << result->path << " != " << rpath;
+    }
+  }());
+}
+
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestSync.cc

@@ -0,0 +1,228 @@
+#include <algorithm>
+#include <fcntl.h>
+#include <folly/Random.h>
+#include <folly/Synchronized.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Task.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <mutex>
+#include <optional>
+#include <string>
+
+#include "client/storage/StorageClient.h"
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/Result.h"
+#include "common/utils/StatusCode.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/storage/Common.h"
+#include "gtest/gtest.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "meta/store/ops/BatchOperation.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestSync : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestSync, KVTypes);
+
+TYPED_TEST(TestSync, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+    auto &fileHelper = cluster.meta().getFileHelper();
+    auto &storage = cluster.meta().getStorageClient();
+
+    for (size_t i = 0; i < 10; i++) {
+      bool dynStripe = i % 2;
+      auto create = co_await meta.create(
+          {SUPER_USER, PathAt(fmt::format("test-file-{}", i)), {}, O_EXCL, p644, std::nullopt, dynStripe});
+      CO_ASSERT_OK(create);
+      auto inode = create->stat;
+      CO_ASSERT_EQ(inode.asFile().length, 0);
+      CO_ASSERT_EQ(inode.asFile().dynStripe != 0, dynStripe);
+
+      // sync directory
+      CO_ASSERT_ERROR(co_await meta.sync({SUPER_USER, InodeId::root(), true, std::nullopt, std::nullopt}),
+                      MetaCode::kNotFile);
+
+      // sync empty file, length should be 0
+      auto result = co_await meta.sync({SUPER_USER, inode.id, true, {}, {}});
+      CO_ASSERT_OK(result);
+      CO_ASSERT_EQ(result->stat.asFile().length, 0);
+
+      // write at random offset, sync and check new length
+#ifdef GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+      for (size_t i = 0; i < 50; i++) {
+        auto offset = folly::Random::rand64(10_GB);
+        auto length = folly::Random::rand64(2_MB);
+#else
+      for (size_t i = 0; i < 10; i++) {
+        auto offset = folly::Random::rand64(100_MB);
+        auto length = folly::Random::rand64(1_MB);
+#endif
+        auto expectedLength = std::max(inode.asFile().length, offset + length);
+        co_await randomWrite(meta, storage, inode, offset, length);
+
+        auto stat = co_await meta.stat({SUPER_USER, inode.id, AtFlags()});
+        CO_ASSERT_OK(stat);
+
+        auto newLength = co_await fileHelper.queryLength(SUPER_USER, stat->stat, nullptr);
+        CO_ASSERT_OK(newLength);
+        CO_ASSERT_EQ(expectedLength, *newLength);
+        inode.asFile().length = expectedLength;
+
+        {
+          // sync file
+          auto result = co_await meta.sync({SUPER_USER, inode.id, true, {}, {}});
+          CO_ASSERT_OK(result);
+          CO_ASSERT_EQ(result->stat.asFile().length, *newLength);
+        }
+
+        // stat file
+        auto result = co_await meta.stat({SUPER_USER, inode.id, AtFlags(AT_SYMLINK_FOLLOW)});
+        CO_ASSERT_OK(result);
+        CO_ASSERT_EQ(result->stat.asFile().length, *newLength);
+      }
+    }
+  }());
+}
+
+TYPED_TEST(TestSync, Concurrent) {
+  auto cluster = this->createMockCluster();
+  auto &meta = cluster.meta().getOperator();
+  auto &fileHelper = cluster.meta().getFileHelper();
+  auto &storage = cluster.meta().getStorageClient();
+
+  // start multiple workers to truncate this file
+  auto worker = [&](Inode &inode) -> CoTask<void> {
+#ifdef GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_
+    for (size_t i = 0; i < folly::Random::rand32(100); i++) {
+      auto offset = folly::Random::rand64(500_MB);
+      auto length = folly::Random::rand64(2_MB);
+#else
+    for (size_t i = 0; i < folly::Random::rand32(10); i++) {
+      auto offset = folly::Random::rand64(500_MB);
+      auto length = folly::Random::rand64(1_MB);
+#endif
+      CO_ASSERT_NE(inode.id, InodeId());
+      CO_ASSERT_TRUE(inode.isFile());
+      auto newLength = std::max(inode.asFile().length, offset + length);
+      if (folly::Random::oneIn(50)) {
+        co_await truncate(meta, storage, inode, newLength);
+      } else {
+        co_await randomWrite(meta, storage, inode, offset, length);
+      }
+    }
+
+    CO_ASSERT_OK(co_await meta.sync({SUPER_USER, inode.id, true, {}, {}}));
+  };
+  folly::CPUThreadPoolExecutor exec(8);
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto create = co_await meta.create({SUPER_USER, PathAt("test-file"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(create);
+    Inode inode = create->stat;
+    CO_ASSERT_TRUE(inode.isFile());
+    CO_ASSERT_EQ(inode.asFile().length, 0);
+
+    std::vector<folly::SemiFuture<folly::Unit>> futures;
+    for (size_t i = 0; i < 64; i++) {
+      futures.push_back(folly::coro::co_invoke(worker, inode).scheduleOn(&exec).start());
+    }
+    for (auto &f : futures) {
+      f.wait();
+    }
+    auto newLength = co_await fileHelper.queryLength(SUPER_USER, inode, nullptr);
+    CO_ASSERT_OK(newLength);
+
+    auto result = co_await meta.stat({SUPER_USER, inode.id, AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_OK(result);
+    CO_ASSERT_TRUE(result->stat.isFile());
+    CO_ASSERT_EQ(result->stat.asFile().length, *newLength);
+  }());
+}
+
+TYPED_TEST(TestSync, ConflictSet) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    MockCluster::Config config;
+    config.set_num_meta(1);
+    auto cluster = this->createMockCluster(config);
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+
+    auto create = co_await meta.create({SUPER_USER, PathAt("test-file"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(create);
+    InodeId inodeId = create->stat.id;
+
+    BatchedOp::Waiter<SyncReq, SyncRsp> waiter(
+        SyncReq(SUPER_USER, inodeId, true, {}, {}, false, VersionedLength{10, 0}));
+    CHECK_CONFLICT_SET(
+        [&](auto &txn) -> CoTask<void> {
+          BatchedOp op(store, inodeId);
+          op.add(waiter);
+          CO_ASSERT_OK(co_await op.run(txn));
+        },
+        (std::vector<String>{MetaTestHelper::getInodeKey(inodeId)}),
+        (std::vector<String>{MetaTestHelper::getInodeKey(inodeId)}),
+        false);
+  }());
+}
+
+TYPED_TEST(TestSync, hint) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    auto create = co_await meta.create({SUPER_USER, PathAt("test-file"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(create);
+    InodeId inodeId = create->stat.id;
+
+    // invalid hint, DFATAL
+    // CO_ASSERT_ERROR(co_await meta.sync({SUPER_USER, inodeId, true, std::nullopt, std::nullopt, false,
+    // VersionedLength{100, 10}}), MetaCode::kFoundBug);
+
+    // sync truncate
+    auto r1 =
+        co_await meta.sync({SUPER_USER, inodeId, true, std::nullopt, std::nullopt, true, VersionedLength{100, 0}});
+    CO_ASSERT_OK(r1);
+    CO_ASSERT_EQ(r1->stat.asFile().getVersionedLength(), (VersionedLength{0, 1}));
+
+    // sync with hint
+    auto r2 =
+        co_await meta.sync({SUPER_USER, inodeId, true, std::nullopt, std::nullopt, false, VersionedLength{1024, 1}});
+    CO_ASSERT_OK(r2);
+    CO_ASSERT_EQ(r2->stat.asFile().getVersionedLength(), (VersionedLength{1024, 1}));
+
+    // sync with outdate hint
+    auto r3 =
+        co_await meta.sync({SUPER_USER, inodeId, true, std::nullopt, std::nullopt, false, VersionedLength{6000, 0}});
+    CO_ASSERT_OK(r3);
+    CO_ASSERT_EQ(r3->stat.asFile().getVersionedLength(), (VersionedLength{0, 2}));
+
+    // sync with hint
+    auto r4 =
+        co_await meta.sync({SUPER_USER, inodeId, true, std::nullopt, std::nullopt, false, VersionedLength{6000, 2}});
+    CO_ASSERT_OK(r4);
+    CO_ASSERT_EQ(r4->stat.asFile().getVersionedLength(), (VersionedLength{6000, 2}));
+
+    // close
+    auto r5 =
+        co_await meta.close({SUPER_USER, inodeId, MetaTestHelper::randomSession(), true, std::nullopt, std::nullopt});
+    CO_ASSERT_OK(r5);
+    CO_ASSERT_EQ(r5->stat.asFile().getVersionedLength(), (VersionedLength{0, 3}));
+  }());
+}
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestTruncate.cc

@@ -0,0 +1,160 @@
+#include <atomic>
+#include <fcntl.h>
+#include <folly/Random.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/experimental/coro/Task.h>
+#include <folly/logging/xlog.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/FaultInjection.h"
+#include "common/utils/Result.h"
+#include "common/utils/StatusCode.h"
+#include "gtest/gtest.h"
+#include "meta/components/FileHelper.h"
+#include "meta/store/Inode.h"
+#include "meta/store/MetaStore.h"
+#include "tests/GtestHelpers.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+namespace {
+
+template <typename KV>
+class TestTruncate : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestTruncate, KVTypes);
+
+TYPED_TEST(TestTruncate, Basic) {
+  auto cluster = this->createMockCluster();
+  auto &meta = cluster.meta().getOperator();
+  auto &fileHelper = cluster.meta().getFileHelper();
+  auto &storage = cluster.meta().getStorageClient();
+  auto truncateAndStat = [&](Inode &inode, uint64_t targetLength) -> CoTask<void> {
+    co_await truncate(meta, storage, inode, targetLength);
+
+    // do stat, should get new length.
+    auto result = co_await meta.stat({SUPER_USER, PathAt("test-file"), AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_OK(result);
+    CO_ASSERT_EQ(result->stat.asFile().length, targetLength);
+
+    // query from file helper, should get same length.
+    auto newLength = co_await fileHelper.queryLength(SUPER_USER, inode, nullptr);
+    CO_ASSERT_OK(newLength);
+    CO_ASSERT_EQ(*newLength, targetLength);
+  };
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto create =
+        co_await meta.create({SUPER_USER, "test-file", {}, O_EXCL, p644, Layout::newEmpty(ChainTableId(1), 1_KB, 128)});
+    CO_ASSERT_OK(create);
+    Inode inode(create->stat);
+    CO_ASSERT_EQ(inode.asFile().length, 0);
+
+    // file length should be zero
+    auto newLength = co_await fileHelper.queryLength(SUPER_USER, inode, nullptr);
+    CO_ASSERT_OK(newLength);
+    CO_ASSERT_EQ(*newLength, 0);
+
+    // truncate file up
+    co_await truncateAndStat(inode, 2_MB + 13);
+    co_await truncateAndStat(inode, 5_MB - 1);
+
+    // truncate file down
+    co_await truncateAndStat(inode, 2_MB - 1023);
+    co_await truncateAndStat(inode, 512_KB - 1);
+    co_await truncateAndStat(inode, 0);
+
+    for (size_t i = 0; i < 10; i++) {
+      co_await truncateAndStat(inode, folly::Random::rand64(2_MB));
+    }
+  }());
+}
+
+TYPED_TEST(TestTruncate, Deperated) {
+  auto cluster = this->createMockCluster();
+  auto &meta = cluster.meta().getOperator();
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto create =
+        co_await meta.create({SUPER_USER, "test-file", {}, O_EXCL, p644, Layout::newEmpty(ChainTableId(1), 1_KB, 1)});
+    CO_ASSERT_OK(create);
+    Inode inode(create->stat);
+    CO_ASSERT_EQ(inode.asFile().length, 0);
+    CO_ASSERT_ERROR(co_await meta.truncate({SUPER_USER, inode.id, 512, 1}), StatusCode::kNotImplemented);
+  }());
+}
+
+TYPED_TEST(TestTruncate, Concurrent) {
+  auto cluster = this->createMockCluster();
+  auto &meta = cluster.meta().getOperator();
+  auto &fileHelper = cluster.meta().getFileHelper();
+  auto &storage = cluster.meta().getStorageClient();
+
+  folly::CPUThreadPoolExecutor exec(8);
+
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    // start multiple workers to truncate this file
+    auto create = co_await meta.create({SUPER_USER, PathAt("test-file"), {}, O_EXCL, p644});
+    CO_ASSERT_OK(create);
+    auto inode = create->stat;
+    CO_ASSERT_EQ(inode.asFile().length, 0);
+
+    auto worker = [&]() -> CoTask<void> {
+      if (folly::Random::oneIn(4)) {
+        co_await truncate(meta, storage, inode, 0);
+      } else {
+        co_await truncate(meta, storage, inode, folly::Random::rand64(10_MB, 20_MB));
+      }
+    };
+
+    std::vector<folly::SemiFuture<folly::Unit>> futures;
+    for (size_t i = 0; i < 64; i++) {
+      futures.push_back(folly::coro::co_invoke(worker).scheduleOn(&exec).start());
+    }
+    for (auto &f : futures) {
+      f.wait();
+    }
+
+    auto result = co_await meta.stat({SUPER_USER, PathAt("test-file"), AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_OK(result);
+    inode = result->stat;
+    auto length = inode.asFile().length;
+    CO_ASSERT_TRUE(length < 20_MB) << length;
+
+    auto newLength = co_await fileHelper.queryLength(SUPER_USER, inode, nullptr);
+    CO_ASSERT_OK(newLength);
+    std::cout << fmt::format("{}", inode.asFile().getVersionedLength()) << " " << *newLength << std::endl;
+    CO_ASSERT_EQ(length, *newLength);
+  }());
+}
+
+// TYPED_TEST(TestTruncate, ConflictSet) {
+//   folly::coro::blockingWait([&]() -> CoTask<void> {
+//     auto cluster = this->createMockCluster();
+//     auto &store = cluster.meta().getStore();
+
+//     auto create = co_await cluster.meta().getOperator().create({SUPER_USER, PathAt("test-file"), {}, O_EXCL, p644});
+//     CO_ASSERT_OK(create);
+//     InodeId inodeId = create->stat.id;
+
+//     CHECK_CONFLICT_SET(
+//         [&](auto &txn) -> CoTask<void> {
+//           auto req = TruncateReq(SUPER_USER, inodeId, 1_MB, 32);
+//           CO_ASSERT_OK(co_await store.truncate(req)->run(txn));
+//         },
+//         (std::vector<String>{MetaTestHelper::getInodeKey(inodeId)}),
+//         (std::vector<String>{MetaTestHelper::getInodeKey(inodeId)}),
+//         false);
+//   }());
+// }
+
+}  // namespace
+}  // namespace hf3fs::meta::server

tests/meta/store/ops/TestUtimes.cc

@@ -0,0 +1,118 @@
+#include <algorithm>
+#include <fcntl.h>
+#include <folly/Random.h>
+#include <folly/experimental/coro/GtestHelpers.h>
+#include <folly/futures/ManualTimekeeper.h>
+#include <gtest/gtest.h>
+#include <optional>
+#include <vector>
+
+#include "common/utils/FaultInjection.h"
+#include "common/utils/StatusCode.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/meta/Common.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/meta/Service.h"
+#include "meta/store/MetaStore.h"
+#include "tests/meta/MetaTestBase.h"
+
+namespace hf3fs::meta::server {
+template <typename KV>
+class TestUtimes : public MetaTestBase<KV> {};
+
+using KVTypes = ::testing::Types<mem::MemKV, fdb::DB>;
+TYPED_TEST_SUITE(TestUtimes, KVTypes);
+
+TYPED_TEST(TestUtimes, Basic) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p777}));
+
+    auto result = co_await meta.stat({SUPER_USER, "file", AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_OK(result);
+    auto &inode = result->stat;
+
+    auto test =
+        [&](const flat::UserInfo &user, PathAt path, UtcTime atime, UtcTime mtime, uint64_t error = 0) -> CoTask<void> {
+      auto gran = cluster.config().mock_meta().time_granularity();
+      atime = (atime == SETATTR_TIME_NOW) ? SETATTR_TIME_NOW : atime.castGranularity(gran);
+      mtime = (atime == SETATTR_TIME_NOW) ? SETATTR_TIME_NOW : mtime.castGranularity(gran);
+      auto before = UtcClock::now().castGranularity(gran);
+      auto result = co_await meta.setAttr(SetAttrReq::utimes(user, path, AtFlags(0), atime, mtime));
+      if (error) {
+        CO_ASSERT_ERROR(result, error);
+        co_return;
+      } else {
+        CO_ASSERT_OK(result);
+      }
+
+      auto sresult = co_await meta.stat({SUPER_USER, "file", AtFlags(AT_SYMLINK_FOLLOW)});
+      CO_ASSERT_OK(sresult);
+      auto &inode = sresult->stat;
+      if (atime != SETATTR_TIME_NOW) {
+        CO_ASSERT_EQ(inode.atime, atime) << fmt::format("{} {}", inode.atime, atime);
+      } else {
+        CO_ASSERT_GE(inode.atime, before);
+      }
+      if (mtime != SETATTR_TIME_NOW) {
+        CO_ASSERT_EQ(inode.mtime, mtime) << fmt::format("{} {}", inode.mtime, mtime);
+      } else {
+        CO_ASSERT_GE(inode.mtime, before);
+      }
+    };
+
+    co_await test(SUPER_USER, "file", UtcClock::now(), UtcClock::now());
+    co_await test(SUPER_USER, "file", SETATTR_TIME_NOW, SETATTR_TIME_NOW);
+    co_await test(SUPER_USER, inode.id, UtcClock::now(), UtcClock::now());
+    co_await test(SUPER_USER, inode.id, SETATTR_TIME_NOW, SETATTR_TIME_NOW);
+
+    flat::UserInfo ua(Uid(1), Gid(1), String());
+    co_await test(ua, "file", SETATTR_TIME_NOW, SETATTR_TIME_NOW);
+    co_await test(ua, "file", UtcClock::now(), UtcClock::now(), MetaCode::kNoPermission);
+    co_await test(ua, inode.id, SETATTR_TIME_NOW, SETATTR_TIME_NOW);
+    co_await test(ua, inode.id, UtcClock::now(), UtcClock::now(), MetaCode::kNoPermission);
+
+    CO_ASSERT_OK(
+        co_await meta.setAttr(SetAttrReq::setPermission(SUPER_USER, "file", AtFlags(0), {}, {}, std::optional(p700))));
+
+    co_await test(ua, "file", SETATTR_TIME_NOW, SETATTR_TIME_NOW, MetaCode::kNoPermission);
+    co_await test(ua, "file", UtcClock::now(), UtcClock::now(), MetaCode::kNoPermission);
+    co_await test(ua, inode.id, SETATTR_TIME_NOW, SETATTR_TIME_NOW, MetaCode::kNoPermission);
+    co_await test(ua, inode.id, UtcClock::now(), UtcClock::now(), MetaCode::kNoPermission);
+  }());
+}
+
+TYPED_TEST(TestUtimes, ConflictSet) {
+  folly::coro::blockingWait([&]() -> CoTask<void> {
+    auto cluster = this->createMockCluster();
+    auto &meta = cluster.meta().getOperator();
+    auto &store = cluster.meta().getStore();
+
+    // create file, directory and symlink
+    CO_ASSERT_OK(co_await meta.create({SUPER_USER, "file", {}, O_RDONLY, p777}));
+
+    auto result = co_await meta.stat({SUPER_USER, "file", AtFlags(AT_SYMLINK_FOLLOW)});
+    CO_ASSERT_OK(result);
+    auto fileId = result->stat.id;
+
+    CHECK_CONFLICT_SET(
+        [&](auto &txn) -> CoTask<void> {
+          auto req = SetAttrReq::utimes(SUPER_USER,
+                                        "file",
+                                        AtFlags(0),
+                                        UtcTime::fromMicroseconds(folly::Random::rand32()),
+                                        UtcTime::fromMicroseconds(folly::Random::rand32()));
+          CO_ASSERT_OK(co_await store.setAttr(req)->run(txn));
+        },
+        (std::vector<String>{
+            MetaTestHelper::getInodeKey(fileId),  // inode
+        }),
+        (std::vector<String>{
+            MetaTestHelper::getInodeKey(fileId),  // inode
+        }),
+        true);
+  }());
+}
+}  // namespace hf3fs::meta::server