Initial commit

src/mgmtd/CMakeLists.txt

@@ -0,0 +1,3 @@
+target_add_lib(mgmtd core-app core-user core-service fdb mgmtd-fbs memory-common)
+target_add_bin(mgmtd_main "mgmtd.cpp" mgmtd)
+

src/mgmtd/MgmtdConfigFetcher.cc

@@ -0,0 +1,55 @@
+#include "MgmtdConfigFetcher.h"
+
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "MgmtdServer.h"
+#include "common/app/ApplicationBase.h"
+#include "common/kv/WithTransaction.h"
+#include "fdb/FDBRetryStrategy.h"
+#include "fdb/HybridKvEngine.h"
+#include "mgmtd/store/MgmtdStore.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+kv::FDBRetryStrategy getRetryStrategy() { return kv::FDBRetryStrategy({1_s, 10, false}); }
+}  // namespace
+
+void MgmtdConfigFetcher::init(const kv::HybridKvEngineConfig &config,
+                              bool useMemKV,
+                              const kv::fdb::FDBConfig &fdbConfig) {
+  kvEngine_ = kv::HybridKvEngine::from(config, useMemKV, fdbConfig);
+}
+
+Result<flat::ConfigInfo> MgmtdConfigFetcher::loadConfigTemplate(flat::NodeType nodeType) {
+  XLOGF_IF(FATAL, nodeType != flat::NodeType::MGMTD, "Unexpected NodeType: {}", static_cast<int>(nodeType));
+  MgmtdStore store;
+  auto handler = [&](kv::IReadOnlyTransaction &txn) -> CoTryTask<flat::ConfigInfo> {
+    auto res = co_await store.loadLatestConfig(txn, flat::NodeType::MGMTD);
+    CO_RETURN_ON_ERROR(res);
+    if (res->has_value()) {
+      co_return std::move(res->value());
+    }
+    co_return flat::ConfigInfo::create();
+  };
+  return folly::coro::blockingWait(
+      kv::WithTransaction(getRetryStrategy()).run(kvEngine_->createReadonlyTransaction(), std::move(handler)));
+}
+
+Result<Void> MgmtdConfigFetcher::completeAppInfo(flat::AppInfo &appInfo) {
+  MgmtdStore store;
+  auto handler = [&](kv::IReadOnlyTransaction &txn) -> CoTryTask<void> {
+    auto res = co_await store.loadNodeInfo(txn, appInfo.nodeId);
+    CO_RETURN_ON_ERROR(res);
+    if (res->has_value()) {
+      appInfo.tags = res->value().tags;
+    }
+    co_return Void{};
+  };
+  return folly::coro::blockingWait(
+      kv::WithTransaction(getRetryStrategy()).run(kvEngine_->createReadonlyTransaction(), std::move(handler)));
+}
+
+Result<Void> MgmtdConfigFetcher::startServer(MgmtdServer &server, const flat::AppInfo &appInfo) {
+  return server.start(appInfo, std::move(kvEngine_));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/MgmtdConfigFetcher.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "common/app/AppInfo.h"
+#include "common/kv/IKVEngine.h"
+#include "fbs/mgmtd/ConfigInfo.h"
+#include "fdb/HybridKvEngineConfig.h"
+
+namespace hf3fs::mgmtd {
+class MgmtdServer;
+struct MgmtdConfigFetcher {
+  template <typename ConfigT>
+  explicit MgmtdConfigFetcher(const ConfigT &cfg) {
+    init(cfg.kv_engine(), cfg.use_memkv(), cfg.fdb());
+  }
+
+  Result<flat::ConfigInfo> loadConfigTemplate(flat::NodeType nodeType);
+  Result<Void> completeAppInfo(flat::AppInfo &appInfo);
+
+  Result<Void> startServer(MgmtdServer &server, const flat::AppInfo &appInfo);
+
+ private:
+  void init(const kv::HybridKvEngineConfig &config, bool useMemKV, const kv::fdb::FDBConfig &fdbConfig);
+
+  std::shared_ptr<kv::IKVEngine> kvEngine_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/MgmtdLauncherConfig.cc

@@ -0,0 +1,15 @@
+#include "MgmtdLauncherConfig.h"
+
+#include "common/app/ApplicationBase.h"
+#include "common/app/Utils.h"
+
+namespace hf3fs::mgmtd {
+void MgmtdLauncherConfig::init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates) {
+  app_detail::initConfigFromFile(*this, filePath, dump, updates);
+
+  auto rv = flat::ReleaseVersion::fromVersionInfo();
+  if (!allow_dev_version() && !rv.getIsReleaseVersion()) {
+    XLOGF(FATAL, "Dev version is not allowed: {}", rv);
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/MgmtdLauncherConfig.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "common/net/ib/IBDevice.h"
+#include "fdb/HybridKvEngineConfig.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdLauncherConfig : public ConfigBase<MgmtdLauncherConfig> {
+  CONFIG_ITEM(cluster_id, "");
+  CONFIG_ITEM(use_memkv, false);        // deprecated
+  CONFIG_OBJ(fdb, kv::fdb::FDBConfig);  // deprecated
+  CONFIG_OBJ(ib_devices, net::IBDevice::Config);
+  CONFIG_OBJ(kv_engine, kv::HybridKvEngineConfig);
+  CONFIG_ITEM(allow_dev_version, true);
+
+ public:
+  using ConfigBase<MgmtdLauncherConfig>::init;
+  void init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/MgmtdServer.cc

@@ -0,0 +1,51 @@
+#include "MgmtdServer.h"
+
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/logging/xlog.h>
+
+#include "common/app/ApplicationBase.h"
+#include "core/app/ServerEnv.h"
+#include "core/service/CoreService.h"
+#include "fdb/HybridKvEngine.h"
+#include "service/MgmtdService.h"
+#include "stubs/common/RealStubFactory.h"
+#include "stubs/mgmtd/MgmtdServiceStub.h"
+
+namespace hf3fs::mgmtd {
+
+MgmtdServer::MgmtdServer(const Config &config)
+    : net::Server(config.base()),
+      config_(config) {}
+
+MgmtdServer::~MgmtdServer() {}
+
+Result<Void> MgmtdServer::beforeStart() {
+  auto env = std::make_shared<core::ServerEnv>();
+  env->setAppInfo(appInfo());
+  XLOGF_IF(FATAL, !kvEngine_, "Should construct kv engine before server");
+  env->setKvEngine(kvEngine_);
+  env->setMgmtdStubFactory(std::make_shared<stubs::RealStubFactory<mgmtd::MgmtdServiceStub>>(serdeCtxCreator()));
+  env->setBackgroundExecutor(&tpg().bgThreadPool());
+  env->setConfigUpdater(ApplicationBase::updateConfig);
+  env->setConfigValidater(ApplicationBase::validateConfig);
+
+  mgmtdOperator_ = std::make_unique<MgmtdOperator>(std::move(env), config_.service());
+  RETURN_ON_ERROR(addSerdeService(std::make_unique<MgmtdService>(*mgmtdOperator_)));
+  RETURN_ON_ERROR(addSerdeService(std::make_unique<core::CoreService>()));
+
+  mgmtdOperator_->start();
+  return Void{};
+}
+
+Result<Void> MgmtdServer::afterStop() {
+  mgmtdOperator_.reset();  // stop all background tasks
+  // todo
+  return Void{};
+}
+
+Result<Void> MgmtdServer::start(const flat::AppInfo &appInfo, std::shared_ptr<kv::IKVEngine> kvEngine) {
+  kvEngine_ = std::move(kvEngine);
+  return net::Server::start(appInfo);
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/MgmtdServer.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "MgmtdConfigFetcher.h"
+#include "MgmtdLauncherConfig.h"
+#include "common/net/Server.h"
+#include "core/app/ServerAppConfig.h"
+#include "core/app/ServerLauncher.h"
+#include "fdb/HybridKvEngineConfig.h"
+#include "mgmtd/service/MgmtdOperator.h"
+#include "service/MgmtdConfig.h"
+
+namespace hf3fs::mgmtd {
+
+class MgmtdServer : public net::Server {
+ public:
+  static constexpr auto kName = "Mgmtd";
+  static constexpr auto kNodeType = flat::NodeType::MGMTD;
+
+  using AppConfig = core::ServerAppConfig;
+  using LauncherConfig = MgmtdLauncherConfig;
+  using RemoteConfigFetcher = MgmtdConfigFetcher;
+  using Launcher = core::ServerLauncher<MgmtdServer>;
+
+  using CommonConfig = ApplicationBase::Config;
+  class Config : public ConfigBase<Config> {
+    CONFIG_OBJ(base, net::Server::Config, [](net::Server::Config &c) {
+      c.set_groups_length(2);
+      c.groups(0).listener().set_listen_port(8000);
+      c.groups(0).set_services({"Mgmtd"});
+
+      c.groups(1).set_network_type(net::Address::TCP);
+      c.groups(1).listener().set_listen_port(9000);
+      c.groups(1).set_use_independent_thread_pool(true);
+      c.groups(1).set_services({"Core"});
+    });
+    CONFIG_OBJ(service, MgmtdConfig);
+  };
+
+  explicit MgmtdServer(const Config &config);
+  ~MgmtdServer() override;
+
+  using net::Server::start;
+  Result<Void> start(const flat::AppInfo &info, std::shared_ptr<kv::IKVEngine> kvEngine);
+
+  Result<Void> beforeStart() final;
+
+  Result<Void> afterStop() final;
+
+ private:
+  const Config &config_;
+
+  std::shared_ptr<kv::IKVEngine> kvEngine_;
+  std::unique_ptr<MgmtdOperator> mgmtdOperator_;
+
+  friend class testing::MgmtdTestHelper;
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdBackgroundRunner.cc

@@ -0,0 +1,87 @@
+#include "MgmtdBackgroundRunner.h"
+
+#include "MgmtdChainsUpdater.h"
+#include "MgmtdClientSessionsChecker.h"
+#include "MgmtdHeartbeatChecker.h"
+#include "MgmtdHeartbeater.h"
+#include "MgmtdLeaseExtender.h"
+#include "MgmtdMetricsUpdater.h"
+#include "MgmtdNewBornChainsChecker.h"
+#include "MgmtdRoutingInfoVersionUpdater.h"
+#include "MgmtdTargetInfoLoader.h"
+#include "MgmtdTargetInfoPersister.h"
+#include "common/utils/BackgroundRunner.h"
+#include "mgmtd/service/MgmtdConfig.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+MgmtdBackgroundRunner::MgmtdBackgroundRunner(MgmtdState &state)
+    : state_(state) {
+  if (state_.env_->backgroundExecutor()) {
+    backgroundRunner_ = std::make_unique<BackgroundRunner>(*state_.env_->backgroundExecutor());
+    heartbeater_ = std::make_unique<MgmtdHeartbeater>(state_);
+    leaseExtender_ = std::make_unique<MgmtdLeaseExtender>(state_);
+    chainsUpdater_ = std::make_unique<MgmtdChainsUpdater>(state_);
+    clientSessionsChecker_ = std::make_unique<MgmtdClientSessionsChecker>(state_);
+    heartbeatChecker_ = std::make_unique<MgmtdHeartbeatChecker>(state_);
+    newBornChainsChecker_ = std::make_unique<MgmtdNewBornChainsChecker>(state_);
+    routingInfoVersionUpdater_ = std::make_unique<MgmtdRoutingInfoVersionUpdater>(state_);
+    metricsUpdater_ = std::make_unique<MgmtdMetricsUpdater>(state_);
+    targetInfoPersister_ = std::make_unique<MgmtdTargetInfoPersister>(state_);
+    targetInfoLoader_ = std::make_unique<MgmtdTargetInfoLoader>(state_);
+  }
+}
+
+MgmtdBackgroundRunner::~MgmtdBackgroundRunner() {}
+
+void MgmtdBackgroundRunner::start() {
+  if (backgroundRunner_) {
+    backgroundRunner_->start(
+        "extendLease",
+        [this] { return leaseExtender_->extend(); },
+        state_.config_.extend_lease_interval_getter());
+    backgroundRunner_->start(
+        "checkClientSessions",
+        [this] { return clientSessionsChecker_->check(); },
+        state_.config_.check_status_interval_getter());
+    backgroundRunner_->start(
+        "checkNewBornChains",
+        [this] { return newBornChainsChecker_->check(); },
+        state_.config_.check_status_interval_getter());
+    backgroundRunner_->start(
+        "checkHeartbeat",
+        [this] { return heartbeatChecker_->check(); },
+        state_.config_.check_status_interval_getter());
+    backgroundRunner_->start(
+        "sendHeartbeat",
+        [this] { return heartbeater_->send(); },
+        state_.config_.send_heartbeat_interval_getter());
+    backgroundRunner_->start(
+        "updateChains",
+        [this, lastUpdateTs = SteadyClock::now()]() mutable { return chainsUpdater_->update(lastUpdateTs); },
+        state_.config_.update_chains_interval_getter());
+    backgroundRunner_->start(
+        "bumpRoutingInfoVersion",
+        [this] { return routingInfoVersionUpdater_->update(); },
+        state_.config_.bump_routing_info_version_interval_getter());
+    backgroundRunner_->start(
+        "updateMetrics",
+        [this] { return metricsUpdater_->update(); },
+        state_.config_.update_metrics_interval_getter());
+    backgroundRunner_->start(
+        "persistTargetInfo",
+        [this] { return targetInfoPersister_->run(); },
+        state_.config_.target_info_persist_interval_getter());
+    backgroundRunner_->start(
+        "loadTargetInfo",
+        [this] { return targetInfoLoader_->run(); },
+        state_.config_.target_info_load_interval_getter());
+  }
+}
+
+CoTask<void> MgmtdBackgroundRunner::stop() {
+  if (backgroundRunner_) {
+    co_await backgroundRunner_->stopAll();
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdBackgroundRunner.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs {
+class BackgroundRunner;
+namespace mgmtd {
+class MgmtdHeartbeater;
+class MgmtdLeaseExtender;
+struct MgmtdState;
+class MgmtdChainsUpdater;
+class MgmtdClientSessionsChecker;
+class MgmtdHeartbeatChecker;
+class MgmtdNewBornChainsChecker;
+class MgmtdRoutingInfoVersionUpdater;
+class MgmtdMetricsUpdater;
+class MgmtdTargetInfoPersister;
+class MgmtdTargetInfoLoader;
+namespace testing {
+class MgmtdTestHelper;
+}
+
+class MgmtdBackgroundRunner {
+ public:
+  explicit MgmtdBackgroundRunner(MgmtdState &state);
+  ~MgmtdBackgroundRunner();
+
+  void start();
+  CoTask<void> stop();
+
+ private:
+  MgmtdState &state_;
+  std::unique_ptr<BackgroundRunner> backgroundRunner_;
+  std::unique_ptr<MgmtdHeartbeater> heartbeater_;
+  std::unique_ptr<MgmtdLeaseExtender> leaseExtender_;
+  std::unique_ptr<MgmtdChainsUpdater> chainsUpdater_;
+  std::unique_ptr<MgmtdClientSessionsChecker> clientSessionsChecker_;
+  std::unique_ptr<MgmtdHeartbeatChecker> heartbeatChecker_;
+  std::unique_ptr<MgmtdNewBornChainsChecker> newBornChainsChecker_;
+  std::unique_ptr<MgmtdRoutingInfoVersionUpdater> routingInfoVersionUpdater_;
+  std::unique_ptr<MgmtdMetricsUpdater> metricsUpdater_;
+  std::unique_ptr<MgmtdTargetInfoPersister> targetInfoPersister_;
+  std::unique_ptr<MgmtdTargetInfoLoader> targetInfoLoader_;
+
+  friend class testing::MgmtdTestHelper;
+};
+}  // namespace mgmtd
+}  // namespace hf3fs

src/mgmtd/background/MgmtdChainsUpdater.cc

@@ -0,0 +1,80 @@
+#include "MgmtdChainsUpdater.h"
+
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", "UpdateChains", "bg"> {
+  String toStringImpl() const final { return "UpdateChains"; }
+
+  CoTryTask<SteadyTime> handle(MgmtdState &state, SteadyTime lastUpdateTs, bool lastAdjustTargetOrderFlag) {
+    auto writerLock = co_await state.coScopedLock<"UpdateChains">();
+
+    robin_hood::unordered_set<flat::ChainId> candidateChains;
+    std::vector<flat::ChainInfo> changedChains;
+    bool needPromoteRoutingInfoVersion = false;
+
+    auto steadyNow = SteadyClock::now();
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      const auto &ri = dataPtr->routingInfo;
+
+      for (const auto &[tid, ti] : ri.getTargets()) {
+        if (ti.ts() > lastUpdateTs) {
+          candidateChains.insert(ti.base().chainId);
+        } else if (!lastAdjustTargetOrderFlag && state.config_.try_adjust_target_order_as_preferred()) {
+          candidateChains.insert(ti.base().chainId);
+        }
+        needPromoteRoutingInfoVersion |= ti.importantInfoChangedTime > lastUpdateTs;
+      }
+
+      for (auto chainId : candidateChains) {
+        dataPtr->appendChangedChains(chainId, changedChains, state.config_.try_adjust_target_order_as_preferred());
+      }
+    }
+
+    if (changedChains.empty() && !needPromoteRoutingInfoVersion) co_return steadyNow;
+
+    auto commitRes =
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          for (const auto &chain : changedChains) {
+            CO_RETURN_ON_ERROR(co_await state.store_.storeChainInfo(txn, chain));
+          }
+          co_return Void{};
+        });
+    CO_RETURN_ON_ERROR(commitRes);
+
+    co_await updateMemoryRoutingInfo(state, *this, [&](RoutingInfo &ri) {
+      ri.applyChainTargetChanges(*this, changedChains, steadyNow);
+    });
+
+    co_return steadyNow;
+  }
+};
+}  // namespace
+
+MgmtdChainsUpdater::MgmtdChainsUpdater(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdChainsUpdater::update(SteadyTime &lastUpdateTs) {
+  Op op;
+
+  auto handler = [&]() -> CoTryTask<SteadyTime> {
+    CO_INVOKE_OP_INFO(op, "background", state_, lastUpdateTs, lastAdjustTargetOrderFlag);
+  };
+
+  auto res = co_await doAsPrimary(state_, std::move(handler));
+  if (res.hasError()) {
+    if (res.error().code() == MgmtdCode::kNotPrimary)
+      XLOGF(INFO, "Mgmtd: self is not primary, skip updateChains");
+    else
+      LOG_OP_ERR(op, "failed: {}", res.error());
+  } else {
+    lastUpdateTs = *res;
+    lastAdjustTargetOrderFlag = state_.config_.try_adjust_target_order_as_preferred();
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdChainsUpdater.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+#include "common/utils/UtcTime.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdChainsUpdater {
+ public:
+  explicit MgmtdChainsUpdater(MgmtdState &state);
+
+  CoTask<void> update(SteadyTime &lastUpdateTs);
+
+ private:
+  bool lastAdjustTargetOrderFlag = false;
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdClientSessionsChecker.cc

@@ -0,0 +1,55 @@
+#include "MgmtdClientSessionsChecker.h"
+
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", "CheckClientSessions", "bg"> {
+  String toStringImpl() const final { return "CheckClientSessions"; }
+  auto handle(MgmtdState &state) -> CoTryTask<void> {
+    auto timeout = state.config_.client_session_timeout().asUs();
+    auto now = SteadyClock::now();
+
+    std::vector<String> timeoutedClients;
+    {
+      auto clientSessionMap = co_await state.clientSessionMap_.coSharedLock();
+      for (const auto &[clientId, sessionData] : *clientSessionMap) {
+        if (sessionData.ts() + timeout < now) timeoutedClients.push_back(clientId);
+      }
+    }
+    if (!timeoutedClients.empty()) {
+      LOG_OP_INFO(*this, "remove timeouted client sessions [{}]", fmt::join(timeoutedClients, ","));
+
+      auto clientSessionMap = co_await state.clientSessionMap_.coLock();
+      for (const auto &clientId : timeoutedClients) {
+        auto it = clientSessionMap->find(clientId);
+        if (it->second.ts() + timeout < now) {
+          clientSessionMap->erase(it);
+        }
+      }
+    }
+    co_return Void{};
+  }
+};
+}  // namespace
+
+MgmtdClientSessionsChecker::MgmtdClientSessionsChecker(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdClientSessionsChecker::check() {
+  Op op;
+  auto handler = [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_); };
+
+  auto res = co_await doAsPrimary(state_, std::move(handler));
+  if (res.hasError()) {
+    if (res.error().code() == MgmtdCode::kNotPrimary)
+      LOG_OP_INFO(op, "self is not primary, skip");
+    else
+      LOG_OP_ERR(op, "failed: {}", res.error());
+  }
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdClientSessionsChecker.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdClientSessionsChecker {
+ public:
+  explicit MgmtdClientSessionsChecker(MgmtdState &state);
+
+  CoTask<void> check();
+
+ private:
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdHeartbeatChecker.cc

@@ -0,0 +1,98 @@
+#include "MgmtdHeartbeatChecker.h"
+
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+flat::NodeInfo onNodeFailed(const flat::NodeInfo &oldNodeInfo, UtcTime) {
+  auto sn = oldNodeInfo;
+  sn.status = flat::NodeStatus::HEARTBEAT_FAILED;
+  return sn;
+}
+
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", "CheckHeartbeat", "bg"> {
+  String toStringImpl() const final { return "CheckHeartbeat"; }
+
+  auto handle(MgmtdState &state) -> CoTryTask<void> {
+    auto heartbeatFailInterval = state.config_.heartbeat_fail_interval().asUs();
+    auto start = state.utcNow();
+
+    auto writerLock = co_await state.coScopedLock<"CheckHeartbeat">();
+
+    auto steadyNow = SteadyClock::now();
+
+    std::vector<flat::NodeId> timeoutedNodeIds;
+    std::vector<flat::NodeInfo> timeoutedNodes;
+    std::vector<flat::TargetId> candidateTargetIds;
+
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      const auto &ri = dataPtr->routingInfo;
+      for (const auto &[nodeId, nodeInfo] : ri.nodeMap) {
+        switch (nodeInfo.base().status) {
+          case flat::NodeStatus::DISABLED:
+          case flat::NodeStatus::HEARTBEAT_FAILED:
+          case flat::NodeStatus::PRIMARY_MGMTD:
+            break;  // do nothing
+          default:
+            if (nodeInfo.ts() + heartbeatFailInterval < steadyNow) {
+              timeoutedNodeIds.push_back(nodeId);
+              timeoutedNodes.push_back(onNodeFailed(nodeInfo.base(), start));
+            }
+        }
+      }
+
+      for (const auto &[tid, ti] : ri.getTargets()) {
+        if (ti.base().localState != flat::LocalTargetState::OFFLINE && ti.ts() + heartbeatFailInterval < steadyNow) {
+          candidateTargetIds.push_back(tid);
+        }
+      }
+
+      if (timeoutedNodes.empty() && candidateTargetIds.empty()) co_return Void{};
+
+      LOG_OP_INFO(*this,
+                  "found timeouted nodes:[{}] targets:[{}]",
+                  fmt::join(timeoutedNodeIds, ","),
+                  fmt::join(candidateTargetIds, ","));
+    }
+
+    if (!candidateTargetIds.empty() || !timeoutedNodes.empty()) {
+      auto dataPtr = co_await state.data_.coLock();
+      auto &ri = dataPtr->routingInfo;
+      auto steadyNow = SteadyClock::now();
+      for (auto tid : candidateTargetIds) {
+        ri.updateTarget(tid, [steadyNow](auto &ti) {
+          ti.base().localState = flat::LocalTargetState::OFFLINE;
+          ti.updateTs(steadyNow);
+        });
+      }
+
+      for (auto &info : timeoutedNodes) ri.nodeMap[info.app.nodeId].base() = std::move(info);
+      ri.routingInfoChanged = true;
+    }
+
+    co_return Void{};
+  }
+};
+}  // namespace
+
+MgmtdHeartbeatChecker::MgmtdHeartbeatChecker(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdHeartbeatChecker::check() {
+  Op op;
+  auto handler = [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_); };
+
+  auto res = co_await doAsPrimary(state_, std::move(handler));
+  if (res.hasError()) {
+    if (res.error().code() == MgmtdCode::kNotPrimary)
+      LOG_OP_INFO(op, "self is not primary, skip");
+    else
+      LOG_OP_ERR(op, "failed: {}", res.error());
+  }
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdHeartbeatChecker.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdHeartbeatChecker {
+ public:
+  explicit MgmtdHeartbeatChecker(MgmtdState &state);
+
+  CoTask<void> check();
+
+ private:
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdHeartbeater.cc

@@ -0,0 +1,93 @@
+#include "MgmtdHeartbeater.h"
+
+#include "common/app/ApplicationBase.h"
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdHeartbeater::SendHeartbeatContext {
+  flat::MgmtdLeaseInfo lease;
+  std::unique_ptr<MgmtdStub> stub;
+  flat::HeartbeatInfo info;
+};
+
+namespace {
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", "SendHeartbeat", "bg"> {
+  String toStringImpl() const final { return "SendHeartbeat"; }
+
+  auto handle(MgmtdState &state, MgmtdHeartbeater::SendHeartbeatContext &sendHeartbeatCtx) -> CoTryTask<void> {
+    auto &info = sendHeartbeatCtx.info;
+    info.configStatus = ApplicationBase::getConfigStatus();
+    auto res = co_await sendHeartbeatCtx.stub->heartbeat(
+        HeartbeatReq::create(state.env_->appInfo().clusterId, info, UtcClock::now()));
+    if (res.hasError()) {
+      if (res.error().code() == MgmtdCode::kHeartbeatVersionStale) {
+        uint64_t v = 0;
+        auto result = scn::scan(String(res.error().message()), "{}", v);
+        if (result) {
+          info.hbVersion = flat::HeartbeatVersion(v + 1);
+        } else {
+          info.hbVersion = flat::HeartbeatVersion(info.hbVersion + 1);
+        }
+      }
+      CO_RETURN_ERROR(res);
+    } else {
+      if (res->config) {
+        auto r = updateSelfConfig(state, *res->config);
+        if (r.hasError()) {
+          LOG_OP_WARN(*this, "Update config failed: {}. version: {}", r.error(), res->config->configVersion);
+        } else {
+          info.configVersion = res->config->configVersion;
+          LOG_OP_INFO(*this, "Update config succeeded and promote config version to {}", res->config->configVersion);
+        }
+      }
+      info.hbVersion = nextVersion(info.hbVersion);
+      co_return Void{};
+    }
+  }
+};
+}  // namespace
+
+MgmtdHeartbeater::MgmtdHeartbeater(MgmtdState &state)
+    : state_(state) {}
+
+MgmtdHeartbeater::~MgmtdHeartbeater() {}
+
+CoTask<void> MgmtdHeartbeater::send() {
+  Op op;
+
+  if (!state_.config_.send_heartbeat()) {
+    sendHeartbeatCtx_.reset();
+    LOG_OP_DBG(op, "disabled, skip");
+    co_return;
+  }
+  auto start = state_.utcNow();
+  auto lease = co_await state_.currentLease(start);
+  if (!lease.has_value()) {
+    sendHeartbeatCtx_.reset();
+    LOG_OP_DBG(op, "primary not found, skip");
+    co_return;
+  } else if (lease->primary.nodeId == state_.selfId()) {
+    sendHeartbeatCtx_.reset();
+    LOG_OP_DBG(op, "self is primary, skip");
+    co_return;
+  }
+
+  if (!sendHeartbeatCtx_ || sendHeartbeatCtx_->lease.leaseStart != lease->leaseStart) {
+    auto addrs = flat::extractAddresses(lease->primary.serviceGroups, "Mgmtd");
+    if (addrs.empty()) {
+      LOG_OP_ERR(op, "primary({}) addr not found, skip", lease->primary.nodeId);
+      co_return;
+    }
+    sendHeartbeatCtx_ = std::make_unique<SendHeartbeatContext>();
+    sendHeartbeatCtx_->lease = *lease;
+    // TODO: consider reuse some facilities of MgmtdClient for auto switching addresses
+    sendHeartbeatCtx_->stub = state_.env_->mgmtdStubFactory()->create(addrs[0]);
+    sendHeartbeatCtx_->info = flat::HeartbeatInfo(state_.env_->appInfo(), flat::MgmtdHeartbeatInfo{});
+  }
+
+  co_await [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_, *sendHeartbeatCtx_); }();
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdHeartbeater.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdHeartbeater {
+ public:
+  explicit MgmtdHeartbeater(MgmtdState &state);
+  ~MgmtdHeartbeater();
+
+  CoTask<void> send();
+
+  struct SendHeartbeatContext;
+
+ private:
+  MgmtdState &state_;
+
+  std::unique_ptr<SendHeartbeatContext> sendHeartbeatCtx_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdLeaseExtender.cc

@@ -0,0 +1,269 @@
+#include "MgmtdLeaseExtender.h"
+
+#include <folly/experimental/coro/Collect.h>
+
+#include "common/utils/StringUtils.h"
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+void addNode(NodeMap &allNodes, flat::NodeInfo nodeInfo) {
+  auto &node = allNodes[nodeInfo.app.nodeId];
+  node.base() = std::move(nodeInfo);
+  node.recordStatusChange(node.base().status, UtcClock::now());
+}
+
+CoTryTask<void> loadRoutingInfoVersion(core::ServiceOperation &ctx,
+                                       MgmtdStore &store,
+                                       kv::IReadWriteTransaction &txn,
+                                       flat::RoutingInfoVersion &newVersion) {
+  auto res = co_await store.loadRoutingInfoVersion(txn);
+  CO_RETURN_ON_ERROR(res);
+  LOG_OP_INFO(ctx, "Load routingInfoVersion {}", *res);
+  newVersion = nextVersion(*res);
+  co_return Void{};
+}
+
+CoTryTask<void> persistNewVersionAndSelfInfo(core::ServiceOperation &ctx,
+                                             MgmtdStore &store,
+                                             kv::IReadWriteTransaction &txn,
+                                             flat::NodeInfo &selfNodeInfo,
+                                             flat::PersistentNodeInfo &selfPersistentNodeInfo,
+                                             flat::RoutingInfoVersion newVersion,
+                                             NodeMap &allNodes) {
+  CO_RETURN_ON_ERROR(co_await store.storeRoutingInfoVersion(txn, newVersion));
+  bool needPersistSelf = true;
+  auto it = allNodes.find(selfNodeInfo.app.nodeId);
+  if (it != allNodes.end()) {
+    XLOGF_IF(FATAL,
+             it->second.base().type != flat::NodeType::MGMTD,
+             "Unexpected self type: {}",
+             toString(it->second.base().type));
+    auto persisted = flat::toPersistentNode(it->second.base());
+    selfNodeInfo.tags = persisted.tags;
+    selfPersistentNodeInfo.tags = persisted.tags;
+    if (persisted == selfPersistentNodeInfo) {
+      needPersistSelf = false;
+    }
+  }
+  if (needPersistSelf) {
+    CO_RETURN_ON_ERROR(co_await store.storeNodeInfo(txn, selfPersistentNodeInfo));
+  }
+  allNodes[selfNodeInfo.app.nodeId].base() = selfNodeInfo;
+  co_return Void{};
+}
+
+CoTryTask<void> loadAllNodes(core::ServiceOperation &ctx,
+                             MgmtdStore &store,
+                             kv::IReadWriteTransaction &txn,
+                             NodeMap &allNodes) {
+  auto loadRes = co_await store.loadAllNodes(txn);
+  CO_RETURN_ON_ERROR(loadRes);
+  LOG_OP_INFO(ctx, "Load {} nodes", loadRes->size());
+  for (auto &sn : *loadRes) {
+    addNode(allNodes, flat::toNode(sn));
+  }
+  co_return Void{};
+}
+
+CoTryTask<void> loadAllConfigs(core::ServiceOperation &ctx,
+                               MgmtdStore &store,
+                               kv::IReadOnlyTransaction &txn,
+                               ConfigMap &configMap) {
+  auto loadRes = co_await store.loadAllConfigs(txn);
+  CO_RETURN_ON_ERROR(loadRes);
+  for (auto &[type, info] : *loadRes) {
+    LOG_OP_INFO(ctx, "Load config {} version {}", magic_enum::enum_name(type), info.configVersion);
+    auto ver = info.configVersion;
+    configMap[type][ver] = std::move(info);
+  }
+  co_return Void{};
+}
+
+CoTryTask<void> loadAllChainTables(core::ServiceOperation &ctx,
+                                   MgmtdStore &store,
+                                   kv::IReadOnlyTransaction &txn,
+                                   ChainTableMap &chainTables) {
+  auto loadRes = co_await store.loadAllChainTables(txn);
+  CO_RETURN_ON_ERROR(loadRes);
+  for (auto &info : *loadRes) {
+    chainTables[info.chainTableId][info.chainTableVersion] = info;
+  }
+  co_return Void{};
+}
+
+CoTryTask<void> loadAllChains(core::ServiceOperation &ctx,
+                              MgmtdStore &store,
+                              kv::IReadOnlyTransaction &txn,
+                              ChainMap &chains,
+                              TargetMap &targetMap) {
+  auto loadRes = co_await store.loadAllChains(txn);
+  CO_RETURN_ON_ERROR(loadRes);
+  auto steadyNow = SteadyClock::now();
+  for (auto &info : *loadRes) {
+    for (const auto &t : info.targets) {
+      targetMap[t.targetId] = TargetInfo(makeTargetInfo(info.chainId, t), steadyNow);
+    }
+    chains[info.chainId] = info;
+  }
+  co_return Void{};
+}
+
+CoTryTask<void> loadAllUniversalTags(core::ServiceOperation &ctx,
+                                     MgmtdStore &store,
+                                     kv::IReadOnlyTransaction &txn,
+                                     UniversalTagsMap &universalTagsMap) {
+  auto loadRes = co_await store.loadAllUniversalTags(txn);
+  CO_RETURN_ON_ERROR(loadRes);
+  for (auto &[id, tags] : *loadRes) {
+    universalTagsMap[id] = std::move(tags);
+  }
+  co_return Void{};
+}
+
+CoTryTask<void> onNewLease(MgmtdState &state, core::ServiceOperation &ctx, MgmtdData &data, UtcTime start) {
+  flat::RoutingInfoVersion newRoutingInfoVersion{1};
+  NodeMap allNodes;
+  ConfigMap allConfigs;
+  magic_enum::enum_for_each<flat::NodeType>(
+      [&](auto type) { allConfigs[type][flat::ConfigVersion(0)] = flat::ConfigInfo{}; });
+  ChainTableMap chainTables;
+  ChainMap chains;
+  TargetMap targetMap;
+  UniversalTagsMap universalTagsMap;
+
+  auto handler = [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+    auto [rivRes, nodesRes, configsRes, chainTablesRes, chainsRes, utagsRes] =
+        co_await folly::coro::collectAll(loadRoutingInfoVersion(ctx, state.store_, txn, newRoutingInfoVersion),
+                                         loadAllNodes(ctx, state.store_, txn, allNodes),
+                                         loadAllConfigs(ctx, state.store_, txn, allConfigs),
+                                         loadAllChainTables(ctx, state.store_, txn, chainTables),
+                                         loadAllChains(ctx, state.store_, txn, chains, targetMap),
+                                         loadAllUniversalTags(ctx, state.store_, txn, universalTagsMap));
+    CO_RETURN_ON_ERROR(rivRes);
+    CO_RETURN_ON_ERROR(nodesRes);
+    CO_RETURN_ON_ERROR(configsRes);
+    CO_RETURN_ON_ERROR(chainTablesRes);
+    CO_RETURN_ON_ERROR(chainsRes);
+    CO_RETURN_ON_ERROR(utagsRes);
+
+    // TODO: check routingInfo integrity
+
+    co_return co_await persistNewVersionAndSelfInfo(ctx,
+                                                    state.store_,
+                                                    txn,
+                                                    state.selfNodeInfo_,
+                                                    state.selfPersistentNodeInfo_,
+                                                    newRoutingInfoVersion,
+                                                    allNodes);
+  };
+  auto loadRes = co_await withReadWriteTxn(state, std::move(handler));
+
+  if (!loadRes.hasError()) {
+    XLOGF_IF(FATAL, !allConfigs.contains(flat::NodeType::MGMTD), "Found no config for MGMTD");
+    const auto &cfg = allConfigs[flat::NodeType::MGMTD].rbegin()->second;
+    if (cfg.configVersion && updateSelfConfig(state, cfg)) {
+      state.selfNodeInfo_.configVersion = cfg.configVersion;
+      allNodes[state.selfId()].base().configVersion = cfg.configVersion;
+    }
+    data.reset(newRoutingInfoVersion,
+               std::move(allNodes),
+               std::move(allConfigs),
+               std::move(chainTables),
+               std::move(chains),
+               std::move(targetMap),
+               std::move(universalTagsMap));
+
+    auto clientSessionMap = co_await state.clientSessionMap_.coLock();
+    clientSessionMap->clear();
+    co_return Void{};
+  } else {
+    // retry in next round
+    CO_RETURN_ERROR(loadRes);
+  }
+}
+
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", "ExtendLease", "bg"> {
+  String toStringImpl() const final { return "ExtendLease"; }
+
+  auto handle(MgmtdState &state) -> CoTryTask<void> {
+    auto handler = [&](kv::IReadWriteTransaction &txn) -> CoTryTask<std::optional<flat::MgmtdLeaseInfo>> {
+      auto checkSelfRes = co_await state.store_.loadNodeInfo(txn, state.selfId());
+      CO_RETURN_ON_ERROR(checkSelfRes);
+
+      if (checkSelfRes->has_value() && flat::findTag(checkSelfRes->value().tags, flat::kDisabledTagKey) != -1) {
+        co_return std::nullopt;
+      }
+
+      auto extendRes = co_await state.store_.extendLease(txn,
+                                                         state.selfPersistentNodeInfo_,
+                                                         state.config_.lease_length().asUs(),
+                                                         state.utcNow(),
+                                                         flat::ReleaseVersion::fromVersionInfo(),
+                                                         state.config_.extend_lease_check_release_version());
+      CO_RETURN_ON_ERROR(extendRes);
+      co_return *extendRes;
+    };
+
+    auto commitRes = co_await withReadWriteTxn(state, std::move(handler), /*expectSelfPrimary=*/false);
+    if (commitRes.hasError()) {
+      co_return makeError(commitRes.error().code(), fmt::format("failed to commit: {}", commitRes.error().message()));
+    }
+
+    auto writerLock = co_await state.coScopedLock<"ExtendLease">();
+    auto start = state.utcNow();
+
+    auto dataPtr = co_await state.data_.coLock();
+
+    if (!commitRes->has_value()) {
+      LOG_OP_INFO(*this, "self is disabled, release lease");
+      dataPtr->lease.lease.reset();
+      co_return Void{};
+    }
+
+    const auto &newLease = commitRes->value();
+    auto &leaseInfo = dataPtr->lease;
+    String leaseChangedReason = [&] {
+      if (!leaseInfo.lease.has_value()) return "prev no lease";
+      if (leaseInfo.lease->primary.nodeId != newLease.primary.nodeId) return "primary id changed";
+      if (leaseInfo.lease->leaseStart != newLease.leaseStart) return "lease start changed";
+      if (leaseInfo.bootstrapping) return "still bootstrapping";
+      return "";
+    }();
+
+    leaseInfo.lease = newLease;
+    leaseInfo.bootstrapping = newLease.primary.nodeId == state.selfId() && !leaseChangedReason.empty();
+
+    if (leaseChangedReason.empty()) {
+      LOG_OP_DBG(*this, "lease not changed");
+    } else {
+      LOG_OP_INFO(*this,
+                  "newLease({}-{}-{}) changed({})",
+                  newLease.primary.nodeId,
+                  newLease.leaseStart.toMicroseconds(),
+                  newLease.leaseEnd.toMicroseconds(),
+                  leaseChangedReason);
+    }
+
+    if (leaseInfo.bootstrapping) {
+      LOG_OP_INFO(*this, "self got lease, loading ...");
+      co_return co_await onNewLease(state, *this, *dataPtr, start);
+    }
+
+    co_return Void{};
+  }
+};
+}  // namespace
+MgmtdLeaseExtender::MgmtdLeaseExtender(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdLeaseExtender::extend() {
+  Op op;
+
+  co_await [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_); }();
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdLeaseExtender.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdLeaseExtender {
+ public:
+  explicit MgmtdLeaseExtender(MgmtdState &state);
+
+  CoTask<void> extend();
+
+ private:
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdMetricsUpdater.cc

@@ -0,0 +1,278 @@
+#include "MgmtdMetricsUpdater.h"
+
+#include "common/utils/StringUtils.h"
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+template <NameWrapper MetricName, typename Map>
+void recordCount(const Map &counts, auto &&instanceF, auto &&tagF) {
+  using Key = typename Map::key_type;
+  static std::map<Key, monitor::ValueRecorder> recorders;
+  auto func = [&](auto &&f, const Key &k) {
+    if constexpr (is_specialization<Key, std::tuple>) {
+      return std::apply(f, k);
+    } else {
+      return f(k);
+    }
+  };
+  for (const auto &[k, v] : counts) {
+    auto it = recorders.find(k);
+    if (it == recorders.end()) {
+      auto instance = func(std::forward<decltype(instanceF)>(instanceF), k);
+      auto tag = func(std::forward<decltype(tagF)>(tagF), k);
+      std::optional<monitor::TagSet> tagSet;
+      if (!instance.empty() || !tag.empty()) {
+        tagSet = monitor::TagSet();
+        if (!instance.empty()) tagSet->addTag("instance", instance);
+        if (!tag.empty()) tagSet->addTag("tag", tag);
+      }
+      it = recorders.try_emplace(k, String(MetricName), std::move(tagSet)).first;
+    }
+    it->second.set(v);
+  }
+}
+
+auto emptyString = [](auto &&...) -> String { return ""; };
+
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", "UpdateMetrics", "bg"> {
+  String toStringImpl() const final { return "UpdateMetrics"; }
+
+  auto handle(MgmtdState &state) -> CoTryTask<void> {
+    auto dataPtr = co_await state.data_.coSharedLock();
+    auto clientSessionMap = co_await state.clientSessionMap_.coSharedLock();
+
+    auto bootstrapping = dataPtr->leaseStartTs + state.config_.bootstrapping_length() > SteadyClock::now();
+
+    reportNodeStatus(*dataPtr);
+    reportConfigStatus(*dataPtr, *clientSessionMap);
+    reportChainByReplicaCountStatus(*dataPtr);
+    reportTargetCount(*dataPtr, bootstrapping);
+    reportRoutingInfoVersion(*dataPtr);
+    reportConfigVersions(*dataPtr);
+    reportLeaseLasting(*dataPtr);
+    reportReleaseVersionStatus(*dataPtr, *clientSessionMap);
+    reportChainStatus(*dataPtr);
+    reportOrphanTargets(*dataPtr);
+    reportChainTargetCount(*dataPtr);
+
+    co_return Void{};
+  }
+
+  void reportNodeStatus(const MgmtdData &data) {
+    std::map<std::tuple<flat::NodeType, flat::NodeStatus>, int64_t> counts;
+    std::map<std::tuple<flat::NodeType, flat::NodeId>, int64_t> nodeStatuses;
+    for (const auto &[_, ni] : data.routingInfo.nodeMap) {
+      auto &base = ni.base();
+      ++counts[std::make_tuple(base.type, base.status)];
+      auto key = std::make_tuple(base.type, base.app.nodeId);
+      nodeStatuses[key] = static_cast<int64_t>(base.status) + 1;  // because PRIMARY_MGMTD = 0
+    }
+    recordCount<"MgmtdService.NodeStatusCount">(
+        counts,
+        [](auto type, auto) { return hf3fs::toString(type); },
+        [](auto, auto status) { return hf3fs::toString(status); });
+
+    recordCount<"MgmtdService.NodeStatus">(
+        nodeStatuses,
+        [](auto type, auto id) { return fmt::format("{}_{}", toStringView(type), id.toUnderType()); },
+        emptyString);
+  }
+
+  void reportConfigStatus(const MgmtdData &data, const ClientSessionMap &clientSessionMap) {
+    std::map<std::tuple<flat::NodeType, ConfigStatus>, int64_t> counts;
+    auto add = [&](auto type, auto version, auto status) {
+      if (version == 0) {
+        status = ConfigStatus::UNKNOWN;
+      } else {
+        auto latestVersion = data.getLatestConfigVersion(type);
+        assert(version <= latestVersion);
+        if (version != latestVersion) {
+          status = ConfigStatus::STALE;
+        }
+      }
+      ++counts[std::make_tuple(type, status)];
+    };
+    for (const auto &[_, ni] : data.routingInfo.nodeMap) {
+      auto &base = ni.base();
+      add(base.type, base.configVersion, base.configStatus);
+    }
+    for (const auto &[_, cs] : clientSessionMap) {
+      auto &base = cs.base();
+      add(base.type, base.configVersion, base.configStatus);
+    }
+    recordCount<"MgmtdService.ConfigStatusCount">(
+        counts,
+        [](auto type, auto) { return hf3fs::toString(type); },
+        [](auto, auto status) { return hf3fs::toString(status); });
+  }
+
+  void reportChainByReplicaCountStatus(const MgmtdData &data) {
+    std::map<size_t, int64_t> counts;
+    for (const auto &[_, ci] : data.routingInfo.chains) {
+      auto rc = ci.targets.size();
+      ++counts[rc];
+    }
+    recordCount<"MgmtdService.ChainCountByReplica">(
+        counts,
+        [](auto count) { return std::to_string(count); },
+        emptyString);
+  }
+
+  void reportChainTargetCount(const MgmtdData &data) {
+    static monitor::ValueRecorder chainCount("MgmtdService.ChainCount");
+    static monitor::ValueRecorder targetCount("MgmtdService.TargetCount");
+
+    chainCount.set(data.routingInfo.chains.size());
+    targetCount.set(data.routingInfo.getTargets().size());
+  }
+
+  void reportTargetCount(const MgmtdData &data, bool bootstrapping) {
+    using PS = flat::PublicTargetState;
+    using LS = flat::LocalTargetState;
+    std::map<std::tuple<PS, LS>, int64_t> counts;
+    std::map<std::tuple<flat::ChainId, flat::TargetId>, int64_t> abnormalChains;
+
+    for (const auto &[cid, ci] : data.routingInfo.chains) {
+      bool abnormal = false;
+      for (const auto &[tid, _] : ci.targets) {
+        const auto &ti = data.routingInfo.getTargets().at(tid);
+        auto ps = ti.base().publicState;
+        auto ls = ti.base().localState;
+        ++counts[std::make_tuple(ps, ls)];
+        abnormal |= (!bootstrapping && ps != PS::SERVING);
+      }
+      if (abnormal) {
+        for (const auto &cti : ci.targets) {
+          abnormalChains[std::make_tuple(cid, cti.targetId)] = static_cast<int64_t>(cti.publicState);
+        }
+      }
+    }
+    recordCount<"MgmtdService.TargetStatusCount">(
+        counts,
+        [](auto ps, auto) { return hf3fs::toString(ps); },
+        [](auto, auto ls) { return hf3fs::toString(ls); });
+
+    recordCount<"MgmtdService.AbnormalChainStatus">(
+        abnormalChains,
+        [](auto cid, auto) { return std::to_string(cid); },
+        [](auto, auto tid) { return std::to_string(tid); });
+  }
+
+  void reportRoutingInfoVersion(const MgmtdData &data) {
+    static monitor::ValueRecorder recorder("MgmtdService.RoutingInfoVersion");
+    recorder.set(data.routingInfo.routingInfoVersion);
+  }
+
+  void reportConfigVersions(const MgmtdData &data) {
+    std::map<flat::NodeType, int64_t> versions;
+    for (const auto &[k, vm] : data.configMap) {
+      auto ver = vm.rbegin()->first;
+      versions[k] = ver;
+    }
+    recordCount<"MgmtdService.ConfigVersions">(
+        versions,
+        [](auto type) { return hf3fs::toString(type); },
+        emptyString);
+  }
+
+  void reportLeaseLasting(const MgmtdData &data) {
+    static monitor::ValueRecorder recorder("MgmtdService.LeaseLasting");
+    auto lasting = Duration(SteadyClock::now() - data.leaseStartTs);
+    recorder.set(lasting.count());
+  }
+
+  void reportReleaseVersionStatus(const MgmtdData &data, const ClientSessionMap &clientSessionMap) {
+    constexpr auto invalidType = static_cast<flat::NodeType>(255U);
+    std::map<std::tuple<flat::NodeType, flat::ReleaseVersion>, int64_t> counts;
+    auto activeNode = flat::selectActiveNode();
+    for (const auto &[_, ni] : data.routingInfo.nodeMap) {
+      if (!activeNode(ni.base())) continue;
+      ++counts[std::make_tuple(ni.base().type, ni.base().app.releaseVersion)];
+      ++counts[std::make_tuple(invalidType, ni.base().app.releaseVersion)];
+    }
+    for (const auto &[_, cs] : clientSessionMap) {
+      ++counts[std::make_tuple(cs.base().type, cs.base().releaseVersion)];
+    }
+    recordCount<"MgmtdService.ReleaseVersionCount">(
+        counts,
+        [](auto type, auto) { return type == invalidType ? "SERVER" : hf3fs::toString(type); },
+        [](auto, auto rv) { return fmt::format("{}", rv); });
+  }
+
+  void reportChainStatus(const MgmtdData &data) {
+    using PS = flat::PublicTargetState;
+    std::map<std::tuple<size_t, String>, int64_t> counts;
+    for (const auto &[_, ci] : data.routingInfo.chains) {
+      auto replicaCount = ci.targets.size();
+      size_t serving = 0, syncing = 0;
+      for (const auto &ti : ci.targets) {
+        switch (ti.publicState) {
+          case PS::SERVING:
+            ++serving;
+            break;
+          case PS::SYNCING:
+            ++syncing;
+            break;
+          default:
+            break;
+        }
+      }
+      String status = [&] {
+        if (serving == replicaCount)
+          return "SERVING-FULL";
+        else if (serving)
+          return "SERVING-PARTIAL";
+        else if (syncing)
+          return "SYNCING";
+        else
+          return "UNAVAILABLE";
+      }();
+      ++counts[std::make_tuple(replicaCount, status)];
+    }
+    recordCount<"MgmtdService.ChainStatus">(
+        counts,
+        [](auto replicaCount, auto) { return std::to_string(replicaCount); },
+        [](auto, auto status) { return status; });
+  }
+
+  void reportOrphanTargets(const MgmtdData &data) {
+    static constexpr auto invalidNodeId = flat::NodeId(0);
+    std::map<flat::NodeId, int64_t> counts;
+    std::map<flat::NodeId, int64_t> usedSizes;
+    for (const auto &[tid, ti] : data.routingInfo.orphanTargetsByTargetId) {
+      XLOGF_IF(FATAL, !ti.nodeId, "Orphan targets should always have nodeId. ti: {}", serde::toJsonString(ti));
+      auto nodeId = *ti.nodeId;
+      ++counts[nodeId];
+      ++counts[invalidNodeId];
+
+      usedSizes[nodeId] += ti.usedSize;
+      usedSizes[invalidNodeId] += ti.usedSize;
+    }
+    recordCount<"MgmtdService.OrphanTargetCount">(
+        counts,
+        [](auto nodeId) { return nodeId == invalidNodeId ? "" : std::to_string(nodeId); },
+        emptyString);
+  }
+};
+}  // namespace
+
+MgmtdMetricsUpdater::MgmtdMetricsUpdater(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdMetricsUpdater::update() {
+  Op op;
+  auto handler = [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_); };
+
+  auto res = co_await doAsPrimary(state_, std::move(handler));
+  if (res.hasError()) {
+    if (res.error().code() == MgmtdCode::kNotPrimary)
+      LOG_OP_INFO(op, "self is not primary, skip");
+    else
+      LOG_OP_ERR(op, "failed: {}", res.error());
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdMetricsUpdater.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdMetricsUpdater {
+ public:
+  explicit MgmtdMetricsUpdater(MgmtdState &state);
+
+  CoTask<void> update();
+
+ private:
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdNewBornChainsChecker.cc

@@ -0,0 +1,71 @@
+#include "MgmtdNewBornChainsChecker.h"
+
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", "CheckNewBornChains", "bg"> {
+  String toStringImpl() const final { return "CheckNewBornChains"; }
+
+  auto handle(MgmtdState &state) -> CoTryTask<void> {
+    auto bootstrapInterval = state.config_.new_chain_bootstrap_interval().asUs();
+
+    SteadyTime leaseStartTs;
+    std::vector<flat::ChainId> candidates;
+    auto steadyNow = SteadyClock::now();
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      leaseStartTs = dataPtr->leaseStartTs;
+
+      const auto &ri = dataPtr->routingInfo;
+
+      for (const auto &[chainId, bornTime] : ri.newBornChains) {
+        if (bornTime + bootstrapInterval <= steadyNow) {
+          candidates.push_back(chainId);
+        }
+      }
+    }
+
+    if (!candidates.empty()) {
+      LOG_OP_DBG(*this, "candidates {}", fmt::join(candidates, ","));
+      auto dataPtr = co_await state.data_.coLock();
+      if (leaseStartTs != dataPtr->leaseStartTs) {
+        LOG_OP_DBG(*this, "lease changed, skip this round");
+        co_return Void{};
+      }
+
+      auto steadyNow = SteadyClock::now();
+      auto &ri = dataPtr->routingInfo;
+      for (auto cid : candidates) {
+        ri.newBornChains.erase(cid);
+        const auto &chain = dataPtr->routingInfo.getChain(cid);
+        for (const auto &cti : chain.targets) {
+          dataPtr->routingInfo.updateTarget(cti.targetId, [&](auto &ti) { ti.updateTs(steadyNow); });
+        }
+      }
+    }
+    co_return Void{};
+  }
+};
+}  // namespace
+
+MgmtdNewBornChainsChecker::MgmtdNewBornChainsChecker(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdNewBornChainsChecker::check() {
+  Op op;
+  auto handler = [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_); };
+
+  auto res = co_await doAsPrimary(state_, std::move(handler));
+  if (res.hasError()) {
+    if (res.error().code() == MgmtdCode::kNotPrimary)
+      LOG_OP_INFO(op, "self is not primary, skip");
+    else
+      LOG_OP_ERR(op, "failed: {}", res.error());
+  }
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdNewBornChainsChecker.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdNewBornChainsChecker {
+ public:
+  explicit MgmtdNewBornChainsChecker(MgmtdState &state);
+
+  CoTask<void> check();
+
+ private:
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdRoutingInfoVersionUpdater.cc

@@ -0,0 +1,44 @@
+#include "MgmtdRoutingInfoVersionUpdater.h"
+
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", "BumpRoutingInfoVersion", "bg"> {
+  String toStringImpl() const final { return "BumpRoutingInfoVersion"; }
+
+  auto handle(MgmtdState &state) -> CoTryTask<void> {
+    auto writerLock = co_await state.coScopedLock<"BumpRoutingInfoVersion">();
+    bool needChange = co_await [&]() -> CoTask<bool> {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      co_return dataPtr->routingInfo.routingInfoChanged;
+    }();
+
+    if (needChange) {
+      CO_RETURN_ON_ERROR(co_await updateStoredRoutingInfo(state, *this));
+      co_await updateMemoryRoutingInfo(state, *this);
+    }
+
+    co_return Void{};
+  }
+};
+}  // namespace
+MgmtdRoutingInfoVersionUpdater::MgmtdRoutingInfoVersionUpdater(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdRoutingInfoVersionUpdater::update() {
+  Op op;
+  auto handler = [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_); };
+
+  auto res = co_await doAsPrimary(state_, std::move(handler));
+  if (res.hasError()) {
+    if (res.error().code() == MgmtdCode::kNotPrimary)
+      LOG_OP_INFO(op, "self is not primary, skip");
+    else
+      LOG_OP_ERR(op, "failed: {}", res.error());
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdRoutingInfoVersionUpdater.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdRoutingInfoVersionUpdater {
+ public:
+  explicit MgmtdRoutingInfoVersionUpdater(MgmtdState &state);
+
+  CoTask<void> update();
+
+ private:
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdTargetInfoLoader.cc

@@ -0,0 +1,120 @@
+#include "MgmtdTargetInfoLoader.h"
+
+#include "common/utils/OptionalUtils.h"
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+
+#define OP_NAME "LoadTargetInfo"
+
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", OP_NAME, "bg"> {
+  Op(SteadyTime &loadedLeaseStart)
+      : loadedLeaseStart_(loadedLeaseStart) {}
+
+  String toStringImpl() const final { return OP_NAME; }
+
+  auto handle(MgmtdState &state) -> CoTryTask<void> {
+    {
+      if ((co_await ensureSelfIsPrimary(state)).hasError()) {
+        co_return Void{};
+      }
+
+      auto dataPtr = co_await state.data_.coSharedLock();
+      if (loadedLeaseStart_ >= dataPtr->leaseStartTs) {
+        co_return Void{};
+      }
+
+      loadedLeaseStart_ = dataPtr->leaseStartTs;
+    }
+
+    flat::TargetId startTid(0);
+    for (;;) {
+      auto handler = [&](kv::IReadOnlyTransaction &txn) -> CoTryTask<std::vector<flat::TargetInfo>> {
+        return state.store_.loadTargetsFrom(txn, startTid);
+      };
+
+      auto res = co_await withReadOnlyTxn(state, std::move(handler));
+      CO_RETURN_ON_ERROR(res);
+
+      LOG_OP_DBG(*this, "load {} targets starting from {}", res->size(), startTid);
+
+      if (res->empty()) break;
+      startTid = flat::TargetId(res->back().targetId + 1);
+
+      if ((co_await ensureSelfIsPrimary(state)).hasError()) {
+        co_return Void{};
+      }
+
+      auto dataPtr = co_await state.data_.coLock();
+      if (loadedLeaseStart_ < dataPtr->leaseStartTs) {
+        co_return Void{};
+      }
+
+      for (auto &loaded : *res) {
+        auto tid = loaded.targetId;
+        auto updater = [this, loaded = std::move(loaded)](TargetInfo &ti) {
+          ti.locationInitLoaded = true;
+          ti.persistedNodeId = loaded.nodeId;
+          ti.persistedDiskIndex = loaded.diskIndex;
+          LOG_OP_DBG(*this,
+                     "TargetInfo of {} loaded, nodeId={}, diskIndex={}",
+                     loaded.targetId,
+                     loaded.nodeId,
+                     OptionalFmt(loaded.diskIndex));
+          if (!ti.base().nodeId) {
+            ti.base().nodeId = loaded.nodeId;
+            ti.base().diskIndex = loaded.diskIndex;
+            LOG_OP_DBG(*this, "Fill TargetInfo of {}", loaded.targetId);
+          }
+        };
+        dataPtr->routingInfo.updateTarget(tid, std::move(updater));
+      }
+    }
+
+    if ((co_await ensureSelfIsPrimary(state)).hasError()) {
+      co_return Void{};
+    }
+
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      if (loadedLeaseStart_ < dataPtr->leaseStartTs) {
+        co_return Void{};
+      }
+
+      for (const auto &[tid, _] : dataPtr->routingInfo.getTargets()) {
+        auto updater = [this](TargetInfo &ti) {
+          if (!ti.locationInitLoaded) {
+            ti.locationInitLoaded = true;
+            LOG_OP_DBG(*this, "TargetInfo of {} not found in kv", ti.base().targetId);
+          }
+        };
+        dataPtr->routingInfo.updateTarget(tid, std::move(updater));
+      }
+    }
+
+    co_return Void{};
+  }
+
+  SteadyTime &loadedLeaseStart_;
+};
+}  // namespace
+MgmtdTargetInfoLoader::MgmtdTargetInfoLoader(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdTargetInfoLoader::run() {
+  Op op(loadedLeaseStart);
+  auto handler = [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_); };
+
+  auto res = co_await doAsPrimary(state_, std::move(handler));
+  if (res.hasError()) {
+    if (res.error().code() == MgmtdCode::kNotPrimary)
+      LOG_OP_INFO(op, "self is not primary, skip");
+    else
+      LOG_OP_ERR(op, "failed: {}", res.error());
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdTargetInfoLoader.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+#include "common/utils/UtcTime.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdTargetInfoLoader {
+ public:
+  explicit MgmtdTargetInfoLoader(MgmtdState &state);
+
+  CoTask<void> run();
+
+ private:
+  SteadyTime loadedLeaseStart;
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdTargetInfoPersister.cc

@@ -0,0 +1,82 @@
+#include "MgmtdTargetInfoPersister.h"
+
+#include "common/utils/OptionalUtils.h"
+#include "core/utils/ServiceOperation.h"
+#include "core/utils/runOp.h"
+#include "mgmtd/service/MgmtdState.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+
+#define OP_NAME "PersistTargetInfo"
+
+struct Op : core::ServiceOperationWithMetric<"MgmtdService", OP_NAME, "bg"> {
+  String toStringImpl() const final { return OP_NAME; }
+
+  auto handle(MgmtdState &state) -> CoTryTask<void> {
+    std::vector<flat::TargetInfo> candidates;
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      const auto &ri = dataPtr->routingInfo;
+      for (const auto &[tid, ti] : ri.getTargets()) {
+        if (ti.locationInitLoaded  // the persisted location is known
+            && ti.base().nodeId    // the actual location is known
+            && (ti.persistedNodeId != ti.base().nodeId ||
+                ti.persistedDiskIndex != ti.base().diskIndex)  // and they are different
+        ) {
+          candidates.push_back(ti.base());
+          if (static_cast<int>(candidates.size()) >= state.config_.target_info_persist_batch()) break;
+        }
+      }
+    }
+
+    if (candidates.empty()) {
+      co_return Void{};
+    }
+
+    auto handler = [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+      for (const auto &ti : candidates) {
+        CO_RETURN_ON_ERROR(co_await state.store_.storeTargetInfo(txn, ti));
+      }
+      co_return Void{};
+    };
+
+    auto res = co_await withReadWriteTxn(state, std::move(handler));
+    CO_RETURN_ON_ERROR(res);
+
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      for (const auto &persisted : candidates) {
+        dataPtr->routingInfo.updateTarget(persisted.targetId, [&](auto &ti) {
+          ti.persistedNodeId = persisted.nodeId;
+          ti.persistedDiskIndex = persisted.diskIndex;
+          LOG_OP_DBG(*this,
+                     "TargetInfo of {} persisted, nodeId={}, diskIndex={}",
+                     persisted.targetId,
+                     persisted.nodeId,
+                     OptionalFmt(persisted.diskIndex));
+        });
+      }
+    }
+
+    co_return Void{};
+  }
+};
+}  // namespace
+MgmtdTargetInfoPersister::MgmtdTargetInfoPersister(MgmtdState &state)
+    : state_(state) {}
+
+CoTask<void> MgmtdTargetInfoPersister::run() {
+  Op op;
+  auto handler = [&]() -> CoTryTask<void> { CO_INVOKE_OP_INFO(op, "background", state_); };
+
+  auto res = co_await doAsPrimary(state_, std::move(handler));
+  if (res.hasError()) {
+    if (res.error().code() == MgmtdCode::kNotPrimary)
+      LOG_OP_INFO(op, "self is not primary, skip");
+    else
+      LOG_OP_ERR(op, "failed: {}", res.error());
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/background/MgmtdTargetInfoPersister.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "common/utils/Coroutine.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdState;
+class MgmtdTargetInfoPersister {
+ public:
+  explicit MgmtdTargetInfoPersister(MgmtdState &state);
+
+  CoTask<void> run();
+
+ private:
+  MgmtdState &state_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/mgmtd.cpp

@@ -0,0 +1,8 @@
+#include "common/app/TwoPhaseApplication.h"
+#include "memory/common/OverrideCppNewDelete.h"
+#include "mgmtd/MgmtdServer.h"
+
+int main(int argc, char *argv[]) {
+  using namespace hf3fs;
+  return TwoPhaseApplication<mgmtd::MgmtdServer>().run(argc, argv);
+}

src/mgmtd/mgmtd.toml

@@ -0,0 +1,7 @@
+[common]
+log = """
+  DBG:normal:err:fatal;
+  normal=file:path=mgmtd.log,async=true;
+  err=file:path=mgmtd.error.log,async=false,level=ERR;
+  fatal=stream:stream=stderr,level=FATAL
+"""

src/mgmtd/ops/EnableDisableNodeOperation.cc

@@ -0,0 +1,90 @@
+#include "EnableDisableNodeOperation.h"
+
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+flat::NodeInfo onNodeEnabled(const flat::NodeInfo &oldNodeInfo, UtcTime) {
+  flat::NodeInfo sn = oldNodeInfo;
+  sn.status = flat::NodeStatus::HEARTBEAT_CONNECTING;
+  [[maybe_unused]] auto ok = flat::removeTag(sn.tags, flat::kDisabledTagKey);
+  assert(ok);
+  return sn;
+}
+
+flat::NodeInfo onNodeDisabled(const flat::NodeInfo &oldNodeInfo, UtcTime) {
+  flat::NodeInfo sn = oldNodeInfo;
+  sn.status = flat::NodeStatus::DISABLED;
+  assert(flat::findTag(sn.tags, flat::kDisabledTagKey) == -1);
+  sn.tags.emplace_back(flat::kDisabledTagKey, "");
+  return sn;
+}
+
+template <NameWrapper method>
+CoTryTask<void> changeNodeStatus(MgmtdState &state, core::ServiceOperation &ctx, flat::NodeId nodeId, bool enable) {
+  auto start = state.utcNow();
+  auto writerLock = co_await state.coScopedLock<method>();
+
+  auto oldNodeRes = co_await [&]() -> CoTryTask<flat::NodeInfo> {
+    auto dataPtr = co_await state.data_.coSharedLock();
+    const auto &ri = dataPtr->routingInfo;
+    auto it = ri.nodeMap.find(nodeId);
+    if (it == ri.nodeMap.end()) CO_RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kNodeNotFound, "");
+    co_return it->second.base();
+  }();
+  CO_RETURN_ON_ERROR(oldNodeRes);
+
+  flat::NodeInfo newNode;
+  if (enable && oldNodeRes->status == flat::NodeStatus::DISABLED) {
+    newNode = onNodeEnabled(*oldNodeRes, start);
+  } else if (!enable && oldNodeRes->status != flat::NodeStatus::DISABLED) {
+    newNode = onNodeDisabled(*oldNodeRes, start);
+  } else {
+    co_return Void{};
+  }
+
+  CO_RETURN_ON_ERROR(
+      co_await updateStoredRoutingInfo(state, ctx, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+        CO_RETURN_ON_ERROR(co_await state.store_.storeNodeInfo(txn, flat::toPersistentNode(newNode)));
+        co_return Void{};
+      }));
+
+  co_await updateMemoryRoutingInfo(state, ctx, [&](auto &ri) {
+    auto &info = ri.nodeMap[nodeId];
+    info.base() = newNode;
+    info.recordStatusChange(newNode.status, UtcClock::now());
+    info.updateTs();
+  });
+  co_return Void{};
+};
+}  // namespace
+
+CoTryTask<EnableNodeRsp> EnableNodeOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  auto nodeId = req.nodeId;
+  if (nodeId == state.selfId()) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Can't enable primary mgmtd");
+  }
+  co_return co_await doAsPrimary(state, [&]() -> CoTryTask<EnableNodeRsp> {
+    CO_RETURN_ON_ERROR(co_await changeNodeStatus<"EnableNode">(state, *this, nodeId, true));
+    co_return EnableNodeRsp::create();
+  });
+}
+
+CoTryTask<DisableNodeRsp> DisableNodeOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+  auto nodeId = req.nodeId;
+  if (nodeId == state.selfId()) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Can't disable primary mgmtd");
+  }
+  co_return co_await doAsPrimary(state, [&]() -> CoTryTask<DisableNodeRsp> {
+    CO_RETURN_ON_ERROR(co_await changeNodeStatus<"DisableNode">(state, *this, nodeId, false));
+    co_return DisableNodeRsp::create();
+  });
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/EnableDisableNodeOperation.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct EnableNodeOperation : core::ServiceOperationWithMetric<"MgmtdService", "EnableNode", "op"> {
+  EnableNodeReq req;
+
+  explicit EnableNodeOperation(EnableNodeReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("EnableNode {}", req.nodeId); }
+
+  CoTryTask<EnableNodeRsp> handle(MgmtdState &state);
+};
+
+struct DisableNodeOperation : core::ServiceOperationWithMetric<"MgmtdService", "DisableNode", "op"> {
+  DisableNodeReq req;
+
+  explicit DisableNodeOperation(DisableNodeReq &&r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("disableNode {}", req.nodeId); }
+
+  CoTryTask<DisableNodeRsp> handle(MgmtdState &state);
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/ExtendClientSessionOperation.cc

@@ -0,0 +1,108 @@
+#include "ExtendClientSessionOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<ExtendClientSessionRsp> ExtendClientSessionOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  const auto &clientId = req.clientId;
+  const auto &sessionData = req.data;
+
+  if (clientId.empty()) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Empty clientId");
+  }
+
+  if (auto uuidRes = Uuid::fromHexString(clientId); uuidRes.hasError()) {
+    LOG_OP_ERR(*this, "ClientId not valid hex uuid. req: {}", serde::toJsonString(req));
+    if (state.config_.only_accept_client_uuid()) {
+      co_return makeError(StatusCode::kInvalidArg, "ClientId not valid hex uuid");
+    }
+  }
+
+  auto handler = [&]() -> CoTryTask<ExtendClientSessionRsp> {
+    std::optional<flat::ConfigInfo> config;
+    std::vector<flat::TagPair> tags;
+    auto nodeType = req.type;
+
+    if (nodeType != flat::NodeType::CLIENT && nodeType != flat::NodeType::FUSE) {
+      CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Invalid node type: {}", toStringView(nodeType));
+    }
+
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      CO_RETURN_ON_ERROR(dataPtr->checkConfigVersion(*this, nodeType, req.configVersion));
+      config = dataPtr->getConfig(nodeType, req.configVersion, /*latest=*/true);
+      if (dataPtr->universalTagsMap.contains(sessionData.universalId)) {
+        tags = dataPtr->universalTagsMap.at(sessionData.universalId);
+      }
+    }
+
+    {
+      auto clientSessionMap = co_await state.clientSessionMap_.coLock();
+      auto &sessionMap = *clientSessionMap;
+      auto it = sessionMap.find(clientId);
+      if (it == sessionMap.end()) {
+        sessionMap[clientId].base() = flat::ClientSession(req);
+      } else {
+        auto &cur = it->second;
+        auto &base = cur.base();
+        if (cur.clientSessionVersion >= req.clientSessionVersion) {
+          LOG_OP_ERR(*this,
+                     "ClientSessoin version stale. clientId:{} client:{} server:{}",
+                     clientId,
+                     req.clientSessionVersion,
+                     cur.clientSessionVersion);
+
+          co_return makeError(MgmtdCode::kClientSessionVersionStale, std::to_string(cur.clientSessionVersion));
+        }
+        auto unexpectedChange = [&]() -> String {
+          if (base.universalId != sessionData.universalId) {
+            return fmt::format("Expected universalId: {}. UniversalId in request: {}",
+                               base.universalId,
+                               sessionData.universalId);
+          }
+          if (base.description != sessionData.description) {
+            return fmt::format("Expected description: {}. Description in request: {}",
+                               base.description,
+                               sessionData.description);
+          }
+          if (base.serviceGroups != sessionData.serviceGroups) {
+            return fmt::format("Expected serviceGroups: {}. ServiceGroups in request: {}",
+                               serde::toJsonString(base.serviceGroups),
+                               serde::toJsonString(sessionData.description));
+          }
+          if (base.releaseVersion != sessionData.releaseVersion) {
+            return fmt::format("Expected releaseVersion: {}. ReleaseVersion in request: {}",
+                               base.releaseVersion,
+                               sessionData.releaseVersion);
+          }
+          if (base.type != nodeType) {
+            return fmt::format("Expected type: {}. Type in request: {}",
+                               toStringView(base.type),
+                               toStringView(nodeType));
+          }
+          if (base.clientStart != req.clientStart) {
+            return fmt::format("Expected clientStart: {}. Type in request: {}",
+                               base.clientStart.YmdHMS(),
+                               req.clientStart.YmdHMS());
+          }
+          return "";
+        }();
+
+        if (!unexpectedChange.empty()) {
+          CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kExtendClientSessionMismatch, "{}", unexpectedChange);
+        }
+        cur.clientSessionVersion = req.clientSessionVersion;
+        cur.base().configVersion = req.configVersion;
+        cur.base().configStatus = req.configStatus;
+        cur.base().lastExtend = UtcClock::now();
+        cur.updateTs();
+      }
+    }
+    co_return ExtendClientSessionRsp::create(std::move(config), std::move(tags));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/ExtendClientSessionOperation.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct ExtendClientSessionOperation : core::ServiceOperationWithMetric<"MgmtdService", "ExtendClientSession", "op"> {
+  ExtendClientSessionReq req;
+
+  explicit ExtendClientSessionOperation(ExtendClientSessionReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final {
+    return fmt::format("ExtendClientSession {}@{}", req.clientId, req.data.universalId);
+  }
+
+  CoTryTask<ExtendClientSessionRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetClientSessionOperation.cc

@@ -0,0 +1,32 @@
+#include "GetClientSessionOperation.h"
+
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<GetClientSessionRsp> GetClientSessionOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  if (req.clientId.empty()) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Empty clientId");
+  }
+
+  const auto bootstrappingLength = state.config_.bootstrapping_length().asUs();
+  GetClientSessionRsp rsp;
+  auto handler = [&]() -> CoTryTask<void> {
+    auto dataPtr = co_await state.data_.coSharedLock();
+    auto clientSessionMap = co_await state.clientSessionMap_.coSharedLock();
+    rsp.bootstrapping = dataPtr->leaseStartTs + bootstrappingLength > SteadyClock::now();
+    auto it = clientSessionMap->find(req.clientId);
+    if (it != clientSessionMap->end()) {
+      rsp.session = it->second.base();
+
+      const auto &universalId = rsp.session->universalId;
+      rsp.referencedTags = dataPtr->universalTagsMap.contains(universalId) ? dataPtr->universalTagsMap.at(universalId)
+                                                                           : std::vector<flat::TagPair>{};
+    }
+    co_return Void{};
+  };
+  CO_RETURN_ON_ERROR(co_await doAsPrimary(state, std::move(handler)));
+  co_return rsp;
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetClientSessionOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct GetClientSessionOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetClientSession", "op"> {
+  GetClientSessionReq req;
+
+  explicit GetClientSessionOperation(GetClientSessionReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("GetClientSession {}", req.clientId); }
+
+  CoTryTask<GetClientSessionRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetConfigOperation.cc

@@ -0,0 +1,19 @@
+#include "GetConfigOperation.h"
+
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<GetConfigRsp> GetConfigOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  auto nodeType = req.nodeType;
+  auto version = req.configVersion;
+
+  auto handler = [&]() -> CoTryTask<GetConfigRsp> {
+    auto configInfo = (co_await state.data_.coSharedLock())->getConfig(nodeType, version, !req.exactVersion);
+    co_return GetConfigRsp::create(std::move(configInfo));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetConfigOperation.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct GetConfigOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetConfig", "op"> {
+  GetConfigReq req;
+
+  explicit GetConfigOperation(GetConfigReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final {
+    return fmt::format("GetConfig {}@{}", magic_enum::enum_name(req.nodeType), req.configVersion);
+  }
+
+  CoTryTask<GetConfigRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetConfigVersionsOperation.cc

@@ -0,0 +1,24 @@
+#include "GetConfigVersionsOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<GetConfigVersionsRsp> GetConfigVersionsOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  auto handler = [&]() -> CoTryTask<GetConfigVersionsRsp> {
+    auto dataPtr = co_await state.data_.coSharedLock();
+    RHStringHashMap<flat::ConfigVersion> versions;
+    for (const auto &[k, vm] : dataPtr->configMap) {
+      auto key = hf3fs::toString(k);
+      XLOGF_IF(FATAL, vm.empty(), "Config version map empty! key: {}", key);
+      auto vit = vm.rbegin();
+      versions[key] = vit->first;
+    }
+    co_return GetConfigVersionsRsp::create(std::move(versions));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetConfigVersionsOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct GetConfigVersionsOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetConfigVersions", "op"> {
+  GetConfigVersionsReq req;
+
+  explicit GetConfigVersionsOperation(GetConfigVersionsReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return "GetConfigVersions"; }
+
+  CoTryTask<GetConfigVersionsRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetPrimaryMgmtdOperation.cc

@@ -0,0 +1,24 @@
+#include "GetPrimaryMgmtdOperation.h"
+
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<GetPrimaryMgmtdRsp> GetPrimaryMgmtdOperation::handle(MgmtdState &state) const {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  auto now = state.utcNow();
+  auto lease = co_await state.currentLease(now);
+  if (lease.has_value()) {
+    co_return GetPrimaryMgmtdRsp::create(std::move(lease->primary));
+  }
+  auto handler = [&](kv::IReadOnlyTransaction &txn) -> CoTryTask<std::optional<flat::MgmtdLeaseInfo>> {
+    co_return co_await state.store_.loadMgmtdLeaseInfo(txn);
+  };
+  auto res = co_await kv::WithTransaction(state.createRetryStrategy())
+                 .run(state.env_->kvEngine()->createReadonlyTransaction(), std::move(handler));
+  CO_RETURN_ON_ERROR(res);
+  if (res->has_value()) {
+    co_return GetPrimaryMgmtdRsp::create(std::move(res->value().primary));
+  }
+  co_return GetPrimaryMgmtdRsp::create(std::nullopt);
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetPrimaryMgmtdOperation.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+
+struct GetPrimaryMgmtdOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetPrimaryMgmtd", "op"> {
+  GetPrimaryMgmtdReq req;
+
+  explicit GetPrimaryMgmtdOperation(GetPrimaryMgmtdReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return "GetPrimaryMgmtd"; }
+
+  CoTryTask<GetPrimaryMgmtdRsp> handle(MgmtdState &state) const;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetRoutingInfoOperation.cc

@@ -0,0 +1,22 @@
+#include "GetRoutingInfoOperation.h"
+
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<GetRoutingInfoRsp> GetRoutingInfoOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  auto handler = [&]() -> CoTryTask<GetRoutingInfoRsp> {
+    auto dataPtr = co_await state.data_.coSharedLock();
+    CO_RETURN_ON_ERROR(dataPtr->checkRoutingInfoVersion(*this, req.routingInfoVersion));
+    auto rsp = GetRoutingInfoRsp::create(dataPtr->getRoutingInfo(req.routingInfoVersion, state.config_));
+    if (rsp.info) {
+      LOG_OP_DBG(*this, "return {}", rsp.info->routingInfoVersion);
+    } else {
+      LOG_OP_DBG(*this, "return nullopt");
+    }
+    co_return rsp;
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetRoutingInfoOperation.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+
+struct GetRoutingInfoOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetRoutingInfo", "op"> {
+  GetRoutingInfoReq req;
+
+  explicit GetRoutingInfoOperation(GetRoutingInfoReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("GetRoutingInfo for {}", req.routingInfoVersion); }
+
+  CoTryTask<GetRoutingInfoRsp> handle(MgmtdState &state);
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetUniversalTagsOperation.cc

@@ -0,0 +1,19 @@
+#include "GetUniversalTagsOperation.h"
+
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<GetUniversalTagsRsp> GetUniversalTagsOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  auto handler = [&]() -> CoTryTask<GetUniversalTagsRsp> {
+    auto dataPtr = co_await state.data_.coSharedLock();
+    const auto &m = dataPtr->universalTagsMap;
+    auto it = m.find(req.universalId);
+    if (it != m.end()) co_return GetUniversalTagsRsp::create(it->second);
+    co_return GetUniversalTagsRsp::create();
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/GetUniversalTagsOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct GetUniversalTagsOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetUniversalTags", "op"> {
+  GetUniversalTagsReq req;
+
+  explicit GetUniversalTagsOperation(GetUniversalTagsReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("GetUniversalTags {}", req.universalId); }
+
+  CoTryTask<GetUniversalTagsRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/HeartbeatOperation.cc

@@ -0,0 +1,202 @@
+#include "HeartbeatOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+flat::NodeInfo onNewNode(const flat::HeartbeatInfo &hb, UtcTime now) {
+  flat::NodeInfo sn;
+  sn.app = hb.app;
+  sn.type = hb.type();
+  sn.status = flat::NodeStatus::HEARTBEAT_CONNECTED;
+  sn.lastHeartbeatTs = now;
+  // sn.tags is empty
+  sn.configVersion = hb.configVersion;
+  return sn;
+}
+
+flat::NodeInfo onNodeChanged(const flat::NodeInfo &oldNodeInfo, const flat::HeartbeatInfo &hb, UtcTime) {
+  flat::NodeInfo sn = oldNodeInfo;
+  sn.app = hb.app;
+  sn.status = flat::NodeStatus::HEARTBEAT_CONNECTED;
+  sn.configVersion = hb.configVersion;
+  sn.configStatus = hb.configStatus;
+  return sn;
+}
+
+Result<flat::NodeInfo> prepareNewNodeInfo(MgmtdState &state,
+                                          core::ServiceOperation &ctx,
+                                          UtcTime now,
+                                          const NodeInfoWrapper *oldNode,
+                                          const flat::HeartbeatInfo &hb) {
+  auto nodeId = hb.app.nodeId;
+  if (!oldNode) {
+    if (state.config_.allow_heartbeat_from_unregistered()) {
+      LOG_OP_INFO(ctx, "auto register");
+      return onNewNode(hb, now);
+    } else {
+      RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kHeartbeatFail, "{} not registered", nodeId);
+    }
+  }
+
+  const auto &oldNodeBase = oldNode->base();
+  if (oldNodeBase.type != hb.type()) {
+    RETURN_AND_LOG_OP_ERR(ctx,
+                          MgmtdCode::kHeartbeatFail,
+                          "{} type mismatch: registered {} got {}",
+                          nodeId,
+                          magic_enum::enum_name(oldNodeBase.type),
+                          magic_enum::enum_name(hb.type()));
+  }
+  if (oldNodeBase.status == flat::NodeStatus::DISABLED) {
+    RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kHeartbeatFail, "{} is marked as DISABLED", nodeId);
+  }
+
+  if (oldNode->lastHbVersion() >= hb.hbVersion) {
+    LOG_OP_WARN(ctx, "{} heartbeat version stale: last {} received {}", nodeId, oldNode->lastHbVersion(), hb.hbVersion);
+    return makeError(MgmtdCode::kHeartbeatVersionStale, fmt::format("{}", oldNode->lastHbVersion().toUnderType()));
+  }
+
+  return onNodeChanged(oldNodeBase, hb, now);
+}
+
+Result<Void> prepareHandleHeartbeat(MgmtdState &state,
+                                    core::ServiceOperation &ctx,
+                                    const MgmtdData &data,
+                                    UtcTime now,
+                                    const flat::HeartbeatInfo &hb,
+                                    const NodeInfoWrapper **oldNodePtr,
+                                    flat::NodeInfo &newNode) {
+  auto nodeId = hb.app.nodeId;
+  RETURN_ON_ERROR(data.checkConfigVersion(ctx, hb.type(), hb.configVersion));
+
+  const auto &ri = data.routingInfo;
+  if (ri.nodeMap.contains(nodeId)) {
+    *oldNodePtr = &ri.nodeMap.at(nodeId);
+  } else {
+    *oldNodePtr = nullptr;
+  }
+
+  auto newNodeRes = prepareNewNodeInfo(state, ctx, now, *oldNodePtr, hb);
+  RETURN_ON_ERROR(newNodeRes);
+  newNode = *newNodeRes;
+
+  if (hb.type() == flat::NodeType::STORAGE) {
+    robin_hood::unordered_set<flat::TargetId> seenTargets;
+    robin_hood::unordered_map<flat::ChainId, std::vector<LocalTargetInfoWithNodeId>> changedChainMap;
+    const auto &shb = hb.asStorage();
+    for (const auto &lti : shb.targets) {
+      if (!seenTargets.insert(lti.targetId).second) {
+        RETURN_AND_LOG_OP_ERR(ctx,
+                              MgmtdCode::kHeartbeatFail,
+                              "Duplicated target id in HeartbeatInfo: {}",
+                              lti.targetId);
+      }
+      using LS = flat::LocalTargetState;
+      switch (lti.localState) {
+        case LS::INVALID:
+          RETURN_AND_LOG_OP_ERR(ctx,
+                                MgmtdCode::kHeartbeatFail,
+                                "Invalid LocalTargetState {} for {}",
+                                toString(lti.localState),
+                                lti.targetId);
+        case LS::OFFLINE:
+        case LS::ONLINE:
+        case LS::UPTODATE:
+          // valid LocalTargetState
+          break;
+      }
+    }
+  }
+
+  return Void{};
+}
+}  // namespace
+
+CoTryTask<HeartbeatRsp> HeartbeatOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  const auto &hb = req.info;
+  auto timestamp = req.timestamp;
+  auto nodeId = hb.app.nodeId;
+
+  auto handler = [&]() -> CoTryTask<HeartbeatRsp> {
+    if (nodeId == state.selfId()) {
+      CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kHeartbeatFail, "Node id {} duplicated", nodeId);
+    }
+    if (nodeId == 0) {
+      CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kHeartbeatFail, "Node id is 0");
+    }
+
+    auto now = state.utcNow();
+    const auto validWindow = state.config_.heartbeat_timestamp_valid_window().asUs();
+    if (validWindow.count() != 0 && (timestamp + validWindow < now || now + validWindow < timestamp)) {
+      CO_RETURN_AND_LOG_OP_ERR(*this,
+                               MgmtdCode::kHeartbeatFail,
+                               "Too much timestamp deviation. now {} timestamp {}",
+                               now.toMicroseconds(),
+                               timestamp.toMicroseconds());
+    }
+
+    auto writerLock = co_await state.coScopedLock<"Heartbeat">();
+
+    const NodeInfoWrapper *oldNode = nullptr;
+    flat::NodeInfo newNode;
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      CO_RETURN_ON_ERROR(prepareHandleHeartbeat(state, *this, *dataPtr, now, hb, &oldNode, newNode));
+    }
+
+    auto persistentNewNode = flat::toPersistentNode(newNode);
+    auto needPersist = oldNode == nullptr || flat::toPersistentNode(oldNode->base()) != persistentNewNode;
+    auto statusChanged = oldNode == nullptr || oldNode->base().status != newNode.status;
+
+    if (needPersist) {
+      CO_RETURN_ON_ERROR(
+          co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+            co_return co_await state.store_.storeNodeInfo(txn, persistentNewNode);
+          }));
+    }
+
+    if (statusChanged) {
+      LOG_OP_INFO(*this,
+                  "ReceiveHeartbeat {} node {} status changed to {}",
+                  magic_enum::enum_name(newNode.type),
+                  newNode.app.nodeId,
+                  magic_enum::enum_name(newNode.status));
+    }
+
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      auto steadyNow = SteadyClock::now();
+      auto &ri = dataPtr->routingInfo;
+      ri.routingInfoChanged |= statusChanged;
+
+      auto &info = ri.nodeMap[nodeId];
+      info.base() = std::move(newNode);
+      info.base().lastHeartbeatTs = now;
+      info.updateTs(steadyNow);
+      info.updateHeartbeatVersion(hb.hbVersion);
+
+      if (hb.type() == flat::NodeType::STORAGE) {
+        dataPtr->routingInfo.localUpdateTargets(nodeId, hb.asStorage().targets, state.config_);
+      }
+
+      if (needPersist) {
+        updateMemoryRoutingInfo(ri, *this);
+      }
+
+      if (statusChanged) {
+        info.recordStatusChange(newNode.status, now);
+      }
+
+      HeartbeatRsp rsp;
+      rsp.config = dataPtr->getConfig(hb.type(), hb.configVersion, /*latest=*/true);
+      co_return rsp;
+    }
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/HeartbeatOperation.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct HeartbeatOperation : core::ServiceOperationWithMetric<"MgmtdService", "Heartbeat", "op"> {
+  HeartbeatReq req;
+
+  explicit HeartbeatOperation(HeartbeatReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final {
+    return fmt::format("Heartbeat from {}@{} type {}",
+                       req.info.app.nodeId,
+                       req.info.app.hostname,
+                       magic_enum::enum_name(req.info.type()));
+  }
+
+  CoTryTask<HeartbeatRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/Include.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "mgmtd/ops/EnableDisableNodeOperation.h"
+#include "mgmtd/ops/ExtendClientSessionOperation.h"
+#include "mgmtd/ops/GetClientSessionOperation.h"
+#include "mgmtd/ops/GetConfigOperation.h"
+#include "mgmtd/ops/GetConfigVersionsOperation.h"
+#include "mgmtd/ops/GetPrimaryMgmtdOperation.h"
+#include "mgmtd/ops/GetRoutingInfoOperation.h"
+#include "mgmtd/ops/GetUniversalTagsOperation.h"
+#include "mgmtd/ops/HeartbeatOperation.h"
+#include "mgmtd/ops/ListClientSessionsOperation.h"
+#include "mgmtd/ops/ListOrphanTargetsOperation.h"
+#include "mgmtd/ops/RegisterNodeOperation.h"
+#include "mgmtd/ops/RotateAsPreferredOrderOperation.h"
+#include "mgmtd/ops/RotateLastSrvOperation.h"
+#include "mgmtd/ops/SetChainTableOperation.h"
+#include "mgmtd/ops/SetChainsOperation.h"
+#include "mgmtd/ops/SetConfigOperation.h"
+#include "mgmtd/ops/SetNodeTagsOperation.h"
+#include "mgmtd/ops/SetPreferredTargetOrderOperation.h"
+#include "mgmtd/ops/SetUniversalTagsOperation.h"
+#include "mgmtd/ops/UnregisterNodeOperation.h"
+#include "mgmtd/ops/UpdateChainOperation.h"

src/mgmtd/ops/ListClientSessionsOperation.cc

@@ -0,0 +1,38 @@
+#include "ListClientSessionsOperation.h"
+
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<ListClientSessionsRsp> ListClientSessionsOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  const auto bootstrappingLength = state.config_.bootstrapping_length().asUs();
+  ListClientSessionsRsp rsp;
+  auto handler = [&]() -> CoTryTask<void> {
+    auto dataPtr = co_await state.data_.coSharedLock();
+    auto clientSessionMap = co_await state.clientSessionMap_.coSharedLock();
+
+    rsp.bootstrapping = dataPtr->leaseStartTs + bootstrappingLength > SteadyClock::now();
+    for (const auto &[clientId, session] : *clientSessionMap) {
+      if (auto uuidRes = Uuid::fromHexString(clientId); uuidRes.hasError()) {
+        LOG_OP_WARN(*this,
+                    "ClientId not valid hex uuid. id: {}. session: {}",
+                    clientId,
+                    serde::toJsonString(session.base()));
+      }
+
+      rsp.sessions.emplace_back(session.base());
+
+      const auto &universalId = session.base().universalId;
+      if (!rsp.referencedTags.contains(universalId)) {
+        rsp.referencedTags[universalId] = dataPtr->universalTagsMap.contains(universalId)
+                                              ? dataPtr->universalTagsMap.at(universalId)
+                                              : std::vector<flat::TagPair>{};
+      }
+    }
+    co_return Void{};
+  };
+  CO_RETURN_ON_ERROR(co_await doAsPrimary(state, std::move(handler)));
+  co_return rsp;
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/ListClientSessionsOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct ListClientSessionsOperation : core::ServiceOperationWithMetric<"MgmtdService", "ListClientSessions", "op"> {
+  ListClientSessionsReq req;
+
+  explicit ListClientSessionsOperation(ListClientSessionsReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("ListClientSessions"); }
+
+  CoTryTask<ListClientSessionsRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/ListOrphanTargetsOperation.cc

@@ -0,0 +1,23 @@
+#include "ListOrphanTargetsOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "mgmtd/service/helpers.h"
+#include "mgmtd/service/updateChain.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<ListOrphanTargetsRsp> ListOrphanTargetsOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+
+  auto handler = [&]() -> CoTryTask<ListOrphanTargetsRsp> {
+    ListOrphanTargetsRsp rsp;
+    auto dataPtr = co_await state.data_.coLock();
+    auto &ri = dataPtr->routingInfo;
+    for (const auto &[_, ti] : ri.orphanTargetsByTargetId) {
+      rsp.targets.push_back(ti);
+    }
+    co_return rsp;
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/ListOrphanTargetsOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct ListOrphanTargetsOperation : core::ServiceOperationWithMetric<"MgmtdService", "ListOrphanTargets", "op"> {
+  ListOrphanTargetsReq req;
+
+  explicit ListOrphanTargetsOperation(ListOrphanTargetsReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("ListOrphanTargets"); }
+
+  CoTryTask<ListOrphanTargetsRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/RegisterNodeOperation.cc

@@ -0,0 +1,47 @@
+#include "RegisterNodeOperation.h"
+
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<RegisterNodeRsp> RegisterNodeOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  auto nodeId = req.nodeId;
+  auto type = req.type;
+  auto handler = [&]() -> CoTryTask<RegisterNodeRsp> {
+    if (nodeId == 0) {
+      CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kRegisterFail, "Node id is 0");
+    }
+
+    auto writerLock = co_await state.coScopedLock<"RegisterNode">();
+
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      if (dataPtr->routingInfo.nodeMap.contains(nodeId))
+        CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kRegisterFail, "Node id {} duplicated", nodeId);
+    }
+
+    flat::NodeInfo newNode;
+    newNode.app.nodeId = nodeId;
+    newNode.type = type;
+
+    auto persistRes =
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          co_return co_await state.store_.storeNodeInfo(txn, flat::toPersistentNode(newNode));
+        });
+    CO_RETURN_ON_ERROR(persistRes);
+
+    co_await updateMemoryRoutingInfo(state, *this, [&](RoutingInfo &ri) {
+      auto &node = ri.nodeMap[nodeId];
+      node.base() = std::move(newNode);
+      node.recordStatusChange(newNode.status, UtcClock::now());
+    });
+
+    co_return RegisterNodeRsp::create();
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/RegisterNodeOperation.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct RegisterNodeOperation : core::ServiceOperationWithMetric<"MgmtdService", "RegisterNode", "op"> {
+  RegisterNodeReq req;
+
+  explicit RegisterNodeOperation(RegisterNodeReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final {
+    return fmt::format("RegisterNode {} as {}", req.nodeId, magic_enum::enum_name(req.type));
+  }
+
+  CoTryTask<RegisterNodeRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/RotateAsPreferredOrderOperation.cc

@@ -0,0 +1,79 @@
+#include "RotateAsPreferredOrderOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "mgmtd/service/helpers.h"
+#include "mgmtd/service/updateChain.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<RotateAsPreferredOrderRsp> RotateAsPreferredOrderOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  if (req.chainId == 0) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Empty chain id");
+  }
+
+  auto handler = [&]() -> CoTryTask<RotateAsPreferredOrderRsp> {
+    auto writerLock = co_await state.coScopedLock<"RotateAsPreferredOrder">();
+
+    flat::ChainInfo chainInfo;
+    std::vector<ChainTargetInfoEx> oldTargets;
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      auto &ri = dataPtr->routingInfo;
+      auto it = ri.chains.find(req.chainId);
+      if (it == ri.chains.end()) {
+        CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kChainNotFound, "chain: {}", req.chainId.toUnderType());
+      }
+      chainInfo = it->second;
+      for (const auto &ti : chainInfo.targets) {
+        auto tit = ri.getTargets().find(ti.targetId);
+        XLOGF_IF(FATAL,
+                 tit == ri.getTargets().end(),
+                 "Target not found. chain: {}. target: {}",
+                 req.chainId,
+                 ti.targetId);
+        oldTargets.emplace_back(tit->second.base());
+      }
+    }
+
+    auto newTargets = rotateAsPreferredOrder(oldTargets, chainInfo.preferredTargetOrder);
+    if (oldTargets == newTargets) {
+      co_return RotateAsPreferredOrderRsp::create(std::move(chainInfo));
+    }
+
+    chainInfo.chainVersion = nextVersion(chainInfo.chainVersion);
+    chainInfo.targets.clear();
+    for (const auto &ti : newTargets) {
+      chainInfo.targets.push_back(ti);
+    }
+
+    auto commitRes =
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          co_return co_await state.store_.storeChainInfo(txn, chainInfo);
+        });
+    CO_RETURN_ON_ERROR(commitRes);
+
+    co_await updateMemoryRoutingInfo(state, *this, [&](RoutingInfo &ri) {
+      auto &oldChain = ri.getChain(chainInfo.chainId);
+      LOG_OP_INFO(*this,
+                  "{} change from {} to {}",
+                  chainInfo.chainId,
+                  serde::toJsonString(oldChain),
+                  serde::toJsonString(chainInfo));
+      oldChain = chainInfo;
+      auto steadyNow = SteadyClock::now();
+      for (const auto &cti : chainInfo.targets) {
+        ri.updateTarget(cti.targetId, [&](auto &ti) {
+          ti.base().publicState = cti.publicState;
+          ti.updateTs(steadyNow);
+        });
+      }
+    });
+
+    co_return RotateAsPreferredOrderRsp::create(std::move(chainInfo));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/RotateAsPreferredOrderOperation.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct RotateAsPreferredOrderOperation
+    : core::ServiceOperationWithMetric<"MgmtdService", "RotateAsPreferredOrder", "op"> {
+  RotateAsPreferredOrderReq req;
+
+  explicit RotateAsPreferredOrderOperation(RotateAsPreferredOrderReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("RotateAsPreferredOrder {}", req.chainId); }
+
+  CoTryTask<RotateAsPreferredOrderRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/RotateLastSrvOperation.cc

@@ -0,0 +1,79 @@
+#include "RotateLastSrvOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "mgmtd/service/helpers.h"
+#include "mgmtd/service/updateChain.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<RotateLastSrvRsp> RotateLastSrvOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  if (req.chainId == 0) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Empty chain id");
+  }
+
+  auto handler = [&]() -> CoTryTask<RotateLastSrvRsp> {
+    auto writerLock = co_await state.coScopedLock<"RotateLastSrv">();
+
+    flat::ChainInfo chainInfo;
+    std::vector<ChainTargetInfoEx> oldTargets;
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      auto &ri = dataPtr->routingInfo;
+      auto it = ri.chains.find(req.chainId);
+      if (it == ri.chains.end()) {
+        CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kChainNotFound, "chain: {}", req.chainId.toUnderType());
+      }
+      chainInfo = it->second;
+      for (const auto &ti : chainInfo.targets) {
+        auto tit = ri.getTargets().find(ti.targetId);
+        XLOGF_IF(FATAL,
+                 tit == ri.getTargets().end(),
+                 "Target not found. chain: {}. target: {}",
+                 req.chainId,
+                 ti.targetId);
+        oldTargets.emplace_back(tit->second.base());
+      }
+    }
+
+    auto newTargets = rotateLastSrv(oldTargets);
+    if (oldTargets == newTargets) {
+      co_return RotateLastSrvRsp::create(std::move(chainInfo));
+    }
+
+    chainInfo.chainVersion = nextVersion(chainInfo.chainVersion);
+    chainInfo.targets.clear();
+    for (const auto &ti : newTargets) {
+      chainInfo.targets.push_back(ti);
+    }
+
+    auto commitRes =
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          co_return co_await state.store_.storeChainInfo(txn, chainInfo);
+        });
+    CO_RETURN_ON_ERROR(commitRes);
+
+    co_await updateMemoryRoutingInfo(state, *this, [&](RoutingInfo &ri) {
+      auto &oldChain = ri.getChain(chainInfo.chainId);
+      LOG_OP_INFO(*this,
+                  "{} change from {} to {}",
+                  chainInfo.chainId,
+                  serde::toJsonString(oldChain),
+                  serde::toJsonString(chainInfo));
+      oldChain = chainInfo;
+      auto steadyNow = SteadyClock::now();
+      for (const auto &cti : chainInfo.targets) {
+        ri.updateTarget(cti.targetId, [&](auto &ti) {
+          ti.base().publicState = cti.publicState;
+          ti.updateTs(steadyNow);
+        });
+      }
+    });
+
+    co_return RotateLastSrvRsp::create(std::move(chainInfo));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/RotateLastSrvOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct RotateLastSrvOperation : core::ServiceOperationWithMetric<"MgmtdService", "RotateLastSrv", "op"> {
+  RotateLastSrvReq req;
+
+  explicit RotateLastSrvOperation(RotateLastSrvReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("RotateLastSrv {}", req.chainId); }
+
+  CoTryTask<RotateLastSrvRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetChainTableOperation.cc

@@ -0,0 +1,89 @@
+#include "SetChainTableOperation.h"
+
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+Result<Void> checkChains(core::ServiceOperation &ctx, const RoutingInfo &ri, const std::vector<flat::ChainId> &chains) {
+  if (chains.empty()) {
+    RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kInvalidChainTable, "Empty chains");
+  }
+
+  size_t expectedTargets = 0;
+  for (auto cid : chains) {
+    auto it = ri.chains.find(cid);
+    if (it == ri.chains.end()) {
+      RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kChainNotFound, "{} not found", cid);
+    }
+    const auto &ci = it->second;
+    if (expectedTargets == 0) {
+      expectedTargets = ci.targets.size();
+    } else if (ci.targets.size() != expectedTargets) {
+      RETURN_AND_LOG_OP_ERR(
+          ctx,
+          MgmtdCode::kInvalidChainTable,
+          "Target count mismatch with first chain. chain id: {}. targets: {}. targets of first chain: {}.",
+          cid.toUnderType(),
+          ci.targets.size(),
+          expectedTargets);
+    }
+  }
+  return Void{};
+}
+}  // namespace
+
+CoTryTask<SetChainTableRsp> SetChainTableOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  auto tableId = req.chainTableId;
+  auto tableVersion = flat::ChainTableVersion(1);
+  auto newChainTable = flat::ChainTable::create(tableId, tableVersion, std::move(req.chains), std::move(req.desc));
+
+  auto handler = [&]() -> CoTryTask<SetChainTableRsp> {
+    auto writerLock = co_await state.coScopedLock<"SetChainTable">();
+
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+
+      CO_RETURN_ON_ERROR(checkChains(*this, dataPtr->routingInfo, newChainTable.chains));
+
+      auto ctit = dataPtr->routingInfo.chainTables.find(tableId);
+      if (ctit != dataPtr->routingInfo.chainTables.end()) {
+        const auto &m = ctit->second;
+        XLOGF_IF(FATAL, m.empty(), "{} has no versions", tableId);
+        const auto &current = m.rbegin()->second;
+        if (newChainTable.chains != current.chains) {
+          newChainTable.chainTableVersion = nextVersion(current.chainTableVersion);
+        }
+        if (newChainTable.desc.empty()) {
+          newChainTable.desc = current.desc;
+        }
+        if (newChainTable == current) {
+          // same as the latest version, do not add new version
+          co_return SetChainTableRsp::create(current.chainTableVersion);
+        }
+      }
+    }
+
+    auto commitRes =
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          co_return co_await state.store_.storeChainTable(txn, newChainTable);
+        });
+    CO_RETURN_ON_ERROR(commitRes);
+
+    co_await updateMemoryRoutingInfo(state, *this, [&](RoutingInfo &ri) {
+      LOG_OP_INFO(*this,
+                  "set ChainTable {} succeeded. new version: {}. new count: {}. desc: {}",
+                  tableId.toUnderType(),
+                  newChainTable.chainTableVersion.toUnderType(),
+                  newChainTable.chains.size(),
+                  newChainTable.desc);
+
+      ri.chainTables[tableId][newChainTable.chainTableVersion] = newChainTable;
+    });
+    co_return SetChainTableRsp::create(newChainTable.chainTableVersion);
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetChainTableOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct SetChainTableOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetChainTable", "op"> {
+  SetChainTableReq req;
+
+  explicit SetChainTableOperation(SetChainTableReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("SetChainTable {}", req.chainTableId); }
+
+  CoTryTask<SetChainTableRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetChainsOperation.cc

@@ -0,0 +1,143 @@
+#include "SetChainsOperation.h"
+
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+flat::ChainInfo makeChainInfo(const flat::ChainSetting &setting) {
+  flat::ChainInfo info;
+  info.chainId = setting.chainId;
+  info.chainVersion = flat::ChainVersion{1};
+  for (const auto &t : setting.targets) {
+    flat::ChainTargetInfo cti;
+    cti.targetId = t.targetId;
+    cti.publicState = flat::PublicTargetState::SERVING;
+    info.targets.push_back(std::move(cti));
+    if (setting.setPreferredTargetOrder) {
+      info.preferredTargetOrder.push_back(cti.targetId);
+    }
+  }
+  return info;
+}
+
+Result<Void> ensureChainNotChanged(core::ServiceOperation &ctx,
+                                   const flat::ChainInfo &oldChain,
+                                   const flat::ChainSetting &newChain) {
+  if (oldChain.chainId != newChain.chainId) {
+    RETURN_AND_LOG_OP_ERR(ctx,
+                          MgmtdCode::kInvalidChainTable,
+                          "Chain id mismatch. old: {}. new: {}",
+                          oldChain.chainId,
+                          newChain.chainId);
+  }
+  if (oldChain.targets.size() != newChain.targets.size()) {
+    RETURN_AND_LOG_OP_ERR(ctx,
+                          MgmtdCode::kInvalidChainTable,
+                          "Target count of chain {} mismatch. old: {}. new: {}",
+                          oldChain.chainId,
+                          oldChain.targets.size(),
+                          newChain.targets.size());
+  }
+  std::vector<flat::TargetId> oldTargets(oldChain.targets.size());
+  for (size_t i = 0; i < oldTargets.size(); ++i) oldTargets[i] = oldChain.targets[i].targetId;
+  std::sort(oldTargets.begin(), oldTargets.end());
+
+  std::vector<flat::TargetId> newTargets(newChain.targets.size());
+  for (size_t i = 0; i < newTargets.size(); ++i) newTargets[i] = newChain.targets[i].targetId;
+  std::sort(newTargets.begin(), newTargets.end());
+
+  if (oldTargets != newTargets) {
+    RETURN_AND_LOG_OP_ERR(ctx,
+                          MgmtdCode::kInvalidChainTable,
+                          "Target mismatch of {}. old: [{}]. new: [{}]",
+                          oldChain.chainId,
+                          fmt::join(oldTargets, ","),
+                          fmt::join(newTargets, ","));
+  }
+  return Void{};
+}
+
+Result<Void> checkChains(core::ServiceOperation &ctx,
+                         const RoutingInfo &ri,
+                         std::span<const flat::ChainSetting> chains,
+                         std::vector<flat::ChainInfo> &newChains,
+                         robin_hood::unordered_set<flat::TargetId> &newTargets) {
+  if (chains.empty()) {
+    RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kInvalidChainTable, "Empty chains");
+  }
+
+  for (const auto &chain : chains) {
+    if (chain.chainId == 0) {
+      RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kInvalidChainTable, "Empty chain id");
+    }
+    if (chain.targets.empty()) {
+      RETURN_AND_LOG_OP_ERR(ctx,
+                            MgmtdCode::kInvalidChainTable,
+                            "Chain contains no targets. chain id: {}.",
+                            chain.chainId.toUnderType());
+    }
+    auto it = ri.chains.find(chain.chainId);
+    if (it != ri.chains.end()) {
+      // existed chain
+      RETURN_ON_ERROR(ensureChainNotChanged(ctx, it->second, chain));
+    } else {
+      // new chain
+      for (const auto &cti : chain.targets) {
+        if (ri.getTargets().contains(cti.targetId) || !newTargets.insert(cti.targetId).second) {
+          RETURN_AND_LOG_OP_ERR(ctx,
+                                MgmtdCode::kInvalidChainTable,
+                                "{} duplicated. chain id: {}",
+                                cti.targetId,
+                                chain.chainId.toUnderType());
+        }
+      }
+      newChains.push_back(makeChainInfo(chain));
+    }
+  }
+  return Void{};
+}
+}  // namespace
+
+CoTryTask<SetChainsRsp> SetChainsOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  const auto &chains = req.chains;
+
+  auto handler = [&]() -> CoTryTask<SetChainsRsp> {
+    auto writerLock = co_await state.coScopedLock<"SetChains">();
+
+    std::vector<flat::ChainInfo> newChains;
+
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+
+      robin_hood::unordered_set<flat::TargetId> newTargets;
+      CO_RETURN_ON_ERROR(
+          checkChains(*this, dataPtr->routingInfo, std::span(chains.begin(), chains.size()), newChains, newTargets));
+    }
+
+    if (newChains.empty()) co_return SetChainsRsp::create();
+
+    auto commitRes =
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          for (const auto &newChain : newChains) {
+            CO_RETURN_ON_ERROR(co_await state.store_.storeChainInfo(txn, newChain));
+            LOG_OP_DBG(*this, "store chain {} succeeded", serde::toJsonString(newChain));
+          }
+          co_return Void{};
+        });
+    CO_RETURN_ON_ERROR(commitRes);
+
+    LOG_OP_DBG(*this, "new chains created: {}", serde::toJsonString(newChains));
+
+    co_await updateMemoryRoutingInfo(state, *this, [&](RoutingInfo &ri) {
+      for (auto &newChain : newChains) {
+        ri.insertNewChain(newChain);
+      }
+    });
+    co_return SetChainsRsp::create();
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetChainsOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct SetChainsOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetChains", "op"> {
+  SetChainsReq req;
+
+  explicit SetChainsOperation(SetChainsReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return "SetChains"; }
+
+  CoTryTask<SetChainsRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetConfigOperation.cc

@@ -0,0 +1,61 @@
+#include "SetConfigOperation.h"
+
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<SetConfigRsp> SetConfigOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  auto nodeType = req.nodeType;
+  auto content = req.content;
+
+  auto handler = [&]() -> CoTryTask<SetConfigRsp> {
+    auto writerLock = co_await state.coScopedLock<"SetConfig">();
+
+    auto oldVersionRes = co_await [&]() -> CoTryTask<flat::ConfigVersion> {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      const auto &cm = dataPtr->configMap;
+      auto it = cm.find(nodeType);
+      if (it == cm.end()) {
+        CO_RETURN_AND_LOG_OP_ERR(*this,
+                                 StatusCode::kInvalidArg,
+                                 "Unknown NodeType {}",
+                                 magic_enum::enum_name(nodeType));
+      }
+      co_return it->second.rbegin()->first;
+    }();
+    CO_RETURN_ON_ERROR(oldVersionRes);
+
+    auto newVersion = nextVersion(*oldVersionRes);
+    auto newConfigInfo = flat::ConfigInfo::create(newVersion, content, req.desc);
+    auto storeHandler = [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+      co_return co_await state.store_.storeConfig(txn, nodeType, newConfigInfo);
+    };
+    CO_RETURN_ON_ERROR(co_await withReadWriteTxn(state, std::move(storeHandler)));
+
+    auto dataPtr = co_await state.data_.coLock();
+    dataPtr->configMap[nodeType][newVersion] = std::move(newConfigInfo);
+    if (nodeType == flat::NodeType::MGMTD) {
+      const auto &updater = state.env_->configUpdater();
+      if (!updater || updater(content, newConfigInfo.genUpdateDesc())) {
+        if (!updater) {
+          LOG_OP_WARN(*this, "blindly promote to {} since no updater found", newVersion);
+        }
+        state.selfNodeInfo_.configVersion = newVersion;
+        XLOGF_IF(FATAL, !dataPtr->routingInfo.nodeMap.contains(state.selfId()), "Self not found in state");
+        auto &self = dataPtr->routingInfo.nodeMap[state.selfId()];
+        XLOGF_IF(FATAL,
+                 self.base().configVersion >= newVersion,
+                 "Self's ConfigVersion {} is larger than newest {}",
+                 self.base().configVersion,
+                 newVersion);
+        self.base().configVersion = newVersion;
+      }
+    }
+    co_return SetConfigRsp::create(newVersion);
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetConfigOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct SetConfigOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetConfig", "op"> {
+  SetConfigReq req;
+
+  explicit SetConfigOperation(SetConfigReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("SetConfig {}", magic_enum::enum_name(req.nodeType)); }
+
+  CoTryTask<SetConfigRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetNodeTagsOperation.cc

@@ -0,0 +1,72 @@
+#include "SetNodeTagsOperation.h"
+
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<SetNodeTagsRsp> SetNodeTagsOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  auto nodeId = req.nodeId;
+  for (const auto &tp : req.tags) {
+    if (tp.key.empty()) {
+      CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kInvalidTag, "tag key is empty");
+    }
+  }
+
+  auto handler = [&]() -> CoTryTask<SetNodeTagsRsp> {
+    auto writerLock = co_await state.coScopedLock<"SetNodeTags">();
+
+    flat::NodeInfo node;
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      const auto &ri = dataPtr->routingInfo;
+      auto it = ri.nodeMap.find(nodeId);
+      if (it == ri.nodeMap.end()) CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kNodeNotFound, "");
+      node = it->second.base();
+    }
+
+    auto updateRes = updateTags(*this, req.mode, node.tags, req.tags);
+    CO_RETURN_ON_ERROR(updateRes);
+
+    XLOGF(DBG,
+          "update = {} node.tags = {} newTags = {} oldTags == newTags? {}",
+          serde::toJsonString(req.tags),
+          serde::toJsonString(node.tags),
+          serde::toJsonString(*updateRes),
+          node.tags == *updateRes ? "true" : "false");
+
+    if (node.tags != *updateRes) {
+      auto oldTags = std::move(node.tags);
+      node.tags = std::move(*updateRes);
+
+      if (findTag(node.tags, flat::kDisabledTagKey) == -1 && node.status == flat::NodeStatus::DISABLED) {
+        node.status = flat::NodeStatus::HEARTBEAT_CONNECTING;
+      }
+      if (findTag(node.tags, flat::kDisabledTagKey) != -1 && node.status != flat::NodeStatus::DISABLED) {
+        node.status = flat::NodeStatus::DISABLED;
+      }
+
+      CO_RETURN_ON_ERROR(
+          co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+            co_return co_await state.store_.storeNodeInfo(txn, flat::toPersistentNode(node));
+          }));
+
+      LOG_OP_INFO(*this, "change tags from {} to {}", serde::toJsonString(oldTags), serde::toJsonString(node.tags));
+
+      co_await updateMemoryRoutingInfo(state, *this, [&](auto &ri) {
+        auto &info = ri.nodeMap[nodeId];
+        info.base() = node;
+        if (nodeId == state.selfId()) {
+          state.selfNodeInfo_.tags = node.tags;
+          state.selfPersistentNodeInfo_.tags = node.tags;
+        }
+      });
+    }
+
+    co_return SetNodeTagsRsp::create(std::move(node));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetNodeTagsOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct SetNodeTagsOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetNodeTags", "op"> {
+  SetNodeTagsReq req;
+
+  explicit SetNodeTagsOperation(SetNodeTagsReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("SetNodeTags {}", req.nodeId); }
+
+  CoTryTask<SetNodeTagsRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetPreferredTargetOrderOperation.cc

@@ -0,0 +1,93 @@
+#include "SetPreferredTargetOrderOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "mgmtd/service/helpers.h"
+#include "mgmtd/service/updateChain.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<SetPreferredTargetOrderRsp> SetPreferredTargetOrderOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  if (req.chainId == 0) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Empty chain id");
+  }
+
+  if (req.preferredTargetOrder.empty()) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Empty preferredTargetOrder");
+  }
+
+  std::set<flat::TargetId> uniqueIds;
+  for (auto id : req.preferredTargetOrder) {
+    uniqueIds.insert(id);
+  }
+
+  if (uniqueIds.size() != req.preferredTargetOrder.size()) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Found duplicated target id in preferredTargetOrder");
+  }
+
+  auto handler = [&]() -> CoTryTask<SetPreferredTargetOrderRsp> {
+    auto writerLock = co_await state.coScopedLock<"SetPreferredTargetOrder">();
+
+    flat::ChainInfo chainInfo;
+    std::vector<ChainTargetInfoEx> oldTargets;
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      auto &ri = dataPtr->routingInfo;
+      auto it = ri.chains.find(req.chainId);
+      if (it == ri.chains.end()) {
+        CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kChainNotFound, "chain: {}", req.chainId.toUnderType());
+      }
+      chainInfo = it->second;
+      for (const auto &ti : chainInfo.targets) {
+        auto tit = ri.getTargets().find(ti.targetId);
+        XLOGF_IF(FATAL,
+                 tit == ri.getTargets().end(),
+                 "Target not found. chain: {}. target: {}",
+                 req.chainId,
+                 ti.targetId);
+        oldTargets.emplace_back(tit->second.base());
+      }
+    }
+
+    for (auto id : uniqueIds) {
+      if (!std::any_of(oldTargets.begin(), oldTargets.end(), [id](const ChainTargetInfoEx &info) {
+            return info.targetId == id;
+          })) {
+        CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "{} not found in chain", id);
+      }
+    }
+
+    if (chainInfo.preferredTargetOrder == req.preferredTargetOrder) {
+      co_return SetPreferredTargetOrderRsp::create(std::move(chainInfo));
+    }
+
+    chainInfo.preferredTargetOrder = req.preferredTargetOrder;
+
+    // do not increase RoutingInfoVersion
+    auto commitRes = co_await withReadWriteTxn(state, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+      co_return co_await state.store_.storeChainInfo(txn, chainInfo);
+    });
+    CO_RETURN_ON_ERROR(commitRes);
+
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      auto &oldChain = dataPtr->routingInfo.getChain(chainInfo.chainId);
+      LOG_OP_INFO(*this,
+                  "{} change preferredTargetOrder from {} to {}",
+                  chainInfo.chainId,
+                  serde::toJsonString(oldChain.preferredTargetOrder),
+                  serde::toJsonString(chainInfo.preferredTargetOrder));
+      oldChain = chainInfo;
+      auto steadyNow = SteadyClock::now();
+      for (auto tid : req.preferredTargetOrder) {
+        dataPtr->routingInfo.updateTarget(tid, [&](auto &ti) { ti.updateTs(steadyNow); });
+      }
+    }
+
+    co_return SetPreferredTargetOrderRsp::create(std::move(chainInfo));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetPreferredTargetOrderOperation.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct SetPreferredTargetOrderOperation
+    : core::ServiceOperationWithMetric<"MgmtdService", "SetPreferredTargetOrder", "op"> {
+  SetPreferredTargetOrderReq req;
+
+  explicit SetPreferredTargetOrderOperation(SetPreferredTargetOrderReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("SetPreferredTargetOrder {}", req.chainId); }
+
+  CoTryTask<SetPreferredTargetOrderRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetUniversalTagsOperation.cc

@@ -0,0 +1,53 @@
+#include "SetUniversalTagsOperation.h"
+
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<SetUniversalTagsRsp> SetUniversalTagsOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  for (const auto &tp : req.tags) {
+    if (tp.key.empty()) {
+      CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kInvalidTag, "tag key is empty");
+    }
+  }
+
+  auto handler = [&]() -> CoTryTask<SetUniversalTagsRsp> {
+    auto writerLock = co_await state.coScopedLock<"SetUniversalTags">();
+
+    std::vector<flat::TagPair> oldTags;
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      const auto &m = dataPtr->universalTagsMap;
+      auto it = m.find(req.universalId);
+      if (it != m.end()) oldTags = it->second;
+    }
+
+    auto updateRes = updateTags(*this, req.mode, oldTags, req.tags);
+    CO_RETURN_ON_ERROR(updateRes);
+    auto &newTags = *updateRes;
+
+    if (oldTags == newTags) {
+      co_return SetUniversalTagsRsp::create(std::move(oldTags));
+    }
+
+    CO_RETURN_ON_ERROR(
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          co_return co_await state.store_.storeUniversalTags(txn, req.universalId, newTags);
+        }));
+
+    LOG_OP_INFO(*this, "change tags from {} to {}", serde::toJsonString(oldTags), serde::toJsonString(newTags));
+
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      auto &m = dataPtr->universalTagsMap;
+      m[req.universalId] = newTags;
+    }
+
+    co_return SetUniversalTagsRsp::create(std::move(newTags));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/SetUniversalTagsOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct SetUniversalTagsOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetUniversalTags", "op"> {
+  SetUniversalTagsReq req;
+
+  explicit SetUniversalTagsOperation(SetUniversalTagsReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("SetUniversalTags {}", req.universalId); }
+
+  CoTryTask<SetUniversalTagsRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/UnregisterNodeOperation.cc

@@ -0,0 +1,60 @@
+#include "UnregisterNodeOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<UnregisterNodeRsp> UnregisterNodeOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  auto nodeId = req.nodeId;
+
+  auto handler = [&]() -> CoTryTask<UnregisterNodeRsp> {
+    auto writerLock = co_await state.coScopedLock<"UnregisterNode">();
+
+    {
+      auto dataPtr = co_await state.data_.coSharedLock();
+      const auto &ri = dataPtr->routingInfo;
+      auto it = ri.nodeMap.find(nodeId);
+      if (it == ri.nodeMap.end()) {
+        CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kNodeNotFound, "{} not found", nodeId);
+      }
+      const auto &node = it->second.base();
+      if (node.status != flat::NodeStatus::DISABLED && node.status != flat::NodeStatus::HEARTBEAT_FAILED) {
+        CO_RETURN_AND_LOG_OP_ERR(*this,
+                                 StatusCode::kInvalidArg,
+                                 "Do not allow to unregister {} of status {}",
+                                 nodeId,
+                                 toStringView(node.status));
+      }
+
+      if (node.type == flat::NodeType::STORAGE) {
+        for (const auto &[tid, ti] : ri.getTargets()) {
+          if (ti.base().nodeId == nodeId) {
+            CO_RETURN_AND_LOG_OP_ERR(*this,
+                                     StatusCode::kInvalidArg,
+                                     "Do not allow to unregister {}: referenced by {} of {}",
+                                     nodeId,
+                                     tid,
+                                     ti.base().chainId);
+          }
+        }
+      }
+    }
+
+    auto persistRes =
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          co_return co_await state.store_.clearNodeInfo(txn, nodeId);
+        });
+    CO_RETURN_ON_ERROR(persistRes);
+
+    co_await updateMemoryRoutingInfo(state, *this, [&](RoutingInfo &ri) { ri.nodeMap.erase(nodeId); });
+
+    co_return UnregisterNodeRsp::create();
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/UnregisterNodeOperation.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct UnregisterNodeOperation : core::ServiceOperationWithMetric<"MgmtdService", "UnregisterNode", "op"> {
+  UnregisterNodeReq req;
+
+  explicit UnregisterNodeOperation(UnregisterNodeReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("UnregisterNode {}", req.nodeId); }
+
+  CoTryTask<UnregisterNodeRsp> handle(MgmtdState &state);
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/UpdateChainOperation.cc

@@ -0,0 +1,123 @@
+#include "UpdateChainOperation.h"
+
+#include "common/utils/StringUtils.h"
+#include "mgmtd/service/helpers.h"
+#include "mgmtd/service/updateChain.h"
+
+namespace hf3fs::mgmtd {
+CoTryTask<UpdateChainRsp> UpdateChainOperation::handle(MgmtdState &state) {
+  CO_RETURN_ON_ERROR(state.validateClusterId(*this, req.clusterId));
+  CO_RETURN_ON_ERROR(co_await state.validateAdmin(*this, req.user));
+
+  if (req.chainId == 0) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Empty chain id");
+  }
+
+  if (req.targetId == 0) {
+    CO_RETURN_AND_LOG_OP_ERR(*this, StatusCode::kInvalidArg, "Empty target id");
+  }
+
+  if (req.mode != UpdateChainReq::Mode::ADD && req.mode != UpdateChainReq::Mode::REMOVE) {
+    CO_RETURN_AND_LOG_OP_ERR(*this,
+                             StatusCode::kInvalidArg,
+                             "Unsupported mode: {}({})",
+                             hf3fs::toStringView(req.mode),
+                             static_cast<int>(req.mode));
+  }
+  bool addTarget = req.mode == UpdateChainReq::Mode::ADD;
+
+  auto handler = [&]() -> CoTryTask<UpdateChainRsp> {
+    auto writerLock = co_await state.coScopedLock<"UpdateChain">();
+
+    flat::ChainInfo chainInfo;
+    {
+      auto dataPtr = co_await state.data_.coLock();
+      auto &ri = dataPtr->routingInfo;
+      auto it = ri.chains.find(req.chainId);
+      if (it == ri.chains.end()) {
+        CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kChainNotFound, "chain: {}", req.chainId.toUnderType());
+      }
+      auto tit = ri.getTargets().find(req.targetId);
+      if (req.mode == UpdateChainReq::Mode::ADD && tit != ri.getTargets().end()) {
+        CO_RETURN_AND_LOG_OP_ERR(*this, MgmtdCode::kTargetExisted, "target: {}", req.targetId.toUnderType());
+      }
+      chainInfo = it->second;
+    }
+
+    if (addTarget) {
+      // already checked req.targetId is not present in any chain,
+      // directly append it into this chain
+      flat::ChainTargetInfo cti;
+      cti.targetId = req.targetId;
+      cti.publicState = flat::PublicTargetState::OFFLINE;
+
+      chainInfo.chainVersion = nextVersion(chainInfo.chainVersion);
+      chainInfo.targets.push_back(cti);
+      if (chainInfo.preferredTargetOrder.size() == chainInfo.targets.size() - 1) {
+        chainInfo.preferredTargetOrder.push_back(req.targetId);
+      }
+    } else {
+      XLOGF_IF(DFATAL, req.mode != UpdateChainReq::Mode::REMOVE, "Invalid mode: {}", static_cast<int>(req.mode));
+      auto tit = std::find_if(chainInfo.targets.begin(), chainInfo.targets.end(), [&](const auto &cti) {
+        return cti.targetId == req.targetId;
+      });
+      if (tit == chainInfo.targets.end()) {
+        CO_RETURN_AND_LOG_OP_ERR(*this,
+                                 MgmtdCode::kTargetNotFound,
+                                 "target {} is not found in chain {}",
+                                 req.targetId.toUnderType(),
+                                 req.chainId.toUnderType());
+      }
+      if (tit->publicState != flat::PublicTargetState::OFFLINE) {
+        CO_RETURN_AND_LOG_OP_ERR(*this,
+                                 StatusCode::kInvalidArg,
+                                 "Do not allow to remove target {} from chain {}: state is {}",
+                                 req.targetId.toUnderType(),
+                                 req.chainId.toUnderType(),
+                                 hf3fs::toStringView(tit->publicState));
+      }
+
+      chainInfo.chainVersion = nextVersion(chainInfo.chainVersion);
+      chainInfo.targets.erase(tit);
+      if (auto it =
+              std::find(chainInfo.preferredTargetOrder.begin(), chainInfo.preferredTargetOrder.end(), req.targetId);
+          it != chainInfo.preferredTargetOrder.end()) {
+        chainInfo.preferredTargetOrder.erase(it);
+      }
+    }
+
+    auto commitRes =
+        co_await updateStoredRoutingInfo(state, *this, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+          if (!addTarget) {
+            CO_RETURN_ON_ERROR(co_await state.store_.clearTargetInfo(txn, req.targetId));
+          }
+          co_return co_await state.store_.storeChainInfo(txn, chainInfo);
+        });
+    CO_RETURN_ON_ERROR(commitRes);
+
+    co_await updateMemoryRoutingInfo(state, *this, [&](RoutingInfo &ri) {
+      auto &oldChain = ri.getChain(req.chainId);
+      LOG_OP_INFO(*this,
+                  "{} change from {} to {}",
+                  chainInfo.chainId,
+                  serde::toJsonString(oldChain),
+                  serde::toJsonString(chainInfo));
+      oldChain = chainInfo;
+      if (addTarget) {
+        XLOGF_IF(DFATAL,
+                 req.targetId != chainInfo.targets.back().targetId,
+                 "targetId mismatch: {} and {}",
+                 req.targetId.toUnderType(),
+                 chainInfo.targets.back().targetId.toUnderType());
+        ri.insertNewTarget(req.chainId, chainInfo.targets.back());
+      } else {
+        ri.removeTarget(req.chainId, req.targetId);
+      }
+    });
+
+    co_return UpdateChainRsp::create(std::move(chainInfo));
+  };
+  co_return co_await doAsPrimary(state, std::move(handler));
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/ops/UpdateChainOperation.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/service/MgmtdState.h"
+
+namespace hf3fs::mgmtd {
+struct UpdateChainOperation : core::ServiceOperationWithMetric<"MgmtdService", "UpdateChain", "op"> {
+  UpdateChainReq req;
+
+  explicit UpdateChainOperation(UpdateChainReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return fmt::format("UpdateChain {}", req.chainId); }
+
+  CoTryTask<UpdateChainRsp> handle(MgmtdState &state);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/ClientSession.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "WithTimestamp.h"
+#include "fbs/mgmtd/ClientSession.h"
+
+namespace hf3fs::mgmtd {
+class ClientSession : public WithTimestamp<flat::ClientSession> {
+ public:
+  using Base = WithTimestamp<flat::ClientSession>;
+  using Base::Base;
+
+  flat::ClientSessionVersion clientSessionVersion{0};
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/LeaseInfo.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "fbs/mgmtd/MgmtdLeaseInfo.h"
+
+namespace hf3fs::mgmtd {
+struct LeaseInfo {
+  std::optional<flat::MgmtdLeaseInfo> lease;
+  bool bootstrapping = false;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/LocalTargetInfoWithNodeId.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "fbs/mgmtd/LocalTargetInfo.h"
+
+namespace hf3fs::mgmtd {
+struct LocalTargetInfoWithNodeId {
+  flat::TargetId targetId{0};
+  flat::NodeId nodeId{0};
+  flat::LocalTargetState localState{flat::LocalTargetState::INVALID};
+
+  LocalTargetInfoWithNodeId() = default;
+  LocalTargetInfoWithNodeId(flat::NodeId nodeId, flat::LocalTargetInfo info)
+      : targetId(info.targetId),
+        nodeId(nodeId),
+        localState(info.localState) {}
+  LocalTargetInfoWithNodeId(flat::TargetId i, flat::NodeId ni, flat::LocalTargetState state)
+      : targetId(i),
+        nodeId(ni),
+        localState(state) {}
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdConfig.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "common/kv/TransactionRetry.h"
+#include "common/utils/ConfigBase.h"
+#include "common/utils/Duration.h"
+#include "core/user/UserCache.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdConfig : ConfigBase<MgmtdConfig> {
+  CONFIG_HOT_UPDATED_ITEM(lease_length, 60_s);
+  CONFIG_HOT_UPDATED_ITEM(extend_lease_interval, 10_s);
+  CONFIG_HOT_UPDATED_ITEM(suspicious_lease_interval, 20_s);
+  CONFIG_HOT_UPDATED_ITEM(heartbeat_timestamp_valid_window, 30_s);
+  CONFIG_HOT_UPDATED_ITEM(allow_heartbeat_from_unregistered, false);
+  CONFIG_HOT_UPDATED_ITEM(check_status_interval, 10_s);
+  CONFIG_HOT_UPDATED_ITEM(heartbeat_fail_interval, 60_s);
+  CONFIG_HOT_UPDATED_ITEM(new_chain_bootstrap_interval, 2_min);
+  CONFIG_HOT_UPDATED_ITEM(send_heartbeat, true);
+  CONFIG_HOT_UPDATED_ITEM(send_heartbeat_interval, 10_s);
+  CONFIG_HOT_UPDATED_ITEM(client_session_timeout, 20_min);
+  CONFIG_HOT_UPDATED_ITEM(bootstrapping_length, 2_min);
+  CONFIG_HOT_UPDATED_ITEM(update_chains_interval, 1_s);
+  CONFIG_HOT_UPDATED_ITEM(validate_lease_on_write, true);
+  CONFIG_HOT_UPDATED_ITEM(bump_routing_info_version_interval, 5_s);
+  // deprecated
+  CONFIG_HOT_UPDATED_ITEM(heartbeat_ignore_unknown_targets, false);
+  CONFIG_HOT_UPDATED_ITEM(heartbeat_ignore_stale_targets, true);
+  CONFIG_HOT_UPDATED_ITEM(retry_times_on_txn_errors, -1);  // -1 means infitely retry, 0 means no retry
+  CONFIG_HOT_UPDATED_ITEM(update_metrics_interval, 1_s);
+  CONFIG_HOT_UPDATED_ITEM(target_info_persist_interval, 1_s);
+  CONFIG_HOT_UPDATED_ITEM(target_info_persist_batch, 1000);
+  CONFIG_HOT_UPDATED_ITEM(target_info_load_interval, 1_s);
+  CONFIG_HOT_UPDATED_ITEM(try_adjust_target_order_as_preferred, false);
+  CONFIG_HOT_UPDATED_ITEM(extend_lease_check_release_version, true);
+  CONFIG_HOT_UPDATED_ITEM(authenticate, false);
+  CONFIG_OBJ(retry_transaction, kv::TransactionRetry);
+  CONFIG_OBJ(user_cache, core::UserCache::Config);
+  CONFIG_HOT_UPDATED_ITEM(enable_routinginfo_cache, true);
+  CONFIG_HOT_UPDATED_ITEM(only_accept_client_uuid, false);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdData.cc

@@ -0,0 +1,172 @@
+#include "MgmtdData.h"
+
+#include "MgmtdConfig.h"
+#include "core/utils/ServiceOperation.h"
+#include "helpers.h"
+#include "updateChain.h"
+
+namespace hf3fs::mgmtd {
+Result<Void> MgmtdData::checkConfigVersion(core::ServiceOperation &ctx,
+                                           flat::NodeType type,
+                                           flat::ConfigVersion version) const {
+  const auto &cm = configMap;
+  auto it = cm.find(type);
+  if (it == cm.end()) {
+    RETURN_AND_LOG_OP_ERR(ctx, StatusCode::kInvalidArg, "unknown NodeType {}", static_cast<int>(type));
+  }
+  const auto &versionedMap = it->second;
+  auto vit = versionedMap.rbegin();
+  if (version > vit->first) {
+    RETURN_AND_LOG_OP_ERR(ctx,
+                          MgmtdCode::kInvalidConfigVersion,
+                          "ConfigVersion {} of {} is newer than server's {}",
+                          version,
+                          magic_enum::enum_name(type),
+                          vit->first);
+  }
+  return Void{};
+}
+
+std::optional<flat::ConfigInfo> MgmtdData::getConfig(flat::NodeType type,
+                                                     flat::ConfigVersion version,
+                                                     bool latest) const {
+  const auto &cm = configMap;
+  auto it = cm.find(type);
+  assert(it != cm.end());
+  const auto &versionedMap = it->second;
+  if (latest) {
+    auto vit = versionedMap.rbegin();
+    const auto &info = vit->second;
+    XLOGF_IF(FATAL,
+             version > info.configVersion,
+             "Should checked that version from user ({}) not larger than from server ({})",
+             version.toUnderType(),
+             info.configVersion.toUnderType());
+    if (version < info.configVersion) {
+      return info;
+    } else {
+      return std::nullopt;
+    }
+  } else {
+    auto vit = versionedMap.find(version);
+    if (vit == versionedMap.end()) {
+      return std::nullopt;
+    } else {
+      return vit->second;
+    }
+  }
+}
+
+flat::ConfigVersion MgmtdData::getLatestConfigVersion(flat::NodeType type) const {
+  auto it = configMap.find(type);
+  assert(it != configMap.end());
+  const auto &versionedMap = it->second;
+  assert(!versionedMap.empty());
+  return versionedMap.rbegin()->first;
+}
+
+Result<Void> MgmtdData::checkRoutingInfoVersion(core::ServiceOperation &ctx, flat::RoutingInfoVersion version) const {
+  const auto &info = routingInfo;
+  if (version > info.routingInfoVersion) {
+    RETURN_AND_LOG_OP_ERR(ctx,
+                          MgmtdCode::kInvalidRoutingInfoVersion,
+                          "RoutingInfoVersion {} is newer than server's ({})",
+                          version,
+                          info.routingInfoVersion);
+  }
+  return Void{};
+}
+
+std::optional<flat::RoutingInfo> MgmtdData::getRoutingInfo(flat::RoutingInfoVersion version,
+                                                           const MgmtdConfig &config) const {
+  const auto &info = routingInfo;
+  assert(version <= info.routingInfoVersion);
+  if (version == info.routingInfoVersion) {
+    return std::nullopt;
+  }
+  bool enableRoutingInfoCache = config.enable_routinginfo_cache();
+  if (version != 0 && enableRoutingInfoCache) {
+    // version == 0 is possible a force refresh, do not read from cache
+    auto cachePtr = routingInfoCache.rlock();
+    if (cachePtr->has_value() && cachePtr->value().routingInfoVersion == info.routingInfoVersion) {
+      return cachePtr->value();
+    }
+  }
+  flat::RoutingInfo res;
+  res.routingInfoVersion = info.routingInfoVersion;
+  res.bootstrapping = bootstrapping(config);
+  for (const auto &[id, nw] : info.nodeMap) {
+    res.nodes[id] = nw.base();
+  }
+  res.chainTables = info.chainTables;
+  res.chains = info.chains;
+  for (const auto &[k, v] : info.getTargets()) {
+    res.targets[k] = v.base();
+  }
+  if (enableRoutingInfoCache) {
+    // always try to update cache even for a force refresh since the result is reusable to other requests
+    auto cachePtr = routingInfoCache.wlock();
+    if (!cachePtr->has_value() || cachePtr->value().routingInfoVersion < info.routingInfoVersion) {
+      *cachePtr = res;
+    }
+  }
+  return res;
+}
+
+void MgmtdData::appendChangedChains(flat::ChainId chainId,
+                                    std::vector<flat::ChainInfo> &changedChains,
+                                    bool tryAdjustTargetOrder) const {
+  if (routingInfo.newBornChains.contains(chainId)) return;
+
+  const auto &oldChain = routingInfo.getChain(chainId);
+
+  std::vector<ChainTargetInfoEx> oldTargets;
+  for (const auto &cti : oldChain.targets) {
+    oldTargets.emplace_back(cti, routingInfo.getTargets().at(cti.targetId).base().localState);
+  }
+
+  auto newTargets = generateNewChain(oldTargets);
+  if (tryAdjustTargetOrder && !oldChain.preferredTargetOrder.empty() &&
+      newTargets.back().publicState == flat::PublicTargetState::SERVING) {
+    newTargets = rotateAsPreferredOrder(newTargets, oldChain.preferredTargetOrder);
+  }
+  if (oldTargets != newTargets) {
+    flat::ChainInfo newChain;
+    newChain.chainId = chainId;
+    newChain.chainVersion = nextVersion(oldChain.chainVersion);
+    newChain.preferredTargetOrder = oldChain.preferredTargetOrder;
+    for (const auto &ti : newTargets) {
+      newChain.targets.push_back(ti);
+    }
+    XLOGF(DBG, "append changed chain {}", serde::toJsonString(newChain));
+    changedChains.push_back(std::move(newChain));
+  }
+}
+
+void MgmtdData::reset(flat::RoutingInfoVersion routingInfoVersion,
+                      NodeMap allNodes,
+                      ConfigMap allConfigs,
+                      ChainTableMap allChainTables,
+                      ChainMap allChains,
+                      TargetMap allTargets,
+                      UniversalTagsMap allUniversalTags) {
+  routingInfo.reset(routingInfoVersion,
+                    std::move(allNodes),
+                    std::move(allChainTables),
+                    std::move(allChains),
+                    std::move(allTargets));
+  {
+    auto cachePtr = routingInfoCache.wlock();
+    cachePtr->reset();
+  }
+
+  configMap = std::move(allConfigs);
+  lease.bootstrapping = false;
+  leaseStartTs = SteadyClock::now();
+  universalTagsMap = std::move(allUniversalTags);
+}
+
+bool MgmtdData::bootstrapping(const MgmtdConfig &config) const {
+  return leaseStartTs + config.bootstrapping_length().asUs() > SteadyClock::now();
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdData.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <folly/Synchronized.h>
+
+#include "ClientSession.h"
+#include "LeaseInfo.h"
+#include "RoutingInfo.h"
+#include "common/utils/RobinHoodUtils.h"
+#include "fbs/mgmtd/ConfigInfo.h"
+#include "fbs/mgmtd/MgmtdTypes.h"
+#include "fbs/mgmtd/RoutingInfo.h"
+
+namespace hf3fs::core {
+struct ServiceOperation;
+}
+
+namespace hf3fs::mgmtd {
+using VersionedConfigMap = std::map<flat::ConfigVersion, flat::ConfigInfo>;
+using ConfigMap = std::map<flat::NodeType, VersionedConfigMap>;
+using UniversalTagsMap = RHStringHashMap<std::vector<flat::TagPair>>;
+
+struct MgmtdConfig;
+
+struct MgmtdData {
+  SteadyTime leaseStartTs;
+  RoutingInfo routingInfo;
+  ConfigMap configMap;
+  LeaseInfo lease;
+  UniversalTagsMap universalTagsMap;
+
+  mutable folly::Synchronized<std::optional<flat::RoutingInfo>> routingInfoCache;
+
+  Result<Void> checkConfigVersion(core::ServiceOperation &ctx, flat::NodeType, flat::ConfigVersion version) const;
+  std::optional<flat::ConfigInfo> getConfig(flat::NodeType, flat::ConfigVersion version, bool latest) const;
+  flat::ConfigVersion getLatestConfigVersion(flat::NodeType) const;
+
+  Result<Void> checkRoutingInfoVersion(core::ServiceOperation &ctx, flat::RoutingInfoVersion version) const;
+  std::optional<flat::RoutingInfo> getRoutingInfo(flat::RoutingInfoVersion version, const MgmtdConfig &config) const;
+
+  void appendChangedChains(flat::ChainId chainId,
+                           std::vector<flat::ChainInfo> &changedChains,
+                           bool tryAdjustTargetOrder) const;
+
+  void reset(flat::RoutingInfoVersion routingInfoVersion,
+             NodeMap allNodes,
+             ConfigMap allConfigs,
+             ChainTableMap allChainTables,
+             ChainMap allChains,
+             TargetMap allTargets,
+             UniversalTagsMap allUniversalTags);
+
+  bool bootstrapping(const MgmtdConfig &config) const;
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdOperator.cc

@@ -0,0 +1,27 @@
+#include "MgmtdOperator.h"
+
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "core/utils/runOp.h"
+#include "mgmtd/ops/Include.h"
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, method, Method, Id, Req, Rsp)  \
+  CoTryTask<Rsp> svc##Operator::method(Req req, const net::PeerInfo &peer) { \
+    Method##Operation op(std::move(req));                                    \
+    CO_INVOKE_OP_INFO(op, peer.str, state_);                                 \
+  }
+
+namespace hf3fs::mgmtd {
+
+MgmtdOperator::MgmtdOperator(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config)
+    : state_(env, config),
+      backgroundRunner_(state_) {}
+
+void MgmtdOperator::start() { backgroundRunner_.start(); }
+
+CoTask<void> MgmtdOperator::stop() { co_await backgroundRunner_.stop(); }
+
+MgmtdOperator::~MgmtdOperator() { folly::coro::blockingWait(stop()); }
+
+#include "fbs/mgmtd/MgmtdServiceDef.h"
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdOperator.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "MgmtdState.h"
+#include "common/net/PeerInfo.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/background/MgmtdBackgroundRunner.h"
+
+namespace hf3fs::mgmtd {
+namespace testing {
+class MgmtdTestHelper;
+}
+
+class MgmtdOperator : public folly::NonCopyableNonMovable {
+ public:
+  MgmtdOperator(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config);
+
+  ~MgmtdOperator();
+
+  // start/stop background tasks
+  void start();
+  CoTask<void> stop();
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(Service, method, Method, Id, Req, Rsp) \
+  CoTryTask<Rsp> method(Req req, const net::PeerInfo &peer);
+
+#include "fbs/mgmtd/MgmtdServiceDef.h"
+
+ private:
+  MgmtdState state_;
+  MgmtdBackgroundRunner backgroundRunner_;
+
+  friend class testing::MgmtdTestHelper;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdService.cc

@@ -0,0 +1,12 @@
+#include "MgmtdService.h"
+
+#include "MgmtdOperator.h"
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype)        \
+  CoTryTask<rsptype> svc##Service::name(serde::CallContext &ctx, const reqtype &req) { \
+    co_return co_await operator_.name(req, net::PeerInfo{ctx.peer()});                 \
+  }
+
+namespace hf3fs::mgmtd {
+#include "fbs/mgmtd/MgmtdServiceDef.h"
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdService.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "common/serde/CallContext.h"
+#include "fbs/mgmtd/MgmtdServiceBase.h"
+
+namespace hf3fs::mgmtd {
+class MgmtdOperator;
+class MgmtdService : public serde::ServiceWrapper<MgmtdService, MgmtdServiceBase> {
+ public:
+  MgmtdService(MgmtdOperator &opr)
+      : operator_(opr) {}
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
+  CoTryTask<rsptype> name(serde::CallContext &ctx, const reqtype &req);
+
+#include "fbs/mgmtd/MgmtdServiceDef.h"
+
+ private:
+  MgmtdOperator &operator_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdState.cc

@@ -0,0 +1,92 @@
+#include "MgmtdState.h"
+
+#include "MgmtdConfig.h"
+#include "core/user/UserToken.h"
+#include "fbs/mgmtd/NodeConversion.h"
+#include "helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+flat::NodeInfo initSelfNodeInfo(core::ServerEnv &env) {
+  flat::NodeInfo selfNodeInfo;
+  const auto &appInfo = env.appInfo();
+  selfNodeInfo.app = appInfo;
+  selfNodeInfo.type = flat::NodeType::MGMTD;
+  selfNodeInfo.status = flat::NodeStatus::PRIMARY_MGMTD;
+  return selfNodeInfo;
+}
+
+void recordWriterLatency(std::string_view method, Duration latency) {
+  static std::map<String, monitor::LatencyRecorder, std::less<>> latencyRecorders;
+  static std::map<String, monitor::CountRecorder, std::less<>> usageRecorders;
+  auto lit = latencyRecorders.find(method);
+  auto uit = usageRecorders.find(method);
+  if (lit == latencyRecorders.end()) {
+    monitor::TagSet tags;
+    tags.addTag("instance", String(method));
+    lit = latencyRecorders.try_emplace(String(method), "MgmtdService.WriterLatency", tags).first;
+    uit = usageRecorders.try_emplace(String(method), "MgmtdService.WriterUsage", tags).first;
+  }
+  lit->second.addSample(latency);
+  uit->second.addSample(latency.count());
+}
+}  // namespace
+
+MgmtdState::MgmtdState(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config)
+    : env_(std::move(env)),
+      config_(config),
+      userStore_(*env_->kvEngine(), config_.retry_transaction(), config_.user_cache()) {
+  selfNodeInfo_ = initSelfNodeInfo(*env_);
+  selfPersistentNodeInfo_ = flat::toPersistentNode(selfNodeInfo_);
+}
+
+MgmtdState::~MgmtdState() {}
+
+UtcTime MgmtdState::utcNow() { return env_->utcTimeGenerator()(); }
+
+Result<Void> MgmtdState::validateClusterId(const core::ServiceOperation &ctx, std::string_view clusterId) {
+  if (clusterId != env_->appInfo().clusterId) {
+    RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kClusterIdMismatch, "Cluster id mismatch");
+  }
+  return Void{};
+}
+
+CoTryTask<void> MgmtdState::validateAdmin(const core::ServiceOperation &ctx, const flat::UserInfo &userInfo) {
+  if (config_.authenticate()) {
+    auto ret = co_await userStore_.getUser(userInfo.token);
+    CO_RETURN_ON_ERROR(ret);
+    if (!ret->admin) {
+      CO_RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kNotAdmin, "");
+    }
+    LOG_OP_INFO(ctx, "Act as admin user {}({})", ret->name, ret->uid.toUnderType());
+  }
+  co_return Void{};
+}
+
+CoTask<std::optional<flat::MgmtdLeaseInfo>> MgmtdState::currentLease(UtcTime now) {
+  auto dataPtr = co_await data_.coSharedLock();
+  const auto &lease = dataPtr->lease;
+  bool canTrustLease =
+      lease.lease.has_value() && now + config_.suspicious_lease_interval().asUs() < lease.lease->leaseEnd;
+  if (canTrustLease && !lease.bootstrapping) {
+    co_return lease.lease;
+  }
+  co_return std::nullopt;
+}
+
+flat::NodeId MgmtdState::selfId() const {
+  assert(env_->appInfo().nodeId != 0);
+  return flat::NodeId{env_->appInfo().nodeId};
+}
+
+MgmtdState::WriterMutexGuard::~WriterMutexGuard() {
+  recordWriterLatency(method_, Duration(SteadyClock::now() - start_));
+}
+
+kv::FDBRetryStrategy MgmtdState::createRetryStrategy() {
+  return kv::FDBRetryStrategy({config_.retry_transaction().max_backoff(),
+                               config_.retry_transaction().max_retry_count(),
+                               /*retryMaybeCommitted=*/true});
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdState.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <folly/experimental/coro/Mutex.h>
+
+#include "MgmtdData.h"
+#include "common/utils/BackgroundRunner.h"
+#include "common/utils/CoroSynchronized.h"
+#include "common/utils/DefaultRetryStrategy.h"
+#include "core/app/ServerEnv.h"
+#include "core/user/UserStoreEx.h"
+#include "fbs/mgmtd/NodeInfo.h"
+#include "fbs/mgmtd/PersistentNodeInfo.h"
+#include "fdb/FDBRetryStrategy.h"
+#include "mgmtd/store/MgmtdStore.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdConfig;
+
+using ClientSessionMap = RHStringHashMap<ClientSession>;
+
+struct MgmtdState {
+  MgmtdState(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config);
+  ~MgmtdState();
+
+  UtcTime utcNow();
+  Result<Void> validateClusterId(const core::ServiceOperation &ctx, std::string_view clusterId);
+  CoTryTask<void> validateAdmin(const core::ServiceOperation &ctx, const flat::UserInfo &userInfo);
+  CoTask<std::optional<flat::MgmtdLeaseInfo>> currentLease(UtcTime now);
+  flat::NodeId selfId() const;
+  kv::FDBRetryStrategy createRetryStrategy();
+
+  const std::shared_ptr<core::ServerEnv> env_;
+  flat::NodeInfo selfNodeInfo_;
+  flat::PersistentNodeInfo selfPersistentNodeInfo_;
+  const MgmtdConfig &config_;
+
+  MgmtdStore store_;
+  core::UserStoreEx userStore_;
+
+  class WriterMutexGuard {
+   public:
+    WriterMutexGuard(std::unique_lock<folly::coro::Mutex> mu, std::string_view m)
+        : mu_(std::move(mu)),
+          method_(m) {}
+    WriterMutexGuard(WriterMutexGuard &&other) = default;
+    ~WriterMutexGuard();
+
+   private:
+    std::unique_lock<folly::coro::Mutex> mu_;
+    std::string_view method_;
+    SteadyTime start_ = SteadyClock::now();
+  };
+
+  template <NameWrapper method>
+  CoTask<WriterMutexGuard> coScopedLock() {
+    auto mu = co_await writerMu_.co_scoped_lock();
+    co_return WriterMutexGuard(std::move(mu), method);
+  }
+
+  CoroSynchronized<MgmtdData> data_;
+  CoroSynchronized<ClientSessionMap> clientSessionMap_;
+
+ private:
+  // logical lock for protecting the whole processing of a writer operation
+  // during which read-modify-write will be performed on `data_`
+  folly::coro::Mutex writerMu_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MockMgmtd.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include <folly/Expected.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/executors/ThreadPoolExecutor.h>
+#include <folly/logging/xlog.h>
+#include <memory>
+#include <optional>
+#include <unistd.h>
+
+#include "common/utils/ConfigBase.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Result.h"
+#include "mgmtd/service/MgmtdConfig.h"
+#include "mgmtd/service/MgmtdOperator.h"
+#include "mgmtd/service/MgmtdService.h"
+
+namespace hf3fs::mgmtd {
+
+class MockMgmtd {
+ public:
+  struct Config : ConfigBase<Config> {
+    CONFIG_OBJ(mgmtd, MgmtdConfig);
+    CONFIG_ITEM(node_id, 1u);
+    CONFIG_ITEM(name, "mock-mgmtd");
+    CONFIG_ITEM(cluster_id, "mock-cluster");
+  };
+
+  static CoTryTask<std::unique_ptr<MockMgmtd>> create(Config config,
+                                                      std::shared_ptr<kv::IKVEngine> kvEngine,
+                                                      CPUExecutorGroup *exec) {
+    flat::ServiceGroupInfo groupInfo;
+    groupInfo.services.emplace(mgmtd::MgmtdService::kServiceName);
+    groupInfo.endpoints.push_back(net::Address::from("TCP://127.0.0.1:8000").value());
+
+    flat::AppInfo info;
+    info.nodeId = flat::NodeId(config.node_id());
+    info.pid = static_cast<uint32_t>(getpid());
+    info.serviceGroups = {groupInfo};
+    info.clusterId = config.cluster_id();
+
+    auto env = std::make_shared<core::ServerEnv>();
+    env->setAppInfo(info);
+    env->setKvEngine(kvEngine);
+    env->setBackgroundExecutor(exec);
+
+    auto mgmtd = std::make_unique<MockMgmtd>();
+    mgmtd->config_ = config;
+    mgmtd->mgmtdOperator_ = std::make_unique<mgmtd::MgmtdOperator>(std::move(env), mgmtd->config_.mgmtd());
+
+    mgmtd->mgmtdOperator_->start();
+    co_return mgmtd;
+  }
+
+  void stop() { mgmtdOperator_.reset(); }
+
+  std::unique_ptr<mgmtd::MgmtdService> getService() { return std::make_unique<mgmtd::MgmtdService>(*mgmtdOperator_); }
+
+  MgmtdOperator &getOperator() { return *mgmtdOperator_; }
+
+ private:
+  Config config_;
+  std::unique_ptr<mgmtd::MgmtdOperator> mgmtdOperator_;
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/NodeInfoWrapper.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "WithTimestamp.h"
+#include "common/utils/SimpleRingBuffer.h"
+#include "fbs/mgmtd/NodeInfo.h"
+#include "fbs/mgmtd/NodeStatusChange.h"
+
+namespace hf3fs::mgmtd {
+class NodeInfoWrapper : public WithTimestamp<flat::NodeInfo> {
+ public:
+  using Base = WithTimestamp<flat::NodeInfo>;
+  using Base::Base;
+
+  void updateHeartbeatVersion(flat::HeartbeatVersion version) { version_ = version; }
+  flat::HeartbeatVersion lastHbVersion() const { return version_; }
+
+  void recordStatusChange(flat::NodeStatus status, UtcTime now) {
+    if (statusChanges_.full()) statusChanges_.pop();
+    statusChanges_.push(flat::NodeStatusChange::create(status, now));
+  }
+
+  std::vector<flat::NodeStatusChange> getAllStatusChanges() {
+    return std::vector<flat::NodeStatusChange>(statusChanges_.begin(), statusChanges_.end());
+  }
+
+ private:
+  flat::HeartbeatVersion version_{0};  // used for rejecting stale heartbeat requests
+  static constexpr size_t kStatusChangeCount = 100;
+
+  SimpleRingBuffer<flat::NodeStatusChange> statusChanges_{kStatusChangeCount};
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/RoutingInfo.cc

@@ -0,0 +1,173 @@
+#include "RoutingInfo.h"
+
+#include "MgmtdConfig.h"
+#include "core/utils/ServiceOperation.h"
+#include "helpers.h"
+
+namespace hf3fs::mgmtd {
+void RoutingInfo::applyChainTargetChanges(core::ServiceOperation &ctx,
+                                          const std::vector<flat::ChainInfo> &changedChains,
+                                          SteadyTime now) {
+  for (const auto &chain : changedChains) {
+    auto &oldChain = getChain(chain.chainId);
+    LOG_OP_INFO(ctx,
+                "{} change from {} to {}",
+                chain.chainId,
+                serde::toJsonString(oldChain),
+                serde::toJsonString(chain));
+    oldChain = chain;
+    for (const auto &cti : chain.targets) {
+      updateTarget(cti.targetId, [&](auto &ti) {
+        ti.base().publicState = cti.publicState;
+        ti.updateTs(now);
+      });
+    }
+  }
+}
+
+flat::ChainInfo &RoutingInfo::getChain(flat::ChainId cid) {
+  XLOGF_IF(FATAL, !chains.contains(cid), "cid = {}", cid.toUnderType());
+  return chains[cid];
+}
+
+const flat::ChainInfo &RoutingInfo::getChain(flat::ChainId cid) const {
+  XLOGF_IF(FATAL, !chains.contains(cid), "cid = {}", cid.toUnderType());
+  return chains.at(cid);
+}
+
+void RoutingInfo::localUpdateTargets(flat::NodeId nodeId,
+                                     const std::vector<flat::LocalTargetInfo> &targets,
+                                     const MgmtdConfig &config) {
+  auto steadyNow = SteadyClock::now();
+
+  auto &orphans = orphanTargetsByNodeId[nodeId];
+  for (auto tid : orphans) {
+    orphanTargetsByTargetId.erase(tid);
+  }
+  orphans.clear();
+
+  for (const auto &lti : targets) {
+    auto it = this->targets.find(lti.targetId);
+    if (it != this->targets.end()) {
+      updateTarget(lti.targetId, [&](auto &ti) {
+        auto &base = ti.base();
+
+        bool shouldIgnore = [&] {
+          if (lti.chainVersion == 0 || !config.heartbeat_ignore_stale_targets()) return false;
+          const auto &chain = getChain(base.chainId);
+          return chain.chainVersion > lti.chainVersion;
+        }();
+
+        if (shouldIgnore) return;
+
+        ti.updateTs(steadyNow);
+
+        bool importantInfoChanged = base.localState != lti.localState || base.nodeId != nodeId;
+        if (importantInfoChanged) {
+          ti.importantInfoChangedTime = steadyNow;
+        }
+
+        base.localState = lti.localState;
+        base.nodeId = nodeId;
+        base.diskIndex = lti.diskIndex;
+        base.usedSize = lti.usedSize;
+      });
+    } else {
+      // only insert to orphans, not targets
+      orphans.insert(lti.targetId);
+      auto &ti = orphanTargetsByTargetId[lti.targetId];
+      ti = flat::TargetInfo();  // ensure clean state
+      ti.targetId = lti.targetId;
+      ti.localState = lti.localState;
+      ti.nodeId = nodeId;
+      ti.diskIndex = lti.diskIndex;
+      ti.usedSize = lti.usedSize;
+    }
+  }
+}
+
+void RoutingInfo::insertNewChain(const flat::ChainInfo &chain) {
+  auto cid = chain.chainId;
+  XLOGF_IF(FATAL, chains.contains(cid), "Insert duplicated chain {}", cid.toUnderType());
+  auto createdTime = SteadyClock::now();
+  chains[cid] = chain;
+  for (const auto &t : chain.targets) {
+    auto tid = t.targetId;
+    XLOGF_IF(FATAL, targets.contains(tid), "Insert duplicated target {}", tid.toUnderType());
+    targets[tid] = TargetInfo(makeTargetInfo(cid, t), createdTime);
+    eraseOrphanTarget(tid);
+  }
+  newBornChains[cid] = createdTime;
+}
+
+void RoutingInfo::insertNewTarget(flat::ChainId cid, const flat::ChainTargetInfo &cti) {
+  auto tid = cti.targetId;
+  XLOGF_IF(FATAL, targets.contains(tid), "Insert duplicated target {}", tid.toUnderType());
+  targets[tid] = TargetInfo(makeTargetInfo(cid, cti), SteadyClock::now());
+  eraseOrphanTarget(tid);
+}
+
+void RoutingInfo::removeTarget(flat::ChainId cid, flat::TargetId tid) {
+  XLOGF_IF(FATAL, !targets.contains(tid), "Try to remove unknown target {}", tid.toUnderType());
+  auto &ti = targets[tid];
+  XLOGF_IF(FATAL,
+           ti.base().publicState != flat::PublicTargetState::OFFLINE,
+           "Try to remove target {} which is not offlined",
+           tid.toUnderType());
+  if (chains.contains(cid)) {
+    auto &chain = getChain(cid);
+    XLOGF_IF(
+        FATAL,
+        std::find_if(chain.targets.begin(), chain.targets.end(), [&](const auto &ti) { return ti.targetId == tid; }) !=
+            chain.targets.end(),
+        "target {} is still present in chain {}",
+        tid.toUnderType(),
+        cid.toUnderType());
+  }
+  targets.erase(tid);
+}
+
+void RoutingInfo::eraseOrphanTarget(flat::TargetId tid) {
+  if (LIKELY(!orphanTargetsByTargetId.contains(tid))) return;
+  const auto &oti = orphanTargetsByTargetId[tid];
+  XLOGF_IF(FATAL, !oti.nodeId.has_value(), "Unexpected orphan target: {}", serde::toJsonString(oti));
+  XLOGF_IF(FATAL,
+           !orphanTargetsByNodeId.contains(*oti.nodeId),
+           "Unexpected orphan target ref: {}",
+           serde::toJsonString(oti));
+  auto &set = orphanTargetsByNodeId[*oti.nodeId];
+  XLOGF_IF(FATAL, !set.contains(tid), "Unexpected orphan target ref: {}", serde::toJsonString(oti));
+  set.erase(tid);
+  if (set.empty()) orphanTargetsByNodeId.erase(*oti.nodeId);
+  orphanTargetsByTargetId.erase(tid);
+}
+
+void RoutingInfo::reset(flat::RoutingInfoVersion routingInfoVersion,
+                        NodeMap allNodes,
+                        ChainTableMap allChainTables,
+                        ChainMap allChains,
+                        TargetMap allTargets) {
+  XLOGF(INFO, "Reset RoutingInfo to routingInfoVersion {}", routingInfoVersion);
+  auto steadyNow = SteadyClock::now();
+
+  nodeMap.clear();
+  chainTables.clear();
+  chains.clear();
+  newBornChains.clear();
+  orphanTargetsByTargetId.clear();
+  orphanTargetsByNodeId.clear();
+
+  for (const auto &[id, sn] : allNodes) {
+    nodeMap.try_emplace(id, sn.base(), steadyNow);
+  }
+  chainTables = std::move(allChainTables);
+  chains = std::move(allChains);
+  targets = std::move(allTargets);
+  this->routingInfoVersion = routingInfoVersion;
+
+  for ([[maybe_unused]] const auto &[cid, _] : chains) {
+    // avoid update chain too early after restart or the chain status may be not stable
+    newBornChains[cid] = steadyNow;
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/RoutingInfo.h

@@ -0,0 +1,80 @@
+#pragma once
+
+#include "LocalTargetInfoWithNodeId.h"
+#include "NodeInfoWrapper.h"
+#include "TargetInfo.h"
+#include "fbs/mgmtd/ChainInfo.h"
+#include "fbs/mgmtd/ChainTable.h"
+#include "fbs/mgmtd/MgmtdTypes.h"
+
+namespace hf3fs::core {
+struct ServiceOperation;
+}
+
+namespace hf3fs::mgmtd {
+struct MgmtdConfig;
+class MgmtdTargetInfoLoader;
+
+namespace testing {
+class MgmtdTestHelper;
+}
+
+using NodeMap = robin_hood::unordered_map<flat::NodeId, NodeInfoWrapper>;
+using ChainTableVersionMap = std::map<flat::ChainTableVersion, flat::ChainTable>;
+using ChainTableMap = robin_hood::unordered_map<flat::ChainTableId, ChainTableVersionMap>;
+using ChainMap = robin_hood::unordered_map<flat::ChainId, flat::ChainInfo>;
+using TargetMap = robin_hood::unordered_map<flat::TargetId, TargetInfo>;
+using NewBornChains = robin_hood::unordered_map<flat::ChainId, SteadyTime>;
+using OrphanTargetsByTargetId = robin_hood::unordered_map<flat::TargetId, flat::TargetInfo>;
+using OrphanTargetsByNodeId = robin_hood::unordered_map<flat::NodeId, robin_hood::unordered_set<flat::TargetId>>;
+
+struct RoutingInfo {
+  bool routingInfoChanged = false;                  // periodically promote routingInfoVersion
+  flat::RoutingInfoVersion routingInfoVersion{1};   // ensure version 0 is less than any valid version
+  NodeMap nodeMap;                                  // persistent
+  ChainTableMap chainTables;                        // persistent
+  ChainMap chains;                                  // persistent
+  NewBornChains newBornChains;                      // temporal
+  OrphanTargetsByTargetId orphanTargetsByTargetId;  // temporal
+  OrphanTargetsByNodeId orphanTargetsByNodeId;      // temporal
+
+  void applyChainTargetChanges(core::ServiceOperation &ctx,
+                               const std::vector<flat::ChainInfo> &changedChains,
+                               SteadyTime now);
+
+  void localUpdateTargets(flat::NodeId nodeId,
+                          const std::vector<flat::LocalTargetInfo> &targets,
+                          const MgmtdConfig &config);
+
+  flat::ChainInfo &getChain(flat::ChainId cid);
+  const flat::ChainInfo &getChain(flat::ChainId cid) const;
+
+  void insertNewChain(const flat::ChainInfo &chain);
+
+  void insertNewTarget(flat::ChainId cid, const flat::ChainTargetInfo &cti);
+
+  void removeTarget(flat::ChainId cid, flat::TargetId tid);
+
+  const TargetMap &getTargets() const { return targets; }
+
+  auto updateTarget(flat::TargetId tid, auto &&func) {
+    XLOGF_IF(DFATAL, !targets.contains(tid), "tid = {}", tid);
+    auto &ti = targets[tid];
+    return func(ti);
+  }
+
+  void reset(flat::RoutingInfoVersion routingInfoVersion,
+             NodeMap allNodes,
+             ChainTableMap allChainTables,
+             ChainMap allChains,
+             TargetMap allTargets);
+
+ private:
+  void eraseOrphanTarget(flat::TargetId tid);
+
+  TargetMap targets;  // derived
+
+  friend class testing::MgmtdTestHelper;
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/TargetInfo.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "WithTimestamp.h"
+#include "fbs/mgmtd/TargetInfo.h"
+
+namespace hf3fs::mgmtd {
+class TargetInfo : public WithTimestamp<flat::TargetInfo> {
+ public:
+  using Base = WithTimestamp<flat::TargetInfo>;
+  using Base::Base;
+
+  bool locationInitLoaded = false;
+  std::optional<flat::NodeId> persistedNodeId;
+  std::optional<uint32_t> persistedDiskIndex;
+
+  SteadyTime importantInfoChangedTime = SteadyClock::now();
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/WithTimestamp.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "common/utils/UtcTime.h"
+#include "fbs/mgmtd/TargetInfo.h"
+
+namespace hf3fs::mgmtd {
+template <typename T>
+class WithTimestamp {
+ public:
+  WithTimestamp()
+      : base_(),
+        ts_(SteadyClock::now()) {}
+
+  explicit WithTimestamp(T info)
+      : base_(std::move(info)),
+        ts_(SteadyClock::now()) {}
+
+  WithTimestamp(T info, SteadyTime ts)
+      : base_(std::move(info)),
+        ts_(ts) {}
+
+  T &base() { return base_; }
+  const T &base() const { return base_; }
+
+  SteadyTime ts() const { return ts_; }
+  void updateTs() { ts_ = SteadyClock::now(); }
+  void updateTs(SteadyTime ts) { ts_ = ts; }
+
+ private:
+  T base_;
+  SteadyTime ts_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/helpers.cc

@@ -0,0 +1,95 @@
+#include "helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTask<void> updateMemoryRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx) {
+  static const auto handler = [](RoutingInfo &) {};
+  co_await updateMemoryRoutingInfo(state, ctx, handler);
+}
+
+CoTryTask<void> updateStoredRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx) {
+  static const auto handler = [](kv::IReadWriteTransaction &) -> CoTryTask<void> { co_return Void{}; };
+  co_return co_await updateStoredRoutingInfo(state, ctx, handler);
+}
+
+flat::TargetInfo makeTargetInfo(flat::ChainId chainId, const flat::ChainTargetInfo &info) {
+  flat::TargetInfo ti;
+  ti.targetId = info.targetId;
+  ti.publicState = info.publicState;
+  ti.localState = flat::LocalTargetState::OFFLINE;
+  ti.chainId = chainId;
+  return ti;
+}
+
+Result<Void> updateSelfConfig(MgmtdState &state, const flat::ConfigInfo &cfg) {
+  XLOGF_IF(FATAL,
+           state.selfNodeInfo_.configVersion > cfg.configVersion,
+           "ConfigVersion in memory ({}) is larger than given ({})",
+           state.selfNodeInfo_.configVersion.toUnderType(),
+           cfg.configVersion.toUnderType());
+  if (state.selfNodeInfo_.configVersion >= cfg.configVersion) return Void{};
+
+  const auto &updater = state.env_->configUpdater();
+  if (!updater) {
+    XLOGF(WARN, "Mgmtd: blindly promote to {} since no listener found", cfg.configVersion);
+    return Void{};
+  }
+  return updater(cfg.content, cfg.genUpdateDesc());
+}
+
+Result<std::vector<flat::TagPair>> updateTags(core::ServiceOperation &op,
+                                              flat::SetTagMode mode,
+                                              const std::vector<flat::TagPair> &oldTags,
+                                              const std::vector<flat::TagPair> &updates) {
+  std::map<std::string_view, std::string_view> tagMap;
+  switch (mode) {
+    case flat::SetTagMode::REPLACE: {
+      for (const auto &tp : updates) tagMap[tp.key] = tp.value;
+      break;
+    }
+    case flat::SetTagMode::UPSERT: {
+      for (const auto &tp : updates) tagMap[tp.key] = tp.value;
+      for (const auto &tp : oldTags) tagMap.try_emplace(tp.key, tp.value);
+      break;
+    }
+    case flat::SetTagMode::REMOVE: {
+      for (const auto &tp : oldTags) tagMap[tp.key] = tp.value;
+      for (const auto &tp : updates) {
+        if (!tp.value.empty()) {
+          RETURN_AND_LOG_OP_ERR(op,
+                                MgmtdCode::kInvalidTag,
+                                "could only pass empty value for REMOVE. key {} value {}",
+                                tp.key,
+                                tp.value);
+        }
+        tagMap.erase(tp.key);
+      }
+      break;
+    }
+  }
+
+  std::vector<flat::TagPair> newTags;
+  for (const auto &[k, v] : tagMap) {
+    newTags.emplace_back(String(k), String(v));
+  }
+  return newTags;
+}
+
+void updateMemoryRoutingInfo(RoutingInfo &alreadyLockedRoutingInfo, core::ServiceOperation &ctx) {
+  auto &ri = alreadyLockedRoutingInfo;
+  ++ri.routingInfoVersion.toUnderType();
+  ri.routingInfoChanged = false;
+  LOG_OP_INFO(ctx, "RoutingInfo: bump memory version to {}", ri.routingInfoVersion);
+}
+
+CoTryTask<Void> ensureSelfIsPrimary(MgmtdState &state) {
+  auto lease = co_await state.currentLease(state.utcNow());
+  if (lease.has_value()) {
+    if (lease->primary.nodeId == state.selfId()) {
+      co_return Void{};
+    } else {
+      co_return makeError(MgmtdCode::kNotPrimary, fmt::format("{}", lease->primary.nodeId.toUnderType()));
+    }
+  }
+  co_return makeError(MgmtdCode::kNotPrimary);
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/helpers.h

@@ -0,0 +1,96 @@
+#include <atomic>
+#include <folly/experimental/coro/Sleep.h>
+
+#include "MgmtdConfig.h"
+#include "MgmtdOperator.h"
+#include "MgmtdState.h"
+#include "common/kv/WithTransaction.h"
+#include "core/utils/ServiceOperation.h"
+
+#define RECORD_LATENCY(latency)                                                                                       \
+  auto FB_ANONYMOUS_VARIABLE(guard) = folly::makeGuard([lat = &(latency), begin = std::chrono::steady_clock::now()] { \
+    lat->addSample(std::chrono::steady_clock::now() - begin);                                                         \
+  })
+
+namespace hf3fs::mgmtd {
+template <typename T>
+inline T nextVersion(T version) {
+  auto v = version + 1;
+  return T(v);
+}
+
+CoTryTask<Void> ensureSelfIsPrimary(MgmtdState &state);
+
+template <typename Handler>
+inline std::invoke_result_t<Handler> doAsPrimary(MgmtdState &state, Handler &&handler) {
+  auto ret = co_await ensureSelfIsPrimary(state);
+  CO_RETURN_ON_ERROR(ret);
+  co_return co_await handler();
+}
+
+void updateMemoryRoutingInfo(RoutingInfo &alreadyLockedRoutingInfo, core::ServiceOperation &ctx);
+
+template <typename Handler>
+inline CoTask<void> updateMemoryRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx, Handler &&handler) {
+  auto dataPtr = co_await state.data_.coLock();
+  auto &ri = dataPtr->routingInfo;
+  updateMemoryRoutingInfo(ri, ctx);
+  handler(ri);
+}
+
+CoTask<void> updateMemoryRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx);
+
+template <typename Handler, typename Result = std::invoke_result_t<Handler, kv::IReadWriteTransaction &>>
+inline Result withReadWriteTxn(MgmtdState &state, Handler &&handler, bool expectSelfPrimary = true) {
+  int maxRetryTimes = state.config_.retry_times_on_txn_errors();
+  constexpr auto retryInterval = std::chrono::milliseconds(1000);
+  auto strategy = state.createRetryStrategy();
+  for (int i = 0;; ++i) {
+    auto res = co_await kv::WithTransaction(strategy).run(
+        state.env_->kvEngine()->createReadWriteTransaction(),
+        [&](kv::IReadWriteTransaction &txn) -> Result {
+          if (expectSelfPrimary && state.config_.validate_lease_on_write()) {
+            CO_RETURN_ON_ERROR(co_await state.store_.ensureLeaseValid(txn, state.selfId(), state.utcNow()));
+          }
+          co_return co_await handler(txn);
+        });
+    if (res || i == maxRetryTimes || StatusCode::typeOf(res.error().code()) != StatusCodeType::Transaction)
+      co_return res;
+    XLOGF(CRITICAL, "Transaction failed: {}\nretryCount: {}", res.error(), i);
+    co_await folly::coro::sleep(retryInterval);
+  }
+}
+
+template <typename Handler, typename Result = std::invoke_result_t<Handler, kv::IReadOnlyTransaction &>>
+inline Result withReadOnlyTxn(MgmtdState &state, Handler &&handler) {
+  auto strategy = state.createRetryStrategy();
+  co_return co_await kv::WithTransaction(strategy).run(
+      state.env_->kvEngine()->createReadonlyTransaction(),
+      [&](kv::IReadOnlyTransaction &txn) -> Result { co_return co_await handler(txn); });
+}
+
+template <typename Handler>
+inline CoTryTask<void> updateStoredRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx, Handler &&handler) {
+  auto dataPtr = co_await state.data_.coSharedLock();
+  auto nextv = nextVersion(dataPtr->routingInfo.routingInfoVersion);
+  co_return co_await withReadWriteTxn(state, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+    CO_RETURN_ON_ERROR(co_await state.store_.storeRoutingInfoVersion(txn, nextv));
+    LOG_OP_INFO(ctx, "RoutingInfo: bump storage version to {}", nextv);
+    co_return co_await handler(txn);
+  });
+}
+
+CoTryTask<void> updateStoredRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx);
+
+flat::TargetInfo makeTargetInfo(flat::ChainId chainId, const flat::ChainTargetInfo &info);
+
+Result<Void> updateSelfConfig(MgmtdState &state, const flat::ConfigInfo &cfg);
+
+using MgmtdStub = mgmtd::IMgmtdServiceStub;
+using MgmtdStubFactory = stubs::IStubFactory<MgmtdStub>;
+
+Result<std::vector<flat::TagPair>> updateTags(core::ServiceOperation &op,
+                                              flat::SetTagMode mode,
+                                              const std::vector<flat::TagPair> &oldTags,
+                                              const std::vector<flat::TagPair> &updates);
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/updateChain.cc

@@ -0,0 +1,188 @@
+#include "updateChain.h"
+
+#include "common/utils/OptionalUtils.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+using PS = enum flat::PublicTargetState;
+using LS = enum flat::LocalTargetState;
+using TargetsByPs = robin_hood::unordered_map<PS, std::vector<ChainTargetInfoEx>>;
+
+void dispatch(TargetsByPs &targetsByPs, ChainTargetInfoEx ti, PS ps, std::string_view reason) {
+  XLOGF_IF(DBG,
+           ti.publicState != ps,
+           "Dispatch {}({}-{}) to {}: {}",
+           ti.targetId,
+           magic_enum::enum_name(ti.publicState),
+           magic_enum::enum_name(ti.localState),
+           magic_enum::enum_name(ps),
+           reason);
+  ti.publicState = ps;
+  targetsByPs[ps].push_back(ti);
+}
+}  // namespace
+
+std::vector<ChainTargetInfoEx> generateNewChain(const std::vector<ChainTargetInfoEx> &oldTargets) {
+  robin_hood::unordered_map<PS, std::vector<ChainTargetInfoEx>> oldTargetsByPs, newTargetsByPs;
+  for (const auto &ti : oldTargets) {
+    oldTargetsByPs[ti.publicState].push_back(ti);
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::SERVING]) {
+    if (ti.localState == LS::ONLINE || ti.localState == LS::UPTODATE) {
+      dispatch(newTargetsByPs, ti, PS::SERVING, "");
+    } else if (newTargetsByPs[PS::LASTSRV].empty()) {
+      // If all SERVING offlined, only the first becomes LASTSRV.
+      // NOTE: in such cases the whole chain has to wait the HEAD for recovering even
+      //       when other replicas are complete.
+      dispatch(newTargetsByPs, ti, PS::LASTSRV, "first SERVING");
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "following SERVINGs");
+    }
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::LASTSRV]) {
+    if (newTargetsByPs[PS::SERVING].empty()) {
+      if (ti.localState == LS::ONLINE || ti.localState == LS::UPTODATE) {
+        dispatch(newTargetsByPs, ti, PS::SERVING, "first LASTSRV");
+      } else {
+        dispatch(newTargetsByPs, ti, PS::LASTSRV, "following LASTSRVs");
+      }
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "Has SERVING");
+    }
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::SYNCING]) {
+    if (ti.localState == LS::UPTODATE) {
+      dispatch(newTargetsByPs, ti, PS::SERVING, "");
+    } else if (ti.localState == LS::ONLINE) {
+      if (!newTargetsByPs[PS::SERVING].empty()) {
+        dispatch(newTargetsByPs, ti, PS::SYNCING, "Has SERVING");
+      } else {
+        dispatch(newTargetsByPs, ti, PS::WAITING, "No SERVING");
+      }
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "");
+    }
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::WAITING]) {
+    if (!newTargetsByPs[PS::SERVING].empty() && newTargetsByPs[PS::SYNCING].empty() && ti.localState == LS::ONLINE) {
+      dispatch(newTargetsByPs, ti, PS::SYNCING, "Has SERVING && No SYNCING");
+    } else if (ti.localState == LS::ONLINE || ti.localState == LS::UPTODATE) {
+      dispatch(newTargetsByPs, ti, PS::WAITING, "");
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "");
+    }
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::OFFLINE]) {
+    if (!newTargetsByPs[PS::SERVING].empty() && newTargetsByPs[PS::SYNCING].empty() && ti.localState == LS::ONLINE) {
+      dispatch(newTargetsByPs, ti, PS::SYNCING, "Has SERVING && No SYNCING");
+    } else if (ti.localState == LS::ONLINE || ti.localState == LS::UPTODATE) {
+      dispatch(newTargetsByPs, ti, PS::WAITING, "");
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "");
+    }
+  }
+
+  if (!newTargetsByPs[PS::SERVING].empty()) {
+    for (auto &ti : newTargetsByPs[PS::LASTSRV]) {
+      dispatch(newTargetsByPs, std::move(ti), PS::OFFLINE, "Has SERVING");
+    }
+    newTargetsByPs[PS::LASTSRV].clear();
+  }
+
+  std::vector<ChainTargetInfoEx> newTargets;
+  for (auto s : {PS::SERVING, PS::LASTSRV, PS::SYNCING, PS::WAITING, PS::OFFLINE}) {
+    const auto &v = newTargetsByPs[s];
+    newTargets.insert(newTargets.end(), v.begin(), v.end());
+  }
+  assert(oldTargets.size() == newTargets.size());
+  return newTargets;
+}
+
+std::vector<ChainTargetInfoEx> rotateAsPreferredOrder(const std::vector<ChainTargetInfoEx> &oldTargets,
+                                                      const std::vector<flat::TargetId> &preferredOrder) {
+  XLOGF_IF(DFATAL,
+           oldTargets.size() < preferredOrder.size(),
+           "oldTargets.size = {}, preferredTargets.size = {}",
+           oldTargets.size(),
+           preferredOrder.size());
+  std::map<flat::TargetId, std::pair<size_t, ChainTargetInfoEx>> oldMapping;
+  for (size_t i = 0; i < oldTargets.size(); ++i) {
+    oldMapping[oldTargets[i].targetId] = std::make_pair(i, oldTargets[i]);
+  }
+  for (size_t i = 0; i < preferredOrder.size(); ++i) {
+    auto tid = preferredOrder[i];
+    XLOGF_IF(DFATAL, !oldMapping.contains(tid), "{} not in oldTargets", tid);
+    const auto &[oldPos, oldInfo] = oldMapping[tid];
+    if (oldPos == i) {
+      continue;
+    }
+    if (oldInfo.publicState != PS::SERVING) {
+      break;
+    }
+    std::vector<ChainTargetInfoEx> newTargets;
+    for (size_t j = 0; j < i; ++j) {
+      newTargets.push_back(oldTargets[j]);
+    }
+    for (size_t j = i + 1; j < oldTargets.size(); ++j) {
+      newTargets.push_back(oldTargets[j]);
+    }
+    auto info = oldTargets[i];
+    info.publicState = PS::OFFLINE;
+    info.localState = LS::OFFLINE;
+    newTargets.push_back(info);
+    return newTargets;
+  }
+  return oldTargets;
+}
+
+std::vector<ChainTargetInfoEx> rotateLastSrv(const std::vector<ChainTargetInfoEx> &oldTargets) {
+  if (oldTargets.size() < 2 || oldTargets[0].publicState != PS::LASTSRV) {
+    return oldTargets;
+  }
+
+  std::vector<ChainTargetInfoEx> newTargets;
+  for (size_t i = 1; i < oldTargets.size(); ++i) newTargets.push_back(oldTargets[i]);
+  newTargets.push_back(oldTargets[0]);
+
+  // possible conversions:
+  // * WAITING -> LASTSRV: it's safer to convert a WAITING to LASTSRV than to SERVING
+  // * OFFLINE -> LASTSRV
+  newTargets[0].publicState = PS::LASTSRV;
+
+  // possible conversions:
+  // * WAITING -> OFFLINE
+  // * OFFLINE -> OFFLINE
+  for (size_t i = 1; i < newTargets.size(); ++i) newTargets[i].publicState = PS::OFFLINE;
+
+  return newTargets;
+}
+
+std::vector<flat::ChainTargetInfo> shutdownChain(const std::vector<flat::ChainTargetInfo> &oldTargets) {
+  std::vector<flat::ChainTargetInfo> newTargets;
+  for (auto ti : oldTargets) {
+    switch (ti.publicState) {
+      case PS::SERVING:
+        ti.publicState = PS::LASTSRV;
+        break;
+      case PS::WAITING:
+      case PS::SYNCING:
+        ti.publicState = PS::OFFLINE;
+        break;
+      case PS::LASTSRV:
+      case PS::OFFLINE:
+        // keep it as is
+        break;
+      case PS::INVALID:
+        XLOGF(FATAL, "Invalid PublicTargetState: {}", ti.targetId);
+    }
+    newTargets.push_back(ti);
+  }
+  return newTargets;
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/updateChain.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "LocalTargetInfoWithNodeId.h"
+#include "fbs/mgmtd/ChainTargetInfo.h"
+#include "fbs/mgmtd/TargetInfo.h"
+
+namespace hf3fs::mgmtd {
+struct ChainTargetInfoEx : public flat::ChainTargetInfo {
+  ChainTargetInfoEx() = default;
+  ChainTargetInfoEx(const flat::ChainTargetInfo &cti, flat::LocalTargetState ls)
+      : flat::ChainTargetInfo(cti),
+        localState(ls) {}
+  explicit ChainTargetInfoEx(const flat::TargetInfo &ti) {
+    targetId = ti.targetId;
+    publicState = ti.publicState;
+    localState = ti.localState;
+  }
+
+  bool operator==(const ChainTargetInfo &other) const { return static_cast<const ChainTargetInfo &>(*this) == other; }
+  bool operator==(const ChainTargetInfoEx &other) const {
+    return *this == static_cast<const ChainTargetInfo &>(other) && localState == other.localState;
+  }
+
+  static ChainTargetInfoEx fromTargetInfo(const flat::TargetInfo &ti) { return ChainTargetInfoEx(ti); }
+
+  static flat::TargetInfo toTargetInfo(const ChainTargetInfoEx &cti) {
+    flat::TargetInfo ti;
+    ti.targetId = cti.targetId;
+    ti.publicState = cti.publicState;
+    ti.localState = cti.localState;
+    return ti;
+  }
+
+  flat::LocalTargetState localState{flat::LocalTargetState::INVALID};
+};
+
+// separate this function just for testing friendly
+std::vector<ChainTargetInfoEx> generateNewChain(const std::vector<ChainTargetInfoEx> &oldTargets);
+
+std::vector<ChainTargetInfoEx> rotateAsPreferredOrder(const std::vector<ChainTargetInfoEx> &oldTargets,
+                                                      const std::vector<flat::TargetId> &preferredOrder);
+
+// If the head of oldTargets is LASTSRV, move it to the tail and let the next target be the new LASTSRV.
+// It's used when a LASTSRV target could not recover in a short time. Admin could let the next target be the new LASTSRV
+// to resume the service. NOTE: it's on risk of losing some data forever.
+std::vector<ChainTargetInfoEx> rotateLastSrv(const std::vector<ChainTargetInfoEx> &oldTargets);
+
+std::vector<flat::ChainTargetInfo> shutdownChain(const std::vector<flat::ChainTargetInfo> &oldTargets);
+}  // namespace hf3fs::mgmtd

src/mgmtd/store/MgmtdStore.cc

@@ -0,0 +1,493 @@
+#include "MgmtdStore.h"
+
+#include <folly/experimental/coro/Collect.h>
+#include <folly/lang/Bits.h>
+
+#include "common/kv/KeyPrefix.h"
+#include "common/utils/MagicEnum.hpp"
+#include "common/utils/SerDeser.h"
+#include "common/utils/StringUtils.h"
+#include "common/utils/Transform.h"
+#include "fbs/mgmtd/NodeConversion.h"
+#include "mgmtd/service/updateChain.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+std::string_view getMgmtdLeaseKey() {
+  static const std::string key = fmt::format("{}MgmtdLease", kv::toStr(kv::KeyPrefix::Single));
+  return key;
+}
+
+std::string_view getRoutingInfoVersionKey() {
+  static const std::string key = fmt::format("{}RoutingInfoVersion", kv::toStr(kv::KeyPrefix::Single));
+  return key;
+}
+
+String getNodeKey(flat::NodeId id) {
+  String buf;
+  Serializer s(buf);
+  s.put(kv::KeyPrefix::NodeTable);
+  s.put(id.toUnderType());
+  return buf;
+}
+
+String getChainTableKey(flat::ChainTableId id, flat::ChainTableVersion ver) {
+  String buf;
+  Serializer s(buf);
+  s.put(kv::KeyPrefix::ChainTable);
+  s.put(id.toUnderType());
+  s.put(ver.toUnderType());
+  return buf;
+}
+
+String getChainInfoKey(flat::ChainId id) {
+  String buf;
+  Serializer s(buf);
+  s.put(kv::KeyPrefix::ChainInfo);
+  s.put(id.toUnderType());
+  return buf;
+}
+
+String getTargetInfoKey(flat::TargetId id) {
+  auto reversedId = folly::Endian::big64(id.toUnderType());
+  String buf;
+  Serializer s(buf);
+  s.put(kv::KeyPrefix::TargetInfo);
+  s.put(reversedId);
+  return buf;
+}
+
+String getKeyForUniversalTags(std::string_view id) {
+  return fmt::format("{}{}", kv::toStr(kv::KeyPrefix::UniversalTags), id);
+}
+
+std::string_view getKeyPrefixForUniversalTags() { return kv::toStr(kv::KeyPrefix::UniversalTags); }
+
+auto getChainTableKeyPrefix() { return kv::toStr(kv::KeyPrefix::ChainTable); }
+
+auto getChainInfoKeyPrefix() { return kv::toStr(kv::KeyPrefix::ChainInfo); }
+
+Result<flat::NodeId> extractNodeIdFromKey(std::string_view key) {
+  Deserializer deser(key);
+  auto prefixRes = deser.get<kv::KeyPrefix>();
+  RETURN_ON_ERROR(prefixRes);
+  if (*prefixRes != kv::KeyPrefix::NodeTable) {
+    return makeError(StatusCode::kDataCorruption, fmt::format("Invalid prefix of node: key == {}", key));
+  }
+  auto idRes = deser.get<flat::NodeId::UnderlyingType>();
+  RETURN_ON_ERROR(idRes);
+  if (!deser.reachEnd()) {
+    return makeError(StatusCode::kDataCorruption, fmt::format("Invalid prefix of node: key == {}", key));
+  }
+  return flat::NodeId{*idRes};
+}
+
+String getConfigKey(flat::NodeType nodeType, flat::ConfigVersion version) {
+  String buf;
+  Serializer s(buf);
+  s.put(kv::KeyPrefix::Config);
+  s.putShortString(toStringView(nodeType));
+
+  // let the latest version be the first
+  auto reversedBigVer = folly::Endian::big64(std::numeric_limits<uint64_t>::max() - version.toUnderType());
+  s.put(reversedBigVer);
+  return buf;
+}
+
+Result<std::tuple<flat::NodeType, flat::ConfigVersion>> decodeConfigKey(std::string_view s) {
+  Deserializer des(s);
+  auto keyPrefix = co_await des.get<kv::KeyPrefix>();
+  if (keyPrefix != kv::KeyPrefix::Config) {
+    co_return makeError(StatusCode::kDataCorruption,
+                        fmt::format("Decode config key failed: prefix mismatch. key: {}. expected prefix: {}.",
+                                    toHexString(s),
+                                    kv::toStr(kv::KeyPrefix::Config)));
+  }
+  auto typeStr = co_await des.getShortString();
+  auto type = magic_enum::enum_cast<flat::NodeType>(typeStr);
+  if (!type) {
+    co_return makeError(
+        StatusCode::kDataCorruption,
+        fmt::format("Decode config key failed: unknown type. key: {}. type: {}", toHexString(s), typeStr));
+  }
+  auto reversedBigVer = co_await des.get<uint64_t>();
+  auto reversedLittleVer = folly::Endian::big64(reversedBigVer);
+  auto ver = std::numeric_limits<uint64_t>::max() - reversedLittleVer;
+  co_return std::make_tuple(*type, flat::ConfigVersion(ver));
+}
+
+std::string_view getConfigKeyPrefix() { return kv::toStr(kv::KeyPrefix::Config); }
+
+String getConfigKeyPrefix(flat::NodeType nodeType) {
+  String buf;
+  Serializer s(buf);
+  s.put(kv::KeyPrefix::Config);
+  s.putShortString(toStringView(nodeType));
+  return buf;
+}
+
+CoTryTask<std::pair<flat::NodeType, flat::ConfigInfo>> unpackConfigInfo(const kv::IReadOnlyTransaction::KeyValue &kv,
+                                                                        std::optional<flat::NodeType> expectedType) {
+  const auto &[k, v] = kv.pair();
+  auto keyDecodeRes = decodeConfigKey(k);
+  CO_RETURN_ON_ERROR(keyDecodeRes);
+  auto [t, version] = *keyDecodeRes;
+  if (expectedType && t != expectedType) {
+    co_return makeError(
+        StatusCode::kDataCorruption,
+        fmt::format("Expect {} in config key but get {}. key: {}", toStringView(*expectedType), toStringView(t), k));
+  }
+  auto valueDecodeRes = flat::ConfigInfo::unpackFrom(v);
+  CO_RETURN_ON_ERROR(valueDecodeRes);
+  if (version != valueDecodeRes->configVersion) {
+    co_return makeError(
+        StatusCode::kDataCorruption,
+        fmt::format("Version mismatch between key and value. version in key: {}. version in value: {}. key: {}",
+                    version.toUnderType(),
+                    valueDecodeRes->configVersion.toUnderType(),
+                    toHexString(k)));
+  }
+  co_return std::make_pair(t, std::move(*valueDecodeRes));
+}
+}  // namespace
+
+CoTryTask<flat::MgmtdLeaseInfo> MgmtdStore::extendLease(kv::IReadWriteTransaction &txn,
+                                                        const flat::PersistentNodeInfo &nodeInfo,
+                                                        std::chrono::microseconds leaseLength,
+                                                        UtcTime now,
+                                                        flat::ReleaseVersion rv,
+                                                        bool checkReleaseVersion) {
+  auto fetchResult = co_await loadMgmtdLeaseInfo(txn);
+  CO_RETURN_ON_ERROR(fetchResult);
+  std::optional<flat::MgmtdLeaseInfo> storedLeaseInfo = std::move(*fetchResult);
+
+  flat::MgmtdLeaseInfo newLeaseInfo(nodeInfo, now, now + leaseLength, rv);
+
+  if (storedLeaseInfo.has_value()) {
+    if (checkReleaseVersion && newLeaseInfo.releaseVersion < storedLeaseInfo->releaseVersion) {
+      // releaseVersion should not rollback
+      co_return *storedLeaseInfo;
+    }
+    auto leaseEnd = storedLeaseInfo->leaseEnd;
+    if (leaseEnd >= now + leaseLength) {
+      // lease is long enough, do nothing
+      co_return *storedLeaseInfo;
+    }
+    if (leaseEnd < now) {
+      // lease already retired, try to start a new lease
+      co_return co_await storeMgmtdLeaseInfo(txn, newLeaseInfo);
+    }
+    if (storedLeaseInfo->primary.nodeId == nodeInfo.nodeId) {
+      newLeaseInfo.leaseStart = storedLeaseInfo->leaseStart;
+      // extend lease
+      co_return co_await storeMgmtdLeaseInfo(txn, newLeaseInfo);
+    }
+    co_return *storedLeaseInfo;
+  } else {
+    co_return co_await storeMgmtdLeaseInfo(txn, newLeaseInfo);
+  }
+}
+
+CoTryTask<void> MgmtdStore::ensureLeaseValid(kv::IReadOnlyTransaction &txn, flat::NodeId nodeId, UtcTime now) {
+  auto fetchResult = co_await loadMgmtdLeaseInfo(txn);
+  CO_RETURN_ON_ERROR(fetchResult);
+  std::optional<flat::MgmtdLeaseInfo> storedLeaseInfo = std::move(*fetchResult);
+  if (storedLeaseInfo.has_value()) {
+    if (storedLeaseInfo->primary.nodeId != nodeId) {
+      co_return makeError(MgmtdCode::kNotPrimary, fmt::format("{}", storedLeaseInfo->primary.nodeId));
+    }
+    if (storedLeaseInfo->leaseEnd < now) co_return makeError(MgmtdCode::kNotPrimary);
+    co_return Void{};
+  } else {
+    co_return makeError(MgmtdCode::kNotPrimary);
+  }
+}
+
+CoTryTask<void> MgmtdStore::storeNodeInfo(kv::IReadWriteTransaction &txn, const flat::PersistentNodeInfo &info) {
+  auto nodeKey = getNodeKey(info.nodeId);
+  co_return co_await info.store(txn, nodeKey);
+}
+
+CoTryTask<void> MgmtdStore::clearNodeInfo(kv::IReadWriteTransaction &txn, flat::NodeId nodeId) {
+  auto nodeKey = getNodeKey(nodeId);
+  co_return co_await txn.clear(nodeKey);
+}
+
+CoTryTask<std::optional<flat::PersistentNodeInfo>> MgmtdStore::loadNodeInfo(kv::IReadOnlyTransaction &txn,
+                                                                            flat::NodeId nodeId,
+                                                                            bool snapshotLoad) {
+  auto nodeKey = getNodeKey(nodeId);
+  if (snapshotLoad)
+    co_return co_await flat::PersistentNodeInfo::snapshotLoad(txn, nodeKey);
+  else
+    co_return co_await flat::PersistentNodeInfo::load(txn, nodeKey);
+}
+
+CoTryTask<std::optional<flat::MgmtdLeaseInfo>> MgmtdStore::loadMgmtdLeaseInfo(kv::IReadOnlyTransaction &txn) {
+  co_return co_await flat::MgmtdLeaseInfo::load(txn, getMgmtdLeaseKey());
+}
+
+CoTryTask<flat::MgmtdLeaseInfo> MgmtdStore::storeMgmtdLeaseInfo(kv::IReadWriteTransaction &txn,
+                                                                const flat::MgmtdLeaseInfo &leaseInfo) {
+  auto res = co_await leaseInfo.store(txn, getMgmtdLeaseKey());
+  CO_RETURN_ON_ERROR(res);
+  co_return leaseInfo;
+}
+
+CoTryTask<flat::RoutingInfoVersion> MgmtdStore::loadRoutingInfoVersion(kv::IReadOnlyTransaction &txn) {
+  auto res = co_await txn.get(getRoutingInfoVersionKey());
+  CO_RETURN_ON_ERROR(res);
+  if (res->has_value()) {
+    Deserializer deser(res->value());
+    auto deserRes = deser.get<flat::RoutingInfoVersion::UnderlyingType>();
+    CO_RETURN_ON_ERROR(deserRes);
+    if (!deser.reachEnd()) {
+      co_return makeError(StatusCode::kDataCorruption, "Parse RoutingInfoVersion failed");
+    }
+    co_return flat::RoutingInfoVersion{*deserRes};
+  }
+  co_return flat::RoutingInfoVersion{0};
+}
+
+CoTryTask<void> MgmtdStore::storeRoutingInfoVersion(kv::IReadWriteTransaction &txn, flat::RoutingInfoVersion version) {
+  String buf;
+  Serializer ser(buf);
+  ser.put(version.toUnderType());
+
+  co_return co_await txn.set(getRoutingInfoVersionKey(), buf);
+}
+
+CoTryTask<std::vector<flat::PersistentNodeInfo>> MgmtdStore::loadAllNodes(kv::IReadOnlyTransaction &txn) {
+  auto prefix = kv::toStr(kv::KeyPrefix::NodeTable);
+  auto listRes = co_await kv::TransactionHelper::listByPrefix(txn, prefix, {});
+  CO_RETURN_ON_ERROR(listRes);
+  std::vector<flat::PersistentNodeInfo> res;
+  for (const auto &kv : *listRes) {
+    const auto &[key, value] = kv.pair();
+    auto keyRes = extractNodeIdFromKey(key);
+    CO_RETURN_ON_ERROR(keyRes);
+    auto valueRes = flat::PersistentNodeInfo::unpackFrom(value);
+    CO_RETURN_ON_ERROR(valueRes);
+    if (*keyRes != valueRes->nodeId) {
+      co_return makeError(StatusCode::kDataCorruption,
+                          fmt::format("NodeId mismatch when load NodeInfo. id in key: {}. id in value: {}.",
+                                      *keyRes,
+                                      valueRes->nodeId));
+    }
+    res.push_back(std::move(*valueRes));
+  }
+  co_return res;
+}
+
+CoTryTask<void> MgmtdStore::storeConfig(kv::IReadWriteTransaction &txn,
+                                        flat::NodeType nodeType,
+                                        const flat::ConfigInfo &info) {
+  auto key = getConfigKey(nodeType, info.configVersion);
+  co_return co_await info.store(txn, key);
+}
+
+CoTryTask<std::vector<std::pair<flat::NodeType, flat::ConfigInfo>>> MgmtdStore::loadAllConfigs(
+    kv::IReadOnlyTransaction &txn) {
+  auto prefix = getConfigKeyPrefix();
+  auto listRes = co_await kv::TransactionHelper::listByPrefix(txn, prefix, {});
+  CO_RETURN_ON_ERROR(listRes);
+  std::vector<std::pair<flat::NodeType, flat::ConfigInfo>> res;
+  for (const auto &kv : *listRes) {
+    auto unpackRes = co_await unpackConfigInfo(kv, std::nullopt);
+    CO_RETURN_ON_ERROR(unpackRes);
+    res.push_back(std::move(*unpackRes));
+  }
+  co_return res;
+}
+
+CoTryTask<std::optional<flat::ConfigInfo>> MgmtdStore::loadLatestConfig(kv::IReadOnlyTransaction &txn,
+                                                                        flat::NodeType type) {
+  auto prefix = getConfigKeyPrefix(type);
+  auto listRes =
+      co_await kv::TransactionHelper::listByPrefix(txn,
+                                                   prefix,
+                                                   kv::TransactionHelper::ListByPrefixOptions().withLimit(1));
+  CO_RETURN_ON_ERROR(listRes);
+  if (listRes->empty()) {
+    co_return std::nullopt;
+  }
+  if (listRes->size() != 1) {
+    co_return makeError(StatusCode::kDataCorruption,
+                        fmt::format("List by limit = 1 but get {} items. prefix: {}", listRes->size(), prefix));
+  }
+  auto unpackRes = co_await unpackConfigInfo(listRes->front(), type);
+  CO_RETURN_ON_ERROR(unpackRes);
+  co_return std::move(unpackRes->second);
+}
+
+CoTryTask<std::optional<flat::ConfigInfo>> MgmtdStore::loadConfig(kv::IReadOnlyTransaction &txn,
+                                                                  flat::NodeType type,
+                                                                  flat::ConfigVersion version) {
+  auto key = getConfigKey(type, version);
+  co_return co_await flat::ConfigInfo::load(txn, key);
+}
+
+CoTryTask<void> MgmtdStore::storeChainTable(kv::IReadWriteTransaction &txn, const flat::ChainTable &chainTable) {
+  auto key = getChainTableKey(chainTable.chainTableId, chainTable.chainTableVersion);
+  co_return co_await chainTable.store(txn, key);
+}
+
+CoTryTask<std::vector<flat::ChainTable>> MgmtdStore::loadAllChainTables(kv::IReadOnlyTransaction &txn) {
+  auto prefix = getChainTableKeyPrefix();
+  auto listRes = co_await kv::TransactionHelper::listByPrefix(txn, prefix, {});
+  CO_RETURN_ON_ERROR(listRes);
+  std::vector<flat::ChainTable> res;
+  for (const auto &kv : *listRes) {
+    auto unpackRes = flat::ChainTable::unpackFrom(kv.value);
+    CO_RETURN_ON_ERROR(unpackRes);
+    res.push_back(std::move(*unpackRes));
+  }
+  co_return res;
+}
+
+CoTryTask<void> MgmtdStore::storeChainInfo(kv::IReadWriteTransaction &txn, const flat::ChainInfo &chainInfo) {
+  auto key = getChainInfoKey(chainInfo.chainId);
+  co_return co_await chainInfo.store(txn, key);
+}
+
+CoTryTask<std::vector<flat::ChainInfo>> MgmtdStore::loadAllChains(kv::IReadOnlyTransaction &txn) {
+  auto prefix = getChainInfoKeyPrefix();
+  auto listRes = co_await kv::TransactionHelper::listByPrefix(txn, prefix, {});
+  CO_RETURN_ON_ERROR(listRes);
+  std::vector<flat::ChainInfo> res;
+  for (const auto &kv : *listRes) {
+    auto unpackRes = flat::ChainInfo::unpackFrom(kv.value);
+    CO_RETURN_ON_ERROR(unpackRes);
+    res.push_back(std::move(*unpackRes));
+  }
+  co_return res;
+}
+
+CoTryTask<void> MgmtdStore::storeRoutingInfo(kv::IReadWriteTransaction &txn, const flat::RoutingInfo &routingInfo) {
+  CO_RETURN_ON_ERROR(co_await storeRoutingInfoVersion(txn, routingInfo.routingInfoVersion));
+  for ([[maybe_unused]] const auto &[id, node] : routingInfo.nodes) {
+    CO_RETURN_ON_ERROR(co_await storeNodeInfo(txn, toPersistentNode(node)));
+  }
+  for ([[maybe_unused]] const auto &[_, vm] : routingInfo.chainTables) {
+    for ([[maybe_unused]] const auto &[_, table] : vm) {
+      CO_RETURN_ON_ERROR(co_await storeChainTable(txn, table));
+    }
+  }
+  for ([[maybe_unused]] const auto &[_, chain] : routingInfo.chains) {
+    CO_RETURN_ON_ERROR(co_await storeChainInfo(txn, chain));
+  }
+  co_return Void{};
+}
+
+CoTryTask<void> MgmtdStore::storeUniversalTags(kv::IReadWriteTransaction &txn,
+                                               std::string_view id,
+                                               const std::vector<flat::TagPair> &tags) {
+  if (id.empty()) {
+    co_return makeError(StatusCode::kInvalidArg, "empty id");
+  }
+  auto key = getKeyForUniversalTags(id);
+  auto value = serde::serialize(tags);
+  co_return co_await txn.set(key, value);
+}
+
+CoTryTask<std::vector<flat::TagPair>> MgmtdStore::loadUniversalTags(kv::IReadOnlyTransaction &txn,
+                                                                    std::string_view id) {
+  if (id.empty()) {
+    co_return makeError(StatusCode::kInvalidArg, "empty id");
+  }
+  auto key = getKeyForUniversalTags(id);
+  auto loadRes = co_await txn.snapshotGet(key);
+  CO_RETURN_ON_ERROR(loadRes);
+
+  std::vector<flat::TagPair> tags;
+  if (loadRes->has_value()) {
+    CO_RETURN_ON_ERROR(serde::deserialize(tags, loadRes->value()));
+  }
+  co_return tags;
+}
+
+CoTryTask<std::vector<std::pair<String, std::vector<flat::TagPair>>>> MgmtdStore::loadAllUniversalTags(
+    kv::IReadOnlyTransaction &txn) {
+  auto prefix = getKeyPrefixForUniversalTags();
+  auto loadRes = co_await kv::TransactionHelper::listByPrefix(txn, prefix, {});
+  CO_RETURN_ON_ERROR(loadRes);
+
+  std::vector<std::pair<String, std::vector<flat::TagPair>>> vec;
+  for (const auto &kv : *loadRes) {
+    const auto &[k, v] = kv;
+    if (k.size() <= prefix.size()) {
+      co_return makeError(
+          StatusCode::kDataCorruption,
+          fmt::format("Key of UniversalTags should be longer than prefix. key: {}. prefix: {}", k, prefix));
+    }
+    auto id = k.substr(prefix.size());
+    std::vector<flat::TagPair> tags;
+    CO_RETURN_ON_ERROR(serde::deserialize(tags, v));
+    vec.emplace_back(std::move(id), std::move(tags));
+  }
+  co_return vec;
+}
+
+CoTryTask<void> MgmtdStore::clearUniversalTags(kv::IReadWriteTransaction &txn, std::string_view id) {
+  if (id.empty()) {
+    co_return makeError(StatusCode::kInvalidArg, "empty id");
+  }
+  auto key = getKeyForUniversalTags(id);
+  co_return co_await txn.clear(key);
+}
+
+CoTryTask<void> MgmtdStore::shutdownAllChains(kv::IReadWriteTransaction &txn) {
+  auto rivRes = co_await loadRoutingInfoVersion(txn);
+  CO_RETURN_ON_ERROR(rivRes);
+
+  ++rivRes->toUnderType();
+  CO_RETURN_ON_ERROR(co_await storeRoutingInfoVersion(txn, *rivRes));
+
+  auto chainsRes = co_await loadAllChains(txn);
+  CO_RETURN_ON_ERROR(chainsRes);
+
+  for (auto &ci : *chainsRes) {
+    auto newTargets = shutdownChain(ci.targets);
+    if (ci.targets != newTargets) {
+      ++ci.chainVersion.toUnderType();
+      ci.targets = std::move(newTargets);
+      CO_RETURN_ON_ERROR(co_await storeChainInfo(txn, ci));
+    }
+  }
+
+  co_return Void{};
+}
+
+CoTryTask<void> MgmtdStore::storeTargetInfo(kv::IReadWriteTransaction &txn, const flat::TargetInfo &ti) {
+  auto key = getTargetInfoKey(ti.targetId);
+  co_return co_await ti.store(txn, key);
+}
+
+CoTryTask<void> MgmtdStore::clearTargetInfo(kv::IReadWriteTransaction &txn, flat::TargetId tid) {
+  auto key = getTargetInfoKey(tid);
+  co_return co_await txn.clear(key);
+}
+
+CoTryTask<std::optional<flat::TargetInfo>> MgmtdStore::loadTargetInfo(kv::IReadOnlyTransaction &txn,
+                                                                      flat::TargetId tid) {
+  auto key = getTargetInfoKey(tid);
+  co_return co_await flat::TargetInfo::snapshotLoad(txn, key);
+}
+
+CoTryTask<std::vector<flat::TargetInfo>> MgmtdStore::loadTargetsFrom(kv::IReadOnlyTransaction &txn,
+                                                                     flat::TargetId tid) {
+  auto beginKey = getTargetInfoKey(tid);
+  auto beginSel = kv::IReadOnlyTransaction::KeySelector(beginKey, true);
+  auto endKey = getTargetInfoKey(flat::TargetId(-1));
+  auto endSel = kv::IReadOnlyTransaction::KeySelector(endKey, true);
+
+  auto res = co_await txn.snapshotGetRange(beginSel, endSel, 0);
+  std::vector<flat::TargetInfo> targets;
+  for (const auto &[_, v] : res->kvs) {
+    auto unpack = flat::TargetInfo::unpackFrom(v);
+    CO_RETURN_ON_ERROR(unpack);
+    targets.push_back(std::move(*unpack));
+  }
+  co_return targets;
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/store/MgmtdStore.h

@@ -0,0 +1,90 @@
+#pragma once
+
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/mgmtd/ChainInfo.h"
+#include "fbs/mgmtd/ChainTable.h"
+#include "fbs/mgmtd/ConfigInfo.h"
+#include "fbs/mgmtd/MgmtdLeaseInfo.h"
+#include "fbs/mgmtd/PersistentNodeInfo.h"
+#include "fbs/mgmtd/RoutingInfo.h"
+
+namespace hf3fs::mgmtd {
+class MgmtdStore {
+ public:
+  MgmtdStore() = default;
+
+  // return current lease
+  CoTryTask<flat::MgmtdLeaseInfo> extendLease(kv::IReadWriteTransaction &txn,
+                                              const flat::PersistentNodeInfo &nodeInfo,
+                                              std::chrono::microseconds leaseLength,
+                                              UtcTime now,
+                                              flat::ReleaseVersion rv = flat::ReleaseVersion::fromVersionInfo(),
+                                              bool checkReleaseVersion = true);
+  CoTryTask<void> ensureLeaseValid(kv::IReadOnlyTransaction &txn, flat::NodeId nodeId, UtcTime now);
+
+  CoTryTask<void> storeNodeInfo(kv::IReadWriteTransaction &txn, const flat::PersistentNodeInfo &info);
+  CoTryTask<void> clearNodeInfo(kv::IReadWriteTransaction &txn, flat::NodeId nodeId);
+  CoTryTask<std::optional<flat::PersistentNodeInfo>> loadNodeInfo(kv::IReadOnlyTransaction &txn,
+                                                                  flat::NodeId nodeId,
+                                                                  bool snapshotLoad = false);
+
+  CoTryTask<flat::RoutingInfoVersion> loadRoutingInfoVersion(kv::IReadOnlyTransaction &txn);
+  CoTryTask<void> storeRoutingInfoVersion(kv::IReadWriteTransaction &txn, flat::RoutingInfoVersion version);
+
+  CoTryTask<std::vector<flat::PersistentNodeInfo>> loadAllNodes(kv::IReadOnlyTransaction &txn);
+
+  CoTryTask<std::optional<flat::MgmtdLeaseInfo>> loadMgmtdLeaseInfo(kv::IReadOnlyTransaction &txn);
+
+  CoTryTask<flat::MgmtdLeaseInfo> storeMgmtdLeaseInfo(kv::IReadWriteTransaction &txn,
+                                                      const flat::MgmtdLeaseInfo &leaseInfo);
+
+  CoTryTask<void> storeConfig(kv::IReadWriteTransaction &txn, flat::NodeType nodeType, const flat::ConfigInfo &info);
+
+  CoTryTask<std::vector<std::pair<flat::NodeType, flat::ConfigInfo>>> loadAllConfigs(kv::IReadOnlyTransaction &txn);
+
+  CoTryTask<std::optional<flat::ConfigInfo>> loadLatestConfig(kv::IReadOnlyTransaction &txn, flat::NodeType type);
+
+  CoTryTask<std::optional<flat::ConfigInfo>> loadConfig(kv::IReadOnlyTransaction &txn,
+                                                        flat::NodeType type,
+                                                        flat::ConfigVersion version);
+
+  CoTryTask<void> storeChainTable(kv::IReadWriteTransaction &txn, const flat::ChainTable &chainTable);
+
+  CoTryTask<std::vector<flat::ChainTable>> loadAllChainTables(kv::IReadOnlyTransaction &txn);
+
+  CoTryTask<void> storeChainInfo(kv::IReadWriteTransaction &txn, const flat::ChainInfo &chainInfo);
+
+  CoTryTask<std::vector<flat::ChainInfo>> loadAllChains(kv::IReadOnlyTransaction &txn);
+
+  // test only
+  CoTryTask<void> storeRoutingInfo(kv::IReadWriteTransaction &txn, const flat::RoutingInfo &routingInfo);
+
+  CoTryTask<void> storeUniversalTags(kv::IReadWriteTransaction &txn,
+                                     std::string_view id,
+                                     const std::vector<flat::TagPair> &tags);
+
+  CoTryTask<std::vector<flat::TagPair>> loadUniversalTags(kv::IReadOnlyTransaction &txn, std::string_view id);
+
+  CoTryTask<std::vector<std::pair<String, std::vector<flat::TagPair>>>> loadAllUniversalTags(
+      kv::IReadOnlyTransaction &txn);
+
+  CoTryTask<void> clearUniversalTags(kv::IReadWriteTransaction &txn, std::string_view id);
+
+  CoTryTask<void> shutdownAllChains(kv::IReadWriteTransaction &txn);
+
+  CoTryTask<std::optional<flat::TargetInfo>> loadTargetInfo(kv::IReadOnlyTransaction &txn, flat::TargetId tid);
+
+  CoTryTask<void> storeTargetInfo(kv::IReadWriteTransaction &txn, const flat::TargetInfo &ti);
+
+  CoTryTask<void> clearTargetInfo(kv::IReadWriteTransaction &txn, flat::TargetId tid);
+
+  CoTryTask<std::vector<flat::TargetInfo>> loadTargetsFrom(kv::IReadOnlyTransaction &txn, flat::TargetId tid);
+};
+}  // namespace hf3fs::mgmtd