Initial commit

src/mgmtd/service/ClientSession.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "WithTimestamp.h"
+#include "fbs/mgmtd/ClientSession.h"
+
+namespace hf3fs::mgmtd {
+class ClientSession : public WithTimestamp<flat::ClientSession> {
+ public:
+  using Base = WithTimestamp<flat::ClientSession>;
+  using Base::Base;
+
+  flat::ClientSessionVersion clientSessionVersion{0};
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/LeaseInfo.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "fbs/mgmtd/MgmtdLeaseInfo.h"
+
+namespace hf3fs::mgmtd {
+struct LeaseInfo {
+  std::optional<flat::MgmtdLeaseInfo> lease;
+  bool bootstrapping = false;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/LocalTargetInfoWithNodeId.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "fbs/mgmtd/LocalTargetInfo.h"
+
+namespace hf3fs::mgmtd {
+struct LocalTargetInfoWithNodeId {
+  flat::TargetId targetId{0};
+  flat::NodeId nodeId{0};
+  flat::LocalTargetState localState{flat::LocalTargetState::INVALID};
+
+  LocalTargetInfoWithNodeId() = default;
+  LocalTargetInfoWithNodeId(flat::NodeId nodeId, flat::LocalTargetInfo info)
+      : targetId(info.targetId),
+        nodeId(nodeId),
+        localState(info.localState) {}
+  LocalTargetInfoWithNodeId(flat::TargetId i, flat::NodeId ni, flat::LocalTargetState state)
+      : targetId(i),
+        nodeId(ni),
+        localState(state) {}
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdConfig.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "common/kv/TransactionRetry.h"
+#include "common/utils/ConfigBase.h"
+#include "common/utils/Duration.h"
+#include "core/user/UserCache.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdConfig : ConfigBase<MgmtdConfig> {
+  CONFIG_HOT_UPDATED_ITEM(lease_length, 60_s);
+  CONFIG_HOT_UPDATED_ITEM(extend_lease_interval, 10_s);
+  CONFIG_HOT_UPDATED_ITEM(suspicious_lease_interval, 20_s);
+  CONFIG_HOT_UPDATED_ITEM(heartbeat_timestamp_valid_window, 30_s);
+  CONFIG_HOT_UPDATED_ITEM(allow_heartbeat_from_unregistered, false);
+  CONFIG_HOT_UPDATED_ITEM(check_status_interval, 10_s);
+  CONFIG_HOT_UPDATED_ITEM(heartbeat_fail_interval, 60_s);
+  CONFIG_HOT_UPDATED_ITEM(new_chain_bootstrap_interval, 2_min);
+  CONFIG_HOT_UPDATED_ITEM(send_heartbeat, true);
+  CONFIG_HOT_UPDATED_ITEM(send_heartbeat_interval, 10_s);
+  CONFIG_HOT_UPDATED_ITEM(client_session_timeout, 20_min);
+  CONFIG_HOT_UPDATED_ITEM(bootstrapping_length, 2_min);
+  CONFIG_HOT_UPDATED_ITEM(update_chains_interval, 1_s);
+  CONFIG_HOT_UPDATED_ITEM(validate_lease_on_write, true);
+  CONFIG_HOT_UPDATED_ITEM(bump_routing_info_version_interval, 5_s);
+  // deprecated
+  CONFIG_HOT_UPDATED_ITEM(heartbeat_ignore_unknown_targets, false);
+  CONFIG_HOT_UPDATED_ITEM(heartbeat_ignore_stale_targets, true);
+  CONFIG_HOT_UPDATED_ITEM(retry_times_on_txn_errors, -1);  // -1 means infitely retry, 0 means no retry
+  CONFIG_HOT_UPDATED_ITEM(update_metrics_interval, 1_s);
+  CONFIG_HOT_UPDATED_ITEM(target_info_persist_interval, 1_s);
+  CONFIG_HOT_UPDATED_ITEM(target_info_persist_batch, 1000);
+  CONFIG_HOT_UPDATED_ITEM(target_info_load_interval, 1_s);
+  CONFIG_HOT_UPDATED_ITEM(try_adjust_target_order_as_preferred, false);
+  CONFIG_HOT_UPDATED_ITEM(extend_lease_check_release_version, true);
+  CONFIG_HOT_UPDATED_ITEM(authenticate, false);
+  CONFIG_OBJ(retry_transaction, kv::TransactionRetry);
+  CONFIG_OBJ(user_cache, core::UserCache::Config);
+  CONFIG_HOT_UPDATED_ITEM(enable_routinginfo_cache, true);
+  CONFIG_HOT_UPDATED_ITEM(only_accept_client_uuid, false);
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdData.cc

@@ -0,0 +1,172 @@
+#include "MgmtdData.h"
+
+#include "MgmtdConfig.h"
+#include "core/utils/ServiceOperation.h"
+#include "helpers.h"
+#include "updateChain.h"
+
+namespace hf3fs::mgmtd {
+Result<Void> MgmtdData::checkConfigVersion(core::ServiceOperation &ctx,
+                                           flat::NodeType type,
+                                           flat::ConfigVersion version) const {
+  const auto &cm = configMap;
+  auto it = cm.find(type);
+  if (it == cm.end()) {
+    RETURN_AND_LOG_OP_ERR(ctx, StatusCode::kInvalidArg, "unknown NodeType {}", static_cast<int>(type));
+  }
+  const auto &versionedMap = it->second;
+  auto vit = versionedMap.rbegin();
+  if (version > vit->first) {
+    RETURN_AND_LOG_OP_ERR(ctx,
+                          MgmtdCode::kInvalidConfigVersion,
+                          "ConfigVersion {} of {} is newer than server's {}",
+                          version,
+                          magic_enum::enum_name(type),
+                          vit->first);
+  }
+  return Void{};
+}
+
+std::optional<flat::ConfigInfo> MgmtdData::getConfig(flat::NodeType type,
+                                                     flat::ConfigVersion version,
+                                                     bool latest) const {
+  const auto &cm = configMap;
+  auto it = cm.find(type);
+  assert(it != cm.end());
+  const auto &versionedMap = it->second;
+  if (latest) {
+    auto vit = versionedMap.rbegin();
+    const auto &info = vit->second;
+    XLOGF_IF(FATAL,
+             version > info.configVersion,
+             "Should checked that version from user ({}) not larger than from server ({})",
+             version.toUnderType(),
+             info.configVersion.toUnderType());
+    if (version < info.configVersion) {
+      return info;
+    } else {
+      return std::nullopt;
+    }
+  } else {
+    auto vit = versionedMap.find(version);
+    if (vit == versionedMap.end()) {
+      return std::nullopt;
+    } else {
+      return vit->second;
+    }
+  }
+}
+
+flat::ConfigVersion MgmtdData::getLatestConfigVersion(flat::NodeType type) const {
+  auto it = configMap.find(type);
+  assert(it != configMap.end());
+  const auto &versionedMap = it->second;
+  assert(!versionedMap.empty());
+  return versionedMap.rbegin()->first;
+}
+
+Result<Void> MgmtdData::checkRoutingInfoVersion(core::ServiceOperation &ctx, flat::RoutingInfoVersion version) const {
+  const auto &info = routingInfo;
+  if (version > info.routingInfoVersion) {
+    RETURN_AND_LOG_OP_ERR(ctx,
+                          MgmtdCode::kInvalidRoutingInfoVersion,
+                          "RoutingInfoVersion {} is newer than server's ({})",
+                          version,
+                          info.routingInfoVersion);
+  }
+  return Void{};
+}
+
+std::optional<flat::RoutingInfo> MgmtdData::getRoutingInfo(flat::RoutingInfoVersion version,
+                                                           const MgmtdConfig &config) const {
+  const auto &info = routingInfo;
+  assert(version <= info.routingInfoVersion);
+  if (version == info.routingInfoVersion) {
+    return std::nullopt;
+  }
+  bool enableRoutingInfoCache = config.enable_routinginfo_cache();
+  if (version != 0 && enableRoutingInfoCache) {
+    // version == 0 is possible a force refresh, do not read from cache
+    auto cachePtr = routingInfoCache.rlock();
+    if (cachePtr->has_value() && cachePtr->value().routingInfoVersion == info.routingInfoVersion) {
+      return cachePtr->value();
+    }
+  }
+  flat::RoutingInfo res;
+  res.routingInfoVersion = info.routingInfoVersion;
+  res.bootstrapping = bootstrapping(config);
+  for (const auto &[id, nw] : info.nodeMap) {
+    res.nodes[id] = nw.base();
+  }
+  res.chainTables = info.chainTables;
+  res.chains = info.chains;
+  for (const auto &[k, v] : info.getTargets()) {
+    res.targets[k] = v.base();
+  }
+  if (enableRoutingInfoCache) {
+    // always try to update cache even for a force refresh since the result is reusable to other requests
+    auto cachePtr = routingInfoCache.wlock();
+    if (!cachePtr->has_value() || cachePtr->value().routingInfoVersion < info.routingInfoVersion) {
+      *cachePtr = res;
+    }
+  }
+  return res;
+}
+
+void MgmtdData::appendChangedChains(flat::ChainId chainId,
+                                    std::vector<flat::ChainInfo> &changedChains,
+                                    bool tryAdjustTargetOrder) const {
+  if (routingInfo.newBornChains.contains(chainId)) return;
+
+  const auto &oldChain = routingInfo.getChain(chainId);
+
+  std::vector<ChainTargetInfoEx> oldTargets;
+  for (const auto &cti : oldChain.targets) {
+    oldTargets.emplace_back(cti, routingInfo.getTargets().at(cti.targetId).base().localState);
+  }
+
+  auto newTargets = generateNewChain(oldTargets);
+  if (tryAdjustTargetOrder && !oldChain.preferredTargetOrder.empty() &&
+      newTargets.back().publicState == flat::PublicTargetState::SERVING) {
+    newTargets = rotateAsPreferredOrder(newTargets, oldChain.preferredTargetOrder);
+  }
+  if (oldTargets != newTargets) {
+    flat::ChainInfo newChain;
+    newChain.chainId = chainId;
+    newChain.chainVersion = nextVersion(oldChain.chainVersion);
+    newChain.preferredTargetOrder = oldChain.preferredTargetOrder;
+    for (const auto &ti : newTargets) {
+      newChain.targets.push_back(ti);
+    }
+    XLOGF(DBG, "append changed chain {}", serde::toJsonString(newChain));
+    changedChains.push_back(std::move(newChain));
+  }
+}
+
+void MgmtdData::reset(flat::RoutingInfoVersion routingInfoVersion,
+                      NodeMap allNodes,
+                      ConfigMap allConfigs,
+                      ChainTableMap allChainTables,
+                      ChainMap allChains,
+                      TargetMap allTargets,
+                      UniversalTagsMap allUniversalTags) {
+  routingInfo.reset(routingInfoVersion,
+                    std::move(allNodes),
+                    std::move(allChainTables),
+                    std::move(allChains),
+                    std::move(allTargets));
+  {
+    auto cachePtr = routingInfoCache.wlock();
+    cachePtr->reset();
+  }
+
+  configMap = std::move(allConfigs);
+  lease.bootstrapping = false;
+  leaseStartTs = SteadyClock::now();
+  universalTagsMap = std::move(allUniversalTags);
+}
+
+bool MgmtdData::bootstrapping(const MgmtdConfig &config) const {
+  return leaseStartTs + config.bootstrapping_length().asUs() > SteadyClock::now();
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdData.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <folly/Synchronized.h>
+
+#include "ClientSession.h"
+#include "LeaseInfo.h"
+#include "RoutingInfo.h"
+#include "common/utils/RobinHoodUtils.h"
+#include "fbs/mgmtd/ConfigInfo.h"
+#include "fbs/mgmtd/MgmtdTypes.h"
+#include "fbs/mgmtd/RoutingInfo.h"
+
+namespace hf3fs::core {
+struct ServiceOperation;
+}
+
+namespace hf3fs::mgmtd {
+using VersionedConfigMap = std::map<flat::ConfigVersion, flat::ConfigInfo>;
+using ConfigMap = std::map<flat::NodeType, VersionedConfigMap>;
+using UniversalTagsMap = RHStringHashMap<std::vector<flat::TagPair>>;
+
+struct MgmtdConfig;
+
+struct MgmtdData {
+  SteadyTime leaseStartTs;
+  RoutingInfo routingInfo;
+  ConfigMap configMap;
+  LeaseInfo lease;
+  UniversalTagsMap universalTagsMap;
+
+  mutable folly::Synchronized<std::optional<flat::RoutingInfo>> routingInfoCache;
+
+  Result<Void> checkConfigVersion(core::ServiceOperation &ctx, flat::NodeType, flat::ConfigVersion version) const;
+  std::optional<flat::ConfigInfo> getConfig(flat::NodeType, flat::ConfigVersion version, bool latest) const;
+  flat::ConfigVersion getLatestConfigVersion(flat::NodeType) const;
+
+  Result<Void> checkRoutingInfoVersion(core::ServiceOperation &ctx, flat::RoutingInfoVersion version) const;
+  std::optional<flat::RoutingInfo> getRoutingInfo(flat::RoutingInfoVersion version, const MgmtdConfig &config) const;
+
+  void appendChangedChains(flat::ChainId chainId,
+                           std::vector<flat::ChainInfo> &changedChains,
+                           bool tryAdjustTargetOrder) const;
+
+  void reset(flat::RoutingInfoVersion routingInfoVersion,
+             NodeMap allNodes,
+             ConfigMap allConfigs,
+             ChainTableMap allChainTables,
+             ChainMap allChains,
+             TargetMap allTargets,
+             UniversalTagsMap allUniversalTags);
+
+  bool bootstrapping(const MgmtdConfig &config) const;
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdOperator.cc

@@ -0,0 +1,27 @@
+#include "MgmtdOperator.h"
+
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "core/utils/runOp.h"
+#include "mgmtd/ops/Include.h"
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, method, Method, Id, Req, Rsp)  \
+  CoTryTask<Rsp> svc##Operator::method(Req req, const net::PeerInfo &peer) { \
+    Method##Operation op(std::move(req));                                    \
+    CO_INVOKE_OP_INFO(op, peer.str, state_);                                 \
+  }
+
+namespace hf3fs::mgmtd {
+
+MgmtdOperator::MgmtdOperator(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config)
+    : state_(env, config),
+      backgroundRunner_(state_) {}
+
+void MgmtdOperator::start() { backgroundRunner_.start(); }
+
+CoTask<void> MgmtdOperator::stop() { co_await backgroundRunner_.stop(); }
+
+MgmtdOperator::~MgmtdOperator() { folly::coro::blockingWait(stop()); }
+
+#include "fbs/mgmtd/MgmtdServiceDef.h"
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdOperator.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "MgmtdState.h"
+#include "common/net/PeerInfo.h"
+#include "fbs/mgmtd/Rpc.h"
+#include "mgmtd/background/MgmtdBackgroundRunner.h"
+
+namespace hf3fs::mgmtd {
+namespace testing {
+class MgmtdTestHelper;
+}
+
+class MgmtdOperator : public folly::NonCopyableNonMovable {
+ public:
+  MgmtdOperator(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config);
+
+  ~MgmtdOperator();
+
+  // start/stop background tasks
+  void start();
+  CoTask<void> stop();
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(Service, method, Method, Id, Req, Rsp) \
+  CoTryTask<Rsp> method(Req req, const net::PeerInfo &peer);
+
+#include "fbs/mgmtd/MgmtdServiceDef.h"
+
+ private:
+  MgmtdState state_;
+  MgmtdBackgroundRunner backgroundRunner_;
+
+  friend class testing::MgmtdTestHelper;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdService.cc

@@ -0,0 +1,12 @@
+#include "MgmtdService.h"
+
+#include "MgmtdOperator.h"
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype)        \
+  CoTryTask<rsptype> svc##Service::name(serde::CallContext &ctx, const reqtype &req) { \
+    co_return co_await operator_.name(req, net::PeerInfo{ctx.peer()});                 \
+  }
+
+namespace hf3fs::mgmtd {
+#include "fbs/mgmtd/MgmtdServiceDef.h"
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdService.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "common/serde/CallContext.h"
+#include "fbs/mgmtd/MgmtdServiceBase.h"
+
+namespace hf3fs::mgmtd {
+class MgmtdOperator;
+class MgmtdService : public serde::ServiceWrapper<MgmtdService, MgmtdServiceBase> {
+ public:
+  MgmtdService(MgmtdOperator &opr)
+      : operator_(opr) {}
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
+  CoTryTask<rsptype> name(serde::CallContext &ctx, const reqtype &req);
+
+#include "fbs/mgmtd/MgmtdServiceDef.h"
+
+ private:
+  MgmtdOperator &operator_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdState.cc

@@ -0,0 +1,92 @@
+#include "MgmtdState.h"
+
+#include "MgmtdConfig.h"
+#include "core/user/UserToken.h"
+#include "fbs/mgmtd/NodeConversion.h"
+#include "helpers.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+flat::NodeInfo initSelfNodeInfo(core::ServerEnv &env) {
+  flat::NodeInfo selfNodeInfo;
+  const auto &appInfo = env.appInfo();
+  selfNodeInfo.app = appInfo;
+  selfNodeInfo.type = flat::NodeType::MGMTD;
+  selfNodeInfo.status = flat::NodeStatus::PRIMARY_MGMTD;
+  return selfNodeInfo;
+}
+
+void recordWriterLatency(std::string_view method, Duration latency) {
+  static std::map<String, monitor::LatencyRecorder, std::less<>> latencyRecorders;
+  static std::map<String, monitor::CountRecorder, std::less<>> usageRecorders;
+  auto lit = latencyRecorders.find(method);
+  auto uit = usageRecorders.find(method);
+  if (lit == latencyRecorders.end()) {
+    monitor::TagSet tags;
+    tags.addTag("instance", String(method));
+    lit = latencyRecorders.try_emplace(String(method), "MgmtdService.WriterLatency", tags).first;
+    uit = usageRecorders.try_emplace(String(method), "MgmtdService.WriterUsage", tags).first;
+  }
+  lit->second.addSample(latency);
+  uit->second.addSample(latency.count());
+}
+}  // namespace
+
+MgmtdState::MgmtdState(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config)
+    : env_(std::move(env)),
+      config_(config),
+      userStore_(*env_->kvEngine(), config_.retry_transaction(), config_.user_cache()) {
+  selfNodeInfo_ = initSelfNodeInfo(*env_);
+  selfPersistentNodeInfo_ = flat::toPersistentNode(selfNodeInfo_);
+}
+
+MgmtdState::~MgmtdState() {}
+
+UtcTime MgmtdState::utcNow() { return env_->utcTimeGenerator()(); }
+
+Result<Void> MgmtdState::validateClusterId(const core::ServiceOperation &ctx, std::string_view clusterId) {
+  if (clusterId != env_->appInfo().clusterId) {
+    RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kClusterIdMismatch, "Cluster id mismatch");
+  }
+  return Void{};
+}
+
+CoTryTask<void> MgmtdState::validateAdmin(const core::ServiceOperation &ctx, const flat::UserInfo &userInfo) {
+  if (config_.authenticate()) {
+    auto ret = co_await userStore_.getUser(userInfo.token);
+    CO_RETURN_ON_ERROR(ret);
+    if (!ret->admin) {
+      CO_RETURN_AND_LOG_OP_ERR(ctx, MgmtdCode::kNotAdmin, "");
+    }
+    LOG_OP_INFO(ctx, "Act as admin user {}({})", ret->name, ret->uid.toUnderType());
+  }
+  co_return Void{};
+}
+
+CoTask<std::optional<flat::MgmtdLeaseInfo>> MgmtdState::currentLease(UtcTime now) {
+  auto dataPtr = co_await data_.coSharedLock();
+  const auto &lease = dataPtr->lease;
+  bool canTrustLease =
+      lease.lease.has_value() && now + config_.suspicious_lease_interval().asUs() < lease.lease->leaseEnd;
+  if (canTrustLease && !lease.bootstrapping) {
+    co_return lease.lease;
+  }
+  co_return std::nullopt;
+}
+
+flat::NodeId MgmtdState::selfId() const {
+  assert(env_->appInfo().nodeId != 0);
+  return flat::NodeId{env_->appInfo().nodeId};
+}
+
+MgmtdState::WriterMutexGuard::~WriterMutexGuard() {
+  recordWriterLatency(method_, Duration(SteadyClock::now() - start_));
+}
+
+kv::FDBRetryStrategy MgmtdState::createRetryStrategy() {
+  return kv::FDBRetryStrategy({config_.retry_transaction().max_backoff(),
+                               config_.retry_transaction().max_retry_count(),
+                               /*retryMaybeCommitted=*/true});
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MgmtdState.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <folly/experimental/coro/Mutex.h>
+
+#include "MgmtdData.h"
+#include "common/utils/BackgroundRunner.h"
+#include "common/utils/CoroSynchronized.h"
+#include "common/utils/DefaultRetryStrategy.h"
+#include "core/app/ServerEnv.h"
+#include "core/user/UserStoreEx.h"
+#include "fbs/mgmtd/NodeInfo.h"
+#include "fbs/mgmtd/PersistentNodeInfo.h"
+#include "fdb/FDBRetryStrategy.h"
+#include "mgmtd/store/MgmtdStore.h"
+
+namespace hf3fs::mgmtd {
+struct MgmtdConfig;
+
+using ClientSessionMap = RHStringHashMap<ClientSession>;
+
+struct MgmtdState {
+  MgmtdState(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config);
+  ~MgmtdState();
+
+  UtcTime utcNow();
+  Result<Void> validateClusterId(const core::ServiceOperation &ctx, std::string_view clusterId);
+  CoTryTask<void> validateAdmin(const core::ServiceOperation &ctx, const flat::UserInfo &userInfo);
+  CoTask<std::optional<flat::MgmtdLeaseInfo>> currentLease(UtcTime now);
+  flat::NodeId selfId() const;
+  kv::FDBRetryStrategy createRetryStrategy();
+
+  const std::shared_ptr<core::ServerEnv> env_;
+  flat::NodeInfo selfNodeInfo_;
+  flat::PersistentNodeInfo selfPersistentNodeInfo_;
+  const MgmtdConfig &config_;
+
+  MgmtdStore store_;
+  core::UserStoreEx userStore_;
+
+  class WriterMutexGuard {
+   public:
+    WriterMutexGuard(std::unique_lock<folly::coro::Mutex> mu, std::string_view m)
+        : mu_(std::move(mu)),
+          method_(m) {}
+    WriterMutexGuard(WriterMutexGuard &&other) = default;
+    ~WriterMutexGuard();
+
+   private:
+    std::unique_lock<folly::coro::Mutex> mu_;
+    std::string_view method_;
+    SteadyTime start_ = SteadyClock::now();
+  };
+
+  template <NameWrapper method>
+  CoTask<WriterMutexGuard> coScopedLock() {
+    auto mu = co_await writerMu_.co_scoped_lock();
+    co_return WriterMutexGuard(std::move(mu), method);
+  }
+
+  CoroSynchronized<MgmtdData> data_;
+  CoroSynchronized<ClientSessionMap> clientSessionMap_;
+
+ private:
+  // logical lock for protecting the whole processing of a writer operation
+  // during which read-modify-write will be performed on `data_`
+  folly::coro::Mutex writerMu_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/MockMgmtd.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include <folly/Expected.h>
+#include <folly/executors/CPUThreadPoolExecutor.h>
+#include <folly/executors/ThreadPoolExecutor.h>
+#include <folly/logging/xlog.h>
+#include <memory>
+#include <optional>
+#include <unistd.h>
+
+#include "common/utils/ConfigBase.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Result.h"
+#include "mgmtd/service/MgmtdConfig.h"
+#include "mgmtd/service/MgmtdOperator.h"
+#include "mgmtd/service/MgmtdService.h"
+
+namespace hf3fs::mgmtd {
+
+class MockMgmtd {
+ public:
+  struct Config : ConfigBase<Config> {
+    CONFIG_OBJ(mgmtd, MgmtdConfig);
+    CONFIG_ITEM(node_id, 1u);
+    CONFIG_ITEM(name, "mock-mgmtd");
+    CONFIG_ITEM(cluster_id, "mock-cluster");
+  };
+
+  static CoTryTask<std::unique_ptr<MockMgmtd>> create(Config config,
+                                                      std::shared_ptr<kv::IKVEngine> kvEngine,
+                                                      CPUExecutorGroup *exec) {
+    flat::ServiceGroupInfo groupInfo;
+    groupInfo.services.emplace(mgmtd::MgmtdService::kServiceName);
+    groupInfo.endpoints.push_back(net::Address::from("TCP://127.0.0.1:8000").value());
+
+    flat::AppInfo info;
+    info.nodeId = flat::NodeId(config.node_id());
+    info.pid = static_cast<uint32_t>(getpid());
+    info.serviceGroups = {groupInfo};
+    info.clusterId = config.cluster_id();
+
+    auto env = std::make_shared<core::ServerEnv>();
+    env->setAppInfo(info);
+    env->setKvEngine(kvEngine);
+    env->setBackgroundExecutor(exec);
+
+    auto mgmtd = std::make_unique<MockMgmtd>();
+    mgmtd->config_ = config;
+    mgmtd->mgmtdOperator_ = std::make_unique<mgmtd::MgmtdOperator>(std::move(env), mgmtd->config_.mgmtd());
+
+    mgmtd->mgmtdOperator_->start();
+    co_return mgmtd;
+  }
+
+  void stop() { mgmtdOperator_.reset(); }
+
+  std::unique_ptr<mgmtd::MgmtdService> getService() { return std::make_unique<mgmtd::MgmtdService>(*mgmtdOperator_); }
+
+  MgmtdOperator &getOperator() { return *mgmtdOperator_; }
+
+ private:
+  Config config_;
+  std::unique_ptr<mgmtd::MgmtdOperator> mgmtdOperator_;
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/NodeInfoWrapper.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "WithTimestamp.h"
+#include "common/utils/SimpleRingBuffer.h"
+#include "fbs/mgmtd/NodeInfo.h"
+#include "fbs/mgmtd/NodeStatusChange.h"
+
+namespace hf3fs::mgmtd {
+class NodeInfoWrapper : public WithTimestamp<flat::NodeInfo> {
+ public:
+  using Base = WithTimestamp<flat::NodeInfo>;
+  using Base::Base;
+
+  void updateHeartbeatVersion(flat::HeartbeatVersion version) { version_ = version; }
+  flat::HeartbeatVersion lastHbVersion() const { return version_; }
+
+  void recordStatusChange(flat::NodeStatus status, UtcTime now) {
+    if (statusChanges_.full()) statusChanges_.pop();
+    statusChanges_.push(flat::NodeStatusChange::create(status, now));
+  }
+
+  std::vector<flat::NodeStatusChange> getAllStatusChanges() {
+    return std::vector<flat::NodeStatusChange>(statusChanges_.begin(), statusChanges_.end());
+  }
+
+ private:
+  flat::HeartbeatVersion version_{0};  // used for rejecting stale heartbeat requests
+  static constexpr size_t kStatusChangeCount = 100;
+
+  SimpleRingBuffer<flat::NodeStatusChange> statusChanges_{kStatusChangeCount};
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/RoutingInfo.cc

@@ -0,0 +1,173 @@
+#include "RoutingInfo.h"
+
+#include "MgmtdConfig.h"
+#include "core/utils/ServiceOperation.h"
+#include "helpers.h"
+
+namespace hf3fs::mgmtd {
+void RoutingInfo::applyChainTargetChanges(core::ServiceOperation &ctx,
+                                          const std::vector<flat::ChainInfo> &changedChains,
+                                          SteadyTime now) {
+  for (const auto &chain : changedChains) {
+    auto &oldChain = getChain(chain.chainId);
+    LOG_OP_INFO(ctx,
+                "{} change from {} to {}",
+                chain.chainId,
+                serde::toJsonString(oldChain),
+                serde::toJsonString(chain));
+    oldChain = chain;
+    for (const auto &cti : chain.targets) {
+      updateTarget(cti.targetId, [&](auto &ti) {
+        ti.base().publicState = cti.publicState;
+        ti.updateTs(now);
+      });
+    }
+  }
+}
+
+flat::ChainInfo &RoutingInfo::getChain(flat::ChainId cid) {
+  XLOGF_IF(FATAL, !chains.contains(cid), "cid = {}", cid.toUnderType());
+  return chains[cid];
+}
+
+const flat::ChainInfo &RoutingInfo::getChain(flat::ChainId cid) const {
+  XLOGF_IF(FATAL, !chains.contains(cid), "cid = {}", cid.toUnderType());
+  return chains.at(cid);
+}
+
+void RoutingInfo::localUpdateTargets(flat::NodeId nodeId,
+                                     const std::vector<flat::LocalTargetInfo> &targets,
+                                     const MgmtdConfig &config) {
+  auto steadyNow = SteadyClock::now();
+
+  auto &orphans = orphanTargetsByNodeId[nodeId];
+  for (auto tid : orphans) {
+    orphanTargetsByTargetId.erase(tid);
+  }
+  orphans.clear();
+
+  for (const auto &lti : targets) {
+    auto it = this->targets.find(lti.targetId);
+    if (it != this->targets.end()) {
+      updateTarget(lti.targetId, [&](auto &ti) {
+        auto &base = ti.base();
+
+        bool shouldIgnore = [&] {
+          if (lti.chainVersion == 0 || !config.heartbeat_ignore_stale_targets()) return false;
+          const auto &chain = getChain(base.chainId);
+          return chain.chainVersion > lti.chainVersion;
+        }();
+
+        if (shouldIgnore) return;
+
+        ti.updateTs(steadyNow);
+
+        bool importantInfoChanged = base.localState != lti.localState || base.nodeId != nodeId;
+        if (importantInfoChanged) {
+          ti.importantInfoChangedTime = steadyNow;
+        }
+
+        base.localState = lti.localState;
+        base.nodeId = nodeId;
+        base.diskIndex = lti.diskIndex;
+        base.usedSize = lti.usedSize;
+      });
+    } else {
+      // only insert to orphans, not targets
+      orphans.insert(lti.targetId);
+      auto &ti = orphanTargetsByTargetId[lti.targetId];
+      ti = flat::TargetInfo();  // ensure clean state
+      ti.targetId = lti.targetId;
+      ti.localState = lti.localState;
+      ti.nodeId = nodeId;
+      ti.diskIndex = lti.diskIndex;
+      ti.usedSize = lti.usedSize;
+    }
+  }
+}
+
+void RoutingInfo::insertNewChain(const flat::ChainInfo &chain) {
+  auto cid = chain.chainId;
+  XLOGF_IF(FATAL, chains.contains(cid), "Insert duplicated chain {}", cid.toUnderType());
+  auto createdTime = SteadyClock::now();
+  chains[cid] = chain;
+  for (const auto &t : chain.targets) {
+    auto tid = t.targetId;
+    XLOGF_IF(FATAL, targets.contains(tid), "Insert duplicated target {}", tid.toUnderType());
+    targets[tid] = TargetInfo(makeTargetInfo(cid, t), createdTime);
+    eraseOrphanTarget(tid);
+  }
+  newBornChains[cid] = createdTime;
+}
+
+void RoutingInfo::insertNewTarget(flat::ChainId cid, const flat::ChainTargetInfo &cti) {
+  auto tid = cti.targetId;
+  XLOGF_IF(FATAL, targets.contains(tid), "Insert duplicated target {}", tid.toUnderType());
+  targets[tid] = TargetInfo(makeTargetInfo(cid, cti), SteadyClock::now());
+  eraseOrphanTarget(tid);
+}
+
+void RoutingInfo::removeTarget(flat::ChainId cid, flat::TargetId tid) {
+  XLOGF_IF(FATAL, !targets.contains(tid), "Try to remove unknown target {}", tid.toUnderType());
+  auto &ti = targets[tid];
+  XLOGF_IF(FATAL,
+           ti.base().publicState != flat::PublicTargetState::OFFLINE,
+           "Try to remove target {} which is not offlined",
+           tid.toUnderType());
+  if (chains.contains(cid)) {
+    auto &chain = getChain(cid);
+    XLOGF_IF(
+        FATAL,
+        std::find_if(chain.targets.begin(), chain.targets.end(), [&](const auto &ti) { return ti.targetId == tid; }) !=
+            chain.targets.end(),
+        "target {} is still present in chain {}",
+        tid.toUnderType(),
+        cid.toUnderType());
+  }
+  targets.erase(tid);
+}
+
+void RoutingInfo::eraseOrphanTarget(flat::TargetId tid) {
+  if (LIKELY(!orphanTargetsByTargetId.contains(tid))) return;
+  const auto &oti = orphanTargetsByTargetId[tid];
+  XLOGF_IF(FATAL, !oti.nodeId.has_value(), "Unexpected orphan target: {}", serde::toJsonString(oti));
+  XLOGF_IF(FATAL,
+           !orphanTargetsByNodeId.contains(*oti.nodeId),
+           "Unexpected orphan target ref: {}",
+           serde::toJsonString(oti));
+  auto &set = orphanTargetsByNodeId[*oti.nodeId];
+  XLOGF_IF(FATAL, !set.contains(tid), "Unexpected orphan target ref: {}", serde::toJsonString(oti));
+  set.erase(tid);
+  if (set.empty()) orphanTargetsByNodeId.erase(*oti.nodeId);
+  orphanTargetsByTargetId.erase(tid);
+}
+
+void RoutingInfo::reset(flat::RoutingInfoVersion routingInfoVersion,
+                        NodeMap allNodes,
+                        ChainTableMap allChainTables,
+                        ChainMap allChains,
+                        TargetMap allTargets) {
+  XLOGF(INFO, "Reset RoutingInfo to routingInfoVersion {}", routingInfoVersion);
+  auto steadyNow = SteadyClock::now();
+
+  nodeMap.clear();
+  chainTables.clear();
+  chains.clear();
+  newBornChains.clear();
+  orphanTargetsByTargetId.clear();
+  orphanTargetsByNodeId.clear();
+
+  for (const auto &[id, sn] : allNodes) {
+    nodeMap.try_emplace(id, sn.base(), steadyNow);
+  }
+  chainTables = std::move(allChainTables);
+  chains = std::move(allChains);
+  targets = std::move(allTargets);
+  this->routingInfoVersion = routingInfoVersion;
+
+  for ([[maybe_unused]] const auto &[cid, _] : chains) {
+    // avoid update chain too early after restart or the chain status may be not stable
+    newBornChains[cid] = steadyNow;
+  }
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/RoutingInfo.h

@@ -0,0 +1,80 @@
+#pragma once
+
+#include "LocalTargetInfoWithNodeId.h"
+#include "NodeInfoWrapper.h"
+#include "TargetInfo.h"
+#include "fbs/mgmtd/ChainInfo.h"
+#include "fbs/mgmtd/ChainTable.h"
+#include "fbs/mgmtd/MgmtdTypes.h"
+
+namespace hf3fs::core {
+struct ServiceOperation;
+}
+
+namespace hf3fs::mgmtd {
+struct MgmtdConfig;
+class MgmtdTargetInfoLoader;
+
+namespace testing {
+class MgmtdTestHelper;
+}
+
+using NodeMap = robin_hood::unordered_map<flat::NodeId, NodeInfoWrapper>;
+using ChainTableVersionMap = std::map<flat::ChainTableVersion, flat::ChainTable>;
+using ChainTableMap = robin_hood::unordered_map<flat::ChainTableId, ChainTableVersionMap>;
+using ChainMap = robin_hood::unordered_map<flat::ChainId, flat::ChainInfo>;
+using TargetMap = robin_hood::unordered_map<flat::TargetId, TargetInfo>;
+using NewBornChains = robin_hood::unordered_map<flat::ChainId, SteadyTime>;
+using OrphanTargetsByTargetId = robin_hood::unordered_map<flat::TargetId, flat::TargetInfo>;
+using OrphanTargetsByNodeId = robin_hood::unordered_map<flat::NodeId, robin_hood::unordered_set<flat::TargetId>>;
+
+struct RoutingInfo {
+  bool routingInfoChanged = false;                  // periodically promote routingInfoVersion
+  flat::RoutingInfoVersion routingInfoVersion{1};   // ensure version 0 is less than any valid version
+  NodeMap nodeMap;                                  // persistent
+  ChainTableMap chainTables;                        // persistent
+  ChainMap chains;                                  // persistent
+  NewBornChains newBornChains;                      // temporal
+  OrphanTargetsByTargetId orphanTargetsByTargetId;  // temporal
+  OrphanTargetsByNodeId orphanTargetsByNodeId;      // temporal
+
+  void applyChainTargetChanges(core::ServiceOperation &ctx,
+                               const std::vector<flat::ChainInfo> &changedChains,
+                               SteadyTime now);
+
+  void localUpdateTargets(flat::NodeId nodeId,
+                          const std::vector<flat::LocalTargetInfo> &targets,
+                          const MgmtdConfig &config);
+
+  flat::ChainInfo &getChain(flat::ChainId cid);
+  const flat::ChainInfo &getChain(flat::ChainId cid) const;
+
+  void insertNewChain(const flat::ChainInfo &chain);
+
+  void insertNewTarget(flat::ChainId cid, const flat::ChainTargetInfo &cti);
+
+  void removeTarget(flat::ChainId cid, flat::TargetId tid);
+
+  const TargetMap &getTargets() const { return targets; }
+
+  auto updateTarget(flat::TargetId tid, auto &&func) {
+    XLOGF_IF(DFATAL, !targets.contains(tid), "tid = {}", tid);
+    auto &ti = targets[tid];
+    return func(ti);
+  }
+
+  void reset(flat::RoutingInfoVersion routingInfoVersion,
+             NodeMap allNodes,
+             ChainTableMap allChainTables,
+             ChainMap allChains,
+             TargetMap allTargets);
+
+ private:
+  void eraseOrphanTarget(flat::TargetId tid);
+
+  TargetMap targets;  // derived
+
+  friend class testing::MgmtdTestHelper;
+};
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/TargetInfo.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "WithTimestamp.h"
+#include "fbs/mgmtd/TargetInfo.h"
+
+namespace hf3fs::mgmtd {
+class TargetInfo : public WithTimestamp<flat::TargetInfo> {
+ public:
+  using Base = WithTimestamp<flat::TargetInfo>;
+  using Base::Base;
+
+  bool locationInitLoaded = false;
+  std::optional<flat::NodeId> persistedNodeId;
+  std::optional<uint32_t> persistedDiskIndex;
+
+  SteadyTime importantInfoChangedTime = SteadyClock::now();
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/WithTimestamp.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "common/utils/UtcTime.h"
+#include "fbs/mgmtd/TargetInfo.h"
+
+namespace hf3fs::mgmtd {
+template <typename T>
+class WithTimestamp {
+ public:
+  WithTimestamp()
+      : base_(),
+        ts_(SteadyClock::now()) {}
+
+  explicit WithTimestamp(T info)
+      : base_(std::move(info)),
+        ts_(SteadyClock::now()) {}
+
+  WithTimestamp(T info, SteadyTime ts)
+      : base_(std::move(info)),
+        ts_(ts) {}
+
+  T &base() { return base_; }
+  const T &base() const { return base_; }
+
+  SteadyTime ts() const { return ts_; }
+  void updateTs() { ts_ = SteadyClock::now(); }
+  void updateTs(SteadyTime ts) { ts_ = ts; }
+
+ private:
+  T base_;
+  SteadyTime ts_;
+};
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/helpers.cc

@@ -0,0 +1,95 @@
+#include "helpers.h"
+
+namespace hf3fs::mgmtd {
+CoTask<void> updateMemoryRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx) {
+  static const auto handler = [](RoutingInfo &) {};
+  co_await updateMemoryRoutingInfo(state, ctx, handler);
+}
+
+CoTryTask<void> updateStoredRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx) {
+  static const auto handler = [](kv::IReadWriteTransaction &) -> CoTryTask<void> { co_return Void{}; };
+  co_return co_await updateStoredRoutingInfo(state, ctx, handler);
+}
+
+flat::TargetInfo makeTargetInfo(flat::ChainId chainId, const flat::ChainTargetInfo &info) {
+  flat::TargetInfo ti;
+  ti.targetId = info.targetId;
+  ti.publicState = info.publicState;
+  ti.localState = flat::LocalTargetState::OFFLINE;
+  ti.chainId = chainId;
+  return ti;
+}
+
+Result<Void> updateSelfConfig(MgmtdState &state, const flat::ConfigInfo &cfg) {
+  XLOGF_IF(FATAL,
+           state.selfNodeInfo_.configVersion > cfg.configVersion,
+           "ConfigVersion in memory ({}) is larger than given ({})",
+           state.selfNodeInfo_.configVersion.toUnderType(),
+           cfg.configVersion.toUnderType());
+  if (state.selfNodeInfo_.configVersion >= cfg.configVersion) return Void{};
+
+  const auto &updater = state.env_->configUpdater();
+  if (!updater) {
+    XLOGF(WARN, "Mgmtd: blindly promote to {} since no listener found", cfg.configVersion);
+    return Void{};
+  }
+  return updater(cfg.content, cfg.genUpdateDesc());
+}
+
+Result<std::vector<flat::TagPair>> updateTags(core::ServiceOperation &op,
+                                              flat::SetTagMode mode,
+                                              const std::vector<flat::TagPair> &oldTags,
+                                              const std::vector<flat::TagPair> &updates) {
+  std::map<std::string_view, std::string_view> tagMap;
+  switch (mode) {
+    case flat::SetTagMode::REPLACE: {
+      for (const auto &tp : updates) tagMap[tp.key] = tp.value;
+      break;
+    }
+    case flat::SetTagMode::UPSERT: {
+      for (const auto &tp : updates) tagMap[tp.key] = tp.value;
+      for (const auto &tp : oldTags) tagMap.try_emplace(tp.key, tp.value);
+      break;
+    }
+    case flat::SetTagMode::REMOVE: {
+      for (const auto &tp : oldTags) tagMap[tp.key] = tp.value;
+      for (const auto &tp : updates) {
+        if (!tp.value.empty()) {
+          RETURN_AND_LOG_OP_ERR(op,
+                                MgmtdCode::kInvalidTag,
+                                "could only pass empty value for REMOVE. key {} value {}",
+                                tp.key,
+                                tp.value);
+        }
+        tagMap.erase(tp.key);
+      }
+      break;
+    }
+  }
+
+  std::vector<flat::TagPair> newTags;
+  for (const auto &[k, v] : tagMap) {
+    newTags.emplace_back(String(k), String(v));
+  }
+  return newTags;
+}
+
+void updateMemoryRoutingInfo(RoutingInfo &alreadyLockedRoutingInfo, core::ServiceOperation &ctx) {
+  auto &ri = alreadyLockedRoutingInfo;
+  ++ri.routingInfoVersion.toUnderType();
+  ri.routingInfoChanged = false;
+  LOG_OP_INFO(ctx, "RoutingInfo: bump memory version to {}", ri.routingInfoVersion);
+}
+
+CoTryTask<Void> ensureSelfIsPrimary(MgmtdState &state) {
+  auto lease = co_await state.currentLease(state.utcNow());
+  if (lease.has_value()) {
+    if (lease->primary.nodeId == state.selfId()) {
+      co_return Void{};
+    } else {
+      co_return makeError(MgmtdCode::kNotPrimary, fmt::format("{}", lease->primary.nodeId.toUnderType()));
+    }
+  }
+  co_return makeError(MgmtdCode::kNotPrimary);
+}
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/helpers.h

@@ -0,0 +1,96 @@
+#include <atomic>
+#include <folly/experimental/coro/Sleep.h>
+
+#include "MgmtdConfig.h"
+#include "MgmtdOperator.h"
+#include "MgmtdState.h"
+#include "common/kv/WithTransaction.h"
+#include "core/utils/ServiceOperation.h"
+
+#define RECORD_LATENCY(latency)                                                                                       \
+  auto FB_ANONYMOUS_VARIABLE(guard) = folly::makeGuard([lat = &(latency), begin = std::chrono::steady_clock::now()] { \
+    lat->addSample(std::chrono::steady_clock::now() - begin);                                                         \
+  })
+
+namespace hf3fs::mgmtd {
+template <typename T>
+inline T nextVersion(T version) {
+  auto v = version + 1;
+  return T(v);
+}
+
+CoTryTask<Void> ensureSelfIsPrimary(MgmtdState &state);
+
+template <typename Handler>
+inline std::invoke_result_t<Handler> doAsPrimary(MgmtdState &state, Handler &&handler) {
+  auto ret = co_await ensureSelfIsPrimary(state);
+  CO_RETURN_ON_ERROR(ret);
+  co_return co_await handler();
+}
+
+void updateMemoryRoutingInfo(RoutingInfo &alreadyLockedRoutingInfo, core::ServiceOperation &ctx);
+
+template <typename Handler>
+inline CoTask<void> updateMemoryRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx, Handler &&handler) {
+  auto dataPtr = co_await state.data_.coLock();
+  auto &ri = dataPtr->routingInfo;
+  updateMemoryRoutingInfo(ri, ctx);
+  handler(ri);
+}
+
+CoTask<void> updateMemoryRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx);
+
+template <typename Handler, typename Result = std::invoke_result_t<Handler, kv::IReadWriteTransaction &>>
+inline Result withReadWriteTxn(MgmtdState &state, Handler &&handler, bool expectSelfPrimary = true) {
+  int maxRetryTimes = state.config_.retry_times_on_txn_errors();
+  constexpr auto retryInterval = std::chrono::milliseconds(1000);
+  auto strategy = state.createRetryStrategy();
+  for (int i = 0;; ++i) {
+    auto res = co_await kv::WithTransaction(strategy).run(
+        state.env_->kvEngine()->createReadWriteTransaction(),
+        [&](kv::IReadWriteTransaction &txn) -> Result {
+          if (expectSelfPrimary && state.config_.validate_lease_on_write()) {
+            CO_RETURN_ON_ERROR(co_await state.store_.ensureLeaseValid(txn, state.selfId(), state.utcNow()));
+          }
+          co_return co_await handler(txn);
+        });
+    if (res || i == maxRetryTimes || StatusCode::typeOf(res.error().code()) != StatusCodeType::Transaction)
+      co_return res;
+    XLOGF(CRITICAL, "Transaction failed: {}\nretryCount: {}", res.error(), i);
+    co_await folly::coro::sleep(retryInterval);
+  }
+}
+
+template <typename Handler, typename Result = std::invoke_result_t<Handler, kv::IReadOnlyTransaction &>>
+inline Result withReadOnlyTxn(MgmtdState &state, Handler &&handler) {
+  auto strategy = state.createRetryStrategy();
+  co_return co_await kv::WithTransaction(strategy).run(
+      state.env_->kvEngine()->createReadonlyTransaction(),
+      [&](kv::IReadOnlyTransaction &txn) -> Result { co_return co_await handler(txn); });
+}
+
+template <typename Handler>
+inline CoTryTask<void> updateStoredRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx, Handler &&handler) {
+  auto dataPtr = co_await state.data_.coSharedLock();
+  auto nextv = nextVersion(dataPtr->routingInfo.routingInfoVersion);
+  co_return co_await withReadWriteTxn(state, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
+    CO_RETURN_ON_ERROR(co_await state.store_.storeRoutingInfoVersion(txn, nextv));
+    LOG_OP_INFO(ctx, "RoutingInfo: bump storage version to {}", nextv);
+    co_return co_await handler(txn);
+  });
+}
+
+CoTryTask<void> updateStoredRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx);
+
+flat::TargetInfo makeTargetInfo(flat::ChainId chainId, const flat::ChainTargetInfo &info);
+
+Result<Void> updateSelfConfig(MgmtdState &state, const flat::ConfigInfo &cfg);
+
+using MgmtdStub = mgmtd::IMgmtdServiceStub;
+using MgmtdStubFactory = stubs::IStubFactory<MgmtdStub>;
+
+Result<std::vector<flat::TagPair>> updateTags(core::ServiceOperation &op,
+                                              flat::SetTagMode mode,
+                                              const std::vector<flat::TagPair> &oldTags,
+                                              const std::vector<flat::TagPair> &updates);
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/updateChain.cc

@@ -0,0 +1,188 @@
+#include "updateChain.h"
+
+#include "common/utils/OptionalUtils.h"
+
+namespace hf3fs::mgmtd {
+namespace {
+using PS = enum flat::PublicTargetState;
+using LS = enum flat::LocalTargetState;
+using TargetsByPs = robin_hood::unordered_map<PS, std::vector<ChainTargetInfoEx>>;
+
+void dispatch(TargetsByPs &targetsByPs, ChainTargetInfoEx ti, PS ps, std::string_view reason) {
+  XLOGF_IF(DBG,
+           ti.publicState != ps,
+           "Dispatch {}({}-{}) to {}: {}",
+           ti.targetId,
+           magic_enum::enum_name(ti.publicState),
+           magic_enum::enum_name(ti.localState),
+           magic_enum::enum_name(ps),
+           reason);
+  ti.publicState = ps;
+  targetsByPs[ps].push_back(ti);
+}
+}  // namespace
+
+std::vector<ChainTargetInfoEx> generateNewChain(const std::vector<ChainTargetInfoEx> &oldTargets) {
+  robin_hood::unordered_map<PS, std::vector<ChainTargetInfoEx>> oldTargetsByPs, newTargetsByPs;
+  for (const auto &ti : oldTargets) {
+    oldTargetsByPs[ti.publicState].push_back(ti);
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::SERVING]) {
+    if (ti.localState == LS::ONLINE || ti.localState == LS::UPTODATE) {
+      dispatch(newTargetsByPs, ti, PS::SERVING, "");
+    } else if (newTargetsByPs[PS::LASTSRV].empty()) {
+      // If all SERVING offlined, only the first becomes LASTSRV.
+      // NOTE: in such cases the whole chain has to wait the HEAD for recovering even
+      //       when other replicas are complete.
+      dispatch(newTargetsByPs, ti, PS::LASTSRV, "first SERVING");
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "following SERVINGs");
+    }
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::LASTSRV]) {
+    if (newTargetsByPs[PS::SERVING].empty()) {
+      if (ti.localState == LS::ONLINE || ti.localState == LS::UPTODATE) {
+        dispatch(newTargetsByPs, ti, PS::SERVING, "first LASTSRV");
+      } else {
+        dispatch(newTargetsByPs, ti, PS::LASTSRV, "following LASTSRVs");
+      }
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "Has SERVING");
+    }
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::SYNCING]) {
+    if (ti.localState == LS::UPTODATE) {
+      dispatch(newTargetsByPs, ti, PS::SERVING, "");
+    } else if (ti.localState == LS::ONLINE) {
+      if (!newTargetsByPs[PS::SERVING].empty()) {
+        dispatch(newTargetsByPs, ti, PS::SYNCING, "Has SERVING");
+      } else {
+        dispatch(newTargetsByPs, ti, PS::WAITING, "No SERVING");
+      }
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "");
+    }
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::WAITING]) {
+    if (!newTargetsByPs[PS::SERVING].empty() && newTargetsByPs[PS::SYNCING].empty() && ti.localState == LS::ONLINE) {
+      dispatch(newTargetsByPs, ti, PS::SYNCING, "Has SERVING && No SYNCING");
+    } else if (ti.localState == LS::ONLINE || ti.localState == LS::UPTODATE) {
+      dispatch(newTargetsByPs, ti, PS::WAITING, "");
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "");
+    }
+  }
+
+  for (const auto &ti : oldTargetsByPs[PS::OFFLINE]) {
+    if (!newTargetsByPs[PS::SERVING].empty() && newTargetsByPs[PS::SYNCING].empty() && ti.localState == LS::ONLINE) {
+      dispatch(newTargetsByPs, ti, PS::SYNCING, "Has SERVING && No SYNCING");
+    } else if (ti.localState == LS::ONLINE || ti.localState == LS::UPTODATE) {
+      dispatch(newTargetsByPs, ti, PS::WAITING, "");
+    } else {
+      dispatch(newTargetsByPs, ti, PS::OFFLINE, "");
+    }
+  }
+
+  if (!newTargetsByPs[PS::SERVING].empty()) {
+    for (auto &ti : newTargetsByPs[PS::LASTSRV]) {
+      dispatch(newTargetsByPs, std::move(ti), PS::OFFLINE, "Has SERVING");
+    }
+    newTargetsByPs[PS::LASTSRV].clear();
+  }
+
+  std::vector<ChainTargetInfoEx> newTargets;
+  for (auto s : {PS::SERVING, PS::LASTSRV, PS::SYNCING, PS::WAITING, PS::OFFLINE}) {
+    const auto &v = newTargetsByPs[s];
+    newTargets.insert(newTargets.end(), v.begin(), v.end());
+  }
+  assert(oldTargets.size() == newTargets.size());
+  return newTargets;
+}
+
+std::vector<ChainTargetInfoEx> rotateAsPreferredOrder(const std::vector<ChainTargetInfoEx> &oldTargets,
+                                                      const std::vector<flat::TargetId> &preferredOrder) {
+  XLOGF_IF(DFATAL,
+           oldTargets.size() < preferredOrder.size(),
+           "oldTargets.size = {}, preferredTargets.size = {}",
+           oldTargets.size(),
+           preferredOrder.size());
+  std::map<flat::TargetId, std::pair<size_t, ChainTargetInfoEx>> oldMapping;
+  for (size_t i = 0; i < oldTargets.size(); ++i) {
+    oldMapping[oldTargets[i].targetId] = std::make_pair(i, oldTargets[i]);
+  }
+  for (size_t i = 0; i < preferredOrder.size(); ++i) {
+    auto tid = preferredOrder[i];
+    XLOGF_IF(DFATAL, !oldMapping.contains(tid), "{} not in oldTargets", tid);
+    const auto &[oldPos, oldInfo] = oldMapping[tid];
+    if (oldPos == i) {
+      continue;
+    }
+    if (oldInfo.publicState != PS::SERVING) {
+      break;
+    }
+    std::vector<ChainTargetInfoEx> newTargets;
+    for (size_t j = 0; j < i; ++j) {
+      newTargets.push_back(oldTargets[j]);
+    }
+    for (size_t j = i + 1; j < oldTargets.size(); ++j) {
+      newTargets.push_back(oldTargets[j]);
+    }
+    auto info = oldTargets[i];
+    info.publicState = PS::OFFLINE;
+    info.localState = LS::OFFLINE;
+    newTargets.push_back(info);
+    return newTargets;
+  }
+  return oldTargets;
+}
+
+std::vector<ChainTargetInfoEx> rotateLastSrv(const std::vector<ChainTargetInfoEx> &oldTargets) {
+  if (oldTargets.size() < 2 || oldTargets[0].publicState != PS::LASTSRV) {
+    return oldTargets;
+  }
+
+  std::vector<ChainTargetInfoEx> newTargets;
+  for (size_t i = 1; i < oldTargets.size(); ++i) newTargets.push_back(oldTargets[i]);
+  newTargets.push_back(oldTargets[0]);
+
+  // possible conversions:
+  // * WAITING -> LASTSRV: it's safer to convert a WAITING to LASTSRV than to SERVING
+  // * OFFLINE -> LASTSRV
+  newTargets[0].publicState = PS::LASTSRV;
+
+  // possible conversions:
+  // * WAITING -> OFFLINE
+  // * OFFLINE -> OFFLINE
+  for (size_t i = 1; i < newTargets.size(); ++i) newTargets[i].publicState = PS::OFFLINE;
+
+  return newTargets;
+}
+
+std::vector<flat::ChainTargetInfo> shutdownChain(const std::vector<flat::ChainTargetInfo> &oldTargets) {
+  std::vector<flat::ChainTargetInfo> newTargets;
+  for (auto ti : oldTargets) {
+    switch (ti.publicState) {
+      case PS::SERVING:
+        ti.publicState = PS::LASTSRV;
+        break;
+      case PS::WAITING:
+      case PS::SYNCING:
+        ti.publicState = PS::OFFLINE;
+        break;
+      case PS::LASTSRV:
+      case PS::OFFLINE:
+        // keep it as is
+        break;
+      case PS::INVALID:
+        XLOGF(FATAL, "Invalid PublicTargetState: {}", ti.targetId);
+    }
+    newTargets.push_back(ti);
+  }
+  return newTargets;
+}
+
+}  // namespace hf3fs::mgmtd

src/mgmtd/service/updateChain.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "LocalTargetInfoWithNodeId.h"
+#include "fbs/mgmtd/ChainTargetInfo.h"
+#include "fbs/mgmtd/TargetInfo.h"
+
+namespace hf3fs::mgmtd {
+struct ChainTargetInfoEx : public flat::ChainTargetInfo {
+  ChainTargetInfoEx() = default;
+  ChainTargetInfoEx(const flat::ChainTargetInfo &cti, flat::LocalTargetState ls)
+      : flat::ChainTargetInfo(cti),
+        localState(ls) {}
+  explicit ChainTargetInfoEx(const flat::TargetInfo &ti) {
+    targetId = ti.targetId;
+    publicState = ti.publicState;
+    localState = ti.localState;
+  }
+
+  bool operator==(const ChainTargetInfo &other) const { return static_cast<const ChainTargetInfo &>(*this) == other; }
+  bool operator==(const ChainTargetInfoEx &other) const {
+    return *this == static_cast<const ChainTargetInfo &>(other) && localState == other.localState;
+  }
+
+  static ChainTargetInfoEx fromTargetInfo(const flat::TargetInfo &ti) { return ChainTargetInfoEx(ti); }
+
+  static flat::TargetInfo toTargetInfo(const ChainTargetInfoEx &cti) {
+    flat::TargetInfo ti;
+    ti.targetId = cti.targetId;
+    ti.publicState = cti.publicState;
+    ti.localState = cti.localState;
+    return ti;
+  }
+
+  flat::LocalTargetState localState{flat::LocalTargetState::INVALID};
+};
+
+// separate this function just for testing friendly
+std::vector<ChainTargetInfoEx> generateNewChain(const std::vector<ChainTargetInfoEx> &oldTargets);
+
+std::vector<ChainTargetInfoEx> rotateAsPreferredOrder(const std::vector<ChainTargetInfoEx> &oldTargets,
+                                                      const std::vector<flat::TargetId> &preferredOrder);
+
+// If the head of oldTargets is LASTSRV, move it to the tail and let the next target be the new LASTSRV.
+// It's used when a LASTSRV target could not recover in a short time. Admin could let the next target be the new LASTSRV
+// to resume the service. NOTE: it's on risk of losing some data forever.
+std::vector<ChainTargetInfoEx> rotateLastSrv(const std::vector<ChainTargetInfoEx> &oldTargets);
+
+std::vector<flat::ChainTargetInfo> shutdownChain(const std::vector<flat::ChainTargetInfo> &oldTargets);
+}  // namespace hf3fs::mgmtd