Initial commit

src/core/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_subdirectory(app)
+add_subdirectory(service)
+add_subdirectory(user)

src/core/README.md

@@ -0,0 +1 @@
+`core` is intended as a base component of `mgmtd`, `meta`, `storage`, and other concrete server components.

src/core/app/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_lib(core-app common mgmtd-client)

src/core/app/LauncherUtils.cc

@@ -0,0 +1,21 @@
+#include "LauncherUtils.h"
+
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "common/utils/SysResource.h"
+#include "common/utils/VersionInfo.h"
+
+namespace hf3fs::core::launcher {
+flat::AppInfo buildBasicAppInfo(flat::NodeId nodeId, const String &clusterId) {
+  auto hostnameResult = SysResource::hostname();
+  XLOGF_IF(FATAL, !hostnameResult, "Get hostname failed: {}", hostnameResult.error());
+
+  flat::AppInfo appInfo;
+  appInfo.nodeId = nodeId;
+  appInfo.clusterId = clusterId;
+  appInfo.hostname = *hostnameResult;
+  appInfo.pid = SysResource::pid();
+  appInfo.releaseVersion = flat::ReleaseVersion::fromVersionInfo();
+  return appInfo;
+}
+}  // namespace hf3fs::core::launcher

src/core/app/LauncherUtils.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "common/app/AppInfo.h"
+
+namespace hf3fs::core::launcher {
+flat::AppInfo buildBasicAppInfo(flat::NodeId nodeId, const String &clusterId);
+}  // namespace hf3fs::core::launcher

src/core/app/MgmtdClientFetcher.cc

@@ -0,0 +1,66 @@
+#include "MgmtdClientFetcher.h"
+
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "common/app/ApplicationBase.h"
+#include "stubs/common/RealStubFactory.h"
+#include "stubs/mgmtd/MgmtdServiceStub.h"
+
+namespace hf3fs::core::launcher {
+MgmtdClientFetcher::MgmtdClientFetcher(String clusterId,
+                                       const net::Client::Config &clientCfg,
+                                       const client::MgmtdClient::Config &mgmtdClientCfg)
+    : clusterId_(std::move(clusterId)),
+      clientCfg_(clientCfg),
+      mgmtdClientCfg_(mgmtdClientCfg) {}
+
+Result<flat::ConfigInfo> MgmtdClientFetcher::loadConfigTemplate(flat::NodeType nodeType) {
+  RETURN_ON_ERROR(ensureClientInited());
+  return folly::coro::blockingWait([&]() -> CoTryTask<flat::ConfigInfo> {
+    XLOGF(INFO, "Start to load config from mgmtd");
+    auto res = co_await mgmtdClient_->getConfig(nodeType, flat::ConfigVersion(0));
+    XLOGF(INFO, "Load config from mgmtd finished. res: {}", res.hasError() ? res.error() : Status::OK);
+    CO_RETURN_ON_ERROR(res);
+    if (!*res) {
+      XLOGF(INFO, "Load empty config from mgmtd");
+      co_return flat::ConfigInfo::create();
+    }
+    XLOGF(INFO, "Load config from mgmtd version: {}", res->value().configVersion);
+    co_return res->value();
+  }());
+}
+
+void MgmtdClientFetcher::stopClient() {
+  if (mgmtdClient_) {
+    folly::coro::blockingWait(mgmtdClient_->stop());
+    mgmtdClient_.reset();
+    client_->stopAndJoin();
+    client_.reset();
+  }
+}
+
+Result<Void> MgmtdClientFetcher::ensureClientInited() {
+  if (!client_) {
+    auto client = std::make_unique<net::Client>(clientCfg_);
+    RETURN_ON_ERROR(client->start("launcher"));
+    client_ = std::move(client);
+  }
+
+  if (!mgmtdClient_) {
+    auto ctxCreator =
+        // client_ may be moved to Server later so here we should capture its raw pointer.
+        [client = client_.get()](net::Address addr) { return client->serdeCtx(addr); };
+    auto stub = std::make_unique<stubs::RealStubFactory<mgmtd::MgmtdServiceStub>>(std::move(ctxCreator));
+
+    auto mgmtdClient = std::make_shared<client::MgmtdClient>(clusterId_, std::move(stub), mgmtdClientCfg_);
+
+    folly::coro::blockingWait(
+        mgmtdClient->start(&client_->tpg().bgThreadPool().randomPick(), /*startBackground=*/false));
+    auto refreshRes = folly::coro::blockingWait(mgmtdClient->refreshRoutingInfo(/*force=*/false));
+    RETURN_ON_ERROR(refreshRes);
+
+    mgmtdClient_ = std::move(mgmtdClient);
+  }
+  return Void{};
+}
+}  // namespace hf3fs::core::launcher

src/core/app/MgmtdClientFetcher.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "LauncherUtils.h"
+#include "client/mgmtd/MgmtdClient.h"
+#include "common/net/Client.h"
+
+namespace hf3fs::core::launcher {
+struct MgmtdClientFetcher {
+  MgmtdClientFetcher(String clusterId,
+                     const net::Client::Config &clientCfg,
+                     const client::MgmtdClient::Config &mgmtdClientCfg);
+
+  template <typename ConfigT>
+  MgmtdClientFetcher(const ConfigT &cfg)
+      : MgmtdClientFetcher(cfg.cluster_id(), cfg.client(), cfg.mgmtd_client()) {}
+
+  virtual ~MgmtdClientFetcher() { stopClient(); }
+
+  Result<flat::ConfigInfo> loadConfigTemplate(flat::NodeType nodeType);
+  Result<Void> ensureClientInited();
+  void stopClient();
+
+  virtual Result<Void> completeAppInfo(flat::AppInfo &appInfo) = 0;
+
+  const String clusterId_;
+  const net::Client::Config &clientCfg_;
+  const client::MgmtdClient::Config &mgmtdClientCfg_;
+  std::unique_ptr<net::Client> client_;
+  std::shared_ptr<client::MgmtdClient> mgmtdClient_;
+};
+}  // namespace hf3fs::core::launcher

src/core/app/ServerAppConfig.cc

@@ -0,0 +1,12 @@
+#include "ServerAppConfig.h"
+
+#include "common/app/ApplicationBase.h"
+
+namespace hf3fs::core {
+void ServerAppConfig::init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates) {
+  auto res = ApplicationBase::initConfig(*this, filePath, dump, updates);
+  XLOGF_IF(FATAL, !res, "Init app config failed: {}. filePath: {}. dump: {}", res.error(), filePath, dump);
+
+  XLOGF_IF(FATAL, !allow_empty_node_id() && node_id() == 0, "node_id is not allowed to be 0");
+}
+}  // namespace hf3fs::core

src/core/app/ServerAppConfig.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "common/app/NodeId.h"
+#include "common/net/ib/IBDevice.h"
+#include "common/utils/ConfigBase.h"
+
+namespace hf3fs::core {
+struct ServerAppConfig : public ConfigBase<ServerAppConfig> {
+  CONFIG_ITEM(node_id, 0);
+  CONFIG_ITEM(allow_empty_node_id, true);
+
+ public:
+  using Base = ConfigBase<ServerAppConfig>;
+  using Base::init;
+
+  void init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates);
+
+  flat::NodeId getNodeId() const { return flat::NodeId(node_id()); }
+};
+}  // namespace hf3fs::core

src/core/app/ServerEnv.cc

@@ -0,0 +1,29 @@
+#include "ServerEnv.h"
+
+namespace hf3fs::core {
+namespace {
+
+template <typename Ptr>
+void setPtr(Ptr &src, Ptr &dst) {
+  if (src.get() != dst.get()) {
+    src.reset();
+    src = std::move(dst);
+  }
+}
+}  // namespace
+
+void ServerEnv::setAppInfo(flat::AppInfo appInfo) { appInfo_ = appInfo; }
+
+void ServerEnv::setKvEngine(std::shared_ptr<kv::IKVEngine> engine) { setPtr(kvEngine_, engine); }
+
+void ServerEnv::setMgmtdStubFactory(std::shared_ptr<MgmtdStubFactory> factory) { setPtr(mgmtdStubFactory_, factory); }
+
+void ServerEnv::setUtcTimeGenerator(UtcTimeGenerator generator) { utcTimeGenerator_ = std::move(generator); }
+
+void ServerEnv::setBackgroundExecutor(CPUExecutorGroup *executor) { backgroundExecutor_ = executor; }
+
+void ServerEnv::setConfigUpdater(ConfigUpdater updater) { configUpdater_ = std::move(updater); }
+
+void ServerEnv::setConfigValidater(ConfigValidater validater) { configValidater_ = std::move(validater); }
+
+}  // namespace hf3fs::core

src/core/app/ServerEnv.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include "common/app/AppInfo.h"
+#include "common/kv/IKVEngine.h"
+#include "common/utils/CPUExecutorGroup.h"
+#include "stubs/common/IStubFactory.h"
+#include "stubs/mgmtd/IMgmtdServiceStub.h"
+
+namespace hf3fs::core {
+class ServerEnv {
+ public:
+  const flat::AppInfo &appInfo() const { return appInfo_; }
+
+  void setAppInfo(flat::AppInfo appInfo);
+
+  const std::shared_ptr<kv::IKVEngine> &kvEngine() const { return kvEngine_; }
+
+  void setKvEngine(std::shared_ptr<kv::IKVEngine> engine);
+
+  using MgmtdStubFactory = stubs::IStubFactory<mgmtd::IMgmtdServiceStub>;
+  const std::shared_ptr<MgmtdStubFactory> &mgmtdStubFactory() const { return mgmtdStubFactory_; }
+
+  void setMgmtdStubFactory(std::shared_ptr<MgmtdStubFactory> factory);
+
+  using UtcTimeGenerator = std::function<UtcTime()>;
+  const UtcTimeGenerator &utcTimeGenerator() const { return utcTimeGenerator_; }
+
+  void setUtcTimeGenerator(UtcTimeGenerator generator);
+
+  CPUExecutorGroup *backgroundExecutor() const { return backgroundExecutor_; }
+
+  void setBackgroundExecutor(CPUExecutorGroup *executor);
+
+  using ConfigUpdater = std::function<Result<Void>(const String &, const String &)>;
+  const ConfigUpdater &configUpdater() const { return configUpdater_; }
+
+  void setConfigUpdater(ConfigUpdater updater);
+
+  using ConfigValidater = std::function<Result<Void>(const String &, const String &)>;
+  const ConfigValidater &configValidater() const { return configValidater_; }
+
+  void setConfigValidater(ConfigValidater validater);
+
+ private:
+  flat::AppInfo appInfo_;
+  std::shared_ptr<kv::IKVEngine> kvEngine_;
+  std::shared_ptr<MgmtdStubFactory> mgmtdStubFactory_;
+  UtcTimeGenerator utcTimeGenerator_ = &UtcClock::now;
+  CPUExecutorGroup *backgroundExecutor_ = nullptr;
+  ConfigUpdater configUpdater_;
+  ConfigValidater configValidater_;
+};
+}  // namespace hf3fs::core

src/core/app/ServerLauncher.cc

@@ -0,0 +1,12 @@
+#include "ServerLauncher.h"
+
+namespace hf3fs::core {
+Result<Void> ServerLauncherBase::parseFlags(int *argc, char ***argv) {
+  static constexpr std::string_view appConfigPrefix = "--app_config.";
+  static constexpr std::string_view launcherConfigPrefix = "--launcher_config.";
+  RETURN_ON_ERROR(ApplicationBase::parseFlags(appConfigPrefix, argc, argv, appConfigFlags_));
+  RETURN_ON_ERROR(ApplicationBase::parseFlags(launcherConfigPrefix, argc, argv, launcherConfigFlags_));
+  return Void{};
+}
+
+}  // namespace hf3fs::core

src/core/app/ServerLauncher.h

@@ -0,0 +1,79 @@
+#pragma once
+
+#include "LauncherUtils.h"
+#include "common/app/ApplicationBase.h"
+#include "common/utils/ConstructLog.h"
+#include "fbs/mgmtd/ConfigInfo.h"
+
+DECLARE_string(app_cfg);
+DECLARE_bool(dump_default_app_cfg);
+
+DECLARE_string(launcher_cfg);
+DECLARE_bool(dump_default_launcher_cfg);
+
+namespace hf3fs::core {
+class ServerLauncherBase {
+ public:
+  Result<Void> parseFlags(int *argc, char ***argv);
+
+ protected:
+  ApplicationBase::ConfigFlags appConfigFlags_;
+  ApplicationBase::ConfigFlags launcherConfigFlags_;
+};
+
+template <typename Server>
+class ServerLauncher : public ServerLauncherBase {
+ public:
+  using AppConfig = typename Server::AppConfig;
+  using LauncherConfig = typename Server::LauncherConfig;
+  using RemoteConfigFetcher = typename Server::RemoteConfigFetcher;
+  static constexpr auto kNodeType = Server::kNodeType;
+
+  ServerLauncher() = default;
+
+  Result<Void> init() {
+    appConfig_.init(FLAGS_app_cfg, FLAGS_dump_default_app_cfg, appConfigFlags_);
+    launcherConfig_.init(FLAGS_launcher_cfg, FLAGS_dump_default_launcher_cfg, launcherConfigFlags_);
+
+    XLOGF(INFO, "Full AppConfig:\n{}", appConfig_.toString());
+    XLOGF(INFO, "Full LauncherConfig:\n{}", launcherConfig_.toString());
+
+    auto ibResult = net::IBManager::start(launcherConfig_.ib_devices());
+    XLOGF_IF(FATAL, !ibResult, "Failed to start IBManager: {}", ibResult.error());
+    XLOGF(INFO, "IBDevice inited");
+
+    fetcher_ = std::make_unique<RemoteConfigFetcher>(launcherConfig_);
+    return Void{};
+  }
+
+  Result<std::pair<String, String>> loadConfigTemplate() {
+    auto res = fetcher_->loadConfigTemplate(kNodeType);
+    RETURN_ON_ERROR(res);
+    return std::make_pair(res->content, res->genUpdateDesc());
+  }
+
+  Result<flat::AppInfo> loadAppInfo() {
+    auto appInfo = launcher::buildBasicAppInfo(appConfig_.getNodeId(), launcherConfig_.cluster_id());
+    RETURN_ON_ERROR(fetcher_->completeAppInfo(appInfo));
+    return appInfo;
+  }
+
+  Result<Void> startServer(Server &server, const flat::AppInfo &appInfo) {
+    if constexpr (requires { fetcher_->startServer(server, appInfo); }) {
+      return fetcher_->startServer(server, appInfo);
+    } else {
+      return server.start(appInfo);
+    }
+  }
+
+  const auto &appConfig() const { return appConfig_; }
+
+  const auto &launcherConfig() const { return launcherConfig_; }
+
+ private:
+  AppConfig appConfig_;
+  LauncherConfig launcherConfig_;
+
+  std::unique_ptr<RemoteConfigFetcher> fetcher_;
+};
+}  // namespace hf3fs::core

src/core/app/ServerLauncherConfig.cc

@@ -0,0 +1,15 @@
+#include "ServerLauncherConfig.h"
+
+#include "common/app/ApplicationBase.h"
+#include "common/app/Utils.h"
+
+namespace hf3fs::core {
+void ServerLauncherConfig::init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates) {
+  app_detail::initConfigFromFile(*this, filePath, dump, updates);
+
+  auto rv = flat::ReleaseVersion::fromVersionInfo();
+  if (!allow_dev_version() && !rv.getIsReleaseVersion()) {
+    XLOGF(FATAL, "Dev version is not allowed: {}", rv);
+  }
+}
+}  // namespace hf3fs::core

src/core/app/ServerLauncherConfig.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "client/mgmtd/MgmtdClient.h"
+#include "common/app/NodeId.h"
+#include "common/net/Client.h"
+#include "common/utils/ConfigBase.h"
+
+namespace hf3fs::core {
+struct ServerLauncherConfig : public ConfigBase<ServerLauncherConfig> {
+  CONFIG_ITEM(cluster_id, "");
+  CONFIG_OBJ(ib_devices, net::IBDevice::Config);
+  CONFIG_OBJ(client, net::Client::Config);
+  CONFIG_OBJ(mgmtd_client, client::MgmtdClient::Config);
+  CONFIG_ITEM(allow_dev_version, true);
+
+ public:
+  using Base = ConfigBase<ServerLauncherConfig>;
+  using Base::init;
+
+  void init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates);
+};
+}  // namespace hf3fs::core

src/core/app/ServerMgmtdClientFetcher.cc

@@ -0,0 +1,18 @@
+#include "ServerMgmtdClientFetcher.h"
+
+#include <folly/experimental/coro/BlockingWait.h>
+
+namespace hf3fs::core::launcher {
+Result<Void> ServerMgmtdClientFetcher::completeAppInfo(flat::AppInfo &appInfo) {
+  RETURN_ON_ERROR(ensureClientInited());
+  return folly::coro::blockingWait([&]() -> CoTryTask<void> {
+    auto routingInfo = mgmtdClient_->getRoutingInfo();
+    if (!routingInfo || !routingInfo->raw()) co_return makeError(MgmtdClientCode::kRoutingInfoNotReady);
+    const auto *ni = routingInfo->raw()->getNode(appInfo.nodeId);
+    if (ni) {
+      appInfo.tags = ni->tags;
+    }
+    co_return Void{};
+  }());
+}
+}  // namespace hf3fs::core::launcher

src/core/app/ServerMgmtdClientFetcher.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "MgmtdClientFetcher.h"
+
+namespace hf3fs::core::launcher {
+struct ServerMgmtdClientFetcher : public MgmtdClientFetcher {
+  using MgmtdClientFetcher::MgmtdClientFetcher;
+  Result<Void> completeAppInfo(flat::AppInfo &appInfo) final;
+};
+}  // namespace hf3fs::core::launcher

src/core/service/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_lib(core-service core-service-fbs)

src/core/service/CoreService.cc

@@ -0,0 +1,14 @@
+#include "CoreService.h"
+
+#include "core/service/ops/Include.h"
+#include "core/utils/runOp.h"
+
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype)        \
+  CoTryTask<rsptype> svc##Service::name(serde::CallContext &ctx, const reqtype &req) { \
+    Name##Operation op(std::move(req));                                                \
+    CO_INVOKE_OP_INFO(op, ctx.peer());                                                 \
+  }
+
+namespace hf3fs::core {
+#include "fbs/core/service/CoreServiceDef.h"
+}  // namespace hf3fs::core

src/core/service/CoreService.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "common/serde/CallContext.h"
+#include "fbs/core/service/CoreServiceBase.h"
+
+namespace hf3fs::core {
+class CoreService : public serde::ServiceWrapper<CoreService, CoreServiceBase> {
+ public:
+#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
+  CoTryTask<rsptype> name(serde::CallContext &ctx, const reqtype &req);
+
+#include "fbs/core/service/CoreServiceDef.h"
+};
+}  // namespace hf3fs::core

src/core/service/ops/EchoOperation.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "core/utils/ServiceOperation.h"
+#include "fbs/core/service/Rpc.h"
+
+namespace hf3fs::core {
+
+struct EchoOperation : ServiceOperationWithMetric<"CoreService", "Echo", "op"> {
+  EchoMessage req;
+
+  explicit EchoOperation(EchoMessage r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return "Echo"; }
+
+  CoTryTask<EchoMessage> handle() { co_return req; }
+};
+
+}  // namespace hf3fs::core

src/core/service/ops/GetConfigOperation.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "common/app/ApplicationBase.h"
+#include "core/utils/ServiceOperation.h"
+#include "fbs/core/service/Rpc.h"
+
+namespace hf3fs::core {
+
+struct GetConfigOperation : ServiceOperationWithMetric<"CoreService", "GetConfig", "op"> {
+  GetConfigReq req;
+
+  explicit GetConfigOperation(GetConfigReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return "GetConfig"; }
+
+  CoTryTask<GetConfigRsp> handle() {
+    auto cfg = ApplicationBase::getConfigString(req.configKey);
+    CO_RETURN_ON_ERROR(cfg);
+    co_return GetConfigRsp::create(std::move(*cfg));
+  }
+};
+
+}  // namespace hf3fs::core

src/core/service/ops/GetLastConfigUpdateRecordOperation.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "common/app/ApplicationBase.h"
+#include "core/utils/ServiceOperation.h"
+#include "fbs/core/service/Rpc.h"
+
+namespace hf3fs::core {
+
+struct GetLastConfigUpdateRecordOperation
+    : ServiceOperationWithMetric<"CoreService", "GetLastConfigUpdateRecord", "op"> {
+  GetLastConfigUpdateRecordReq req;
+
+  explicit GetLastConfigUpdateRecordOperation(GetLastConfigUpdateRecordReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return "GetLastConfigUpdateRecord"; }
+
+  CoTryTask<GetLastConfigUpdateRecordRsp> handle() {
+    auto res = ApplicationBase::getLastConfigUpdateRecord();
+    co_return GetLastConfigUpdateRecordRsp::create(std::move(res));
+  }
+};
+
+}  // namespace hf3fs::core

src/core/service/ops/HotUpdateConfigOperation.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "common/app/ApplicationBase.h"
+#include "core/utils/ServiceOperation.h"
+#include "fbs/core/service/Rpc.h"
+
+namespace hf3fs::core {
+
+struct HotUpdateConfigOperation : ServiceOperationWithMetric<"CoreService", "HotUpdateConfig", "op"> {
+  HotUpdateConfigReq req;
+
+  explicit HotUpdateConfigOperation(HotUpdateConfigReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return "HotUpdateConfig"; }
+
+  CoTryTask<HotUpdateConfigRsp> handle() {
+    auto res = ApplicationBase::hotUpdateConfig(req.update, req.render);
+    CO_RETURN_ON_ERROR(res);
+    co_return HotUpdateConfigRsp::create();
+  }
+};
+
+}  // namespace hf3fs::core

src/core/service/ops/Include.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include "EchoOperation.h"
+#include "GetConfigOperation.h"
+#include "GetLastConfigUpdateRecordOperation.h"
+#include "HotUpdateConfigOperation.h"
+#include "RenderConfigOperation.h"
+#include "ShutdownOperation.h"

src/core/service/ops/RenderConfigOperation.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "common/app/ApplicationBase.h"
+#include "core/utils/ServiceOperation.h"
+#include "fbs/core/service/Rpc.h"
+
+namespace hf3fs::core {
+
+struct RenderConfigOperation : ServiceOperationWithMetric<"CoreService", "RenderConfig", "op"> {
+  RenderConfigReq req;
+
+  explicit RenderConfigOperation(RenderConfigReq r)
+      : req(std::move(r)) {}
+
+  String toStringImpl() const final { return "RenderConfig"; }
+
+  CoTryTask<RenderConfigRsp> handle() {
+    auto res = ApplicationBase::renderConfig(req.configTemplate, req.testUpdate, req.isHotUpdate);
+    CO_RETURN_ON_ERROR(res);
+    auto &[content, updateRes] = *res;
+    if (updateRes) {
+      co_return RenderConfigRsp::create(std::move(content), Status(StatusCode::kOK), std::move(*updateRes));
+    } else {
+      co_return RenderConfigRsp::create(std::move(content), std::move(updateRes.error()));
+    }
+  }
+};
+
+}  // namespace hf3fs::core

src/core/service/ops/ShutdownOperation.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "common/app/ApplicationBase.h"
+#include "common/utils/suicide.h"
+#include "core/utils/ServiceOperation.h"
+#include "fbs/core/service/Rpc.h"
+
+namespace hf3fs::core {
+
+struct ShutdownOperation : ServiceOperationWithMetric<"CoreService", "Shutdown", "op"> {
+  ShutdownReq req_;
+
+  explicit ShutdownOperation(ShutdownReq req)
+      : req_(std::move(req)) {}
+
+  String toStringImpl() const final { return "Shutdown"; }
+
+  CoTryTask<ShutdownRsp> handle() {
+    CO_RETURN_ON_ERROR(suicide(req_.graceful));
+    co_return ShutdownRsp::create();
+  }
+};
+
+}  // namespace hf3fs::core

src/core/user/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_lib(core-user core-user-fbs fdb)

src/core/user/UserCache.h

@@ -0,0 +1,120 @@
+#pragma once
+
+#include "common/utils/ConfigBase.h"
+#include "common/utils/LockManager.h"
+#include "common/utils/RobinHood.h"
+#include "common/utils/UtcTime.h"
+#include "fbs/core/user/User.h"
+
+namespace hf3fs::core {
+using flat::Gid;
+using flat::Uid;
+using flat::UserAttr;
+using flat::UserInfo;
+
+class UserCache {
+ public:
+  struct Config : ConfigBase<Config> {
+    CONFIG_ITEM(buckets, 127u, ConfigCheckers::isPositivePrime<uint32_t>);
+    CONFIG_HOT_UPDATED_ITEM(exist_ttl, 5_min);
+    CONFIG_HOT_UPDATED_ITEM(inexist_ttl, 10_s);
+  };
+
+  explicit UserCache(const Config &cfg)
+      : cfg_(cfg),
+        lockManager_(cfg.buckets()),
+        buckets_(cfg.buckets()) {}
+
+  struct GetResult {
+    std::optional<UserAttr> attr;
+    bool marked = false;
+
+    static GetResult unmarkedNotExist() { return {std::nullopt, false}; }
+    static GetResult markedNotExist() { return {std::nullopt, true}; }
+    static GetResult exist(UserAttr attr) { return {std::move(attr), false}; }
+  };
+
+  GetResult get(Uid id) {
+    auto lock = lockManager_.lock(id);
+    auto now = SteadyClock::now();
+    auto &bucket = buckets_[lockManager_.idx(id)];
+
+    auto it = bucket.find(id);
+    if (it == bucket.end()) return GetResult::unmarkedNotExist();
+
+    auto &entry = it->second;
+    if (expired(entry, now)) {
+      bucket.erase(it);
+      return GetResult::unmarkedNotExist();
+    }
+
+    if (entry.attr)
+      return GetResult::exist(*entry.attr);
+    else
+      return GetResult::markedNotExist();
+  }
+
+  void set(Uid id, std::optional<UserAttr> attr) {
+    auto lock = lockManager_.lock(id);
+    auto &bucket = buckets_[lockManager_.idx(id)];
+    auto &entry = bucket[id];
+    auto now = SteadyClock::now();
+    entry.setTs = now;
+    entry.attr = std::move(attr);
+  }
+
+  void clear(Uid id) {
+    auto lock = lockManager_.lock(id);
+    auto &bucket = buckets_[lockManager_.idx(id)];
+    bucket.erase(id);
+  }
+
+  void clear() {
+    for (uint32_t i = 0; i < lockManager_.numBuckets(); ++i) {
+      auto lock = lockManager_.lock_at(i);
+      auto &bucket = buckets_[i];
+      bucket.clear();
+    }
+  }
+
+  void clearRetired() {
+    std::vector<Uid> uids;
+    for (uint32_t i = 0; i < lockManager_.numBuckets(); ++i) {
+      auto lock = lockManager_.lock_at(i);
+      auto &bucket = buckets_[i];
+      auto now = SteadyClock::now();
+      for (const auto &[id, entry] : bucket) {
+        if (expired(entry, now)) uids.push_back(id);
+      }
+    }
+    if (!uids.empty()) {
+      for (uint32_t i = 0; i < lockManager_.numBuckets(); ++i) {
+        auto lock = lockManager_.lock_at(i);
+        auto &bucket = buckets_[i];
+        auto now = SteadyClock::now();
+        for (auto id : uids) {
+          auto it = bucket.find(id);
+          if (expired(it->second, now)) bucket.erase(it);
+        }
+      }
+    }
+  }
+
+ private:
+  struct Entry {
+    std::optional<UserAttr> attr;
+    SteadyTime setTs;
+  };
+
+  bool expired(const Entry &entry, SteadyTime now) const {
+    auto timeout = entry.attr ? cfg_.exist_ttl().asUs() : cfg_.inexist_ttl().asUs();
+    return now - entry.setTs >= timeout;
+  }
+
+  using Bucket = robin_hood::unordered_node_map<Uid, Entry>;
+
+  const Config &cfg_;
+  UniqueLockManager lockManager_;
+  std::vector<Bucket> buckets_;
+};
+}  // namespace hf3fs::core

src/core/user/UserStore.cc

@@ -0,0 +1,180 @@
+#include "UserStore.h"
+
+#include <fmt/core.h>
+#include <folly/logging/xlog.h>
+
+#include "UserToken.h"
+#include "common/kv/KeyPrefix.h"
+#include "common/serde/Serde.h"
+#include "common/utils/Result.h"
+#include "common/utils/SerDeser.h"
+#include "core/user/UserCache.h"
+#include "fdb/FDBTransaction.h"
+
+namespace hf3fs::core {
+namespace {
+String packUid(Uid uid) {
+  static constexpr auto prefix = kv::KeyPrefix::User;
+  String buf;
+  buf.reserve(sizeof(prefix) + sizeof(Uid::UnderlyingType));
+  Serializer ser(buf);
+  ser.put(prefix);
+  ser.put(uid.toUnderType());
+  return buf;
+}
+
+Result<Void> ensureTokenMatchUid(std::string_view token, flat::Uid uid) {
+  auto newUid = decodeUidFromUserToken(token);
+  RETURN_ON_ERROR(newUid);
+  if (*newUid != uid) {
+    return makeError(StatusCode::kInvalidFormat, "Could not set token from other uid");
+  }
+  return Void{};
+}
+}  // namespace
+
+CoTryTask<std::optional<UserAttr>> UserStore::getUser(IReadOnlyTransaction &txn, Uid uid) {
+  auto key = packUid(uid);
+  auto getResult = co_await txn.snapshotGet(key);
+  CO_RETURN_ON_ERROR(getResult);
+  if (!getResult->has_value()) {
+    co_return std::nullopt;
+  }
+  auto value = **getResult;
+  XLOGF(DBG,
+        "key value {:02x} {:02x}",
+        fmt::join(key.data(), key.data(), ","),
+        fmt::join(value.data(), value.data(), ","));
+  UserAttr attr;
+  CO_RETURN_ON_ERROR(serde::deserialize(attr, **getResult));
+  attr.uid = uid;  // old versions of UserAttr didn't persist uid
+  co_return attr;
+}
+
+CoTryTask<UserAttr> UserStore::addUser(IReadWriteTransaction &txn,
+                                       Uid uid,
+                                       String name,
+                                       std::vector<Gid> groups,
+                                       bool isRootUser,
+                                       bool isAdmin,
+                                       std::string_view token) {
+  auto key = packUid(uid);
+  auto getResult = co_await txn.snapshotGet(key);
+  CO_RETURN_ON_ERROR(getResult);
+  if (getResult->has_value()) co_return makeError(MetaCode::kExists);
+
+  UserAttr attr;
+  attr.name = std::move(name);
+  attr.uid = uid;
+  attr.gid = flat::Gid(uid);
+  attr.groups = std::move(groups);
+  attr.root = isRootUser;
+  attr.admin = isAdmin;
+
+  if (!token.empty()) {
+    CO_RETURN_ON_ERROR(ensureTokenMatchUid(token, uid));
+    attr.token = token;
+  } else {
+    auto tokenResult = co_await encodeUserToken(uid, txn);
+    CO_RETURN_ON_ERROR(tokenResult);
+    attr.token = *tokenResult;
+  }
+
+  CO_RETURN_ON_ERROR(co_await txn.set(key, serde::serialize(attr)));
+  co_return attr;
+}
+
+CoTryTask<void> UserStore::removeUser(IReadWriteTransaction &txn, Uid uid) {
+  auto fetchResult = co_await getUser(txn, uid);
+  CO_RETURN_ON_ERROR(fetchResult);
+  if (!fetchResult->has_value()) {
+    co_return makeError(MetaCode::kNotFound, fmt::format("Uid {} not found", uid.toUnderType()));
+  }
+  auto key = packUid(uid);
+  co_return co_await txn.clear(key);
+}
+
+CoTryTask<UserAttr> UserStore::setUserAttr(IReadWriteTransaction &txn,
+                                           Uid uid,
+                                           std::string_view newName,
+                                           const std::vector<Gid> *gids,
+                                           std::optional<bool> isRootUser,
+                                           std::optional<bool> isAdmin) {
+  auto fetchResult = co_await getUser(txn, uid);
+  CO_RETURN_ON_ERROR(fetchResult);
+  if (!fetchResult->has_value()) {
+    co_return makeError(MetaCode::kNotFound, fmt::format("Uid {} not found", uid.toUnderType()));
+  }
+
+  const auto &user = **fetchResult;
+  auto newUser = user;
+
+  if (!newName.empty()) {
+    newUser.name = newName;
+  }
+
+  if (gids) {
+    newUser.groups = *gids;
+  }
+  if (isRootUser) {
+    newUser.root = *isRootUser;
+  }
+  if (isAdmin) {
+    newUser.admin = *isAdmin;
+  }
+  if (newUser != user) {
+    CO_RETURN_ON_ERROR(co_await txn.set(packUid(uid), serde::serialize(newUser)));
+  }
+  co_return newUser;
+}
+
+CoTryTask<UserAttr> UserStore::setUserToken(IReadWriteTransaction &txn,
+                                            Uid uid,
+                                            std::string_view newToken,
+                                            bool invalidateExistedToken,
+                                            size_t maxTokens,
+                                            Duration tokenLifetimeExtendLength) {
+  auto fetchResult = co_await getUser(txn, uid);
+  CO_RETURN_ON_ERROR(fetchResult);
+  if (!fetchResult->has_value()) {
+    co_return makeError(MetaCode::kNotFound, fmt::format("Uid {} not found", uid.toUnderType()));
+  }
+
+  const auto &user = **fetchResult;
+  auto newUser = user;
+
+  String newTokenStr(newToken);
+  if (!newTokenStr.empty()) {
+    CO_RETURN_ON_ERROR(ensureTokenMatchUid(newTokenStr, uid));
+  } else {
+    auto tokenResult = co_await encodeUserToken(uid, txn);
+    CO_RETURN_ON_ERROR(tokenResult);
+    newTokenStr = std::move(*tokenResult);
+  }
+  CO_RETURN_ON_ERROR(
+      newUser.addNewToken(newTokenStr, UtcClock::now(), invalidateExistedToken, maxTokens, tokenLifetimeExtendLength));
+  if (newUser != user) {
+    CO_RETURN_ON_ERROR(co_await txn.set(packUid(uid), serde::serialize(newUser)));
+  }
+  co_return newUser;
+}
+
+CoTryTask<std::vector<UserAttr>> UserStore::listUsers(IReadOnlyTransaction &txn) {
+  auto listRes = co_await kv::TransactionHelper::listByPrefix(txn, kv::toStr(kv::KeyPrefix::User), {});
+  CO_RETURN_ON_ERROR(listRes);
+
+  std::vector<UserAttr> res;
+  for (const auto &kv : *listRes) {
+    kv::KeyPrefix keyPrefix;
+    Uid::UnderlyingType uid;
+    CO_RETURN_ON_ERROR(Deserializer::deserRawArgs(kv.key, keyPrefix, uid));
+
+    UserAttr attr;
+    CO_RETURN_ON_ERROR(serde::deserialize(attr, kv.value));
+    attr.uid = Uid(uid);
+    res.push_back(std::move(attr));
+  }
+  co_return res;
+}
+
+}  // namespace hf3fs::core

src/core/user/UserStore.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "common/kv/ITransaction.h"
+#include "core/user/UserCache.h"
+#include "fbs/core/user/User.h"
+
+namespace hf3fs::core {
+using flat::Gid;
+using flat::Uid;
+using flat::UserAttr;
+using flat::UserInfo;
+using hf3fs::kv::IReadOnlyTransaction;
+using hf3fs::kv::IReadWriteTransaction;
+
+class UserStore {
+ public:
+  UserStore() = default;
+
+  CoTryTask<std::optional<UserAttr>> getUser(IReadOnlyTransaction &txn, Uid uid);
+
+  CoTryTask<UserAttr> addUser(IReadWriteTransaction &txn,
+                              Uid uid,
+                              String name,
+                              std::vector<Gid> groups,
+                              bool isRootUser,
+                              bool isAdmin = false,
+                              std::string_view token = {});
+
+  CoTryTask<void> removeUser(IReadWriteTransaction &txn, Uid uid);
+
+  CoTryTask<UserAttr> setUserAttr(IReadWriteTransaction &txn,
+                                  Uid uid,
+                                  std::string_view newName,
+                                  const std::vector<Gid> *gids,
+                                  std::optional<bool> isRootUser,
+                                  std::optional<bool> isAdmin);
+
+  CoTryTask<UserAttr> setUserToken(IReadWriteTransaction &txn,
+                                   Uid uid,
+                                   std::string_view newToken,
+                                   bool invalidateExistedToken,
+                                   size_t maxTokens,
+                                   Duration tokenLifetimeExtendLength);
+
+  CoTryTask<std::vector<UserAttr>> listUsers(IReadOnlyTransaction &txn);
+};
+}  // namespace hf3fs::core

src/core/user/UserStoreEx.cc

@@ -0,0 +1,83 @@
+#include "UserStoreEx.h"
+
+#include "UserToken.h"
+#include "common/kv/IKVEngine.h"
+#include "common/kv/WithTransaction.h"
+#include "fdb/FDBRetryStrategy.h"
+
+namespace hf3fs::core {
+UserStoreEx::UserStoreEx(kv::IKVEngine &kvEngine,
+                         const kv::TransactionRetry &transactionRetry,
+                         const UserCache::Config &cacheCfg)
+    : kvEngine_(kvEngine),
+      transactionRetry_(transactionRetry),
+      userCache_(cacheCfg) {}
+
+CoTryTask<flat::UserAttr> UserStoreEx::getUser(std::string_view token) {
+  auto uid = decodeUidFromUserToken(token);
+  if (UNLIKELY(uid.hasError())) {
+    co_return makeError(StatusCode::kAuthenticationFail, "Token decode failed");
+  }
+  auto user = co_await getUser(*uid);
+  CO_RETURN_ON_ERROR(user);
+  if (auto res = user->validateToken(token, UtcClock::now()); res.hasError()) {
+    XLOGF(ERR, "Token validate failed: {}, uid {}", res.error(), *uid);
+    co_return makeError(StatusCode::kAuthenticationFail, "Token validate failed");
+  }
+  co_return *user;
+}
+
+CoTryTask<void> UserStoreEx::authenticate(flat::UserInfo &userInfo) {
+  std::vector<Gid> gids;
+  auto userFromToken = co_await getUser(userInfo.token);
+  CO_RETURN_ON_ERROR(userFromToken);
+
+  if (userFromToken->root) {
+    XLOGF(DBG, "Root user {}, uid {}, euid {}", userFromToken->name, userFromToken->uid, userInfo.uid);
+    if (userInfo.uid != userFromToken->uid) {
+      // root users can act as anyone
+      userFromToken = co_await getUser(userInfo.uid);
+      CO_RETURN_ON_ERROR(userFromToken);
+    }
+  } else {
+    if (userInfo.uid != userFromToken->uid || userInfo.gid != userFromToken->gid) {
+      auto msg = fmt::format("User not match, userInfo {}, (uid {}, gid {})",
+                             userInfo,
+                             userFromToken->uid.toUnderType(),
+                             userFromToken->gid.toUnderType());
+      XLOGF(ERR, "{}", msg);
+      co_return makeError(StatusCode::kAuthenticationFail, std::move(msg));
+    }
+  }
+
+  userInfo.groups = std::move(userFromToken->groups);
+  co_return Void{};
+}
+
+CoTryTask<flat::UserAttr> UserStoreEx::getUser(flat::Uid uid) {
+  auto getResult = userCache_.get(uid);
+  if (getResult.attr) {
+    co_return *getResult.attr;
+  }
+  if (getResult.marked) {
+    co_return makeError(StatusCode::kAuthenticationFail, fmt::format("Uid {} not found", uid.toUnderType()));
+  }
+  auto strategy = kv::FDBRetryStrategy(
+      {transactionRetry_.max_backoff(), transactionRetry_.max_retry_count(), /*retryMaybeCommitted=*/true});
+  auto result = co_await kv::WithTransaction(strategy).run(
+      kvEngine_.createReadonlyTransaction(),
+      [&](kv::IReadOnlyTransaction &txn) -> CoTryTask<std::optional<UserAttr>> {
+        co_return co_await userStore_.getUser(txn, uid);
+      });
+  if (result.hasError()) {
+    CO_RETURN_ERROR(result);
+  } else {
+    userCache_.set(uid, *result);
+    if (result->has_value()) {
+      co_return **result;
+    } else {
+      co_return makeError(StatusCode::kAuthenticationFail, fmt::format("Uid {} not found", uid.toUnderType()));
+    }
+  }
+}
+}  // namespace hf3fs::core

src/core/user/UserStoreEx.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "UserCache.h"
+#include "UserStore.h"
+#include "common/kv/IKVEngine.h"
+#include "common/kv/TransactionRetry.h"
+
+namespace hf3fs::core {
+
+class UserStoreEx {
+ public:
+  UserStoreEx(kv::IKVEngine &kvEngine, const kv::TransactionRetry &transactionRetry, const UserCache::Config &cacheCfg);
+
+  CoTryTask<void> authenticate(flat::UserInfo &userInfo);
+  CoTryTask<flat::UserAttr> getUser(std::string_view token);
+  CoTryTask<flat::UserAttr> getUser(flat::Uid uid);
+
+  UserCache &cache() { return userCache_; }
+
+ private:
+  kv::IKVEngine &kvEngine_;
+  const kv::TransactionRetry &transactionRetry_;
+  UserStore userStore_;
+  UserCache userCache_;
+};
+}  // namespace hf3fs::core

src/core/user/UserToken.cc

@@ -0,0 +1,100 @@
+#include "UserToken.h"
+
+#include <folly/Random.h>
+#include <folly/base64.h>
+#include <folly/hash/Checksum.h>
+#include <folly/lang/Bits.h>
+
+#include "fdb/FDBTransaction.h"
+
+namespace hf3fs::core {
+namespace {
+const uint16_t magicNum = 0xF1DB;
+// clang-format off
+constexpr std::array<uint8_t, 18> mapping = {
+  12, 6, 0,
+  13, 7, 1,
+  14, 8, 2,
+  15, 9, 3,
+  16, 10, 4,
+  17, 11, 5,
+};
+// clang-format on
+}  // namespace
+
+String encodeUserToken(uint32_t uid, uint64_t randomv) {
+  static_assert(folly::Endian::order == folly::Endian::Order::LITTLE);
+
+  std::array<uint8_t, 18> tmp;
+  auto *p = tmp.data();
+
+  static_assert(sizeof(magicNum) == 2);
+  std::memcpy(p, &magicNum, sizeof(magicNum));
+  p += sizeof(magicNum);
+
+  static_assert(sizeof(uid) == 4);
+  std::memcpy(p, &uid, sizeof(uid));
+  p += sizeof(uid);
+
+  static_assert(sizeof(randomv) == 8);
+  std::memcpy(p, &randomv, sizeof(randomv));
+  p += sizeof(randomv);
+
+  auto crc = folly::crc32(tmp.data(), 14, 0);
+  static_assert(sizeof(crc) == 4);
+  std::memcpy(p, &crc, sizeof(crc));
+
+  std::array<char, 18> buffer;
+  for (size_t i = 0; i < 18; ++i) {
+    buffer[mapping[i]] = tmp[i];
+  }
+  return folly::base64Encode(std::string_view(buffer.data(), buffer.size()));
+}
+
+Result<std::pair<uint32_t, uint64_t>> decodeUserToken(std::string_view token) {
+  try {
+    auto decoded = folly::base64Decode(token);
+    if (decoded.size() != 18) return makeError(StatusCode::kInvalidFormat, "Decode token fail: invalid format");
+
+    std::array<uint8_t, 18> tmp;
+    for (size_t i = 0; i < 18; ++i) {
+      tmp[i] = decoded[mapping[i]];
+    }
+
+    uint16_t mn = 0;
+    uint32_t uid = 0, crc = 0;
+    uint64_t timestamp = 0;
+    auto *p = tmp.data();
+    std::memcpy(&mn, p, sizeof(mn));
+    p += sizeof(mn);
+    std::memcpy(&uid, p, sizeof(uid));
+    p += sizeof(uid);
+    std::memcpy(&timestamp, p, sizeof(timestamp));
+    p += sizeof(timestamp);
+    std::memcpy(&crc, p, sizeof(crc));
+
+    if (mn != magicNum) {
+      return makeError(StatusCode::kInvalidFormat, "Decode token fail: invalid format");
+    }
+
+    auto expectedCrc = folly::crc32(tmp.data(), 14, 0);
+    if (crc != expectedCrc) {
+      return makeError(StatusCode::kInvalidFormat, "Decode token fail: invalid crc");
+    }
+    return std::make_pair(uid, timestamp);
+  } catch (const folly::base64_decode_error &e) {
+    return makeError(StatusCode::kInvalidFormat, "Decode token fail: {}", e.what());
+  }
+}
+
+Result<flat::Uid> decodeUidFromUserToken(std::string_view token) {
+  auto r = decodeUserToken(token);
+  RETURN_ON_ERROR(r);
+  auto [uid, ts] = *r;
+  return flat::Uid{uid};
+}
+
+CoTryTask<String> encodeUserToken(uint32_t uid, kv::IReadOnlyTransaction &txn) {
+  co_return encodeUserToken(uid, folly::Random::secureRand64());
+}
+}  // namespace hf3fs::core

src/core/user/UserToken.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "common/kv/ITransaction.h"
+#include "common/utils/Coroutine.h"
+#include "common/utils/Result.h"
+#include "fbs/core/user/User.h"
+
+namespace hf3fs::core {
+String encodeUserToken(uint32_t uid, uint64_t timestamp);
+CoTryTask<String> encodeUserToken(uint32_t uid, kv::IReadOnlyTransaction &txn);
+
+Result<std::pair<uint32_t, uint64_t>> decodeUserToken(std::string_view token);
+Result<flat::Uid> decodeUidFromUserToken(std::string_view token);
+}  // namespace hf3fs::core

src/core/utils/ServiceOperation.h

@@ -0,0 +1,101 @@
+#pragma once
+
+#include <fmt/format.h>
+#include <folly/experimental/symbolizer/Symbolizer.h>
+#include <folly/logging/xlog.h>
+
+#include "common/monitor/Recorder.h"
+#include "common/utils/NameWrapper.h"
+#include "common/utils/String.h"
+
+#define RETURN_AND_LOG_OP_ERR(...) RETURN_AND_LOG_OP_ERR_IMPL(return, __VA_ARGS__)
+#define CO_RETURN_AND_LOG_OP_ERR(...) RETURN_AND_LOG_OP_ERR_IMPL(co_return, __VA_ARGS__)
+
+#define RETURN_AND_LOG_OP_ERR_IMPL(RETURN, op, code, s, ...)            \
+  do {                                                                  \
+    auto _msg = fmt::format(s __VA_OPT__(, ) __VA_ARGS__);              \
+    LOG_OP_ERR((op), "error: {} {}", StatusCode::toString(code), _msg); \
+    RETURN makeError(code, _msg);                                       \
+  } while (false)
+
+#define LOG_OP_DBG(...) LOG_OP_IMPL(DBG, __VA_ARGS__)
+#define LOG_OP_INFO(...) LOG_OP_IMPL(INFO, __VA_ARGS__)
+#define LOG_OP_WARN(...) LOG_OP_IMPL(WARN, __VA_ARGS__)
+#define LOG_OP_ERR(...) LOG_OP_IMPL(ERR, __VA_ARGS__)
+#define LOG_OP_CRITICAL(...) LOG_OP_IMPL(CRITICAL, __VA_ARGS__)
+#define LOG_OP_FATAL(op, fmt, ...)                  \
+  XLOGF(FATAL,                                      \
+        "{} " fmt ". stack trace:\n{}",             \
+        (op).toString() __VA_OPT__(, ) __VA_ARGS__, \
+        folly::symbolizer::getStackTraceStr())
+
+#define LOG_OP_IMPL(LEVEL, op, fmt, ...) XLOGF(LEVEL, "{} " fmt, (op).toString() __VA_OPT__(, ) __VA_ARGS__)
+
+namespace hf3fs::core {
+struct ServiceOperation {
+  virtual ~ServiceOperation() = default;
+
+  String toString() const {
+    if (!cache) {
+      cache = fmt::format("[{}Op {} No.{}]", serviceNameImpl(), toStringImpl(), reqId);
+    }
+    return *cache;
+  }
+
+  template <typename T>
+  requires(std::is_base_of_v<ServiceOperation, T>) T &as() {
+    auto *p = dynamic_cast<T *>(this);
+    XLOG_IF(FATAL, !p, "invalid dynamic cast from ServiceOperation to {}", typeid(T).name());
+    return *p;
+  }
+
+  template <typename T>
+  requires(std::is_base_of_v<ServiceOperation, T>) const T &as() const {
+    auto *p = dynamic_cast<const T *>(this);
+    XLOG_IF(FATAL, !p, "invalid dynamic cast from ServiceOperation to {}", typeid(T).name());
+    return *p;
+  }
+
+ private:
+  virtual String serviceNameImpl() const = 0;
+  virtual String toStringImpl() const = 0;
+
+  static int64_t nextReqId() {
+    static std::atomic<int64_t> id{1};
+    return id.fetch_add(1, std::memory_order_acq_rel);
+  }
+
+  const int64_t reqId = nextReqId();
+  mutable std::optional<String> cache;
+};
+
+template <NameWrapper ServiceName, NameWrapper OpName, NameWrapper MetricNamePrefix>
+struct ServiceOperationWithMetric : ServiceOperation {
+  static constexpr std::string_view serviceName = ServiceName;
+  static constexpr std::string_view opName = OpName;
+  static constexpr std::string_view metricNamePrefix = MetricNamePrefix;
+
+  static_assert(!serviceName.empty());
+  static_assert(!opName.empty());
+  static_assert(!metricNamePrefix.empty());
+
+  static const monitor::TagSet &getTagSet() {
+    static auto tagSet = [] {
+      monitor::TagSet ts;
+      ts.addTag("instance", String(opName));
+      return ts;
+    }();
+    return tagSet;
+  }
+
+  static monitor::SimpleOperationRecorder::Guard startRecord() {
+    static monitor::SimpleOperationRecorder recorder(fmt::format("{}.{}", serviceName, metricNamePrefix),
+                                                     getTagSet(),
+                                                     /*recordErrorCode=*/true);
+    return recorder.record();
+  }
+
+  String serviceNameImpl() const final { return String(serviceName); }
+};
+
+}  // namespace hf3fs::core

src/core/utils/runOp.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "common/utils/UtcTime.h"
+
+#define CO_INVOKE_OP(NORMAL_PATH_LOG_LEVEL, op, peer, ...)                          \
+  LOG_OP_##NORMAL_PATH_LOG_LEVEL(op, "start execution. from:{}", peer);             \
+  auto guard = op.startRecord();                                                    \
+  auto result = co_await op.handle(__VA_ARGS__);                                    \
+  if (result.hasError()) {                                                          \
+    guard.reportWithCode(result.error().code());                                    \
+    LOG_OP_ERR(op, "failed: {}. latency: {}", result.error(), *guard.latency());    \
+    CO_RETURN_ERROR(result);                                                        \
+  } else {                                                                          \
+    guard.reportWithCode(StatusCode::kOK);                                          \
+    LOG_OP_##NORMAL_PATH_LOG_LEVEL(op, "succeeded. latency: {}", *guard.latency()); \
+    co_return result;                                                               \
+  }
+
+#define CO_INVOKE_OP_INFO(...) CO_INVOKE_OP(INFO, __VA_ARGS__)
+#define CO_INVOKE_OP_DBG(...) CO_INVOKE_OP(DBG, __VA_ARGS__)