Initial commit

src/tools/CMakeLists.txt

@@ -0,0 +1,3 @@
+target_add_bin(hf3fs-admin "admin.cc" admin-commands)
+
+add_subdirectory(commands)

src/tools/admin.cc

@@ -0,0 +1,99 @@
+#include <folly/experimental/coro/BlockingWait.h>
+#include <folly/logging/xlog.h>
+
+#include "client/meta/MetaClient.h"
+#include "client/mgmtd/MgmtdClientForClient.h"
+#include "common/logging/LogInit.h"
+#include "common/net/Client.h"
+#include "common/net/ib/IBDevice.h"
+#include "common/utils/ConfigBase.h"
+#include "common/utils/SysResource.h"
+#include "stubs/common/RealStubFactory.h"
+#include "stubs/mgmtd/MgmtdServiceStub.h"
+#include "tools/commands/Commands.h"
+
+DEFINE_bool(set_dir_layout, false, "Set default file layout for given directory");
+DEFINE_bool(create_with_layout, false, "Create file/directory with given file layout");
+DEFINE_bool(as_super, false, "Execute commands as super user");
+
+using namespace hf3fs;
+using namespace hf3fs::client;
+using namespace hf3fs::flat;
+using namespace hf3fs::stubs;
+using namespace hf3fs::tools;
+using hf3fs::meta::client::MetaClient;
+
+class Config : public ConfigBase<Config> {
+  CONFIG_OBJ(meta, MetaClient::Config);
+  CONFIG_OBJ(mgmtd, MgmtdClientForClient::Config);
+
+  CONFIG_OBJ(client, net::Client::Config);
+  CONFIG_OBJ(ib_devices, net::IBDevice::Config);
+
+  CONFIG_ITEM(cluster_id, "");
+};
+
+class TokenConfig : public ConfigBase<TokenConfig> {
+  CONFIG_ITEM(token, "");
+};
+
+int main(int argc, char **argv) {
+  google::SetCommandLineOptionWithMode("logging",
+                                       "CRITICAL:out:err; out=stream:stream=stdout; err=stream:stream=stderr,level=ERR",
+                                       google::SET_FLAGS_DEFAULT);
+
+  auto home = std::string(getenv("HOME"));
+  XLOGF(DBG, "home dir {}", home);
+  auto cfg = home + "/.hf3fs/admin.toml";
+  google::SetCommandLineOptionWithMode("cfg", cfg.c_str(), google::SET_FLAGS_DEFAULT);
+
+  Config config;
+  auto initResult = config.init(&argc, &argv);
+  XLOGF_IF(FATAL, !initResult, "Load config from flags failed with {}", initResult.error().describe());
+
+  auto ibResult = hf3fs::net::IBManager::start(config.ib_devices());
+  XLOGF_IF(FATAL, !ibResult, "Failed to start IBManager: {}.", ibResult.error().describe());
+  SCOPE_EXIT { hf3fs::net::IBManager::stop(); };
+
+  net::Client client(config.client());
+  client.start();
+
+  auto makeCtx = [&client](net::Address addr) { return client.serdeCtx(addr); };
+  // TODO: fill in real client id
+  auto mgmtdClient =
+      std::make_shared<MgmtdClientForClient>(config.cluster_id(),
+                                             std::make_unique<RealStubFactory<mgmtd::MgmtdServiceStub>>(makeCtx),
+                                             config.mgmtd());
+  auto physicalHostnameRes = SysResource::hostname(/*physicalMachineName=*/true);
+  XLOGF_IF(FATAL, !physicalHostnameRes, "Get physical hostname failed: {}", physicalHostnameRes.error());
+
+  auto containerHostnameRes = SysResource::hostname(/*physicalMachineName=*/false);
+  XLOGF_IF(FATAL, !containerHostnameRes, "Get container hostname failed: {}", containerHostnameRes.error());
+
+  auto clientId = ClientId::random(*physicalHostnameRes);
+  auto makeSerdeCtx = [&client](net::Address addr) { return client.serdeCtx(addr); };
+  auto metaClient = std::make_unique<MetaClient>(clientId,
+                                                 config.meta(),
+                                                 std::make_unique<MetaClient::StubFactory>(makeSerdeCtx),
+                                                 mgmtdClient,
+                                                 nullptr,
+                                                 false);
+
+  folly::coro::blockingWait(mgmtdClient->start(&client.tpg().bgThreadPool().randomPick()));
+
+  TokenConfig tokenConfig;
+  tokenConfig.atomicallyUpdate(Path(home + "/.hf3fs/admin_token.toml"), false);
+
+  UserInfo ui(Uid(FLAGS_as_super ? 0 : getuid()),
+              Gid(FLAGS_as_super ? 0 : getgid()),
+              {} /* groups */,
+              tokenConfig.token());
+
+  XLOGF(DBG, "uid {} gid {}", ui.uid, ui.gid);
+
+  if (FLAGS_set_dir_layout) {
+    setDirLayout(*metaClient, ui);
+  } else if (FLAGS_create_with_layout) {
+    createWithLayout(*metaClient, ui);
+  }
+}

src/tools/commands/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_lib(admin-commands meta-client mgmtd-client)

src/tools/commands/Commands.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include "client/meta/MetaClient.h"
+
+namespace hf3fs::tools {
+void setDirLayout(meta::client::MetaClient &metaClient, const flat::UserInfo &ui);
+void createWithLayout(meta::client::MetaClient &metaClient, const flat::UserInfo &ui);
+}  // namespace hf3fs::tools

src/tools/commands/CreateWithLayout.cc

@@ -0,0 +1,29 @@
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "Commands.h"
+#include "Layout.h"
+
+DEFINE_string(path, "", "Path of file/directory to set layout");
+DEFINE_bool(create_dir, false, "True to create a directory, false to create a file");
+DEFINE_bool(recursive, false, "If to recursively create intermediate directories when creating a directory");
+
+namespace hf3fs::tools {
+void createWithLayout(meta::client::MetaClient &metaClient, const flat::UserInfo &ui) {
+  auto layout = layoutFromFlags();
+  if (FLAGS_create_dir) {
+    auto res = folly::coro::blockingWait(
+        metaClient
+            .mkdirs(ui, meta::InodeId::root(), Path(FLAGS_path), meta::Permission(0755), FLAGS_recursive, layout));
+    XLOGF_IF(FATAL, !res, "failed to create directory {} with layout error {}", FLAGS_path, res.error().describe());
+  } else {
+    auto res = folly::coro::blockingWait(metaClient.create(ui,
+                                                           meta::InodeId::root(),
+                                                           Path(FLAGS_path),
+                                                           std::nullopt,
+                                                           meta::Permission(0644),
+                                                           O_CREAT | O_EXCL | O_RDONLY,
+                                                           layout));
+    XLOGF_IF(FATAL, !res, "failed to create file {} with layout error {}", FLAGS_path, res.error().describe());
+  }
+}
+}  // namespace hf3fs::tools

src/tools/commands/Layout.cc

@@ -0,0 +1,38 @@
+#include "Layout.h"
+
+DEFINE_int32(chain_table_id, 0, "Chain table id for the file layout");
+DEFINE_int32(chain_table_ver, 0, "Chain table for the file layout");
+DEFINE_string(chunk_size, "0", "Chunk size for the file layout");
+DEFINE_int32(stripe_size, 0, "Stripe size for the file layout");
+DEFINE_string(chain_index_list, "", "List of chain indexList for the file layout");
+
+namespace hf3fs::tools {
+meta::Layout layoutFromFlags() {
+  auto chainTable = flat::ChainTableId(FLAGS_chain_table_id);
+  auto chainTableVersion = flat::ChainTableVersion(FLAGS_chain_table_ver);
+  auto cs = Size::from(FLAGS_chunk_size);
+  XLOGF_IF(FATAL, !cs, "invalid chunk size {}", FLAGS_chunk_size);
+  auto chunkSize = (uint64_t)*cs;
+  if (FLAGS_chain_index_list.empty()) {
+    return meta::Layout::newEmpty(chainTable, chunkSize, FLAGS_stripe_size);
+  } else {
+    std::vector<std::string> idxs;
+    folly::split(',', FLAGS_chain_index_list, idxs, true);
+    std::vector<uint32_t> chains;
+    chains.reserve(idxs.size());
+    for (auto &&idx : idxs) {
+      char *p;
+      auto cidx = strtoul(idx.c_str(), &p, 10);
+      XLOGF_IF(FATAL, !cidx && *p, "invalid chain index {}, the whole list {}", cidx, FLAGS_chain_index_list);
+      XLOGF(DBG, "chain index without table id {}", cidx);
+      chains.push_back(cidx);
+    }
+
+    for (auto cid : chains) {
+      XLOGF(DBG, "chain indexList {}", cid);
+    }
+
+    return meta::Layout::newChainList(chainTable, chainTableVersion, chunkSize, chains);
+  }
+}
+}  // namespace hf3fs::tools

src/tools/commands/Layout.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <folly/String.h>
+
+#include "fbs/meta/Schema.h"
+
+namespace hf3fs::tools {
+meta::Layout layoutFromFlags();
+}  // namespace hf3fs::tools

src/tools/commands/SetDirLayout.cc

@@ -0,0 +1,14 @@
+#include <folly/experimental/coro/BlockingWait.h>
+
+#include "Commands.h"
+#include "Layout.h"
+
+DEFINE_string(dir_path, "", "Path of directory to set default layout");
+
+namespace hf3fs::tools {
+void setDirLayout(meta::client::MetaClient &metaClient, const flat::UserInfo &ui) {
+  auto layout = layoutFromFlags();
+  auto res = folly::coro::blockingWait(metaClient.setLayout(ui, meta::InodeId::root(), Path(FLAGS_dir_path), layout));
+  XLOGF_IF(FATAL, !res, "failed to set layout of directory {} error {}", FLAGS_dir_path, res.error().describe());
+}
+}  // namespace hf3fs::tools