DeepSeek/3FS
1
0
Fork 0
mirror of https://github.com/deepseek-ai/3FS synced 2025-06-26 18:16:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

cli: reuse env KVEngine in dump-*

Browse Source 
This allows us to issue multiple command in one process. The --fdb-cluster-file flag is removed to reuse the fdb config from config file.

Resolve the SIGFAULT if the specified fdb_cluster_file from flag does not exist.
This commit is contained in:
胡玮文
2025-03-11 00:57:25 +08:00
parent 3a30c53bf2
commit e26f5a901a
8 changed files with 17 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/client/cli/admin/DumpDirEntries.cc
View File

@@ -25,7 +25,6 @@ namespace {
auto getParser() {
argparse::ArgumentParser parser("dump-dentries");
parser.add_argument("-n", "--num-dentries-perfile").default_value(uint32_t{10'000'000}).scan<'u', uint32_t>();
parser.add_argument("-f", "--fdb-cluster-file").default_value(std::string{"./fdb.cluster"});
parser.add_argument("-d", "--dentry-dir").default_value(std::string{"dentries"});
parser.add_argument("-t", "--threads").default_value(uint32_t(4)).scan<'u', uint32_t>();
return parser;
@@ -37,14 +36,13 @@ CoTryTask<Dispatcher::OutputTable> dumpDirEntries(IEnv &ienv,
auto &env = dynamic_cast<AdminEnv &>(ienv);
ENSURE_USAGE(args.empty());
ENSURE_USAGE(env.mgmtdClientGetter);
ENSURE_USAGE(env.kvEngineGetter);
const auto &fdbClusterFile = parser.get<std::string>("fdb-cluster-file");
const auto &numEntriesPerFile = parser.get<uint32_t>("num-dentries-perfile");
const auto &dentryDir = parser.get<std::string>("dentry-dir");
const auto threads = parser.get<uint32_t>("threads");
ENSURE_USAGE(threads > 0);
ENSURE_USAGE(!fdbClusterFile.empty());
ENSURE_USAGE(!dentryDir.empty());
ENSURE_USAGE(numEntriesPerFile > 0);
@@ -55,13 +53,13 @@ CoTryTask<Dispatcher::OutputTable> dumpDirEntries(IEnv &ienv,
Dispatcher::OutputTable table;
auto dumpRes =
co_await dumpDirEntriesFromFdb(fdbClusterFile, numEntriesPerFile, dentryDir, std::max(uint32_t(1), threads));
co_await dumpDirEntriesFromFdb(env.kvEngineGetter(), numEntriesPerFile, dentryDir, std::max(uint32_t(1), threads));
if (!dumpRes) co_return makeError(dumpRes.error());
co_return table;
}
} // namespace
CoTryTask<Void> dumpDirEntriesFromFdb(const std::string fdbClusterFile,
CoTryTask<Void> dumpDirEntriesFromFdb(std::shared_ptr<kv::IKVEngine> kvEngine,
const uint32_t numEntriesPerDir,
const std::string dentryDir,
const uint32_t threads) {
@@ -77,10 +75,9 @@ CoTryTask<Void> dumpDirEntriesFromFdb(const std::string fdbClusterFile,
XLOGF(CRITICAL, "Saving directory entries to directory: {}", dentryDir);
meta::server::MetaScan::Options options;
options.fdb_cluster_file = fdbClusterFile;
options.threads = 8;
options.coroutines = 32;
auto scan = std::make_unique<meta::server::MetaScan>(options);
auto scan = std::make_unique<meta::server::MetaScan>(options, kvEngine);
auto exec = std::make_unique<folly::CPUThreadPoolExecutor>(16);
time_t timestamp = UtcClock::secondsSinceEpoch();

3
src/client/cli/admin/DumpDirEntries.h
View File

@@ -3,6 +3,7 @@
#include <cstdint>
#include <vector>
#include "common/kv/IKVEngine.h"
#include "common/serde/Serde.h"
#include "fbs/meta/Schema.h"
namespace hf3fs::client::cli {
@@ -21,7 +22,7 @@ struct DirEntryTable {
};
static_assert(serde::Serializable<DirEntryTable>);
CoTryTask<Void> dumpDirEntriesFromFdb(const std::string fdbClusterFile,
CoTryTask<Void> dumpDirEntriesFromFdb(std::shared_ptr<kv::IKVEngine> kvEngine,
const uint32_t numEntriesPerDir,
const std::string dentryDir,
const uint32_t threads);

11
src/client/cli/admin/DumpInodes.cc
View File

@@ -30,7 +30,6 @@ namespace {
auto getParser() {
argparse::ArgumentParser parser("dump-inodes");
parser.add_argument("-n", "--num-inodes-perfile").default_value(uint32_t{10'000'000}).scan<'u', uint32_t>();
parser.add_argument("-f", "--fdb-cluster-file").default_value(std::string{"./fdb.cluster"});
parser.add_argument("-i", "--inode-dir").default_value(std::string{"inodes"});
parser.add_argument("-q", "--parquet-format").default_value(false).implicit_value(true);
parser.add_argument("-a", "--all-inodes").default_value(false).implicit_value(true);
@@ -44,8 +43,8 @@ CoTryTask<Dispatcher::OutputTable> dumpInodes(IEnv &ienv,
auto &env = dynamic_cast<AdminEnv &>(ienv);
ENSURE_USAGE(args.empty());
ENSURE_USAGE(env.mgmtdClientGetter);
ENSURE_USAGE(env.kvEngineGetter);
const auto &fdbClusterFile = parser.get<std::string>("fdb-cluster-file");
const auto &numInodesPerFile = parser.get<uint32_t>("num-inodes-perfile");
const auto &inodeDir = parser.get<std::string>("inode-dir");
const auto &parquetFormat = parser.get<bool>("parquet-format");
@@ -53,7 +52,6 @@ CoTryTask<Dispatcher::OutputTable> dumpInodes(IEnv &ienv,
const auto &threads = parser.get<uint32_t>("threads");
ENSURE_USAGE(threads > 0);
ENSURE_USAGE(!fdbClusterFile.empty());
if (boost::filesystem::exists(inodeDir)) {
XLOGF(CRITICAL, "Output directory for inodes already exists: {}", inodeDir);
@@ -61,7 +59,7 @@ CoTryTask<Dispatcher::OutputTable> dumpInodes(IEnv &ienv,
}
Dispatcher::OutputTable table;
auto dumpRes = co_await dumpInodesFromFdb(fdbClusterFile,
auto dumpRes = co_await dumpInodesFromFdb(env.kvEngineGetter(),
numInodesPerFile,
inodeDir,
parquetFormat,
@@ -73,7 +71,7 @@ CoTryTask<Dispatcher::OutputTable> dumpInodes(IEnv &ienv,
} // namespace
CoTryTask<Void> dumpInodesFromFdb(const std::string fdbClusterFile,
CoTryTask<Void> dumpInodesFromFdb(std::shared_ptr<kv::IKVEngine> kvEngine,
const uint32_t numInodesPerFile,
const std::string inodeDir,
const bool parquetFormat,
@@ -90,10 +88,9 @@ CoTryTask<Void> dumpInodesFromFdb(const std::string fdbClusterFile,
XLOGF(CRITICAL, "Saving inodes to directory: {}", inodeDir);
meta::server::MetaScan::Options options;
options.fdb_cluster_file = fdbClusterFile;
options.threads = 8;
options.coroutines = 32;
auto scan = std::make_unique<meta::server::MetaScan>(options);
auto scan = std::make_unique<meta::server::MetaScan>(options, kvEngine);
auto exec = std::make_unique<folly::CPUThreadPoolExecutor>(16);
time_t timestamp = UtcClock::secondsSinceEpoch();

3
src/client/cli/admin/DumpInodes.h
View File

@@ -1,5 +1,6 @@
#pragma once
#include "common/kv/IKVEngine.h"
#include "common/serde/Serde.h"
#include "common/utils/Coroutine.h"
#include "fbs/meta/Schema.h"
@@ -25,7 +26,7 @@ static_assert(serde::Serializable<InodeTable>);
std::vector<Path> listFilesFromPath(const Path path);
CoTryTask<Void> dumpInodesFromFdb(const std::string fdbClusterFile,
CoTryTask<Void> dumpInodesFromFdb(std::shared_ptr<kv::IKVEngine> kvEngine,
const uint32_t numInodesPerFile,
const std::string inodeDir,
const bool parquetFormat = false,

5
src/client/cli/admin/RemoveChunks.cc
View File

@@ -16,7 +16,6 @@ namespace {
auto getParser() {
argparse::ArgumentParser parser("remove-chunks");
parser.add_argument("-n", "--num-inodes-perfile").default_value(uint32_t{10'000'000}).scan<'u', uint32_t>();
parser.add_argument("-f", "--fdb-cluster-file").default_value(std::string{"./fdb.cluster"});
parser.add_argument("-i", "--inode-dir").default_value(std::string{"inodes2"});
parser.add_argument("-o", "--orphaned-path").default_value(std::string{"orphaned"});
parser.add_argument("-r", "--do-remove").default_value(false).implicit_value(true);
@@ -30,11 +29,11 @@ CoTryTask<Dispatcher::OutputTable> removeChunks(IEnv &ienv,
auto &env = dynamic_cast<AdminEnv &>(ienv);
ENSURE_USAGE(args.empty());
ENSURE_USAGE(env.mgmtdClientGetter);
ENSURE_USAGE(env.kvEngineGetter);
Dispatcher::OutputTable table;
// dump latest inodes
const auto &fdbClusterFile = parser.get<std::string>("fdb-cluster-file");
const auto &numInodesPerFile = parser.get<uint32_t>("num-inodes-perfile");
const auto &inodeDir = parser.get<std::string>("inode-dir");
const auto &parquetFormat = parser.get<bool>("parquet-format");
@@ -44,7 +43,7 @@ CoTryTask<Dispatcher::OutputTable> removeChunks(IEnv &ienv,
co_return makeError(StatusCode::kInvalidArg);
}
auto dumpRes = co_await dumpInodesFromFdb(fdbClusterFile, numInodesPerFile, inodeDir, parquetFormat);
auto dumpRes = co_await dumpInodesFromFdb(env.kvEngineGetter(), numInodesPerFile, inodeDir, parquetFormat);
if (!dumpRes) co_return makeError(dumpRes.error());
// load the inode dump