#pragma once #include #include #include "client/mgmtd/MgmtdClientForClient.h" #include "client/mgmtd/MgmtdClientForServer.h" #include "client/storage/StorageClient.h" #include "fbs/mgmtd/MgmtdTypes.h" #include "fbs/mgmtd/RoutingInfo.h" #include "storage/service/StorageServer.h" #include "tests/GtestHelpers.h" #include "tests/client/ClientWithConfig.h" #include "tests/client/ServerWithConfig.h" #include "tests/mgmtd/MgmtdTestHelper.h" namespace hf3fs::test { class FakeMgmtdClient : public hf3fs::client::IMgmtdClientForClient, public hf3fs::client::IMgmtdClientForServer { public: FakeMgmtdClient(std::shared_ptr routingInfo) { setRoutingInfo(routingInfo); } std::shared_ptr getRoutingInfo() override { return routingInfo_.load(); } void setRoutingInfo(std::shared_ptr routingInfo) { routingInfo_.store(std::make_shared(routingInfo, SteadyClock::now())); for (auto &listener : listeners_) { listener(routingInfo_.load()); } } CoTryTask refreshRoutingInfo(bool) override { co_return Void{}; } CoTryTask heartbeat() override { co_return makeError(StatusCode::kNotImplemented); } Result triggerHeartbeat() override { return makeError(StatusCode::kNotImplemented); } CoTryTask> getConfig(flat::NodeType, flat::ConfigVersion) override { co_return makeError(StatusCode::kNotImplemented); } CoTryTask> getConfigVersions() override { co_return makeError(StatusCode::kNotImplemented); } CoTryTask> getUniversalTags(const String &universalId) override { co_return makeError(StatusCode::kNotImplemented); } void setAppInfoForHeartbeat(flat::AppInfo) override {} bool addRoutingInfoListener(String, RoutingInfoListener listener) override { listeners_.push_back(listener); return true; } bool removeRoutingInfoListener(std::string_view) override { return true; } void setConfigListener(ConfigListener) override {} void updateHeartbeatPayload(HeartbeatPayload) override {} CoTryTask extendClientSession() override { co_return makeError(StatusCode::kNotImplemented); } void setClientSessionPayload(ClientSessionPayload) override {} CoTryTask listClientSessions() override { co_return makeError(StatusCode::kNotImplemented); } CoTryTask getClientSession(const String &) override { co_return makeError(StatusCode::kNotImplemented); } // The diamond inheritance forbids directly upcasting from FakeMgmtdClient to ICommonMgmtdClient hf3fs::client::ICommonMgmtdClient *asCommon() { hf3fs::client::IMgmtdClientForServer *s = this; return s; } private: folly::atomic_shared_ptr routingInfo_; std::vector listeners_; }; struct SystemSetupConfig : public ConfigBase { CONFIG_ITEM(chunk_size, 128_KB); CONFIG_ITEM(num_chains, 1u); CONFIG_ITEM(num_replicas, 1u); CONFIG_ITEM(num_storage_nodes, 1u); CONFIG_ITEM(data_paths, (std::vector{})); CONFIG_ITEM(client_config, hf3fs::Path()); CONFIG_ITEM(server_config, hf3fs::Path()); CONFIG_ITEM(storage_endpoints, (std::map{})); CONFIG_ITEM(service_level, 0u); CONFIG_ITEM(listen_port, 0u); CONFIG_ITEM(client_impl_type, storage::client::StorageClient::ImplementationType::RPC); CONFIG_ITEM(meta_store_type, kv::KVStore::Type::LevelDB); CONFIG_ITEM(use_fake_mgmtd_client, true); CONFIG_ITEM(start_storage_server, true); CONFIG_ITEM(use_temp_path, true); public: std::vector getChunkSizeList() const { auto chunkSize = chunk_size(); std::vector chunkSizeList; for (auto size : std::vector{chunkSize / 4, chunkSize / 2, chunkSize, 2 * chunkSize, 4 * chunkSize}) { if (size >= 512 && (size & size - 1) == 0 /*pow of 2*/) { chunkSizeList.push_back(size); } } return chunkSizeList; } public: SystemSetupConfig() = default; SystemSetupConfig(uint32_t chunkSize, uint32_t numChains, uint32_t numReplicas, uint32_t numStorageNodes, std::vector dataPaths, hf3fs::Path clientConfig = hf3fs::Path(), hf3fs::Path serverConfig = hf3fs::Path(), std::map storageEndpoints = {}, uint32_t serviceLevel = 0, uint32_t listenPort = 0, storage::client::StorageClient::ImplementationType clientImplType = storage::client::StorageClient::ImplementationType::RPC, kv::KVStore::Type metaStoreType = kv::KVStore::Type::LevelDB, bool useFakeMgmtdClient = true, bool startStorageServer = true, bool useTempPath = true) { set_chunk_size(chunkSize); set_num_chains(numChains); set_num_replicas(numReplicas); set_num_storage_nodes(numStorageNodes); set_data_paths(dataPaths); set_client_config(clientConfig); set_server_config(serverConfig); std::map map{}; for (auto &[nodeId, addr] : storageEndpoints) { map[std::to_string(nodeId)] = addr; } set_storage_endpoints(map); set_service_level(serviceLevel); set_listen_port(listenPort); set_client_impl_type(clientImplType); set_meta_store_type(metaStoreType); set_use_fake_mgmtd_client(useFakeMgmtdClient); set_start_storage_server(startStorageServer); set_use_temp_path(useTempPath); } static std::string prettyPrintConfig(const testing::TestParamInfo &info) { return fmt::format("{}replicas_{}nodes_client{}_metastore{}_fakemgmtd{}", info.param.num_replicas(), info.param.num_storage_nodes(), static_cast(info.param.client_impl_type()), static_cast(info.param.meta_store_type()), info.param.use_fake_mgmtd_client()); } }; using NodeTargetMap = std::map>; class UnitTestFabric { public: static NodeTargetMap buildNodeTargetMap(uint32_t numNodes, uint32_t numTargetsPerNode); /* Suppose numChains = 4, numReplicas = 3 and nodeTargets is a 2-d array: ---------------------------------------------- node1 | target1001 | target1002 | target1003 node2 | target2001 | target2002 | target2003 node3 | target3001 | target3002 | target3003 node4 | target4001 | target4002 | target4003 ---------------------------------------------- Then this function generates the following routing table: ----------------------------------------------- chain1 | target1001 | target2001 | target3001 chain2 | target4001 | target1002 | target2002 chain3 | target3002 | target4002 | target1003 chain4 | target2003 | target3003 | target4003 ----------------------------------------------- */ static std::map buildRepliaChainMap(uint32_t numChains, uint32_t numReplicas, NodeTargetMap nodeTargets); static std::shared_ptr createRoutingInfo( const std::map &replicaChains, const std::map &nodeEndpoints); /* Set up and tear down */ public: UnitTestFabric(const SystemSetupConfig &setupConfig) : setupConfig_(setupConfig), requestExe_(std::max(1U, std::thread::hardware_concurrency() / 2), std::make_shared("UnitTestFabric")) {} bool setUpStorageSystem(); void tearDownStorageSystem(); std::unique_ptr createStorageServer(size_t nodeIndex); std::shared_ptr getRoutingInfo(); bool updateRoutingInfo(std::function callback); auto copyRoutingInfo() { return std::make_shared(*rawRoutingInfo_); } void setTargetOffline(hf3fs::flat::RoutingInfo &routingInfo, uint32_t targetIndex, uint32_t chainIndex = 0); bool stopAndRemoveStorageServer(uint32_t serverIndex); bool stopAndRemoveStorageServer(storage::NodeId nodeId); /* Helper functions */ storage::IOResult writeToChunk(storage::ChainId chainId, storage::ChunkId chunkId, std::span chunkData, uint32_t offset = 0, uint32_t length = 0, const storage::client::WriteOptions &options = storage::client::WriteOptions()); bool writeToChunks(storage::ChainId chainId, storage::ChunkId chunkBegin, storage::ChunkId chunkEnd, std::span chunkData, uint32_t offset = 0, uint32_t length = 0, const storage::client::WriteOptions &options = storage::client::WriteOptions(), std::vector *results = nullptr); storage::IOResult readFromChunk(storage::ChainId chainId, storage::ChunkId chunkId, std::span chunkData, uint32_t offset = 0, uint32_t length = 0, const storage::client::ReadOptions &options = storage::client::ReadOptions()); bool readFromChunks(storage::ChainId chainId, storage::ChunkId chunkBegin, storage::ChunkId chunkEnd, std::vector> &chunkData, uint32_t offset = 0, uint32_t length = 0, const storage::client::ReadOptions &options = storage::client::ReadOptions(), std::vector *results = nullptr); protected: // the id of the only table static flat::ChainTableId kTableId() { static const auto v = flat::ChainTableId(1); return v; } // the initial routing table version static const flat::RoutingInfoVersion kRoutingVersion() { static const auto v = flat::RoutingInfoVersion(10); return v; } // the initial chain version static const storage::ChainVer kInitChainVer() { static const auto v = storage::ChainVer(1); return v; } constexpr static auto kClusterId = "test"; char const *const kTempDataFolderPrefix = "temp_storage_data"; ClientId clientId_ = ClientId::random(); storage::ChainId firstChainId_; std::vector chainIds_; NodeTargetMap nodeTargets_; std::vector storageNodeIds_; std::map replicaChains_; std::shared_ptr rawRoutingInfo_; std::vector> tmpDataPaths_; std::vector targetsConfigs_; std::vector serverConfigs_; std::vector> storageServers_; MgmtdServerWithConfig mgmtdServer_{kClusterId, flat::NodeId(10000)}; SystemSetupConfig setupConfig_; storage::client::StorageClient::Config clientConfig_; net::Client::Config netClientConfig_; net::Client client_{netClientConfig_}; hf3fs::client::MgmtdClientForClient::Config mgmtdClientConfig_; std::unique_ptr mgmtdForClient_; std::shared_ptr storageClient_; folly::CPUThreadPoolExecutor requestExe_; }; } // namespace hf3fs::test