DeepSeek/3FS
1
0
Fork 0
mirror of https://github.com/deepseek-ai/3FS synced 2025-06-26 18:16:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
dev
2025-02-27 21:53:53 +08:00
commit 815e55e4c0
1291 changed files with 185445 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/storage/service/CMakeLists.txt Normal file
View File

@@ -0,0 +1 @@
target_add_test(test_storage_service test-fabric-lib)

80
tests/storage/service/TestDumpMeta.cc Normal file
View File

@@ -0,0 +1,80 @@
#include <boost/filesystem/operations.hpp>
#include <thread>
#include "common/serde/Serde.h"
#include "common/utils/FileUtils.h"
#include "kv/KVStore.h"
#include "kv/MemDBStore.h"
#include "storage/service/StorageServer.h"
#include "tests/GtestHelpers.h"
#include "tests/lib/Helper.h"
#include "tests/lib/UnitTestFabric.h"
namespace hf3fs::storage::test {
namespace {
using namespace hf3fs::test;
class TestDumpMeta : public UnitTestFabric, public ::testing::Test {
protected:
TestDumpMeta()
: UnitTestFabric(SystemSetupConfig{128_KB /*chunkSize*/,
1 /*numChains*/,
1 /*numReplicas*/,
1 /*numStorageNodes*/,
{folly::fs::temp_directory_path()} /*dataPaths*/,
hf3fs::Path() /*clientConfig*/,
hf3fs::Path() /*serverConfig*/,
{} /*storageEndpoints*/,
0 /*serviceLevel*/,
0 /*listenPort*/,
client::StorageClient::ImplementationType::RPC,
kv::KVStore::Type::RocksDB,
false,
true}) {}
void SetUp() override {
// init ib device
net::IBDevice::Config ibConfig;
auto ibResult = net::IBManager::start(ibConfig);
ASSERT_OK(ibResult);
ASSERT_TRUE(setUpStorageSystem());
}
void TearDown() override { tearDownStorageSystem(); }
};
TEST_F(TestDumpMeta, Normal) {
auto chunkId = ChunkId{0u, 0u};
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
auto writeRes = writeToChunk(chainIds_.front(), chunkId, chunkData);
ASSERT_OK(writeRes.lengthInfo);
folly::test::TemporaryDirectory tmpPath;
serverConfigs_[0].dump_worker().set_dump_root_path(tmpPath.path());
serverConfigs_[0].dump_worker().set_dump_interval(100_ms);
std::this_thread::sleep_for(2_s);
stopAndRemoveStorageServer(0);
std::vector<Path> files;
for (auto &filePath : boost::filesystem::recursive_directory_iterator(tmpPath.path())) {
if (boost::filesystem::is_regular_file(filePath)) {
files.push_back(filePath);
}
}
ASSERT_EQ(files.size(), 1);
XLOGF(CRITICAL, "dump files: {}", serde::toJsonString(files));
auto readResult = loadFile(files.front());
ASSERT_OK(readResult);
std::map<ChunkId, ChunkMetadata> metas;
ASSERT_TRUE(serde::deserialize(metas, *readResult));
ASSERT_EQ(metas.size(), 1);
ASSERT_EQ(metas.begin()->first, chunkId);
XLOGF(CRITICAL, "dump metas: {}", serde::toJsonString(metas));
}
} // namespace
} // namespace hf3fs::storage::test

209
tests/storage/service/TestIncorrectRoutingInfo.cc Normal file
View File

@@ -0,0 +1,209 @@
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/Collect.h>
#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/storage/StorageClient.h"
#include "common/net/Client.h"
#include "tests/lib/Helper.h"
#include "tests/lib/UnitTestFabric.h"
namespace hf3fs::storage {
namespace {
using namespace hf3fs::test;
class TestIncorrectRoutingInfo : public UnitTestFabric, public ::testing::Test {
protected:
TestIncorrectRoutingInfo()
: UnitTestFabric(SystemSetupConfig{
128_KB /*chunkSize*/,
1 /*numChains*/,
3 /*numReplicas*/,
3 /*numStorageNodes*/,
{folly::fs::temp_directory_path()} /*dataPaths*/,
hf3fs::Path() /*clientConfig*/,
hf3fs::Path() /*serverConfig*/,
{} /*storageEndpoints*/,
0 /*serviceLevel*/,
0 /*listenPort*/,
client::StorageClient::ImplementationType::RPC,
kv::KVStore::Type::RocksDB,
true /*useFakeMgmtdClient*/,
}) {}
void SetUp() override {
// init ib device
net::IBDevice::Config ibConfig;
auto ibResult = net::IBManager::start(ibConfig);
ASSERT_OK(ibResult);
ASSERT_TRUE(setUpStorageSystem());
clientConfig_.retry().set_max_retry_time(10_s);
}
void TearDown() override { tearDownStorageSystem(); }
};
TEST_F(TestIncorrectRoutingInfo, MismatchChainVersion) {
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
// create a test chunk
auto ioResult = writeToChunk(chainId, chunkId, chunkData);
ASSERT_RESULT_EQ(chunkData.size(), ioResult.lengthInfo);
// get the first chain
auto newRoutingInfo = copyRoutingInfo();
auto &chainTable = *newRoutingInfo->getChainTable(kTableId());
auto &firstChain = newRoutingInfo->chains[chainTable.chains.front()];
ASSERT_EQ(chainId, firstChain.chainId);
auto chainVersion = firstChain.chainVersion;
for (int32_t deltaVer : {-1, 1}) {
// change version of the first chain
firstChain.chainVersion = flat::ChainVersion(chainVersion + deltaVer);
// only update client's routing info
auto fakeClient = dynamic_cast<FakeMgmtdClient *>(mgmtdForClient_.get());
newRoutingInfo->routingInfoVersion++;
fakeClient->setRoutingInfo(newRoutingInfo);
ASSERT_OK(folly::coro::blockingWait(mgmtdForClient_->refreshRoutingInfo(/*force=*/false)));
// try to read/write with mismatch chain version
auto readRes = readFromChunk(chainId, chunkId, chunkData, 0, chunkData.size());
ASSERT_ERROR(readRes.lengthInfo, StorageClientCode::kRoutingVersionMismatch);
auto writeRes = writeToChunk(chainId, chunkId, chunkData, 0, chunkData.size());
ASSERT_ERROR(writeRes.lengthInfo, StorageClientCode::kRoutingVersionMismatch);
}
}
TEST_F(TestIncorrectRoutingInfo, WriteNotSentToHeadTarget) {
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
// send write to the second target on chain
client::WriteOptions options;
options.targetSelection().set_mode(client::TargetSelectionMode::ManualMode);
options.targetSelection().set_targetIndex(1);
auto ioResult = writeToChunk(chainId, chunkId, chunkData, 0, chunkData.size(), options);
ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
}
TEST_F(TestIncorrectRoutingInfo, RemoveNotSentToHeadTarget) {
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
// create a test chunk
auto ioResult = writeToChunk(chainId, chunkId, chunkData);
ASSERT_RESULT_EQ(chunkData.size(), ioResult.lengthInfo);
// send remove to the second target on chain
client::WriteOptions options;
options.targetSelection().set_mode(client::TargetSelectionMode::ManualMode);
options.targetSelection().set_targetIndex(1);
auto removeOp = storageClient_->createRemoveOp(chainId,
ChunkId(chunkId, 0),
ChunkId(chunkId, 0xFF),
1 /*maxNumChunkIdsToProcess*/);
folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo(), options));
ASSERT_ERROR(removeOp.result.statusCode, StorageClientCode::kRoutingError);
}
TEST_F(TestIncorrectRoutingInfo, WorkingHeadDetectedAsOffline) {
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
ASSERT_OK(regRes);
auto ioBuffer = std::move(*regRes);
size_t numWriteIOs = 100;
std::vector<client::WriteIO> writeIOs;
// create write IOs to one chunk
for (size_t writeIndex = 0; writeIndex < numWriteIOs; writeIndex++) {
auto writeIO = storageClient_->createWriteIO(chainId,
chunkId,
0 /*offset*/,
chunkData.size() /*length*/,
setupConfig_.chunk_size() /*chunkSize*/,
&chunkData[0],
&ioBuffer);
writeIOs.push_back(std::move(writeIO));
}
// issue the write requests
flat::UserInfo dummyUserInfo{};
auto options = client::WriteOptions();
std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> writeTasks;
for (auto &writeIO : writeIOs) {
auto writeTask = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
writeTasks.push_back(std::move(writeTask));
}
// set head target to offline state but it's actually working
auto newRoutingInfo = copyRoutingInfo();
setTargetOffline(*newRoutingInfo, 0);
// let all storage servers except the host of head targets know the latest routing info
for (size_t serverIndex = 1; serverIndex < storageServers_.size(); serverIndex++) {
auto &storageServer = storageServers_[serverIndex];
auto client = RoutingStoreHelper::getMgmtdClient(*storageServer);
auto fakeClient = dynamic_cast<FakeMgmtdClient *>(client.get());
fakeClient->setRoutingInfo(newRoutingInfo);
RoutingStoreHelper::refreshRoutingInfo(*storageServer);
}
// client does not konw the head is marked offline yet, so let it retry for a short time
std::this_thread::sleep_for(500_ms);
#if defined(__has_feature)
#if !__has_feature(thread_sanitizer)
for (auto &writeIO : writeIOs) {
// some writes are completed but others are still under retry
if (writeIO.result.lengthInfo) {
ASSERT_RESULT_EQ(writeIO.length, writeIO.result.lengthInfo);
} else {
switch (writeIO.statusCode()) {
case StorageClientCode::kCommError:
case StorageClientCode::kTimeout:
case StorageClientCode::kRoutingVersionMismatch:
XLOGF(INFO, "Write IO {} length info: {}", fmt::ptr(&writeIO), writeIO.result.lengthInfo);
break;
default:
ASSERT_EQ(StorageClientCode::kNotInitialized, writeIO.result.lengthInfo.error().code())
<< fmt::format("Write IO {} length info: {}", fmt::ptr(&writeIO), writeIO.result.lengthInfo);
}
}
}
#endif
#endif
// let client know the latest routing info
auto fakeClient = dynamic_cast<FakeMgmtdClient *>(mgmtdForClient_.get());
fakeClient->setRoutingInfo(newRoutingInfo);
ASSERT_OK(folly::coro::blockingWait(mgmtdForClient_->refreshRoutingInfo(/*force=*/false)));
// wait until all write tasks completed
folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));
// check all write IOs have succeeded
std::set<ChunkVer> updateVersions;
for (const auto &writeIO : writeIOs) {
ASSERT_OK(writeIO.result.lengthInfo);
// check commit version == update version
ASSERT_EQ(writeIO.result.commitVer, writeIO.result.updateVer);
// check the update versions are in range [1..numWriteIOs]
ASSERT_LE(1, writeIO.result.updateVer);
ASSERT_LE(writeIO.result.updateVer, numWriteIOs);
updateVersions.insert(writeIO.result.updateVer);
}
// check the update versions are unique
ASSERT_EQ(numWriteIOs, updateVersions.size());
}
} // namespace
} // namespace hf3fs::storage

679
tests/storage/service/TestSingleProcessCluster.cc Normal file
View File

@@ -0,0 +1,679 @@
#include <filesystem>
#include <folly/experimental/coro/Collect.h>
#include "common/serde/Serde.h"
#include "common/utils/FileUtils.h"
#include "common/utils/SysResource.h"
#include "tests/GtestHelpers.h"
#include "tests/lib/Helper.h"
#include "tests/lib/UnitTestFabric.h"
namespace hf3fs::storage {
namespace {
using namespace hf3fs::test;
class TestSingleProcessCluster : public UnitTestFabric, public ::testing::TestWithParam<SystemSetupConfig> {
protected:
TestSingleProcessCluster()
: UnitTestFabric(GetParam()) {}
void SetUp() override {
#if defined(__has_feature)
#if __has_feature(thread_sanitizer)
GTEST_SKIP();
#endif
#endif
// init ib device
net::IBDevice::Config ibConfig;
auto ibResult = net::IBManager::start(ibConfig);
ASSERT_OK(ibResult);
ASSERT_TRUE(setUpStorageSystem());
for (auto nodeId : storageNodeIds_) {
waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED);
waitStorageTargetStatus(nodeTargets_[nodeId], flat::LocalTargetState::UPTODATE, flat::PublicTargetState::SERVING);
}
mgmtdServer_.config.service().set_check_status_interval(200_ms);
mgmtdServer_.config.service().set_heartbeat_fail_interval(1_s);
clientConfig_.retry().set_init_wait_time(200_ms);
clientConfig_.retry().set_max_wait_time(5_s);
clientConfig_.retry().set_max_retry_time(60_s);
ASSERT_LT(clientConfig_.retry().init_wait_time(), clientConfig_.retry().max_wait_time());
}
void TearDown() override { tearDownStorageSystem(); }
void checkStorageNodeStatus(const std::vector<flat::NodeId> &nodeIds, flat::NodeStatus status) {
for (const auto &nodeId : nodeIds) {
ASSERT_TRUE(nodeTargets_.count(nodeId));
auto nodeInfo = rawRoutingInfo_->getNode(nodeId);
ASSERT_NE(nullptr, nodeInfo);
ASSERT_EQ(status, nodeInfo->status);
}
}
void checkStorageTargetStatus(const std::vector<flat::TargetId> &targetIds,
flat::LocalTargetState localStatus,
flat::PublicTargetState publicStatus) {
for (const auto &targetId : targetIds) {
auto targetInfo = rawRoutingInfo_->getTarget(targetId);
ASSERT_NE(nullptr, targetInfo);
ASSERT_EQ(localStatus, targetInfo->localState);
ASSERT_EQ(publicStatus, targetInfo->publicState);
}
}
void waitStorageNodeStatus(const std::vector<flat::NodeId> &nodeIds,
flat::NodeStatus status,
size_t maxRetries = 200) {
flat::NodeInfo *nodeInfo = nullptr;
XLOGF(INFO,
"# Waiting storage nodes {} moving to status {}",
serde::toJsonString(nodeIds),
magic_enum::enum_name(status));
for (size_t retry = 0; retry < maxRetries; retry++) {
if (retry > 0) std::this_thread::sleep_for(mgmtdServer_.config.service().check_status_interval());
auto routingInfo = getRoutingInfo();
if (routingInfo) {
bool allMatch = true;
for (const auto &nodeId : nodeIds) {
ASSERT_TRUE(nodeTargets_.count(nodeId));
nodeInfo = routingInfo->getNode(nodeId);
ASSERT_NE(nullptr, nodeInfo);
if (nodeInfo->status != status) {
XLOGF(INFO,
"#{} Waiting storage node {} moving to status {}, current status {}, routing info: {}",
retry,
nodeId,
magic_enum::enum_name(status),
magic_enum::enum_name(nodeInfo->status),
routingInfo->routingInfoVersion);
allMatch = false;
break;
}
}
if (allMatch) return;
}
}
ASSERT_NE(nullptr, nodeInfo);
ASSERT_EQ(status, nodeInfo->status) << "node id " << nodeInfo->app.nodeId;
}
void waitStorageTargetStatus(const std::vector<flat::TargetId> &targetIds,
flat::LocalTargetState localStatus,
flat::PublicTargetState publicStatus,
size_t maxRetries = 500) {
flat::TargetInfo *targetInfo = nullptr;
XLOGF(INFO,
"# wait storage targets {} moving to status {}/{}",
serde::toJsonString(targetIds),
magic_enum::enum_name(localStatus),
magic_enum::enum_name(publicStatus));
for (size_t retry = 0; retry < maxRetries; retry++) {
if (retry > 0) std::this_thread::sleep_for(mgmtdServer_.config.service().check_status_interval());
auto routingInfo = getRoutingInfo();
if (routingInfo) {
bool allMatch = true;
for (const auto &targetId : targetIds) {
targetInfo = routingInfo->getTarget(targetId);
ASSERT_NE(nullptr, targetInfo);
if (localStatus != targetInfo->localState || publicStatus != targetInfo->publicState) {
XLOGF(INFO,
"#{} Waiting storage target {} moving to status {}/{}, current status {}/{}, routing info: {}",
retry,
targetId,
magic_enum::enum_name(localStatus),
magic_enum::enum_name(publicStatus),
magic_enum::enum_name(targetInfo->localState),
magic_enum::enum_name(targetInfo->publicState),
routingInfo->routingInfoVersion);
allMatch = false;
break;
}
}
if (allMatch) return;
}
}
ASSERT_NE(nullptr, targetInfo);
ASSERT_EQ(localStatus, targetInfo->localState) << "target id " << targetInfo->targetId;
ASSERT_EQ(publicStatus, targetInfo->publicState) << "target id " << targetInfo->targetId;
}
void readAndCompareAllReplicas(const ChainId &chainId,
const ChunkId &chunkBegin,
size_t numChunks,
const std::vector<IOResult> &expectedResults) {
const ChunkId chunkEnd(chunkBegin, numChunks);
std::vector<std::vector<std::vector<uint8_t>>> chunkData(setupConfig_.num_replicas());
std::vector<std::vector<IOResult>> ioResults(setupConfig_.num_replicas());
for (size_t replicaIndex = 0; replicaIndex < setupConfig_.num_replicas(); replicaIndex++) {
client::ReadOptions options;
options.targetSelection().set_mode(client::TargetSelectionMode::ManualMode);
options.targetSelection().set_targetIndex(replicaIndex);
auto readRes = readFromChunks(chainId,
chunkBegin,
chunkEnd,
chunkData[replicaIndex],
0,
setupConfig_.chunk_size(),
options,
&ioResults[replicaIndex]);
ASSERT_TRUE(readRes);
}
for (size_t chunkIndex = 0; chunkIndex < numChunks; chunkIndex++) {
for (size_t replicaIndex = 0; replicaIndex < setupConfig_.num_replicas(); replicaIndex++) {
ASSERT_EQ(chunkData[0][chunkIndex], chunkData[replicaIndex][chunkIndex]);
ASSERT_EQ(expectedResults[chunkIndex].commitVer, ioResults[replicaIndex][chunkIndex].commitVer);
ASSERT_EQ(expectedResults[chunkIndex].updateVer, ioResults[replicaIndex][chunkIndex].updateVer);
ASSERT_EQ(expectedResults[chunkIndex].checksum, ioResults[replicaIndex][chunkIndex].checksum);
ASSERT_EQ(expectedResults[chunkIndex].commitChainVer, ioResults[replicaIndex][chunkIndex].commitChainVer);
}
}
}
};
TEST_P(TestSingleProcessCluster, StorageFailureDetected) {
// stop the first storage server
flat::NodeId firstNodeId{1};
stopAndRemoveStorageServer(firstNodeId);
waitStorageTargetStatus(nodeTargets_[firstNodeId], flat::LocalTargetState::OFFLINE, flat::PublicTargetState::OFFLINE);
waitStorageNodeStatus({firstNodeId}, flat::NodeStatus::HEARTBEAT_FAILED);
}
TEST_P(TestSingleProcessCluster, StorageFailureDetectedBeforeWrite) {
// stop the first storage server
flat::NodeId firstNodeId{1};
stopAndRemoveStorageServer(firstNodeId);
waitStorageTargetStatus(nodeTargets_[firstNodeId], flat::LocalTargetState::OFFLINE, flat::PublicTargetState::OFFLINE);
size_t numChunks = 10;
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
for (const auto &chainId : chainIds_) {
storage::ChunkId chunkBegin(1, 0);
storage::ChunkId chunkEnd(1, numChunks);
auto writeRes = writeToChunks(chainId, chunkBegin, chunkEnd, chunkData);
ASSERT_TRUE(writeRes);
}
}
#if false
TEST_P(TestSingleProcessCluster, DISABLED_StorageWriteFailed) {
clientConfig_.retry().set_init_wait_time(1_s);
clientConfig_.retry().set_max_wait_time(1_s);
clientConfig_.retry().set_max_retry_time(5_s);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
auto writeRes = writeToChunk(chainIds_.front(), ChunkId{0u, 0u}, chunkData);
ASSERT_OK(writeRes.lengthInfo);
auto flagPath = fmt::format("/tmp/storage_main_write_failed.{}", SysResource::pid());
ASSERT_OK(hf3fs::storeToFile(flagPath, "100"));
auto guard = folly::makeGuard([&] { std::filesystem::remove(flagPath); });
// write the special chunk and receive fail.
writeRes = writeToChunk(chainIds_.front(), ChunkId{0u, 0u}, chunkData);
ASSERT_FALSE(writeRes.lengthInfo);
waitStorageTargetStatus(nodeTargets_[flat::NodeId{1}],
flat::LocalTargetState::OFFLINE,
flat::PublicTargetState::OFFLINE);
}
#endif
TEST_P(TestSingleProcessCluster, StorageFailAndRestart) {
// stop the first storage server
uint32_t nodeIndex = 0;
auto firstNodeId = storageNodeIds_[nodeIndex];
stopAndRemoveStorageServer(nodeIndex);
waitStorageNodeStatus({firstNodeId}, flat::NodeStatus::HEARTBEAT_FAILED);
waitStorageTargetStatus(nodeTargets_[firstNodeId], flat::LocalTargetState::OFFLINE, flat::PublicTargetState::OFFLINE);
size_t numChunks = 10;
storage::ChunkId chunkBegin(1, 0);
storage::ChunkId chunkEnd(1, numChunks);
std::vector<std::vector<IOResult>> writeResults(chainIds_.size());
for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size());
std::vector<uint8_t> chunkData(folly::Random::rand32(1, setupConfig_.chunk_size() - offset + 1),
(uint8_t)folly::Random::rand32());
auto writeRes = writeToChunks(chainIds_[chainIndex],
chunkBegin,
chunkEnd,
chunkData,
offset,
chunkData.size(),
client::WriteOptions(),
&writeResults[chainIndex]);
ASSERT_TRUE(writeRes);
}
// restart storage server
auto storageServer = createStorageServer(nodeIndex);
ASSERT_NE(storageServer, nullptr);
storageServers_.insert(storageServers_.begin() + nodeIndex, std::move(storageServer));
waitStorageNodeStatus({firstNodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED);
waitStorageTargetStatus(nodeTargets_[firstNodeId],
flat::LocalTargetState::UPTODATE,
flat::PublicTargetState::SERVING);
// read chunk data from all replicas
for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
readAndCompareAllReplicas(chainIds_[chainIndex], chunkBegin, numChunks, writeResults[chainIndex]);
}
}
TEST_P(TestSingleProcessCluster, StorageRepeatedFailAndRestart) {
uint32_t maxLoops = setupConfig_.num_storage_nodes();
for (uint32_t testLoop = 1; testLoop <= maxLoops; testLoop++) {
uint64_t chunkIdPrefix = testLoop;
size_t numChunks = 10;
storage::ChunkId chunkBegin(chunkIdPrefix, 0);
storage::ChunkId chunkEnd(chunkIdPrefix, numChunks);
std::vector<std::vector<IOResult>> writeResults(chainIds_.size());
uint32_t nodeIndex = folly::Random::rand32(0, setupConfig_.num_storage_nodes());
auto nodeId = storageNodeIds_[nodeIndex];
XLOGF(INFO, "Test loop {}, stopping #{} {}", testLoop, nodeIndex, nodeId);
stopAndRemoveStorageServer(nodeIndex);
waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_FAILED, 100 /*maxRetries*/);
waitStorageTargetStatus(nodeTargets_[nodeId],
flat::LocalTargetState::OFFLINE,
flat::PublicTargetState::OFFLINE,
500 /*maxRetries*/);
XLOGF(INFO, "Test loop {}, writing chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size());
std::vector<uint8_t> chunkData(folly::Random::rand32(1, setupConfig_.chunk_size() - offset + 1),
(uint8_t)folly::Random::rand32());
auto writeRes = writeToChunks(chainIds_[chainIndex],
chunkBegin,
chunkEnd,
chunkData,
offset,
chunkData.size(),
client::WriteOptions(),
&writeResults[chainIndex]);
ASSERT_TRUE(writeRes);
}
// restart storage server
XLOGF(INFO, "Test loop {}, restarting #{} {}", testLoop, nodeIndex, nodeId);
auto storageServer = createStorageServer(nodeIndex);
ASSERT_NE(storageServer, nullptr);
storageServers_.insert(storageServers_.begin() + nodeIndex, std::move(storageServer));
waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED, 200 /*maxRetries*/);
waitStorageTargetStatus(nodeTargets_[nodeId],
flat::LocalTargetState::UPTODATE,
flat::PublicTargetState::SERVING,
1000 /*maxRetries*/);
XLOGF(INFO, "Test loop {}, reading chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
// read chunk data from all replicas
for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
readAndCompareAllReplicas(chainIds_[chainIndex], chunkBegin, numChunks, writeResults[chainIndex]);
}
}
}
TEST_P(TestSingleProcessCluster, MultiStoragesRepeatedFailAndRestart) {
uint32_t nodeIndexStart = folly::Random::rand32(0, setupConfig_.num_storage_nodes());
uint32_t maxLoops = setupConfig_.num_storage_nodes();
for (uint32_t testLoop = 1; testLoop <= maxLoops; testLoop++) {
uint64_t chunkIdPrefix = testLoop;
size_t numChunks = 10;
storage::ChunkId chunkBegin(chunkIdPrefix, 0);
storage::ChunkId chunkEnd(chunkIdPrefix, numChunks);
uint32_t nodeIndex = (nodeIndexStart + testLoop) % setupConfig_.num_storage_nodes();
auto nodeId = storageNodeIds_[nodeIndex];
auto routingInfo = getRoutingInfo();
ASSERT_NE(nullptr, routingInfo);
auto nodeInfo = routingInfo->getNode(nodeId);
ASSERT_NE(nullptr, nodeInfo);
bool stoppedStorage = false;
bool writeFailed = false;
if (nodeInfo->status == flat::NodeStatus::HEARTBEAT_CONNECTED) {
XLOGF(INFO, "Test loop {}, stopping #{} {}", testLoop, nodeIndex, nodeId);
stoppedStorage = stopAndRemoveStorageServer(nodeId);
}
XLOGF(INFO, "Test loop {}, writing chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
for (const auto &chainId : chainIds_) {
uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size());
std::vector<uint8_t> chunkData(folly::Random::rand32(1, setupConfig_.chunk_size() - offset + 1),
(uint8_t)folly::Random::rand32());
auto writeRes = writeToChunks(chainId, chunkBegin, chunkEnd, chunkData, offset, chunkData.size());
auto routingInfo = getRoutingInfo();
auto chainInfo = routingInfo->getChain(chainId);
ASSERT_NE(nullptr, chainInfo);
if (chainInfo->targets.front().publicState == flat::PublicTargetState::SERVING) {
// check write result if the chain has serving target
ASSERT_TRUE(writeRes);
} else {
XLOGF(INFO, "Test loop {}, failed to write to chain: {:?}", testLoop, *chainInfo);
writeFailed = true;
break;
}
}
// restart storage server
if (stoppedStorage) {
XLOGF(INFO, "Test loop {}, restarting #{} {}", testLoop, nodeIndex, nodeId);
auto storageServer = createStorageServer(nodeIndex);
ASSERT_NE(storageServer, nullptr);
storageServers_.insert(storageServers_.begin() + nodeIndex, std::move(storageServer));
}
if (writeFailed) {
// read chunks from last test loop if write in current loop failed
chunkBegin = storage::ChunkId(chunkIdPrefix - 1, 0);
chunkEnd = storage::ChunkId(chunkIdPrefix - 1, numChunks);
}
// read chunk data from all serving replicas
XLOGF(INFO, "Test loop {}, reading chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
for (const auto &chainId : chainIds_) {
for (size_t retryIndex = 0;; retryIndex++) {
XLOGF(INFO,
"Test loop {}, #{} retry, reading chunks in range {} - {}",
testLoop,
retryIndex,
chunkBegin,
chunkEnd);
std::vector<std::vector<uint8_t>> chunkData;
storage::client::ReadOptions options;
options.targetSelection().set_mode(storage::client::TargetSelectionMode::RoundRobin);
auto readRes = readFromChunks(chainId, chunkBegin, chunkEnd, chunkData, 0, setupConfig_.chunk_size(), options);
auto chainInfo = routingInfo->getChain(chainId);
ASSERT_NE(nullptr, chainInfo);
if (chainInfo->targets.front().publicState == flat::PublicTargetState::SERVING) {
// check read result if the chain has serving target
ASSERT_TRUE(readRes);
break;
} else {
// otherwise wait the storage server restart and retry
waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED, 200 /*maxRetries*/);
waitStorageTargetStatus(nodeTargets_[nodeId],
flat::LocalTargetState::UPTODATE,
flat::PublicTargetState::SERVING,
1000 /*maxRetries*/);
}
}
}
}
}
TEST_P(TestSingleProcessCluster, ConcurrentStorageSyncAndWrite) {
uint32_t nodeIndexStart = folly::Random::rand32(0, setupConfig_.num_storage_nodes());
uint32_t maxLoops = setupConfig_.num_storage_nodes();
for (uint32_t testLoop = 1; testLoop <= maxLoops; testLoop++) {
uint64_t chunkIdPrefix = testLoop;
size_t numChunks = 20;
storage::ChunkId chunkBegin(chunkIdPrefix, 0);
storage::ChunkId chunkEnd(chunkIdPrefix, numChunks);
uint32_t nodeIndex = (nodeIndexStart + testLoop) % setupConfig_.num_storage_nodes();
auto nodeId = storageNodeIds_[nodeIndex];
XLOGF(INFO, "Test loop {}, writing chunks in range {} - {}", testLoop, chunkBegin, chunkEnd);
std::atomic<bool> runWrite = true;
std::atomic<bool> writeOK = true;
std::vector<std::jthread> backgroundWorkers;
backgroundWorkers.reserve(chainIds_.size());
std::vector<std::vector<IOResult>> writeResults(chainIds_.size());
for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
const auto &chainId = chainIds_[chainIndex];
auto &ioResults = writeResults[chainIndex];
backgroundWorkers.emplace_back([&, this]() {
for (uint32_t chunkVer = 1; runWrite && writeOK; chunkVer++) {
uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size());
std::vector<uint8_t> chunkData(folly::Random::rand32(1, setupConfig_.chunk_size() - offset + 1),
(uint8_t)folly::Random::rand32());
ioResults.clear();
auto writeRes = writeToChunks(chainId,
chunkBegin,
chunkEnd,
chunkData,
offset,
chunkData.size(),
client::WriteOptions(),
&ioResults);
if (!writeRes) writeOK = false;
ASSERT_TRUE(writeRes);
for (const auto &result : ioResults) {
writeRes = result.lengthInfo && *result.lengthInfo == chunkData.size() && chunkVer == result.commitVer;
if (!writeRes) writeOK = false;
ASSERT_TRUE(writeRes);
}
}
});
}
// write a few chunks before restart
auto waitTime = mgmtdServer_.config.service().heartbeat_fail_interval() * 2;
std::this_thread::sleep_for(waitTime);
ASSERT_TRUE(writeOK);
auto routingInfo = getRoutingInfo();
ASSERT_NE(nullptr, routingInfo);
auto nodeInfo = routingInfo->getNode(nodeId);
ASSERT_NE(nullptr, nodeInfo);
if (nodeInfo->status == flat::NodeStatus::HEARTBEAT_CONNECTED) {
XLOGF(INFO, "Test loop {}, try to stop #{} {}", testLoop, nodeIndex, nodeId);
auto stoppedStorage = stopAndRemoveStorageServer(nodeId);
ASSERT_TRUE(writeOK);
if (stoppedStorage) {
if (folly::Random::randBool(0.5)) { // wait until mgmtd knows the failure
waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_FAILED);
XLOGF(INFO, "Test loop {}, stopped #{} {}", testLoop, nodeIndex, nodeId);
ASSERT_TRUE(writeOK);
}
// restart the storage service
XLOGF(INFO, "Test loop {}, try to start #{} {}", testLoop, nodeIndex, nodeId);
auto storageServer = createStorageServer(nodeIndex);
ASSERT_NE(storageServer, nullptr);
storageServers_.insert(storageServers_.begin() + nodeIndex, std::move(storageServer));
ASSERT_TRUE(writeOK);
waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED);
XLOGF(INFO, "Test loop {}, started #{} {}", testLoop, nodeIndex, nodeId);
ASSERT_TRUE(writeOK);
}
}
std::this_thread::sleep_for(waitTime);
ASSERT_TRUE(writeOK);
runWrite = false;
backgroundWorkers.clear();
ASSERT_TRUE(writeOK);
// read chunk data from the serving replicas
for (const auto &chainId : chainIds_) {
XLOGF(INFO, "Test loop {}, chain {}, reading chunks in range {} - {}", testLoop, chainId, chunkBegin, chunkEnd);
std::vector<std::vector<uint8_t>> chunkData;
auto readRes = readFromChunks(chainId, chunkBegin, chunkEnd, chunkData, 0, setupConfig_.chunk_size());
ASSERT_TRUE(readRes);
}
waitStorageNodeStatus({nodeId}, flat::NodeStatus::HEARTBEAT_CONNECTED, 200 /*maxRetries*/);
waitStorageTargetStatus(nodeTargets_[nodeId],
flat::LocalTargetState::UPTODATE,
flat::PublicTargetState::SERVING,
1000 /*maxRetries*/);
// read chunk data from all replicas
for (size_t chainIndex = 0; chainIndex < chainIds_.size(); chainIndex++) {
readAndCompareAllReplicas(chainIds_[chainIndex], chunkBegin, numChunks, writeResults[chainIndex]);
}
}
}
TEST_P(TestSingleProcessCluster, StorageResetUncommittedChunks) {
size_t numChunks = 5;
std::vector<uint8_t> chunkData(setupConfig_.chunk_size() - 1, 0x86);
auto chainId = chainIds_.front();
auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
ASSERT_OK(regRes);
auto ioBuffer = std::move(*regRes);
// create write IOs
std::vector<client::WriteIO> writeIOs;
for (size_t i = 0; i < numChunks; ++i) {
auto writeIO = storageClient_->createWriteIO(chainId,
ChunkId{1, i},
1 /*offset*/,
chunkData.size() /*length*/,
setupConfig_.chunk_size() /*chunkSize*/,
&chunkData[0],
&ioBuffer);
writeIOs.push_back(std::move(writeIO));
}
clientConfig_.retry().set_init_wait_time(1_s);
clientConfig_.retry().set_max_wait_time(1_s);
clientConfig_.retry().set_max_retry_time(1_s);
flat::UserInfo dummyUserInfo{};
std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> ioTasks;
storage::client::WriteOptions options;
for (auto &writeIO : writeIOs) {
auto task = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
ioTasks.push_back(std::move(task));
}
// stop tail server.
auto lastNodeId = storageNodeIds_.back();
stopAndRemoveStorageServer(lastNodeId);
std::this_thread::sleep_for(100_ms);
// restart head server.
folly::coro::blockingWait(folly::coro::collectAllRange(std::move(ioTasks)));
auto succ = std::count_if(writeIOs.begin(), writeIOs.end(), [](auto &w) { return bool(w.result.lengthInfo); });
XLOGF(INFO, "write chunks succ: {}", succ);
ASSERT_LE(succ, numChunks);
auto firstNodeId = storageNodeIds_.front();
stopAndRemoveStorageServer(firstNodeId);
auto storageServer = createStorageServer(0);
ASSERT_NE(storageServer, nullptr);
storageServers_.insert(storageServers_.end(), std::move(storageServer));
waitStorageTargetStatus(nodeTargets_[flat::NodeId{1}],
flat::LocalTargetState::UPTODATE,
flat::PublicTargetState::SERVING,
1000);
clientConfig_.retry().set_init_wait_time(1_s);
clientConfig_.retry().set_max_wait_time(1_s);
clientConfig_.retry().set_max_retry_time(5_s);
std::vector<std::vector<uint8_t>> readResult;
ASSERT_TRUE(readFromChunks(chainId, ChunkId{1, 0}, ChunkId{1, numChunks}, readResult, 1, chunkData.size()));
ASSERT_EQ(readResult.size(), numChunks);
for (auto &line : readResult) {
ASSERT_EQ(line, chunkData);
}
// write again.
ASSERT_TRUE(writeToChunks(chainId, ChunkId{1, 0}, ChunkId{1, numChunks}, chunkData));
}
SystemSetupConfig testTwoReplicas = {
32_KB /*chunkSize*/,
16 /*numChains*/,
2 /*numReplicas*/,
2 /*numStorageNodes*/,
{folly::fs::temp_directory_path()} /*dataPaths*/,
"" /*clientConfig*/,
"" /*serverConfig*/,
{} /*storageEndpoints*/,
0 /*serviceLevel*/,
0 /*listenPort*/,
storage::client::StorageClient::ImplementationType::RPC /*clientImplType*/,
kv::KVStore::Type::LevelDB /*metaStoreType*/,
false /*useFakeMgmtdClient*/,
true /*startStorageServer*/,
};
SystemSetupConfig testThreeReplicas = {
32_KB /*chunkSize*/,
16 /*numChains*/,
3 /*numReplicas*/,
3 /*numStorageNodes*/,
{folly::fs::temp_directory_path()} /*dataPaths*/,
"" /*clientConfig*/,
"" /*serverConfig*/,
{} /*storageEndpoints*/,
0 /*serviceLevel*/,
0 /*listenPort*/,
storage::client::StorageClient::ImplementationType::RPC /*clientImplType*/,
kv::KVStore::Type::LevelDB /*metaStoreType*/,
false /*useFakeMgmtdClient*/,
true /*startStorageServer*/,
};
INSTANTIATE_TEST_SUITE_P(TwoReplicas,
TestSingleProcessCluster,
::testing::Values(testTwoReplicas),
SystemSetupConfig::prettyPrintConfig);
INSTANTIATE_TEST_SUITE_P(ThreeReplicas,
TestSingleProcessCluster,
::testing::Values(testThreeReplicas),
SystemSetupConfig::prettyPrintConfig);
} // namespace
} // namespace hf3fs::storage

165
tests/storage/service/TestStorageService.cc Normal file
View File

@@ -0,0 +1,165 @@
#include <algorithm>
#include <chrono>
#include <thread>
#include "client/storage/StorageClient.h"
#include "common/kv/mem/MemKVEngine.h"
#include "common/net/Client.h"
#include "common/serde/Serde.h"
#include "fbs/mgmtd/RoutingInfo.h"
#include "fbs/storage/Service.h"
#include "storage/service/StorageOperator.h"
#include "storage/service/StorageServer.h"
#include "storage/store/ChunkMetadata.h"
#include "tests/GtestHelpers.h"
#include "tests/client/ClientWithConfig.h"
#include "tests/client/ServerWithConfig.h"
#include "tests/lib/Helper.h"
#include "tests/lib/UnitTestFabric.h"
#include "tests/mgmtd/MgmtdTestHelper.h"
namespace hf3fs::storage {
namespace {
using namespace hf3fs::test;
class TestStorageForward : public UnitTestFabric, public ::testing::Test {
protected:
TestStorageForward()
: UnitTestFabric(SystemSetupConfig{
128_KB /*chunkSize*/,
1 /*numChains*/,
3 /*numReplicas*/,
3 /*numStorageNodes*/,
{folly::fs::temp_directory_path()} /*dataPaths*/,
hf3fs::Path() /*clientConfig*/,
hf3fs::Path() /*serverConfig*/,
{} /*storageEndpoints*/,
0 /*serviceLevel*/,
0 /*listenPort*/,
client::StorageClient::ImplementationType::RPC /*clientImplType*/,
kv::KVStore::Type::RocksDB /*metaStoreType*/,
true /*useFakeMgmtdClient*/,
}) {}
void SetUp() override {
net::IBDevice::Config ibConfig;
auto ibResult = net::IBManager::start(ibConfig);
ASSERT_OK(ibResult);
ASSERT_TRUE(setUpStorageSystem());
}
void TearDown() override { tearDownStorageSystem(); }
};
TEST_F(TestStorageForward, Write) {
// register a block of memory
std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
ASSERT_OK(regRes);
// create write IO
auto ioBuffer = std::move(*regRes);
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
auto writeIO = storageClient_->createWriteIO(chainId,
chunkId,
0 /*offset*/,
setupConfig_.chunk_size() /*length*/,
setupConfig_.chunk_size() /*chunkSize*/,
&memoryBlock[0],
&ioBuffer);
client::WriteOptions options;
storageServers_.back()->stopAndJoin();
storageServers_.pop_back();
ASSERT_TRUE(updateRoutingInfo([&](auto &routingInfo) {
setTargetOffline(routingInfo, 0);
setTargetOffline(routingInfo, 1);
}));
folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
ASSERT_OK(writeIO.result.lengthInfo);
ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
}
TEST_F(TestStorageForward, WriteFailed) {
// register a block of memory
std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
ASSERT_OK(regRes);
// create write IO
auto ioBuffer = std::move(*regRes);
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
auto writeIO = storageClient_->createWriteIO(chainId,
chunkId,
0 /*offset*/,
setupConfig_.chunk_size() /*length*/,
setupConfig_.chunk_size() /*chunkSize*/,
&memoryBlock[0],
&ioBuffer);
client::WriteOptions options;
storageServers_.back()->stopAndJoin();
storageServers_.pop_back();
clientConfig_.retry().set_max_retry_time(10_s);
folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
ASSERT_FALSE(writeIO.result.lengthInfo);
}
TEST_F(TestStorageForward, WriteAndRead) {
// register a block of memory
std::vector<uint8_t> memoryBlock(setupConfig_.chunk_size(), 0xFF);
memoryBlock[0] = 0x01;
memoryBlock[1] = 0x02;
memoryBlock[2] = 0x03;
auto regRes = storageClient_->registerIOBuffer(&memoryBlock[0], memoryBlock.size());
ASSERT_OK(regRes);
// create write IO
auto ioBuffer = std::move(*regRes);
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
auto writeIO = storageClient_->createWriteIO(chainId,
chunkId,
0 /*offset*/,
3,
setupConfig_.chunk_size() /*chunkSize*/,
&memoryBlock[0],
&ioBuffer);
client::WriteOptions options;
folly::coro::blockingWait(storageClient_->write(writeIO, flat::UserInfo(), options));
ASSERT_OK(writeIO.result.lengthInfo);
ASSERT_EQ(writeIO.length, writeIO.result.lengthInfo.value());
{
auto readIO = storageClient_->createReadIO(chainId, chunkId, 0, 4096, &memoryBlock[0], &ioBuffer);
folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), client::ReadOptions{}));
ASSERT_OK(readIO.result.lengthInfo);
ASSERT_EQ(*readIO.result.lengthInfo, 3);
ASSERT_EQ(memoryBlock[0], 0x01);
ASSERT_EQ(memoryBlock[1], 0x02);
ASSERT_EQ(memoryBlock[2], 0x03);
}
{
auto readIO = storageClient_->createReadIO(chainId, chunkId, 1, 4096, &memoryBlock[0], &ioBuffer);
folly::coro::blockingWait(storageClient_->read(readIO, flat::UserInfo(), client::ReadOptions{}));
ASSERT_OK(readIO.result.lengthInfo);
ASSERT_EQ(*readIO.result.lengthInfo, 2);
ASSERT_EQ(memoryBlock[0], 0x02);
ASSERT_EQ(memoryBlock[1], 0x03);
}
}
} // namespace
} // namespace hf3fs::storage

302
tests/storage/service/TestStorageServiceFailStop.cc Normal file
View File

@@ -0,0 +1,302 @@
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/Collect.h>
#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/storage/StorageClient.h"
#include "common/net/Client.h"
#include "tests/lib/UnitTestFabric.h"
namespace hf3fs::storage {
namespace {
using namespace hf3fs::test;
using SystemFailureConfig =
std::tuple<uint32_t /*numReplicas*/, uint32_t /*failedTargetIndex*/, bool /*useFakeMgmtdClient*/>;
std::string prettyPrintConfig(const testing::TestParamInfo<SystemFailureConfig> &info) {
return fmt::format("{}of{}failed_fakemgmtd{}",
std::get<1>(info.param) + 1,
std::get<0>(info.param),
std::get<2>(info.param));
}
class TestStorageServiceFailStop : public UnitTestFabric, public ::testing::TestWithParam<SystemFailureConfig> {
protected:
TestStorageServiceFailStop()
: UnitTestFabric(SystemSetupConfig{
128_KB /*chunkSize*/,
1 /*numChains*/,
std::get<0>(GetParam()) /*numReplicas*/,
std::get<0>(GetParam()) /*numStorageNodes*/,
{folly::fs::temp_directory_path()} /*dataPaths*/,
hf3fs::Path() /*clientConfig*/,
hf3fs::Path() /*serverConfig*/,
{} /*storageEndpoints*/,
0 /*serviceLevel*/,
0 /*listenPort*/,
client::StorageClient::ImplementationType::RPC,
kv::KVStore::Type::RocksDB,
std::get<2>(GetParam()) /*useFakeMgmtdClient*/,
}),
failedTargetIndex_(std::get<1>(GetParam())) {}
void SetUp() override {
// init ib device
net::IBDevice::Config ibConfig;
auto ibResult = net::IBManager::start(ibConfig);
ASSERT_OK(ibResult);
ASSERT_TRUE(setUpStorageSystem());
}
void TearDown() override { tearDownStorageSystem(); }
protected:
uint32_t failedTargetIndex_;
};
TEST_P(TestStorageServiceFailStop, FailureUndetected) {
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
// stop the target
ASSERT_TRUE(stopAndRemoveStorageServer(failedTargetIndex_));
// write fails after the target stops undetectedly
clientConfig_.retry().set_max_retry_time(10_s);
auto ioResult = writeToChunk(chainId, chunkId, chunkData);
ASSERT_TRUE(ioResult.lengthInfo.hasError());
if (failedTargetIndex_ == 0) {
switch (ioResult.lengthInfo.error().code()) {
case StorageClientCode::kCommError:
case StorageClientCode::kTimeout:
break;
default:
ASSERT_TRUE(ioResult.lengthInfo.error().code());
}
} else
ASSERT_EQ(StorageClientCode::kResourceBusy, ioResult.lengthInfo.error().code());
}
TEST_P(TestStorageServiceFailStop, FailureDetectedBeforeWrite) {
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
// stop the target
ASSERT_TRUE(stopAndRemoveStorageServer(failedTargetIndex_));
// set the target to offline state
ASSERT_TRUE(updateRoutingInfo([&](auto &routingInfo) { setTargetOffline(routingInfo, failedTargetIndex_); }));
// write succeeds after the failure is detected
if (storageServers_.empty()) clientConfig_.retry().set_max_retry_time(10_s);
auto ioResult = writeToChunk(chainId, chunkId, chunkData, 0 /*offset*/, chunkData.size());
if (storageServers_.empty()) {
// only single replica
ASSERT_TRUE(ioResult.lengthInfo.hasError());
ASSERT_EQ(StorageClientCode::kNotAvailable, ioResult.lengthInfo.error().code());
} else {
// multiple replicas
ASSERT_OK(ioResult.lengthInfo);
ASSERT_EQ(chunkData.size(), ioResult.lengthInfo.value());
}
}
TEST_P(TestStorageServiceFailStop, FailureDetectedDuringRetry) {
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
ASSERT_OK(regRes);
// create write IO
auto ioBuffer = std::move(*regRes);
auto writeIO = storageClient_->createWriteIO(chainId,
chunkId,
0 /*offset*/,
chunkData.size() /*length*/,
setupConfig_.chunk_size() /*chunkSize*/,
&chunkData[0],
&ioBuffer);
// stop the target
ASSERT_TRUE(stopAndRemoveStorageServer(failedTargetIndex_));
// issue the write request
flat::UserInfo dummyUserInfo{};
auto options = client::WriteOptions();
options.retry().set_max_retry_time(5_s);
auto writeTask = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
// retry for a short time
std::this_thread::sleep_for(2000_ms);
// check the request failed with communication error
#if defined(__has_feature)
#if !__has_feature(thread_sanitizer)
ASSERT_FALSE(writeIO.result.lengthInfo);
if (failedTargetIndex_ == 0) {
switch (writeIO.result.lengthInfo.error().code()) {
case StorageClientCode::kCommError:
case StorageClientCode::kTimeout:
break;
default:
ASSERT_TRUE(writeIO.result.lengthInfo.error().code());
}
} else
ASSERT_EQ(StorageClientCode::kResourceBusy, writeIO.result.lengthInfo.error().code());
#endif
#endif
// set the target to offline state
ASSERT_TRUE(updateRoutingInfo([&](auto &routingInfo) { setTargetOffline(routingInfo, failedTargetIndex_); }));
writeTask.wait();
if (storageServers_.empty()) {
// only single replica
ASSERT_TRUE(writeIO.result.lengthInfo.hasError());
ASSERT_EQ(StorageClientCode::kNotAvailable, writeIO.result.lengthInfo.error().code());
} else {
// multiple replicas
ASSERT_OK(writeIO.result.lengthInfo);
ASSERT_EQ(chunkData.size(), writeIO.result.lengthInfo.value());
// read and check chunk data
std::vector<uint8_t> readData(chunkData.size());
auto readRes = readFromChunk(chainId, chunkId, readData);
ASSERT_OK(readRes.lengthInfo);
ASSERT_EQ(1, readRes.commitVer);
ASSERT_EQ(1, readRes.updateVer);
ASSERT_EQ(chunkData.size(), *readRes.lengthInfo);
ASSERT_EQ(chunkData, readData);
}
}
TEST_P(TestStorageServiceFailStop, ConcurrentWrites) {
auto chainId = firstChainId_;
ChunkId chunkId(1 /*high*/, 1 /*low*/);
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
ASSERT_OK(regRes);
auto ioBuffer = std::move(*regRes);
size_t numWriteIOs = 10;
std::vector<client::WriteIO> writeIOs;
for (size_t writeIndex = 0; writeIndex < numWriteIOs; writeIndex++) {
// create write IO
auto writeIO = storageClient_->createWriteIO(chainId,
chunkId,
0 /*offset*/,
chunkData.size() /*length*/,
setupConfig_.chunk_size() /*chunkSize*/,
&chunkData[0],
&ioBuffer);
writeIOs.push_back(std::move(writeIO));
}
// stop the target
ASSERT_TRUE(stopAndRemoveStorageServer(failedTargetIndex_));
// issue the write request
flat::UserInfo dummyUserInfo{};
auto options = client::WriteOptions();
options.retry().set_max_retry_time(10_s);
std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> writeTasks;
for (auto &writeIO : writeIOs) {
auto writeTask = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
writeTasks.push_back(std::move(writeTask));
}
// retry for a short time
std::this_thread::sleep_for(1500_ms);
// check the request failed with communication error
#if defined(__has_feature)
#if !__has_feature(thread_sanitizer)
for (auto &writeIO : writeIOs) {
ASSERT_FALSE(writeIO.result.lengthInfo);
if (failedTargetIndex_ == 0) {
switch (writeIO.result.lengthInfo.error().code()) {
case StorageClientCode::kCommError:
case StorageClientCode::kTimeout:
break;
default:
ASSERT_TRUE(writeIO.result.lengthInfo.error().code());
}
} else
ASSERT_EQ(StorageClientCode::kResourceBusy, writeIO.result.lengthInfo.error().code());
}
#endif
#endif
// set the target to offline state
ASSERT_TRUE(updateRoutingInfo([&](auto &routingInfo) { setTargetOffline(routingInfo, failedTargetIndex_); }));
// wait until all write IOs completed
folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));
if (!storageServers_.empty()) {
std::set<ChunkVer> updateVersions;
for (const auto &writeIO : writeIOs) {
ASSERT_OK(writeIO.result.lengthInfo);
// check commit version == update version
ASSERT_EQ(writeIO.result.commitVer, writeIO.result.updateVer);
// check the update versions are in range [1..numWriteIOs]
ASSERT_LE(1, writeIO.result.updateVer);
ASSERT_LE(writeIO.result.updateVer, numWriteIOs);
updateVersions.insert(writeIO.result.updateVer);
}
// check the update versions are unique
ASSERT_EQ(numWriteIOs, updateVersions.size());
}
for (auto &writeIO : writeIOs) {
if (storageServers_.empty()) {
// only single replica
ASSERT_TRUE(writeIO.result.lengthInfo.hasError());
ASSERT_EQ(StorageClientCode::kNotAvailable, writeIO.result.lengthInfo.error().code());
} else {
// multiple replicas
ASSERT_OK(writeIO.result.lengthInfo);
ASSERT_EQ(chunkData.size(), writeIO.result.lengthInfo.value());
// read and check chunk data
std::vector<uint8_t> readData(chunkData.size());
auto readRes = readFromChunk(chainId, chunkId, readData);
ASSERT_OK(readRes.lengthInfo);
ASSERT_EQ(numWriteIOs, readRes.commitVer);
ASSERT_EQ(numWriteIOs, readRes.updateVer);
ASSERT_EQ(chunkData.size(), *readRes.lengthInfo);
ASSERT_EQ(chunkData, readData);
}
}
}
INSTANTIATE_TEST_SUITE_P(SingleReplica,
TestStorageServiceFailStop,
::testing::Combine(::testing::Values(1) /*numReplicas*/,
::testing::Values(0) /*failedTargetIndex*/,
::testing::Values(true /*useFakeMgmtdClient*/)),
prettyPrintConfig);
INSTANTIATE_TEST_SUITE_P(TwoReplicas,
TestStorageServiceFailStop,
::testing::Combine(::testing::Values(2) /*numReplicas*/,
::testing::Values(0, 1) /*failedTargetIndex*/,
::testing::Values(true /*useFakeMgmtdClient*/)),
prettyPrintConfig);
INSTANTIATE_TEST_SUITE_P(ThreeReplicas,
TestStorageServiceFailStop,
::testing::Combine(::testing::Values(3) /*numReplicas*/,
::testing::Values(0, 1, 2) /*failedTargetIndex*/,
::testing::Values(true /*useFakeMgmtdClient*/)),
prettyPrintConfig);
} // namespace
} // namespace hf3fs::storage