mirror of
https://github.com/deepseek-ai/3FS
synced 2025-06-26 18:16:45 +00:00
Initial commit
This commit is contained in:
dev
209
tests/storage/service/TestIncorrectRoutingInfo.cc
Normal file
209
tests/storage/service/TestIncorrectRoutingInfo.cc
Normal file
@@ -0,0 +1,209 @@
|
||||
#include <folly/experimental/coro/BlockingWait.h>
|
||||
#include <folly/experimental/coro/Collect.h>
|
||||
|
||||
#include "client/mgmtd/ICommonMgmtdClient.h"
|
||||
#include "client/storage/StorageClient.h"
|
||||
#include "common/net/Client.h"
|
||||
#include "tests/lib/Helper.h"
|
||||
#include "tests/lib/UnitTestFabric.h"
|
||||
|
||||
namespace hf3fs::storage {
|
||||
namespace {
|
||||
|
||||
using namespace hf3fs::test;
|
||||
|
||||
class TestIncorrectRoutingInfo : public UnitTestFabric, public ::testing::Test {
|
||||
protected:
|
||||
TestIncorrectRoutingInfo()
|
||||
: UnitTestFabric(SystemSetupConfig{
|
||||
128_KB /*chunkSize*/,
|
||||
1 /*numChains*/,
|
||||
3 /*numReplicas*/,
|
||||
3 /*numStorageNodes*/,
|
||||
{folly::fs::temp_directory_path()} /*dataPaths*/,
|
||||
hf3fs::Path() /*clientConfig*/,
|
||||
hf3fs::Path() /*serverConfig*/,
|
||||
{} /*storageEndpoints*/,
|
||||
0 /*serviceLevel*/,
|
||||
0 /*listenPort*/,
|
||||
client::StorageClient::ImplementationType::RPC,
|
||||
kv::KVStore::Type::RocksDB,
|
||||
true /*useFakeMgmtdClient*/,
|
||||
}) {}
|
||||
|
||||
void SetUp() override {
|
||||
// init ib device
|
||||
net::IBDevice::Config ibConfig;
|
||||
auto ibResult = net::IBManager::start(ibConfig);
|
||||
ASSERT_OK(ibResult);
|
||||
ASSERT_TRUE(setUpStorageSystem());
|
||||
clientConfig_.retry().set_max_retry_time(10_s);
|
||||
}
|
||||
|
||||
void TearDown() override { tearDownStorageSystem(); }
|
||||
};
|
||||
|
||||
TEST_F(TestIncorrectRoutingInfo, MismatchChainVersion) {
|
||||
auto chainId = firstChainId_;
|
||||
ChunkId chunkId(1 /*high*/, 1 /*low*/);
|
||||
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
|
||||
|
||||
// create a test chunk
|
||||
auto ioResult = writeToChunk(chainId, chunkId, chunkData);
|
||||
ASSERT_RESULT_EQ(chunkData.size(), ioResult.lengthInfo);
|
||||
|
||||
// get the first chain
|
||||
auto newRoutingInfo = copyRoutingInfo();
|
||||
auto &chainTable = *newRoutingInfo->getChainTable(kTableId());
|
||||
auto &firstChain = newRoutingInfo->chains[chainTable.chains.front()];
|
||||
ASSERT_EQ(chainId, firstChain.chainId);
|
||||
auto chainVersion = firstChain.chainVersion;
|
||||
|
||||
for (int32_t deltaVer : {-1, 1}) {
|
||||
// change version of the first chain
|
||||
firstChain.chainVersion = flat::ChainVersion(chainVersion + deltaVer);
|
||||
// only update client's routing info
|
||||
auto fakeClient = dynamic_cast<FakeMgmtdClient *>(mgmtdForClient_.get());
|
||||
newRoutingInfo->routingInfoVersion++;
|
||||
fakeClient->setRoutingInfo(newRoutingInfo);
|
||||
ASSERT_OK(folly::coro::blockingWait(mgmtdForClient_->refreshRoutingInfo(/*force=*/false)));
|
||||
// try to read/write with mismatch chain version
|
||||
auto readRes = readFromChunk(chainId, chunkId, chunkData, 0, chunkData.size());
|
||||
ASSERT_ERROR(readRes.lengthInfo, StorageClientCode::kRoutingVersionMismatch);
|
||||
auto writeRes = writeToChunk(chainId, chunkId, chunkData, 0, chunkData.size());
|
||||
ASSERT_ERROR(writeRes.lengthInfo, StorageClientCode::kRoutingVersionMismatch);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestIncorrectRoutingInfo, WriteNotSentToHeadTarget) {
|
||||
auto chainId = firstChainId_;
|
||||
ChunkId chunkId(1 /*high*/, 1 /*low*/);
|
||||
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
|
||||
|
||||
// send write to the second target on chain
|
||||
client::WriteOptions options;
|
||||
options.targetSelection().set_mode(client::TargetSelectionMode::ManualMode);
|
||||
options.targetSelection().set_targetIndex(1);
|
||||
auto ioResult = writeToChunk(chainId, chunkId, chunkData, 0, chunkData.size(), options);
|
||||
ASSERT_ERROR(ioResult.lengthInfo, StorageClientCode::kRoutingError);
|
||||
}
|
||||
|
||||
TEST_F(TestIncorrectRoutingInfo, RemoveNotSentToHeadTarget) {
|
||||
auto chainId = firstChainId_;
|
||||
ChunkId chunkId(1 /*high*/, 1 /*low*/);
|
||||
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
|
||||
|
||||
// create a test chunk
|
||||
auto ioResult = writeToChunk(chainId, chunkId, chunkData);
|
||||
ASSERT_RESULT_EQ(chunkData.size(), ioResult.lengthInfo);
|
||||
|
||||
// send remove to the second target on chain
|
||||
client::WriteOptions options;
|
||||
options.targetSelection().set_mode(client::TargetSelectionMode::ManualMode);
|
||||
options.targetSelection().set_targetIndex(1);
|
||||
auto removeOp = storageClient_->createRemoveOp(chainId,
|
||||
ChunkId(chunkId, 0),
|
||||
ChunkId(chunkId, 0xFF),
|
||||
1 /*maxNumChunkIdsToProcess*/);
|
||||
folly::coro::blockingWait(storageClient_->removeChunks(std::span(&removeOp, 1), flat::UserInfo(), options));
|
||||
ASSERT_ERROR(removeOp.result.statusCode, StorageClientCode::kRoutingError);
|
||||
}
|
||||
|
||||
TEST_F(TestIncorrectRoutingInfo, WorkingHeadDetectedAsOffline) {
|
||||
auto chainId = firstChainId_;
|
||||
ChunkId chunkId(1 /*high*/, 1 /*low*/);
|
||||
std::vector<uint8_t> chunkData(setupConfig_.chunk_size(), 0xFF);
|
||||
|
||||
auto regRes = storageClient_->registerIOBuffer(&chunkData[0], chunkData.size());
|
||||
ASSERT_OK(regRes);
|
||||
auto ioBuffer = std::move(*regRes);
|
||||
|
||||
size_t numWriteIOs = 100;
|
||||
std::vector<client::WriteIO> writeIOs;
|
||||
|
||||
// create write IOs to one chunk
|
||||
for (size_t writeIndex = 0; writeIndex < numWriteIOs; writeIndex++) {
|
||||
auto writeIO = storageClient_->createWriteIO(chainId,
|
||||
chunkId,
|
||||
0 /*offset*/,
|
||||
chunkData.size() /*length*/,
|
||||
setupConfig_.chunk_size() /*chunkSize*/,
|
||||
&chunkData[0],
|
||||
&ioBuffer);
|
||||
writeIOs.push_back(std::move(writeIO));
|
||||
}
|
||||
|
||||
// issue the write requests
|
||||
flat::UserInfo dummyUserInfo{};
|
||||
auto options = client::WriteOptions();
|
||||
|
||||
std::vector<folly::SemiFuture<folly::Expected<folly::Unit, hf3fs::Status>>> writeTasks;
|
||||
|
||||
for (auto &writeIO : writeIOs) {
|
||||
auto writeTask = storageClient_->write(writeIO, dummyUserInfo, options).scheduleOn(&requestExe_).start();
|
||||
writeTasks.push_back(std::move(writeTask));
|
||||
}
|
||||
|
||||
// set head target to offline state but it's actually working
|
||||
auto newRoutingInfo = copyRoutingInfo();
|
||||
setTargetOffline(*newRoutingInfo, 0);
|
||||
|
||||
// let all storage servers except the host of head targets know the latest routing info
|
||||
for (size_t serverIndex = 1; serverIndex < storageServers_.size(); serverIndex++) {
|
||||
auto &storageServer = storageServers_[serverIndex];
|
||||
auto client = RoutingStoreHelper::getMgmtdClient(*storageServer);
|
||||
auto fakeClient = dynamic_cast<FakeMgmtdClient *>(client.get());
|
||||
fakeClient->setRoutingInfo(newRoutingInfo);
|
||||
RoutingStoreHelper::refreshRoutingInfo(*storageServer);
|
||||
}
|
||||
|
||||
// client does not konw the head is marked offline yet, so let it retry for a short time
|
||||
std::this_thread::sleep_for(500_ms);
|
||||
|
||||
#if defined(__has_feature)
|
||||
#if !__has_feature(thread_sanitizer)
|
||||
for (auto &writeIO : writeIOs) {
|
||||
// some writes are completed but others are still under retry
|
||||
if (writeIO.result.lengthInfo) {
|
||||
ASSERT_RESULT_EQ(writeIO.length, writeIO.result.lengthInfo);
|
||||
} else {
|
||||
switch (writeIO.statusCode()) {
|
||||
case StorageClientCode::kCommError:
|
||||
case StorageClientCode::kTimeout:
|
||||
case StorageClientCode::kRoutingVersionMismatch:
|
||||
XLOGF(INFO, "Write IO {} length info: {}", fmt::ptr(&writeIO), writeIO.result.lengthInfo);
|
||||
break;
|
||||
default:
|
||||
ASSERT_EQ(StorageClientCode::kNotInitialized, writeIO.result.lengthInfo.error().code())
|
||||
<< fmt::format("Write IO {} length info: {}", fmt::ptr(&writeIO), writeIO.result.lengthInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// let client know the latest routing info
|
||||
auto fakeClient = dynamic_cast<FakeMgmtdClient *>(mgmtdForClient_.get());
|
||||
fakeClient->setRoutingInfo(newRoutingInfo);
|
||||
ASSERT_OK(folly::coro::blockingWait(mgmtdForClient_->refreshRoutingInfo(/*force=*/false)));
|
||||
|
||||
// wait until all write tasks completed
|
||||
folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));
|
||||
|
||||
// check all write IOs have succeeded
|
||||
std::set<ChunkVer> updateVersions;
|
||||
for (const auto &writeIO : writeIOs) {
|
||||
ASSERT_OK(writeIO.result.lengthInfo);
|
||||
// check commit version == update version
|
||||
ASSERT_EQ(writeIO.result.commitVer, writeIO.result.updateVer);
|
||||
// check the update versions are in range [1..numWriteIOs]
|
||||
ASSERT_LE(1, writeIO.result.updateVer);
|
||||
ASSERT_LE(writeIO.result.updateVer, numWriteIOs);
|
||||
updateVersions.insert(writeIO.result.updateVer);
|
||||
}
|
||||
// check the update versions are unique
|
||||
ASSERT_EQ(numWriteIOs, updateVersions.size());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace hf3fs::storage
|
||||
Reference in New Issue
Block a user