Initial commit

This commit is contained in:
dev
2025-02-27 21:53:53 +08:00
commit 815e55e4c0
1291 changed files with 185445 additions and 0 deletions

View File

@@ -0,0 +1,35 @@
#include "common/utils/Address.h"
#include "common/utils/ConfigBase.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::net::test {
namespace {
TEST(TestAddress, Normal) {
constexpr std::string_view str = "IPoIB://192.168.1.1:8000";
auto addr = Address::fromString(str);
ASSERT_TRUE(addr.isTCP());
ASSERT_EQ(addr.ipStr(), "192.168.1.1");
ASSERT_EQ(addr.port, 8000);
ASSERT_EQ(addr.type, Address::IPoIB);
ASSERT_EQ(addr.str(), str);
addr = Address::fromString("tcp://127.0.0.1:8888");
ASSERT_TRUE(addr);
ASSERT_EQ(addr.type, Address::TCP);
ASSERT_EQ(addr.port, 8888);
}
static_assert(config::IsPrimitive<Address>, "address is not primitive");
auto kDefaultAddr = Address::from("tcp://192.168.1.1:8000").value();
class Config : public ConfigBase<Config> {
CONFIG_ITEM(addr, Address{kDefaultAddr});
};
TEST(TestAddress, Config) {
Config cfg;
ASSERT_EQ(cfg.toString(), "addr = 'TCP://192.168.1.1:8000'");
}
} // namespace
} // namespace hf3fs::net::test

View File

@@ -0,0 +1,22 @@
#include <folly/experimental/coro/GtestHelpers.h>
#include "common/utils/AtomicValue.h"
namespace hf3fs::test {
namespace {
using namespace std::chrono_literals;
TEST(TestAtomicValue, Normal) {
AtomicValue<uint64_t> a;
ASSERT_EQ(a, 0);
a = 1;
ASSERT_EQ(a, 1);
AtomicValue<uint64_t> b = a;
ASSERT_EQ(b, 1);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,79 @@
#include <chrono>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <gtest/gtest.h>
#include <thread>
#include "common/utils/BoundedQueue.h"
#include "common/utils/Coroutine.h"
namespace hf3fs::test {
namespace {
using namespace std::chrono_literals;
TEST(TestBoundedQueue, Normal) {
BoundedQueue<int> queue(4);
ASSERT_TRUE(queue.empty());
queue.enqueue(1);
queue.enqueue(2);
queue.enqueue(3);
queue.enqueue(4);
ASSERT_TRUE(queue.full());
ASSERT_FALSE(queue.try_enqueue(5));
// 1. sync.
{
std::jthread dequeue([&] {
std::this_thread::sleep_for(100ms);
ASSERT_EQ(queue.dequeue(), 1);
});
auto start = std::chrono::steady_clock::now();
queue.enqueue(5);
auto elapsed = std::chrono::steady_clock::now() - start;
ASSERT_LE(50ms, elapsed);
ASSERT_LE(elapsed, 150ms);
}
// 2. async.
{
std::jthread dequeue([&] {
std::this_thread::sleep_for(100ms);
ASSERT_EQ(folly::coro::blockingWait(queue.co_dequeue()), 2);
});
auto start = std::chrono::steady_clock::now();
folly::coro::blockingWait(queue.co_enqueue(6));
auto elapsed = std::chrono::steady_clock::now() - start;
ASSERT_LE(50ms, elapsed);
ASSERT_LE(elapsed, 150ms);
}
// 3. try enqueue/dequeue.
{
ASSERT_FALSE(queue.try_enqueue(7));
ASSERT_EQ(queue.try_dequeue(), std::optional<int>{3});
ASSERT_TRUE(queue.try_enqueue(7));
int value;
ASSERT_TRUE(queue.try_dequeue(value));
ASSERT_EQ(value, 4);
ASSERT_EQ(queue.size(), 3);
queue.dequeue(value);
ASSERT_EQ(value, 5);
folly::coro::blockingWait(queue.co_dequeue(value));
ASSERT_EQ(value, 6);
ASSERT_EQ(queue.try_dequeue(), std::optional<int>{7});
ASSERT_EQ(queue.try_dequeue(), std::optional<int>{});
ASSERT_FALSE(queue.try_dequeue());
}
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,165 @@
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/Invoke.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/experimental/coro/Timeout.h>
#include <gtest/gtest.h>
#include "common/utils/CoLockManager.h"
#include "common/utils/Duration.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
using namespace std::chrono_literals;
TEST(TestCoLockManager, Normal) {
CoLockManager lockManager;
{
folly::coro::Baton baton;
auto guard = lockManager.lock(baton, "A", "no wait");
folly::coro::blockingWait(guard.lock());
}
{
folly::coro::Baton baton;
auto guard = lockManager.lock(baton, "A", "no wait");
folly::coro::blockingWait(guard.lock());
}
folly::CPUThreadPoolExecutor executor(1);
folly::coro::co_invoke([&]() -> CoTask<void> {
folly::coro::Baton baton;
auto guard = lockManager.lock(baton, "B", "first");
CO_ASSERT_TRUE(guard.locked());
co_await guard.lock();
co_await folly::coro::sleep(200ms);
co_return;
})
.scheduleOn(&executor)
.start();
folly::coro::co_invoke([&]() -> CoTask<void> {
co_await folly::coro::sleep(100ms);
folly::coro::Baton baton;
auto start = RelativeTime::now();
auto guard = lockManager.lock(baton, "B", "second");
CO_ASSERT_FALSE(guard.locked());
co_await guard.lock();
auto elapsed = RelativeTime::now() - start;
CO_ASSERT_TRUE(70_ms <= elapsed);
CO_ASSERT_TRUE(elapsed <= 130_ms);
co_await folly::coro::sleep(200ms);
co_return;
})
.scheduleOn(&executor)
.start();
{
std::this_thread::sleep_for(300ms);
folly::coro::Baton baton;
auto start = RelativeTime::now();
auto guard = lockManager.lock(baton, "B", "third");
ASSERT_FALSE(guard.locked());
folly::coro::blockingWait(guard.lock());
auto elapsed = RelativeTime::now() - start;
ASSERT_NEAR(elapsed.asMs().count(), 100, 30);
}
}
TEST(TestCoLockManager, Concurrent) {
CoLockManager lockManager;
folly::CPUThreadPoolExecutor executor(16);
constexpr auto N = 20000;
auto cnt = 0;
auto current = 0;
folly::coro::Baton finish;
for (auto i = 0; i < N; ++i) {
folly::coro::co_invoke([&]() -> CoTask<void> {
folly::coro::Baton baton;
auto guard = lockManager.lock(baton, "A");
co_await guard.lock();
++current;
CO_ASSERT_EQ(current, 1);
if (++cnt == N) {
finish.post();
}
--current;
})
.scheduleOn(&executor)
.start();
}
auto result = folly::coro::blockingWait(
folly::coro::co_awaitTry(folly::coro::timeout([&]() -> CoTask<void> { co_await finish; }(), 5s)));
ASSERT_FALSE(result.hasException());
}
TEST(TestCoLockManager, Concurrent2) {
CoLockManager lockManager;
constexpr auto N = 20000;
auto cnt = 0;
auto current = 0;
folly::coro::Baton finish;
std::vector<std::thread> threads(16);
for (auto &thread : threads) {
thread = std::thread([&] {
for (auto i = 0; i < N / 16; ++i) {
folly::coro::Baton baton;
auto guard = lockManager.lock(baton, "A");
if (!guard.locked()) {
folly::coro::blockingWait(guard.lock());
}
++current;
ASSERT_EQ(current, 1);
if (++cnt == N) {
finish.post();
}
--current;
}
});
}
auto result = folly::coro::blockingWait(
folly::coro::co_awaitTry(folly::coro::timeout([&]() -> CoTask<void> { co_await finish; }(), 5s)));
ASSERT_FALSE(result.hasException());
for (auto &thread : threads) {
thread.join();
}
}
TEST(TestCoLockManager, Concurrent3) {
CoLockManager lockManager;
constexpr auto N = 20000;
auto cnt = 0;
auto current = 0;
folly::coro::Baton finish;
std::vector<std::thread> threads(16);
for (auto &thread : threads) {
thread = std::thread([&] {
for (auto i = 0; i < N / 16;) {
folly::coro::Baton baton;
auto guard = lockManager.tryLock(baton, "A");
if (!guard.locked()) {
continue;
}
++current;
ASSERT_EQ(current, 1);
if (++cnt == N) {
finish.post();
}
--current;
++i;
}
});
}
auto result = folly::coro::blockingWait(
folly::coro::co_awaitTry(folly::coro::timeout([&]() -> CoTask<void> { co_await finish; }(), 5s)));
ASSERT_FALSE(result.hasException());
for (auto &thread : threads) {
thread.join();
}
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,44 @@
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/Timeout.h>
#include "common/utils/ConcurrencyLimiter.h"
#include "common/utils/Duration.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::net::test {
namespace {
TEST(TestConcurrencyLimiter, Normal) {
ConcurrencyLimiterConfig config;
ConcurrencyLimiter<std::string> limiter(config);
for (auto c = 1; c <= 4; c *= 2) {
config.set_max_concurrency(c);
constexpr auto N = 20000;
std::atomic<size_t> cnt = 0;
std::atomic<size_t> current = 0;
folly::coro::Baton finish;
std::vector<std::thread> threads(16);
for (auto &thread : threads) {
thread = std::thread([&] {
for (auto i = 0; i < N / 16; ++i) {
auto guard = folly::coro::blockingWait(limiter.lock("A"));
++current;
ASSERT_LE(current, c);
if (++cnt == N) {
finish.post();
}
--current;
}
});
}
auto result = folly::coro::blockingWait(
folly::coro::co_awaitTry(folly::coro::timeout([&]() -> CoTask<void> { co_await finish; }(), 5_s)));
ASSERT_FALSE(result.hasException());
for (auto &thread : threads) {
thread.join();
}
}
}
} // namespace
} // namespace hf3fs::net::test

View File

@@ -0,0 +1,522 @@
#include <chrono>
#include <folly/experimental/TestUtil.h>
#include <folly/logging/xlog.h>
#include <gtest/gtest.h>
#include <type_traits>
#include "common/utils/ConfigBase.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
enum class Channel { Red, Green, Blue };
class Config : public ConfigBase<Config> {
CONFIG_HOT_UPDATED_ITEM(string, "ing");
CONFIG_HOT_UPDATED_ITEM(channel, Channel::Red);
CONFIG_HOT_UPDATED_ITEM(optional, std::optional<std::string>{});
CONFIG_SECT(sect, {
CONFIG_HOT_UPDATED_ITEM(val, 100l, [](const int64_t &val) { return val < 200; });
CONFIG_HOT_UPDATED_ITEM(foo, "foo");
CONFIG_HOT_UPDATED_ITEM(score, 0.0);
CONFIG_HOT_UPDATED_ITEM(ok, false);
CONFIG_HOT_UPDATED_ITEM(succ, true);
CONFIG_SECT(sub, {
CONFIG_HOT_UPDATED_ITEM(val, 100l);
CONFIG_HOT_UPDATED_ITEM(foo, "foo");
});
});
CONFIG_SECT(vec, {
CONFIG_HOT_UPDATED_ITEM(int_vec, std::vector<int64_t>({1, 2}), [](const std::vector<int64_t> &v) {
return v.size() <= 2;
});
CONFIG_HOT_UPDATED_ITEM(str_vec, std::vector<std::string>({"foo", "foo"}));
CONFIG_HOT_UPDATED_ITEM(double_vec, std::vector<double>({1.0, 2.0}));
CONFIG_HOT_UPDATED_ITEM(bool_vec, std::vector<bool>({true, false}));
CONFIG_HOT_UPDATED_ITEM(enum_vec, std::vector<Channel>({Channel::Red, Channel::Green}));
});
};
static_assert(std::is_same_v<decltype(Config{}.sect().foo()), const std::string &>, "ok");
static_assert(std::is_same_v<decltype(Config{}.sect().val()), int64_t>, "ok");
static_assert(std::is_same_v<decltype(Config{}.vec().int_vec()), const std::vector<int64_t> &>, "ok");
class BigConfig : public ConfigBase<BigConfig> {
CONFIG_OBJ(a, Config);
CONFIG_OBJ(b, Config, [](Config &c) { c.set_string("replaced"); });
};
class ArrayConfig : public ConfigBase<ArrayConfig> {
CONFIG_OBJ_ARRAY(cfgs, Config, 4);
} array;
TEST(TestConfig, Normal) {
Config cfg;
ASSERT_EQ(cfg.string(), "ing");
ASSERT_FALSE(cfg.optional().has_value());
ASSERT_EQ(cfg.sect().foo(), "foo");
ASSERT_EQ(cfg.sect().val(), 100);
ASSERT_EQ(cfg.sect().ok(), false);
ASSERT_EQ(cfg.sect().succ(), true);
ASSERT_EQ(cfg.sect().sub().val(), 100);
toml::table result = toml::parse(R"(
optional = "has value"
[sect]
foo = "bar"
val = 123
score = 1
ok = true
[sect.sub]
val = 200
)");
ASSERT_EQ(cfg.sect().foo(), "foo");
ASSERT_EQ(cfg.sect().val(), 100);
ASSERT_EQ(cfg.sect().ok(), false);
ASSERT_EQ(cfg.sect().sub().val(), 100);
ASSERT_TRUE(cfg.update(result));
ASSERT_TRUE(cfg.optional().has_value());
ASSERT_EQ(cfg.optional().value(), "has value");
ASSERT_EQ(cfg.sect().foo(), "bar");
ASSERT_EQ(cfg.sect().val(), 123);
ASSERT_EQ(cfg.sect().ok(), true);
ASSERT_EQ(cfg.sect().sub().val(), 200);
ASSERT_TRUE(cfg.sect().set_val(150));
ASSERT_FALSE(cfg.sect().set_val(200));
// support copy.
auto other = cfg;
ASSERT_EQ(other.sect().foo(), "bar");
ASSERT_EQ(other.sect().val(), 150);
ASSERT_TRUE(cfg.update(result));
ASSERT_EQ(cfg.sect().val(), 123);
ASSERT_EQ(other.sect().val(), 150);
}
TEST(TestConfig, Vector) {
Config cfg;
ASSERT_TRUE(cfg.vec().enum_vec() == std::vector({Channel::Red, Channel::Green}));
toml::table result = toml::parse(R"(
[vec]
int_vec = [1, 2]
str_vec = ["foo", "bar"]
enum_vec = ["Red", "Blue"]
)");
ASSERT_TRUE(cfg.update(result));
std::vector<int64_t> expected_int_vec{1, 2};
ASSERT_TRUE(cfg.vec().int_vec() == expected_int_vec);
std::vector<std::string> expected_str_vec{"foo", "bar"};
ASSERT_TRUE(cfg.vec().str_vec() == expected_str_vec);
std::vector expected_enum_vec{Channel::Red, Channel::Blue};
ASSERT_TRUE(cfg.vec().enum_vec() == expected_enum_vec);
result = toml::parse(R"(
[vec]
int_vec = [1, 2, 3]
)");
ASSERT_FALSE(cfg.update(result));
result = toml::parse(R"(
[vec]
enum_vec = ["Yellow"]
)");
ASSERT_FALSE(cfg.update(result));
}
TEST(TestConfig, ParseValue) {
Config cfg;
// 1. empty config.
ASSERT_TRUE(cfg.update(toml::parse("")));
// 2. redundant entries.
ASSERT_FALSE(cfg.update(toml::parse(R"(
something = "else"
)")));
ASSERT_FALSE(cfg.update(toml::parse(R"(
[sect]
something = "else"
)")));
ASSERT_FALSE(cfg.update(toml::parse(R"(
[some]
thing = "else"
)")));
// 3. invalid values.
ASSERT_FALSE(cfg.update(toml::parse(R"(
[sect]
val = 200
)")));
// 4. invalid enum.
ASSERT_FALSE(cfg.update(toml::parse(R"(
channel = "Yellow"
)")));
}
TEST(TestConfig, Nested) {
BigConfig cfg;
ASSERT_EQ(cfg.a().channel(), Channel::Red);
ASSERT_EQ(cfg.a().string(), "ing");
ASSERT_EQ(cfg.b().string(), "replaced");
cfg.b().set_channel(Channel::Green);
ASSERT_EQ(cfg.a().channel(), Channel::Red);
ASSERT_EQ(cfg.b().channel(), Channel::Green);
BigConfig other;
ASSERT_TRUE(other.update(toml::parse(cfg.toString())));
ASSERT_EQ(other.a().channel(), Channel::Red);
ASSERT_EQ(other.b().channel(), Channel::Green);
}
TEST(TestConfig, ToToml) {
Config cfg;
cfg.set_string("set string with \"quotes\"");
cfg.sect().set_ok(false);
cfg.vec().set_str_vec({"1", "2", "3"});
cfg.set_channel(Channel::Blue);
XLOGF(INFO, "toString: {}", cfg.toString());
toml::table result = toml::parse(cfg.toString());
Config other;
ASSERT_TRUE(other.update(result));
ASSERT_EQ(cfg.string(), other.string());
ASSERT_EQ(cfg.sect().ok(), other.sect().ok());
ASSERT_EQ(cfg.vec().str_vec(), other.vec().str_vec());
ASSERT_EQ(cfg.toString(), other.toString());
ASSERT_EQ(cfg.channel(), other.channel());
}
// TEST(TestConfig, Init) {
// char *argvArr[] = {
// const_cast<char *>("./test"),
// const_cast<char *>("--config.channel=Blue"),
// const_cast<char *>("--config.string"),
// const_cast<char *>("a long string"),
// const_cast<char *>("--config.sect.val"),
// const_cast<char *>("123"),
// };
// int argc = ARRAY_SIZE(argvArr);
// Config cfg;
// auto *argv = argvArr; // decay char *[] to char **
// ASSERT_TRUE(cfg.init(&argc, &argv, false));
// ASSERT_EQ(cfg.channel(), Channel::Blue);
// ASSERT_EQ(cfg.string(), "a long string");
// ASSERT_EQ(cfg.sect().val(), 123);
// }
TEST(TestConfig, ArrayOfTable) {
ASSERT_EQ(array.cfgs_length(), 1u);
ASSERT_EQ(array.cfgs(0).channel(), Channel::Red);
toml::table table = toml::parse(R"(
[[cfgs]]
string = "A"
channel = "Blue"
[[cfgs]]
string = "B"
channel = "Red"
[cfgs.sect]
val = 121
)");
ASSERT_TRUE(array.update(table));
ASSERT_EQ(array.cfgs_length(), 2u);
ASSERT_EQ(array.cfgs(0).string(), "A");
ASSERT_EQ(array.cfgs(0).channel(), Channel::Blue);
ASSERT_EQ(array.cfgs(0).sect().val(), 100);
ASSERT_EQ(array.cfgs(1).string(), "B");
ASSERT_EQ(array.cfgs(1).channel(), Channel::Red);
ASSERT_EQ(array.cfgs(1).sect().val(), 121);
XLOGF(INFO, "toString: {}", array.toString());
}
TEST(TestConfig, CheckIsParsedFromString) {
Config cfg;
ASSERT_TRUE(cfg.find("string"));
ASSERT_TRUE(cfg.find("string").value()->isParsedFromString());
ASSERT_TRUE(cfg.find("channel"));
ASSERT_TRUE(cfg.find("channel").value()->isParsedFromString());
ASSERT_TRUE(cfg.find("optional"));
ASSERT_TRUE(cfg.find("optional").value()->isParsedFromString());
ASSERT_FALSE(cfg.find("sect"));
ASSERT_FALSE(cfg.find("not_found"));
ASSERT_TRUE(cfg.find("sect.foo"));
ASSERT_TRUE(cfg.find("sect.foo").value()->isParsedFromString());
ASSERT_TRUE(cfg.find("sect.ok"));
ASSERT_FALSE(cfg.find("sect.ok").value()->isParsedFromString());
}
TEST(TestConfig, HotUpdated) {
Config cfg;
auto now = std::chrono::steady_clock::now;
cfg.set_string(cfg.sect().foo());
std::jthread read([&] {
auto start = now();
while (now() - start <= std::chrono::milliseconds(100)) {
auto clone = cfg.clone();
ASSERT_EQ(clone.string(), clone.sect().foo());
clone.copy(cfg);
ASSERT_EQ(clone.string(), clone.sect().foo());
}
});
std::jthread update([&] {
auto start = now();
while (now() - start <= std::chrono::milliseconds(100)) {
ASSERT_TRUE(cfg.update(toml::parse(fmt::format(R"(
string = "{0}"
[sect]
foo = "{0}")",
now().time_since_epoch().count()))));
}
});
}
TEST(TestConfig, InlineTable) {
class Config : public ConfigBase<Config> {
CONFIG_HOT_UPDATED_ITEM(map, (std::map<std::string, int>{}));
CONFIG_HOT_UPDATED_ITEM(string_to_string, (std::map<std::string, std::string>{}));
} cfg;
ASSERT_TRUE(cfg.map().empty());
ASSERT_TRUE(cfg.string_to_string().empty());
toml::table table = toml::parse(R"(
map = { one = 1, two = 2 }
[string_to_string]
hello = 'world'
language = 'C++'
)");
ASSERT_TRUE(cfg.update(table));
ASSERT_EQ(cfg.map().at("one"), 1);
ASSERT_EQ(cfg.map().at("two"), 2);
ASSERT_EQ(cfg.string_to_string().at("hello"), "world");
ASSERT_EQ(cfg.string_to_string().at("language"), "C++");
XLOGF(INFO, "toString: {}", cfg.toString());
}
TEST(TestConfig, Set) {
class Config : public ConfigBase<Config> {
CONFIG_HOT_UPDATED_ITEM(set, std::set<std::string>{});
} cfg;
ASSERT_TRUE(cfg.set().empty());
toml::table table = toml::parse(R"(
set = ["one", "two"]
)");
ASSERT_TRUE(cfg.update(table));
ASSERT_EQ(cfg.set().count("one"), 1);
ASSERT_EQ(cfg.set().count("two"), 1);
ASSERT_EQ(cfg.set().count("three"), 0);
XLOGF(INFO, "toString: {}", cfg.toString());
ASSERT_EQ(cfg.toString(), "set = [ 'one', 'two' ]");
}
TEST(TestConfig, AtomicallyUpdate) {
std::string_view str = R"(
[sect]
val = 123
)";
Config cfg;
ASSERT_TRUE(cfg.atomicallyUpdate(str));
ASSERT_EQ(cfg.sect().val(), 123);
cfg.sect().set_val(100);
folly::test::TemporaryFile file;
::write(file.fd(), str.data(), str.length());
ASSERT_TRUE(cfg.atomicallyUpdate(file.path()));
ASSERT_EQ(cfg.sect().val(), 123);
}
TEST(TestConfig, VariantType) {
class Config : public ConfigBase<Config> {
CONFIG_SECT(type1, {
CONFIG_HOT_UPDATED_ITEM(a, 0);
CONFIG_HOT_UPDATED_ITEM(b, 0);
CONFIG_HOT_UPDATED_ITEM(c, 0);
});
CONFIG_SECT(type2, {
CONFIG_HOT_UPDATED_ITEM(x, 0);
CONFIG_HOT_UPDATED_ITEM(y, 0);
CONFIG_HOT_UPDATED_ITEM(z, 0);
});
CONFIG_VARIANT_TYPE("type1");
};
Config config;
XLOGF(INFO, "{}", config.toString());
ASSERT_OK(config.validate());
ASSERT_TRUE(config.set_type("type2"));
XLOGF(INFO, "{}", config.toString());
ASSERT_OK(config.validate());
ASSERT_FALSE(config.set_type("type3"));
}
TEST(TestConfig, UpdateCallback) {
Config config;
auto cnt = 0;
auto guard = config.addCallbackGuard([&] { ++cnt; });
ASSERT_EQ(cnt, 0);
ASSERT_OK(config.update(toml::parse(R"( string = "else" )")));
ASSERT_EQ(cnt, 1);
guard->dismiss();
ASSERT_OK(config.update(toml::parse(R"( string = "else" )")));
ASSERT_EQ(cnt, 1);
guard = config.addCallbackGuard();
guard->setCallback([&] { cnt += 7; });
ASSERT_OK(config.update(toml::parse(R"( string = "else" )")));
ASSERT_EQ(cnt, 8);
}
TEST(TestConfig, HotUpdatedMacro) {
class Config : public ConfigBase<Config> {
CONFIG_SECT(cold, {
CONFIG_ITEM(a, 0);
CONFIG_ITEM(b, (std::vector<int>{1, 2, 3}));
CONFIG_ITEM(c, (std::map<std::string, int>{{"one", 1}, {"two", 2}}));
});
CONFIG_SECT(hot, {
CONFIG_HOT_UPDATED_ITEM(a, 0);
CONFIG_HOT_UPDATED_ITEM(b, (std::vector<int>{1, 2, 3}));
CONFIG_HOT_UPDATED_ITEM(c, (std::map<std::string, int>{{"one", 1}, {"two", 2}}));
});
};
Config config;
// hot update but keep value, succ.
ASSERT_OK(config.update(toml::parse(R"(
[cold]
a = 0
b = [1, 2, 3]
c = { one = 1, two = 2 }
)")));
// hot update and change value, fail.
ASSERT_FALSE(config.update(toml::parse(R"(
[cold]
a = 1
)")));
ASSERT_FALSE(config.update(toml::parse(R"(
[cold]
b = [1, 2]
)")));
ASSERT_FALSE(config.update(toml::parse(R"(
[cold]
c = { one = 1 }
)")));
// not a hot update. succ.
ASSERT_OK(config.update(toml::parse(R"(
[cold]
a = 1
)"),
false));
ASSERT_OK(config.update(toml::parse(R"(
[cold]
b = [1, 2]
)"),
false));
ASSERT_OK(config.update(toml::parse(R"(
[cold]
c = { one = 1 }
)"),
false));
ASSERT_EQ(config.cold().a(), 1);
ASSERT_EQ(config.cold().b(), (std::vector<int>{1, 2}));
ASSERT_EQ(config.cold().c(), (std::map<std::string, int>{{"one", 1}}));
// support hot updated. succ.
ASSERT_OK(config.update(toml::parse(R"(
[hot]
a = 0
b = [1, 2, 3]
c = { one = 1, two = 2 }
)")));
ASSERT_OK(config.update(toml::parse(R"(
[hot]
a = 1
)")));
ASSERT_OK(config.update(toml::parse(R"(
[hot]
b = [1, 2]
)")));
ASSERT_OK(config.update(toml::parse(R"(
[hot]
c = { one = 1 }
)")));
ASSERT_OK(config.update(toml::parse(R"(
[hot]
a = 1
)"),
false));
ASSERT_OK(config.update(toml::parse(R"(
[hot]
b = [1, 2]
)"),
false));
ASSERT_OK(config.update(toml::parse(R"(
[hot]
c = { one = 1 }
)"),
false));
}
TEST(TestConfig, DiffWith) {
auto stringfy = [](const config::IConfig::ItemDiff &diff) {
return fmt::format("{}: {} -> {}", diff.key, diff.left, diff.right);
};
config::IConfig::ItemDiff diffs[3];
Config c0;
Config c1;
ASSERT_EQ(c0.diffWith(c1, std::span(diffs)), 0);
c1.set_optional("nonnull");
c1.sect().set_foo("abc");
c1.sect().sub().set_val(101);
auto diffCnt = c0.diffWith(c1, std::span(diffs));
ASSERT_EQ(diffCnt, 3);
ASSERT_EQ(stringfy(diffs[0]), "optional: nullopt -> 'nonnull'");
ASSERT_EQ(stringfy(diffs[1]), "sect.foo: 'foo' -> 'abc'");
ASSERT_EQ(stringfy(diffs[2]), "sect.sub.val: 100 -> 101");
c1.sect().sub().set_foo("");
ASSERT_EQ(c0.diffWith(c1, std::span(diffs)), 3);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,42 @@
#include <folly/executors/CPUThreadPoolExecutor.h>
#include "common/utils/CoroSynchronized.h"
#include "common/utils/CountDownLatch.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
folly::CPUThreadPoolExecutor &getExecutor() {
static folly::CPUThreadPoolExecutor executor(4, std::make_shared<folly::NamedThreadFactory>("Test"));
return executor;
}
TEST(CoroSynchronizedTest, testInc) {
CoroSynchronized<int64_t> count(0);
constexpr int64_t countPerAdder = 10000;
constexpr int64_t adderCount = 10;
CountDownLatch latch(adderCount);
auto adder = [&]() -> CoTask<void> {
for (int64_t i = 0; i < countPerAdder; ++i) {
auto ptr = co_await count.coLock();
++*ptr;
}
latch.countDown();
};
for (int64_t i = 0; i < adderCount; ++i) {
adder().scheduleOn(&getExecutor()).start();
}
auto result = folly::coro::blockingWait([&]() -> CoTask<int64_t> {
co_await latch.wait();
auto ptr = co_await count.coSharedLock();
co_return *ptr;
}());
ASSERT_EQ(result, adderCount * countPerAdder);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,49 @@
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/BlockingWait.h>
#include "common/utils/CoroutinesPool.h"
#include "common/utils/CountDownLatch.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
void testCoroutinesPool(bool enableWorkStealing) {
struct Job {
int cnt = 0;
};
folly::CPUThreadPoolExecutor executor(4);
CoroutinesPool<Job>::Config config;
config.set_enable_work_stealing(enableWorkStealing);
CoroutinesPool<Job> pool(config, &executor);
SCOPE_EXIT { pool.stopAndJoin(); };
int n = 100;
CountDownLatch latch(2 * n);
std::atomic<int> sum = 0;
pool.start([&sum, &latch](Job job) -> CoTask<void> {
sum += job.cnt;
latch.countDown();
co_return;
});
for (auto i = 0; i < n; ++i) {
pool.enqueueSync(Job{i + 1});
folly::coro::blockingWait(pool.enqueue(Job{i}));
}
folly::coro::blockingWait(latch.wait());
ASSERT_EQ(sum, 10000);
}
TEST(TestCoroutinesPool, Normal) { testCoroutinesPool(false); }
TEST(TestCoroutinesPool, WorkStealing) { testCoroutinesPool(true); }
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,52 @@
#include <folly/experimental/coro/BlockingWait.h>
#include <gtest/gtest.h>
#include "common/utils/DefaultRetryStrategy.h"
namespace hf3fs::tests {
namespace {
using namespace std::chrono_literals;
struct MockSleeper {
CoTask<void> operator()(std::chrono::milliseconds d) {
records.push_back(d);
co_return;
}
std::vector<std::chrono::milliseconds> records;
};
DefaultRetryStrategy<MockSleeper> getDefaultRetryStrategy() {
return DefaultRetryStrategy<MockSleeper>(RetryConfig{10ms, 100ms, 5});
}
TEST(DefaultRetryStrategy, basic) {
folly::coro::blockingWait([&]() -> CoTask<void> {
{
auto retryStrategy = getDefaultRetryStrategy();
// kDataCorruption is unretryable
auto r = co_await retryStrategy.onError(Status(StatusCode::kDataCorruption));
EXPECT_TRUE(r.hasError());
EXPECT_EQ(r.error().code(), StatusCode::kDataCorruption);
EXPECT_EQ(retryStrategy.getSleeper().records.size(), 0);
}
{
auto retryStrategy = getDefaultRetryStrategy();
// kThrottled is retryable
for (int i = 0; i < 5; ++i) {
auto r = co_await retryStrategy.onError(Status(TransactionCode::kThrottled));
EXPECT_TRUE(!r.hasError());
}
{
auto r = co_await retryStrategy.onError(Status(TransactionCode::kThrottled));
EXPECT_TRUE(r.hasError());
EXPECT_EQ(r.error().code(), TransactionCode::kThrottled);
}
std::vector<std::chrono::milliseconds> expected = {10ms, 20ms, 40ms, 80ms, 100ms};
EXPECT_EQ(retryStrategy.getSleeper().records, expected);
}
}());
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,121 @@
#include "common/net/Allocator.h"
#include "common/utils/DownwardBytes.h"
#include "common/utils/Size.h"
#include "tests/GtestHelpers.h"
namespace hf3fs {
namespace {
std::vector<std::pair<size_t, bool>> logger;
struct Allocator {
public:
static constexpr size_t kDefaultSize = 1_MB;
static std::uint8_t *allocate(size_t size) {
size = std::max(size, kDefaultSize);
logger.emplace_back(size, true);
return new uint8_t[size];
}
static void deallocate(const uint8_t *buf, size_t size) {
size = std::max(size, kDefaultSize);
logger.emplace_back(size, false);
delete[] buf;
}
};
TEST(TestDownwardBytes, Normal) {
ASSERT_EQ(logger.size(), 0);
{
DownwardBytes<Allocator> bytes;
ASSERT_EQ(bytes.capacity(), 1_MB);
ASSERT_EQ(logger.size(), 1);
ASSERT_EQ(logger.back().first, 1_MB);
ASSERT_TRUE(logger.back().second);
}
ASSERT_EQ(logger.size(), 2);
ASSERT_EQ(logger.back().first, 1_MB);
ASSERT_FALSE(logger.back().second);
{
DownwardBytes<Allocator> bytes;
auto old = bytes.data();
ASSERT_EQ(bytes.capacity(), 1_MB);
ASSERT_EQ(logger.size(), 3);
ASSERT_EQ(logger.back().first, 1_MB);
ASSERT_TRUE(logger.back().second);
std::vector<uint8_t> empty(512_KB, 'A');
bytes.append(empty.data(), empty.size());
ASSERT_EQ(bytes.data() + 512_KB, old);
ASSERT_EQ(bytes.data()[0], 'A');
ASSERT_EQ(bytes.data()[512_KB - 1], 'A');
ASSERT_EQ(bytes.capacity(), 1_MB);
ASSERT_EQ(bytes.size(), 512_KB);
ASSERT_EQ(logger.size(), 3);
std::fill(empty.begin(), empty.end(), 'B');
bytes.append(empty.data(), empty.size());
ASSERT_EQ(bytes.data() + 1_MB, old);
ASSERT_EQ(bytes.data()[0], 'B');
ASSERT_EQ(bytes.data()[512_KB - 1], 'B');
ASSERT_EQ(bytes.data()[512_KB], 'A');
ASSERT_EQ(bytes.data()[1_MB - 1], 'A');
ASSERT_EQ(bytes.capacity(), 1_MB);
ASSERT_EQ(bytes.size(), 1_MB);
ASSERT_EQ(logger.size(), 3);
std::fill(empty.begin(), empty.end(), 'C');
bytes.append(empty.data(), 1);
ASSERT_NE(bytes.data() + 1_MB + 1, old);
ASSERT_EQ(bytes.data()[0], 'C');
ASSERT_EQ(bytes.data()[1], 'B');
ASSERT_EQ(bytes.data()[512_KB], 'B');
ASSERT_EQ(bytes.data()[512_KB + 1], 'A');
ASSERT_EQ(bytes.data()[1_MB], 'A');
ASSERT_EQ(bytes.capacity(), 2_MB);
ASSERT_EQ(bytes.size(), 1_MB + 1);
ASSERT_EQ(logger.size(), 5);
ASSERT_EQ(logger[3].first, 2_MB);
ASSERT_TRUE(logger[3].second);
ASSERT_EQ(logger[4].first, 1_MB);
ASSERT_FALSE(logger[4].second);
uint32_t offset, capacity;
auto buf = bytes.release(offset, capacity);
ASSERT_EQ(logger.size(), 5);
ASSERT_EQ(bytes.data(), nullptr);
ASSERT_EQ(bytes.size(), 0);
ASSERT_EQ(bytes.capacity(), 0);
ASSERT_EQ(offset, 1_MB - 1);
ASSERT_EQ(capacity, 2_MB);
ASSERT_EQ(buf[offset], 'C');
Allocator::deallocate(buf, capacity);
}
ASSERT_EQ(logger.size(), 6);
ASSERT_EQ(logger.back().first, 2_MB);
ASSERT_FALSE(logger.back().second);
{
DownwardBytes<Allocator> bytes;
ASSERT_EQ(bytes.capacity(), 1_MB);
ASSERT_EQ(logger.size(), 7);
ASSERT_EQ(logger.back().first, 1_MB);
ASSERT_TRUE(logger.back().second);
std::vector<uint8_t> empty(2_MB, 'A');
bytes.append(empty.data(), empty.size());
ASSERT_EQ(logger.size(), 9);
ASSERT_EQ(logger.back().first, 1_MB);
ASSERT_FALSE(logger.back().second);
ASSERT_EQ(logger[7].first, 2_MB);
ASSERT_TRUE(logger[7].second);
}
ASSERT_EQ(logger.size(), 10);
ASSERT_EQ(logger.back().first, 2_MB);
ASSERT_FALSE(logger.back().second);
}
} // namespace
} // namespace hf3fs

View File

@@ -0,0 +1,76 @@
#include <folly/logging/xlog.h>
#include "common/utils/ConfigBase.h"
#include "common/utils/Duration.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
static_assert(config::IsPrimitive<Duration>, "duration is not primitive");
class Config : public ConfigBase<Config> {
CONFIG_HOT_UPDATED_ITEM(c0, Duration::from("0s").value());
CONFIG_HOT_UPDATED_ITEM(c1, 0_ns);
};
TEST(Duration, testDefaultConstruct) {
Duration config;
ASSERT_EQ(config.asMs(), std::chrono::milliseconds{0});
}
TEST(Duration, testParse) {
Duration c0 = Duration::from("100ns").value();
ASSERT_EQ(c0.toString(), "100ns");
Duration c1 = Duration::from("2us").value();
ASSERT_EQ(c1.toString(), "2us");
Duration c2 = Duration::from("1001ms").value();
ASSERT_EQ(c2.toString(), "1s 1ms");
Duration c3 = Duration::from("45s ").value();
ASSERT_EQ(c3.toString(), "45s");
Duration c5 = Duration::from("08h").value();
ASSERT_EQ(c5.toString(), "8h");
Duration c6 = Duration::from("1day").value();
ASSERT_EQ(c6.toString(), "1day");
ASSERT_FALSE(Duration::from("10mon"));
ASSERT_FALSE(Duration::from(""));
ASSERT_FALSE(Duration::from("100"));
ASSERT_FALSE(Duration::from("ns"));
ASSERT_FALSE(Duration::from("1.1s"));
ASSERT_ERROR(Duration::from("1ns2ms"), StatusCode::kInvalidConfig);
ASSERT_ERROR(Duration::from("1ns2ns"), StatusCode::kInvalidConfig);
ASSERT_ERROR(Duration::from("1ns2k"), StatusCode::kInvalidConfig);
ASSERT_ERROR(Duration::from("1ms2.1us"), StatusCode::kInvalidConfig);
ASSERT_ERROR(Duration::from("ms2us"), StatusCode::kInvalidConfig);
ASSERT_ERROR(Duration::from("1day2sec"), StatusCode::kInvalidConfig);
ASSERT_EQ(Duration::from("1day 2s")->toString(), "1day 2s");
ASSERT_EQ(Duration::from("1day 2h 2001 ms")->toString(), "1day 2h 2s 1ms");
}
TEST(Duration, testConfig) {
Config cfg;
ASSERT_EQ(cfg.c0(), Duration::from("0s").value());
ASSERT_EQ(cfg.c0(), cfg.c1());
cfg.set_c0(Duration::from("10ms").value());
ASSERT_EQ(cfg.c0(), Duration::from("10ms").value());
toml::table result = toml::parse(R"(
c0 = "5s"
c1 = "10min"
)");
ASSERT_TRUE(cfg.update(result));
ASSERT_EQ(cfg.c0(), Duration::from("5s").value());
ASSERT_EQ(cfg.c1(), Duration::from("10min").value());
XLOGF(INFO, "toString: {}", cfg.toString());
result = toml::parse(cfg.toString());
Config other;
ASSERT_TRUE(other.update(result));
ASSERT_EQ(cfg.c0(), other.c0());
ASSERT_EQ(cfg.c1(), other.c1());
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,77 @@
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/Invoke.h>
#include <folly/experimental/coro/Sleep.h>
#include "common/utils/Duration.h"
#include "common/utils/DynamicCoroutinesPool.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
TEST(TestDynamicCoroutinesPool, Normal) {
DynamicCoroutinesPool::Config config;
config.set_coroutines_num(8);
DynamicCoroutinesPool pool(config);
ASSERT_OK(pool.start());
ASSERT_OK(pool.stopAndJoin());
DynamicCoroutinesPool{config};
}
TEST(TestDynamicCoroutinesPool, ManyTasks) {
constexpr auto N = 200;
constexpr auto M = 64;
DynamicCoroutinesPool::Config config;
config.set_coroutines_num(M);
DynamicCoroutinesPool pool(config);
ASSERT_OK(pool.start());
folly::coro::Baton baton;
std::atomic<uint32_t> cnt{};
std::atomic<uint32_t> current{};
for (auto i = 0; i < N; ++i) {
pool.enqueue(folly::coro::co_invoke([&]() -> CoTask<void> {
auto now = ++current;
CO_ASSERT_LE(now, M);
co_await folly::coro::sleep(50_ms);
if (++cnt == N) {
baton.post();
}
--current;
}));
}
folly::coro::blockingWait(baton);
}
TEST(TestDynamicCoroutinesPool, HotUpdated) {
constexpr auto N = 64;
DynamicCoroutinesPool::Config config;
config.set_coroutines_num(1);
DynamicCoroutinesPool pool(config);
ASSERT_OK(pool.start());
folly::coro::Baton baton;
std::atomic<uint32_t> cnt{};
std::atomic<uint32_t> current{};
for (auto i = 0; i < N; ++i) {
pool.enqueue(folly::coro::co_invoke([&]() -> CoTask<void> {
auto now = ++current;
config.set_coroutines_num(now + 1);
config.update(toml::table{}, true);
co_await folly::coro::sleep(50_ms);
if (++cnt == N) {
baton.post();
}
--current;
}));
}
folly::coro::blockingWait(baton);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,225 @@
#include <atomic>
#include <chrono>
#include <fmt/core.h>
#include <folly/Random.h>
#include <folly/concurrency/UnboundedQueue.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/executors/IOThreadPoolExecutor.h>
#include <folly/experimental/coro/AsyncGenerator.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/BoundedQueue.h>
#include <folly/experimental/coro/CurrentExecutor.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <folly/experimental/coro/Invoke.h>
#include <folly/experimental/coro/ScopeExit.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/experimental/coro/Task.h>
#include <folly/experimental/coro/UnboundedQueue.h>
#include <folly/futures/Future.h>
#include <folly/io/async/Request.h>
#include <gtest/gtest.h>
#include <optional>
#include <type_traits>
#include <vector>
#include "common/utils/Coroutine.h"
#include "common/utils/FaultInjection.h"
namespace hf3fs::test {
namespace {
TEST(TestFaultInjection, Basic) {
static constexpr size_t kLoops = 100000;
auto fi = FaultInjection::get();
ASSERT_EQ(fi, nullptr);
for (size_t i = 0; i < kLoops; i++) {
auto inject = FAULT_INJECTION();
ASSERT_FALSE(inject);
}
FAULT_INJECTION_SET(1, -1); // 1% probility and unlimited times
ASSERT_EQ(FaultInjection::get()->getProbability(), 1);
ASSERT_EQ(FaultInjectionFactor::get(), 1);
size_t cnt = 0;
for (size_t i = 0; i < kLoops; i++) {
if (FAULT_INJECTION()) {
cnt++;
}
}
ASSERT_NE(cnt, 0);
std::cout << "Inject " << cnt << " faults during " << kLoops << " loops, " << (double)cnt / kLoops * 100
<< "%, expect 1%" << std::endl;
{
// disable fault injection in current scope
FAULT_INJECTION_SET(0, -1);
size_t cnt = 0;
for (size_t i = 0; i < kLoops; i++) {
if (FAULT_INJECTION()) {
cnt++;
}
}
ASSERT_EQ(cnt, 0);
}
ASSERT_EQ(FaultInjection::get()->getProbability(), 1);
ASSERT_EQ(FaultInjectionFactor::get(), 1);
// test set factor scope.
{
FAULT_INJECTION_SET_FACTOR(2);
ASSERT_EQ(FaultInjectionFactor::get(), 2);
{
FAULT_INJECTION_SET_FACTOR(3);
ASSERT_EQ(FaultInjectionFactor::get(), 3);
}
ASSERT_EQ(FaultInjectionFactor::get(), 2);
FAULT_INJECTION_SET_FACTOR(4);
ASSERT_EQ(FaultInjectionFactor::get(), 4);
}
ASSERT_EQ(FaultInjectionFactor::get(), 1);
{
FAULT_INJECTION_SET_FACTOR(10);
ASSERT_EQ(FaultInjectionFactor::get(), 10);
size_t cnt = 0;
for (size_t i = 0; i < kLoops; i++) {
if (FAULT_INJECTION()) {
cnt++;
}
}
ASSERT_NE(cnt, 0);
std::cout << "Set factor of current scope to 10, inject " << cnt << " faults during " << kLoops << " loops, "
<< (double)cnt / kLoops * 100 << "%, expect 0.1%" << std::endl;
}
ASSERT_EQ(FaultInjectionFactor::get(), 1);
{
size_t cnt = 0;
FAULT_INJECTION_SET_FACTOR(100); // should be ignored
for (size_t i = 0; i < kLoops; i++) {
if (FAULT_INJECTION_WITH_FACTOR(5)) {
cnt++;
}
}
ASSERT_NE(cnt, 0);
std::cout << "Inject with factor 5, inject " << cnt << " faults during " << kLoops << " loops, "
<< (double)cnt / kLoops * 100 << "%, expect 0.2%" << std::endl;
}
{
FAULT_INJECTION_SET(10, 10);
size_t cnt = 0;
for (size_t i = 0; i < kLoops; i++) {
if (FAULT_INJECTION()) {
cnt++;
}
}
ASSERT_EQ(cnt, 10);
}
}
TEST(TestFaultInjection, Coroutine) {
ASSERT_EQ(FaultInjection::get(), nullptr);
folly::coro::blockingWait([]() -> folly::coro::Task<void> {
folly::RequestContext::create();
FAULT_INJECTION_SET(10, -1);
co_return;
}());
ASSERT_EQ(FaultInjection::get(), nullptr);
FAULT_INJECTION_SET(10, -1);
auto fi = FaultInjection::get();
ASSERT_NE(fi, nullptr);
folly::coro::blockingWait([]() -> folly::coro::Task<void> {
folly::RequestContext::create();
FAULT_INJECTION_SET(10, -1);
co_return;
}());
ASSERT_EQ(FaultInjection::get(), fi);
{
folly::CPUThreadPoolExecutor executor(4);
folly::coro::co_invoke([fi]() -> folly::coro::Task<void> {
CO_ASSERT_EQ(FaultInjection::get(), fi);
co_return;
})
.scheduleOn(&executor)
.start()
.wait();
executor.join();
}
{
FAULT_INJECTION_SET(10, 1000);
folly::CPUThreadPoolExecutor executor(4);
std::vector<folly::SemiFuture<size_t>> futures;
for (int i = 0; i < 10; i++) {
auto future = folly::coro::co_invoke([&]() -> folly::coro::Task<size_t> {
size_t cnts = 0;
for (int i = 0; i < 100000; i++) {
if (FAULT_INJECTION()) {
cnts++;
}
if (folly::Random::oneIn(100)) co_await folly::coro::co_reschedule_on_current_executor_t();
}
co_return cnts;
})
.scheduleOn(&executor)
.start();
futures.push_back(std::move(future));
}
auto results = folly::collectAll(futures.begin(), futures.end()).wait().result().value();
size_t cnts = 0;
for (auto &result : results) {
cnts += result.value();
}
executor.join();
ASSERT_EQ(cnts, 1000);
}
}
TEST(TestFaultInjection, Coroutine1) {
folly::CPUThreadPoolExecutor exec(1);
auto coro1 = [](FaultInjection *ptr) -> CoTask<void> {
CO_ASSERT_EQ(FaultInjection::get(), ptr);
co_return;
};
auto coro2 = [&]() -> CoTask<void> {
CO_ASSERT_EQ(FaultInjection::get(), nullptr);
FAULT_INJECTION_SET(10, 1);
co_await coro1(FaultInjection::get()).scheduleOn(&exec).start();
co_return;
};
coro2().scheduleOn(&exec).start().wait();
sleep(1);
}
TEST(TestFaultInjection, Coroutine2) {
folly::CPUThreadPoolExecutor exec(1);
auto coro1 = []() -> CoTask<void> {
CO_ASSERT_EQ(folly::RequestContext::try_get(), nullptr);
CO_ASSERT_EQ(FaultInjection::get(), nullptr);
folly::RequestContext::create();
FAULT_INJECTION_SET(10, 1);
CO_ASSERT_NE(folly::RequestContext::try_get(), nullptr);
CO_ASSERT_NE(FaultInjection::get(), nullptr);
co_return;
};
auto coro2 = [&]() -> CoTask<void> {
CO_ASSERT_EQ(folly::RequestContext::try_get(), nullptr);
CO_ASSERT_EQ(FaultInjection::get(), nullptr);
co_await coro1().scheduleOn(&exec).start();
CO_ASSERT_EQ(folly::RequestContext::try_get(), nullptr);
CO_ASSERT_EQ(FaultInjection::get(), nullptr);
co_return;
};
coro2().scheduleOn(&exec).start().wait();
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,24 @@
#include <gtest/gtest.h>
#include "common/utils/FdWrapper.h"
namespace hf3fs::test {
namespace {
TEST(TestFdWrapper, Normal) {
int p[2];
ASSERT_EQ(::pipe(p), 0);
FdWrapper fd;
ASSERT_EQ(fd, -1);
fd = p[0];
fd = FdWrapper{p[1]}; // p[0] is closed.
ASSERT_EQ(fd, p[1]);
fd = FdWrapper{}; // p[1] is closed.
ASSERT_EQ(fd, -1);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,14 @@
#include <fmt/chrono.h>
#include <fmt/format.h>
#include <fmt/ranges.h>
#include <gtest/gtest.h>
namespace hf3fs::tests {
namespace {
TEST(Fmt, testFormat) {
ASSERT_EQ(fmt::format("Hello {}", "world"), "Hello world");
ASSERT_EQ(fmt::format("{}", std::vector<std::string>{"\naan"}), "[\"\\naan\"]");
ASSERT_EQ(fmt::format("{:%S}", std::chrono::milliseconds(1234)), "01.234");
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,27 @@
#include <folly/hash/Checksum.h>
#include <folly/logging/xlog.h>
#include "tests/GtestHelpers.h"
namespace hf3fs::tests {
namespace {
TEST(Folly, CRC32C) {
std::string a = "hello";
auto crc1 = folly::crc32c(reinterpret_cast<const uint8_t *>(a.data()), a.size(), 0);
std::string b = "world";
auto crc2 = folly::crc32c(reinterpret_cast<const uint8_t *>(b.data()), b.size(), 0);
auto x = folly::crc32c_combine(crc1, crc2, b.size());
auto y = folly::crc32c(reinterpret_cast<const uint8_t *>(b.data()), b.size(), crc1);
ASSERT_EQ(x, y);
auto crc32c1 = ~0x14298C12; // 1MB zero.
auto crc32c2 = ~0x527D5351; // one zero.
auto out = folly::crc32c_combine(~crc32c1, crc32c2, 1);
XLOGF(INFO, "{:08X} {:08X}", out, ~out);
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,16 @@
#include <cstdint>
#include <folly/hash/Hash.h>
#include "common/utils/RobinHoodUtils.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
TEST(HashCombine, testUint64) {
for (uint64_t x = 0; x < 64; ++x) {
auto h = folly::hash::hash_combine_generic(RobinHoodHasher{}, x);
fmt::print("x = {} h = {} h % 256 = {}\n", x, h, h % 256);
}
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,175 @@
#include <algorithm>
#include <atomic>
#include <chrono>
#include <fmt/core.h>
#include <folly/Random.h>
#include <folly/Synchronized.h>
#include <folly/Unit.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/CurrentExecutor.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <folly/experimental/coro/Invoke.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/futures/Future.h>
#include <gtest/gtest.h>
#include <memory>
#include <mutex>
#include <vector>
#include "common/kv/mem/MemKVEngine.h"
#include "common/utils/Coroutine.h"
#include "common/utils/FaultInjection.h"
#include "common/utils/IdAllocator.h"
#include "fdb/FDBRetryStrategy.h"
#include "fdb/FDBTransaction.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
kv::FDBRetryStrategy retryStrategy() { return kv::FDBRetryStrategy({.retryMaybeCommitted = true}); }
TEST(TestIdAllocator, Basic) {
folly::coro::blockingWait([&]() -> CoTask<void> {
kv::MemKVEngine engine;
IdAllocator<kv::FDBRetryStrategy> allocator(engine, retryStrategy(), "test", 1);
for (size_t i = 1; i <= 1000; i++) {
auto result = co_await allocator.allocate();
CO_ASSERT_OK(result);
CO_ASSERT_EQ(result.value(), i);
}
}());
}
TEST(TestIdAllocator, MultiThreads) {
kv::MemKVEngine engine;
IdAllocator<kv::FDBRetryStrategy> allocator(engine, retryStrategy(), "test", 1);
std::atomic<size_t> failed = 0;
folly::Synchronized<std::vector<uint64_t>, std::mutex> allocated;
std::vector<folly::SemiFuture<folly::Unit>> futures;
folly::CPUThreadPoolExecutor exec(8);
for (size_t i = 0; i < 128; i++) {
auto task = [&]() -> CoTask<void> {
for (size_t i = 1; i <= 1000; i++) {
auto result = co_await allocator.allocate();
if (result.hasError()) {
CO_ASSERT_ERROR(result, TransactionCode::kConflict);
failed++;
} else {
allocated.lock()->push_back(result.value());
}
co_await folly::coro::co_reschedule_on_current_executor;
}
co_return;
};
futures.push_back(folly::coro::co_invoke(task).scheduleOn(&exec).start());
}
for (auto &f : futures) {
f.wait();
}
auto lock = allocated.lock();
std::sort(lock->begin(), lock->end());
fmt::print("success {}, failed {}, first {}, last {}\n",
lock->size(),
failed.load(),
*lock->begin(),
*lock->rbegin());
for (size_t i = 1; i < lock->size(); i++) {
ASSERT_GT(lock->at(i), lock->at(i - 1));
}
}
TEST(TestIdAllocator, Sharded) {
folly::coro::blockingWait([&]() -> CoTask<void> {
kv::MemKVEngine engine;
IdAllocator<kv::FDBRetryStrategy> allocator(engine, retryStrategy(), "test", 8);
std::vector<uint64_t> allocated;
for (size_t i = 1; i <= 1000; i++) {
auto result = co_await allocator.allocate();
CO_ASSERT_OK(result);
allocated.push_back(result.value());
}
std::sort(allocated.begin(), allocated.end());
for (size_t i = 1; i < allocated.size(); i++) {
CO_ASSERT_GT(allocated.at(i), allocated.at(i - 1));
}
auto status = co_await allocator.status();
CO_ASSERT_OK(status);
std::sort(status->begin(), status->end());
fmt::print("shard usage: min {}, max {}\n", *status->begin(), *status->rbegin());
}());
}
TEST(TestIdAllocator, ShardedMultiThreads) {
kv::MemKVEngine engine;
std::vector<std::unique_ptr<IdAllocator<kv::FDBRetryStrategy>>> vec;
for (int i = 0; i < 8; i++) {
vec.emplace_back(std::make_unique<IdAllocator<kv::FDBRetryStrategy>>(engine, retryStrategy(), "test", 32));
}
std::atomic<size_t> failed = 0;
folly::Synchronized<std::vector<uint64_t>, std::mutex> allocated;
std::vector<folly::SemiFuture<folly::Unit>> futures;
folly::CPUThreadPoolExecutor exec(8);
for (size_t i = 0; i < 128; i++) {
auto task = [&, &allocator = vec[i % vec.size()]]() -> CoTask<void> {
for (size_t i = 1; i <= 1000; i++) {
auto result = co_await allocator->allocate();
if (result.hasError()) {
CO_ASSERT_ERROR(result, TransactionCode::kConflict);
failed++;
} else {
allocated.lock()->push_back(result.value());
}
co_await folly::coro::co_reschedule_on_current_executor;
}
co_return;
};
futures.push_back(folly::coro::co_invoke(task).scheduleOn(&exec).start());
}
for (auto &f : futures) {
f.wait();
}
auto lock = allocated.lock();
std::sort(lock->begin(), lock->end());
fmt::print("success {}, failed {}, min {}, max {}\n", lock->size(), failed.load(), *lock->begin(), *lock->rbegin());
for (size_t i = 1; i < lock->size(); i++) {
ASSERT_GT(lock->at(i), lock->at(i - 1));
}
auto status = vec[0]->status().scheduleOn(&exec).start().wait().value();
ASSERT_OK(status);
std::sort(status->begin(), status->end());
fmt::print("shard usage: min {}, max {}\n", *status->begin(), *status->rbegin());
}
TEST(TestIdAllocator, FaultInjection) {
folly::coro::blockingWait([&]() -> CoTask<void> {
FAULT_INJECTION_SET(2, -1);
kv::MemKVEngine engine;
IdAllocator<kv::FDBRetryStrategy> allocator(engine, retryStrategy(), "test", 8);
std::set<uint64_t> set;
size_t fail = 0;
for (size_t i = 1; i <= 1000; i++) {
auto result = co_await allocator.allocate();
if (result.hasValue()) {
CO_ASSERT_FALSE(set.contains(result.value()));
set.insert(result.value());
} else {
fail++;
}
}
std::cout << "success " << set.size() << ", fail " << fail << std::endl;
}());
}
} // namespace hf3fs::test

View File

@@ -0,0 +1,23 @@
#include <fmt/format.h>
#include <limits>
#include <set>
#include "common/utils/Int128.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
TEST(TestInt128, Normal) {
auto value = std::numeric_limits<uint128_t>::max();
ASSERT_EQ(fmt::format("{}", value), "340282366920938463463374607431768211455");
std::set<uint128_t> set;
set.insert(value);
ASSERT_EQ(set.size(), 1ul);
ASSERT_TRUE(set.count(value));
ASSERT_FALSE(set.count(0));
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,85 @@
#include <fcntl.h>
#include <liburing.h>
#include <liburing/io_uring.h>
#include "common/utils/FdWrapper.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
TEST(TestLibUring, Normal) {
constexpr auto N = 1024;
FdWrapper fd{::open("/dev/zero", O_RDONLY)};
ASSERT_TRUE(fd);
struct io_uring ring;
ASSERT_EQ(io_uring_queue_init(N, &ring, 0), 0);
std::string buf(N, 'A');
struct iovec iovec;
iovec.iov_base = buf.data();
iovec.iov_len = N;
struct io_uring_sqe *sqe = io_uring_get_sqe(&ring);
ASSERT_NE(sqe, nullptr);
io_uring_prep_readv(sqe, fd, &iovec, 1, 0);
io_uring_sqe_set_data(sqe, nullptr);
ASSERT_EQ(io_uring_submit(&ring), 1);
struct io_uring_cqe *cqe;
ASSERT_EQ(::io_uring_wait_cqe(&ring, &cqe), 0);
ASSERT_EQ(cqe->res, N);
::io_uring_cqe_seen(&ring, cqe);
ASSERT_EQ(buf, std::string(N, '\0'));
::io_uring_queue_exit(&ring);
}
TEST(TestLibUring, SQPolling) {
if (getuid()) {
// skip if non-root.
GTEST_SKIP_("Skip SQPolling if non-root");
}
constexpr auto N = 256;
FdWrapper fd{::open("/dev/zero", O_RDONLY)};
ASSERT_TRUE(fd);
struct io_uring ring {};
struct io_uring_params params {};
params.flags = IORING_SETUP_SQPOLL;
params.sq_thread_idle = 50;
ASSERT_EQ(io_uring_queue_init_params(N, &ring, &params), 0);
int files[1] = {fd};
ASSERT_EQ(io_uring_register_files(&ring, files, 1), 0);
std::string buf(N, 'A');
struct iovec iovec;
iovec.iov_base = buf.data();
iovec.iov_len = N;
ASSERT_EQ(io_uring_register_buffers(&ring, &iovec, 1), 0);
struct io_uring_sqe *sqe = io_uring_get_sqe(&ring);
ASSERT_NE(sqe, nullptr);
io_uring_prep_read_fixed(sqe, 0, iovec.iov_base, N, 0, 0);
sqe->flags |= IOSQE_FIXED_FILE;
io_uring_sqe_set_data(sqe, nullptr);
ASSERT_EQ(io_uring_submit(&ring), 1);
struct io_uring_cqe *cqe;
ASSERT_EQ(::io_uring_wait_cqe(&ring, &cqe), 0);
ASSERT_EQ(cqe->res, N);
::io_uring_cqe_seen(&ring, cqe);
ASSERT_EQ(buf, std::string(N, '\0'));
::io_uring_queue_exit(&ring);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,136 @@
#include <chrono>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/logging/xlog.h>
#include <mutex>
#include <thread>
#include "common/utils/LockManager.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
using namespace std::chrono_literals;
TEST(TestLockManager, Normal) {
UniqueLockManager lockManager;
auto lock1 = lockManager.lock(123);
auto lock2 = lockManager.lock(std::string{"yes"});
auto lock3 = lockManager.try_lock(124);
ASSERT_TRUE(lock3.owns_lock());
auto lock4 = lockManager.try_lock(123);
ASSERT_FALSE(lock4.owns_lock());
lock1.unlock();
auto lock5 = lockManager.try_lock(123);
ASSERT_TRUE(lock5.owns_lock());
auto lock6 = lockManager.try_lock(std::string{"yes"});
ASSERT_FALSE(lock6.owns_lock());
}
TEST(TestLockManager, SharedLock) {
SharedLockManager lockManager;
auto lock = lockManager.lock_shared(123);
std::jthread([&] { auto anotherLock = lockManager.lock_shared(123); });
}
TEST(TestLockManager, FairSharedLock) {
SharedLockManager lockManager;
std::jthread read1([&] {
// 0ms
auto start = std::chrono::steady_clock::now();
auto lock = lockManager.lock_shared(1);
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start);
XLOGF(INFO, "read1 lock wait {}ms", elapsed.count());
std::this_thread::sleep_for(100ms);
// 100ms
std::this_thread::sleep_for(100ms);
// 200ms
std::this_thread::sleep_for(100ms);
// 300ms
lock.unlock();
std::this_thread::sleep_for(100ms);
// 400ms
lock.lock();
std::this_thread::sleep_for(100ms);
// 500ms
std::this_thread::sleep_for(100ms);
// 600ms
});
std::jthread read2([&] {
// 0ms
std::this_thread::sleep_for(100ms);
// 100ms
std::this_thread::sleep_for(100ms);
// 200ms
auto start = std::chrono::steady_clock::now();
auto lock = lockManager.lock_shared(1);
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start);
XLOGF(INFO, "read2 lock wait {}ms", elapsed.count());
std::this_thread::sleep_for(100ms);
// 300ms
std::this_thread::sleep_for(100ms);
// 400ms
std::this_thread::sleep_for(100ms);
// 500ms
lock.unlock();
});
std::jthread write([&] {
// 0ms
std::this_thread::sleep_for(100ms);
// 100ms
auto start = std::chrono::steady_clock::now();
auto lock = lockManager.lock(1);
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start);
XLOGF(INFO, "write lock wait {}ms", elapsed.count());
ASSERT_NEAR(elapsed.count(), 200, 50);
});
}
TEST(TestLockManager, CoMutex) {
CoUniqueLockManager lockManager;
folly::coro::blockingWait([&]() -> folly::coro::Task<void> {
auto lock1 = co_await lockManager.co_scoped_lock(123);
auto lock2 = co_await lockManager.co_scoped_lock(std::string{"yes"});
auto lock3 = lockManager.try_lock(124);
CO_ASSERT_TRUE(lock3.owns_lock());
auto lock4 = lockManager.try_lock(123);
CO_ASSERT_FALSE(lock4.owns_lock());
lock1.unlock();
auto lock5 = lockManager.try_lock(123);
CO_ASSERT_TRUE(lock5.owns_lock());
auto lock6 = lockManager.try_lock(std::string{"yes"});
CO_ASSERT_FALSE(lock6.owns_lock());
}());
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,82 @@
#include <gtest/gtest.h>
#include "common/utils/LruCache.h"
namespace hf3fs::test {
namespace {
TEST(TestLruCache, Normal) {
LruCache<int, int> cache(4);
ASSERT_TRUE(cache.empty());
ASSERT_EQ(cache.size(), 0);
ASSERT_EQ(cache.begin(), cache.end());
cache[1] = 1;
cache[2] = 2;
cache[3] = 3;
cache[4] = 4;
ASSERT_FALSE(cache.empty());
ASSERT_EQ(cache.size(), 4);
ASSERT_NE(cache.begin(), cache.end());
auto it = cache.begin();
ASSERT_EQ(it->second, 4);
++it;
ASSERT_EQ(it->second, 3);
++it;
ASSERT_EQ(it->second, 2);
++it;
ASSERT_EQ(it->second, 1);
++it;
ASSERT_EQ(it, cache.end());
cache[1] = 1;
ASSERT_EQ(cache.front().second, 1);
ASSERT_EQ(cache.back().second, 2);
ASSERT_TRUE(cache.emplace(5, 5).second);
ASSERT_EQ(cache.front().second, 5);
ASSERT_EQ(cache.back().second, 3);
ASSERT_EQ(cache.size(), 4);
cache.promote(cache.find(1));
ASSERT_EQ(cache.front().second, 1);
ASSERT_EQ(cache.back().second, 3);
cache.obsolete(cache.find(1));
ASSERT_EQ(cache.front().second, 5);
ASSERT_EQ(cache.back().second, 1);
it = cache.erase(5);
ASSERT_EQ(it, cache.begin());
it = cache.erase(cache.find(1));
ASSERT_EQ(it, cache.end());
ASSERT_EQ(cache.size(), 2);
auto idx = 0;
for (auto &pair : cache) {
ASSERT_EQ(pair.first, 4 - idx++);
}
}
TEST(TestLruCache, ManuallyRemove) {
constexpr auto N = 4;
constexpr auto M = 8;
LruCache<int, int> cache(N, false);
for (auto i = 0; i < M; ++i) {
cache[i] = i;
}
ASSERT_EQ(cache.size(), M);
cache.evictObsoletedIf([](int, int value) { return value < 2; });
ASSERT_EQ(cache.size(), 6);
ASSERT_EQ(cache.back().second, 2);
cache.evictObsoleted();
ASSERT_EQ(cache.size(), N);
ASSERT_EQ(cache.back().second, 4);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,31 @@
#include <gtest/gtest.h>
#include <memory>
#include "common/utils/MPSCQueue.h"
namespace hf3fs::test {
namespace {
TEST(TestMPSCQueue, Normal) {
constexpr auto N = 4096u;
MPSCQueue<size_t> que(N);
for (auto i = 0u; i < N; ++i) {
auto item = std::make_unique<size_t>(i);
ASSERT_TRUE(que.push(&item));
ASSERT_TRUE(item == nullptr);
}
auto more = std::make_unique<size_t>(0);
ASSERT_EQ(que.push(&more).code(), StatusCode::kQueueFull);
ASSERT_TRUE(more != nullptr);
for (auto i = 0u; i < N; ++i) {
std::unique_ptr<size_t> item;
ASSERT_TRUE(que.pop(&item));
ASSERT_TRUE(item != nullptr);
}
ASSERT_EQ(que.pop(&more).code(), StatusCode::kQueueEmpty);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,100 @@
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/Collect.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <folly/experimental/coro/Sleep.h>
#include <gtest/gtest.h>
#include <thread>
#include "common/utils/Coroutine.h"
#include "common/utils/Semaphore.h"
#include "common/utils/SemaphoreGuard.h"
#include "common/utils/Size.h"
#include "memory/common/GlobalMemoryAllocator.h"
namespace hf3fs::test {
namespace {
TEST(TestMemoryAllocator, Basic) {
size_t totalSize = 512_MB;
size_t blockSize = 16_KB;
std::vector<void *> memBlocks;
for (size_t allocatedSize = 0; allocatedSize < totalSize; allocatedSize += blockSize) {
auto mem = memory::allocate(blockSize);
memBlocks.push_back(mem);
}
char logbuf[512] = {0};
memory::logstatus(logbuf, sizeof(logbuf));
XLOGF(INFO, "Memory allocator log: {}", logbuf);
for (auto mem : memBlocks) {
memory::deallocate(mem);
}
memory::logstatus(logbuf, sizeof(logbuf));
XLOGF(INFO, "Memory allocator log: {}", logbuf);
}
TEST(TestMemoryAllocator, AlignedAlloc) {
static size_t kPageSize = sysconf(_SC_PAGESIZE);
size_t totalSize = 512_MB;
size_t blockSize = 100_KB;
std::vector<void *> memBlocks;
for (size_t allocatedSize = 0; allocatedSize < totalSize; allocatedSize += blockSize) {
auto mem = memory::memalign(kPageSize, blockSize);
memBlocks.push_back(mem);
}
char logbuf[512] = {0};
memory::logstatus(logbuf, sizeof(logbuf));
XLOGF(INFO, "Memory allocator log: {}", logbuf);
for (auto mem : memBlocks) {
memory::deallocate(mem);
}
memory::logstatus(logbuf, sizeof(logbuf));
XLOGF(INFO, "Memory allocator log: {}", logbuf);
}
/* Set the following environment variables before run the profiling test
export JE_MALLOC_CONF="prof:true,prof_active:false"
export MEMORY_ALLOCATOR_LIB_PATH=build/src/memory/jemalloc/libjemalloc_wrapper.so
*/
TEST(TestMemoryAllocator, Profiling) {
size_t totalSize = 512_MB;
size_t blockSize = 16_KB;
std::vector<void *> memBlocks;
auto enabled = memory::profiling(true /*active*/, "prof");
// skip this test if profiling is not supported
if (!enabled) return;
for (size_t allocatedSize = 0; allocatedSize < totalSize; allocatedSize += blockSize) {
auto mem = memory::allocate(blockSize);
memBlocks.push_back(mem);
}
// re-enable to dump profiling
ASSERT_TRUE(memory::profiling(true /*active*/, nullptr));
// disable and dump profiling
ASSERT_TRUE(memory::profiling(false /*active*/, "prof"));
// enable profiling
enabled = memory::profiling(true /*active*/, "prof");
for (auto mem : memBlocks) {
memory::deallocate(mem);
}
// disable and dump profiling
ASSERT_TRUE(memory::profiling(false /*active*/, nullptr));
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,71 @@
#include <gtest/gtest.h>
#include "common/utils/ObjectPool.h"
namespace hf3fs::test {
namespace {
TEST(TestObjectPool, Normal) {
constexpr auto X = 123;
static size_t destructTimes = 0;
struct A {
~A() { ++destructTimes; }
int x = X;
};
A *rawPtr = nullptr;
{
auto a = ObjectPool<A>::get();
ASSERT_NE(a.get(), nullptr);
ASSERT_EQ(a->x, X);
a->x = 0;
rawPtr = a.get();
ASSERT_EQ(destructTimes, 0);
a.reset();
ASSERT_EQ(destructTimes, 1);
}
ASSERT_EQ(destructTimes, 1);
ASSERT_EQ(ObjectPool<A>::get().get(), rawPtr);
ASSERT_EQ(ObjectPool<A>::get()->x, X);
}
TEST(TestObjectPool, Allocate) {
constexpr auto N = 1000000;
static size_t constructTimes = 0;
static size_t destructTimes = 0;
struct A {
A() { ++constructTimes; }
~A() { ++destructTimes; }
};
{
std::vector<ObjectPool<A>::Ptr> vec{N};
for (auto &item : vec) {
item = ObjectPool<A>::get();
}
ASSERT_EQ(constructTimes, N);
ASSERT_EQ(destructTimes, 0);
}
ASSERT_EQ(destructTimes, N);
}
TEST(TestObjectPool, AllocateAndRelease) {
constexpr auto N = 1000000;
static size_t constructTimes = 0;
static size_t destructTimes = 0;
struct A {
A() { ++constructTimes; }
~A() { ++destructTimes; }
};
for (auto i = 0; i < N; ++i) {
ObjectPool<A>::get();
}
ASSERT_EQ(constructTimes, N);
ASSERT_EQ(destructTimes, N);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,47 @@
#include <chrono>
#include <folly/Executor.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <gtest/gtest.h>
#include <thread>
#include "common/utils/Coroutine.h"
#include "common/utils/PriorityUnboundedQueue.h"
namespace hf3fs::test {
namespace {
TEST(TestPriorityBoundedQueue, Normal) {
PriorityUnboundedQueue<int> queue(3);
ASSERT_TRUE(queue.size() == 0);
queue.addWithPriority(1, folly::Executor::LO_PRI);
queue.addWithPriority(2, folly::Executor::HI_PRI);
queue.addWithPriority(3, folly::Executor::MID_PRI);
queue.addWithPriority(4, folly::Executor::HI_PRI);
ASSERT_EQ(queue.dequeue(), 2);
ASSERT_EQ(queue.dequeue(), 4);
ASSERT_EQ(queue.dequeue(), 3);
ASSERT_EQ(queue.dequeue(), 1);
}
TEST(TestPriorityBoundedQueue, Many) {
PriorityUnboundedQueue<int> queue(3);
ASSERT_TRUE(queue.size() == 0);
std::jthread a([&]() {
for (size_t i = 0; i < 2000000; i++) {
ASSERT_EQ(queue.dequeue(), 1);
}
});
std::jthread b([&]() {
for (size_t i = 0; i < 1000000; i++) {
queue.addWithPriority(1, folly::Executor::HI_PRI);
queue.addWithPriority(1, folly::Executor::LO_PRI);
}
});
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,115 @@
#include <chrono>
#include <folly/logging/xlog.h>
#include <gtest/gtest.h>
#include <mutex>
#include <thread>
#include "common/utils/ReentrantLockManager.h"
namespace hf3fs::test {
namespace {
using namespace std::chrono_literals;
TEST(TestReentrantLockManager, Normal) {
ReentrantLockManager lockManager;
lockManager.init();
auto lock1 = lockManager.writeLock(123);
auto lock2 = lockManager.writeLock(456);
auto lock3 = lockManager.tryWriteLock(124);
ASSERT_TRUE(lock3.writeLocked());
auto lock4 = lockManager.tryWriteLock(123);
ASSERT_FALSE(lock4.writeLocked());
lock1.unlock();
auto lock5 = lockManager.tryWriteLock(123);
ASSERT_TRUE(lock5.writeLocked());
auto lock6 = lockManager.tryWriteLock(456);
ASSERT_FALSE(lock6.writeLocked());
}
TEST(TestReentrantLockManager, SharedLock) {
ReentrantLockManager lockManager;
lockManager.init();
auto lock = lockManager.readLock(123);
std::jthread([&] { auto anotherLock = lockManager.readLock(123); });
}
TEST(TestReentrantLockManager, FairSharedLock) {
ReentrantLockManager lockManager;
lockManager.init();
std::jthread read1([&] {
// 0ms
auto start = std::chrono::steady_clock::now();
auto lock = lockManager.readLock(1);
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start);
XLOGF(INFO, "read1 lock wait {}ms", elapsed.count());
std::this_thread::sleep_for(100ms);
// 100ms
std::this_thread::sleep_for(100ms);
// 200ms
std::this_thread::sleep_for(100ms);
// 300ms
lock.unlock();
std::this_thread::sleep_for(100ms);
// 400ms
lock = lockManager.readLock(1);
std::this_thread::sleep_for(100ms);
// 500ms
std::this_thread::sleep_for(100ms);
// 600ms
});
std::jthread read2([&] {
// 0ms
std::this_thread::sleep_for(100ms);
// 100ms
std::this_thread::sleep_for(100ms);
// 200ms
auto start = std::chrono::steady_clock::now();
auto lock = lockManager.readLock(1);
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start);
XLOGF(INFO, "read2 lock wait {}ms", elapsed.count());
std::this_thread::sleep_for(100ms);
// 300ms
std::this_thread::sleep_for(100ms);
// 400ms
std::this_thread::sleep_for(100ms);
// 500ms
lock.unlock();
});
std::jthread write([&] {
// 0ms
std::this_thread::sleep_for(100ms);
// 100ms
auto start = std::chrono::steady_clock::now();
auto lock = lockManager.writeLock(1);
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - start);
XLOGF(INFO, "write lock wait {}ms", elapsed.count());
ASSERT_NEAR(elapsed.count(), 200, 50);
});
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,27 @@
#include "common/app/AppInfo.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
TEST(TestReleaseVersion, testFormat) {
auto rv = flat::ReleaseVersion::fromVersionInfoV0();
ASSERT_EQ(fmt::format("Version: v{}", rv), VersionInfo::fullV0());
rv = flat::ReleaseVersion::fromVersionInfo();
ASSERT_EQ(fmt::format("Version: v{}", rv), VersionInfo::full());
}
TEST(TestReleaseVersion, testDeserializeFromV0) {
auto rv = flat::ReleaseVersion::fromV0(0, 1, 4, 0xabcdef12, 1688016296, 54321);
auto str = serde::serialize(rv);
auto unpackRes = flat::ReleaseVersion::unpackFrom(str);
ASSERT_TRUE(unpackRes);
auto newRv = *unpackRes;
ASSERT_EQ(newRv.toString(), "0.1.4-54321-20230629-abcdef12");
auto rv2 = flat::ReleaseVersion::fromVersionInfo();
ASSERT_TRUE(rv2 > rv);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,187 @@
#include <cstdlib>
#include "common/utils/RenderConfig.h"
#include "common/utils/Toml.hpp"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
String toPrettyToml(const String &s) {
auto t = toml::parse(s);
std::stringstream ss;
ss << toml::toml_formatter(t, toml::toml_formatter::default_flags & ~toml::format_flags::indentation);
return ss.str();
}
TEST(RenderConfig, testNormalV0) {
flat::AppInfo app;
app.nodeId = flat::NodeId(2);
app.hostname = "hostname";
app.pid = 100;
app.tags = {flat::TagPair("k1", "v1"), flat::TagPair("k2"), flat::TagPair("k3", "v3")};
app.releaseVersion = flat::ReleaseVersion::fromV0(0, 1, 2, 0, 0, 100);
::setenv("TEST_RENDER_CONFIG", "abc", 1);
String rawConfig = R"(
[server]
{% if app.nodeId in [1, 3, 5] %}
queue_size = 10
{% elif hasTagValue(app, "k1", "v1") %}
queue_size = 20
{% else %}
queue_size = 30
{% endif %}
{% if app.hostname is startsWith("host") %}
zzz = 1
{% else %}
zzz = 0
{% endif %}
xxx = {{ "\"abc\"" if app is hasTag("k2") else "\"def\""}}
port0 = {{ 10003 % 2 }}
port1 = {{ 10007 % 2 }}
version_less_than = {{ relaseVersionCompare(app, "<", "0.1.2") }}
version_less_than_or_equal_to = {{ relaseVersionCompare(app, "<=", "0.1.2-100") }}
version_equal_to = {{ relaseVersionCompare(app, "==", "0.1.2") }}
version_greater_than = {{ releaseVersionCompare(app, ">", "0.1.2-100") }}
version_greater_than_or_equal_to = {{ releaseVersionCompare(app, ">=", "0.1.2") }}
version_not_equal_to = {{ releaseVersionCompare(app, "!=", "0.1.2-100") }}
yyy = "{{ env.TEST_RENDER_CONFIG }}"
log_level = '{{ env.HF3FS_LOG_LEVEL if env.HF3FS_LOG_LEVEL else "INFO" }}'
# TODO: implement startswith
{% if app.hostname == "hostname" %}
enable_fast_mode = true
{% endif %})";
auto res = renderConfig(rawConfig, &app);
ASSERT_OK(res);
auto resConfig = toPrettyToml(res.value());
auto expected = toPrettyToml(R"(
[server]
xxx = 'abc'
yyy = 'abc'
zzz = 1
log_level = 'INFO'
version_less_than = false
version_less_than_or_equal_to = true
version_equal_to = true
version_greater_than_or_equal_to = true
version_greater_than = false
version_not_equal_to = false
port0 = 1
port1 = 1
queue_size = 20 # hit second case
enable_fast_mode = true)");
ASSERT_EQ(resConfig, expected);
}
TEST(RenderConfig, testTemplateNotParsed) {
flat::AppInfo app;
app.nodeId = flat::NodeId(2);
app.hostname = "hostname";
app.pid = 100;
app.tags = {flat::TagPair("k1", "v1"), flat::TagPair("k2"), flat::TagPair("k3", "v3")};
app.releaseVersion = flat::ReleaseVersion::fromV0(0, 1, 2, 0, 0, 0);
::setenv("TEST_RENDER_CONFIG", "abc", 1);
String rawConfig = R"(
[server]
{% if app.nodeId in (1, 3, 5) %}
queue_size = 10
{% elif app.hostname not in ('hostname') %}
queue_size = 10
{% elif hasTagValue(app, "k1", "v1") %}
queue_size = 20
{% else %}
queue_size = 30
{% endif %}
xxx = {{ "\"abc\"" if app is hasTag("k2") else "\"def\""}}
{% if app.releaseVersion.major == 0 and app.releaseVersion.minor == 1 and app.releaseVersion.patch == 0 %}
yyy = 1
{% endif %}
version_less_than = {{ relaseVersionCompare(app, "<", "0.1.2") }}
version_less_than_or_equal_to = {{ relaseVersionCompare(app, "<=", "0.1.2") }}
version_equal_to = {{ relaseVersionCompare(app, "==", "0.1.2") }}
version_greater_than = {{ relaseVersionCompare(app, ">", "0.1.2") }}
version_greater_than_or_equal_to = {{ relaseVersionCompare(app, ">=", "0.1.2") }}
version_not_equal_to = {{ relaseVersionCompare(app, "!=", "0.1.2") }}
yyy = "{{ env.TEST_RENDER_CONFIG }}"
log_level = '{{ env.HF3FS_LOG_LEVEL if env.HF3FS_LOG_LEVEL else "INFO" }}'
# TODO: implement startswith
{% if app.hostname == "hostname" %}
enable_fast_mode = true
{% endif %})";
auto res = renderConfig(rawConfig, &app);
ASSERT_ERROR(res, StatusCode::kInvalidConfig);
ASSERT_TRUE(res.error().message().find("app.hostname not in") != std::string_view::npos);
}
#define ASSERT_REDNER_EQ(raw, app, expected) \
do { \
auto &&_raw = (raw); \
auto &&_app = (app); \
auto &&_expected = (expected); \
auto _res = renderConfig(_raw, _app); \
ASSERT_OK(_res); \
auto _resConfig = toPrettyToml(_res.value()); \
ASSERT_EQ(_resConfig, _expected); \
} while (0)
TEST(RenderConfig, testNormalV1) {
String config0 = R"(
x = {{ app.nodeId in [1, 2, 3, 4, 5] }}
{% set port = app.nodeId % 2 %}
{% if port == 0 %}
target_paths = ["/storage/data1/hf3fs", "/storage/data2/hf3fs", "/storage/data3/hf3fs", "/storage/data4/hf3fs", "/storage/data5/hf3fs", "/storage/data6/hf3fs", "/storage/data7/hf3fs", "/storage/data8/hf3fs"]
{% else %}
target_paths = ["/storage/data9/hf3fs", "/storage/data10/hf3fs", "/storage/data11/hf3fs", "/storage/data12/hf3fs", "/storage/data13/hf3fs", "/storage/data14/hf3fs", "/storage/data15/hf3fs", "/storage/data16/hf3fs"]
{% endif %}
)";
String config1 = R"(
x = {{ app.nodeId in range(1, 6) }}
target_paths = [ {% for i in range(8) %} "/storage/data{{i + 8 * (app.nodeId % 2) + 1}}/hf3fs", {% endfor %} ]
)";
flat::AppInfo app0;
app0.nodeId = flat::NodeId(0);
auto expected0 = toPrettyToml(R"(
x = false
target_paths = ["/storage/data1/hf3fs", "/storage/data2/hf3fs", "/storage/data3/hf3fs", "/storage/data4/hf3fs", "/storage/data5/hf3fs", "/storage/data6/hf3fs", "/storage/data7/hf3fs", "/storage/data8/hf3fs"]
)");
ASSERT_REDNER_EQ(config0, &app0, expected0);
ASSERT_REDNER_EQ(config1, &app0, expected0);
flat::AppInfo app1;
app1.nodeId = flat::NodeId(1);
auto expected1 = toPrettyToml(R"(
x = true
target_paths = ["/storage/data9/hf3fs", "/storage/data10/hf3fs", "/storage/data11/hf3fs", "/storage/data12/hf3fs", "/storage/data13/hf3fs", "/storage/data14/hf3fs", "/storage/data15/hf3fs", "/storage/data16/hf3fs"]
)");
ASSERT_REDNER_EQ(config0, &app1, expected1);
ASSERT_REDNER_EQ(config1, &app1, expected1);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,103 @@
#include <atomic>
#include <chrono>
#include <fmt/core.h>
#include <folly/Random.h>
#include <folly/ScopeGuard.h>
#include <folly/concurrency/UnboundedQueue.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/executors/IOThreadPoolExecutor.h>
#include <folly/experimental/coro/AsyncGenerator.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/BoundedQueue.h>
#include <folly/experimental/coro/CurrentExecutor.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <folly/experimental/coro/Invoke.h>
#include <folly/experimental/coro/ScopeExit.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/experimental/coro/Task.h>
#include <folly/experimental/coro/UnboundedQueue.h>
#include <folly/fibers/Semaphore.h>
#include <folly/futures/Future.h>
#include <folly/io/async/Request.h>
#include <gtest/gtest.h>
#include <optional>
#include <type_traits>
#include <vector>
#include "common/utils/Coroutine.h"
namespace hf3fs::test {
namespace {
TEST(TestRequestContext, FibersBaton) {
folly::CPUThreadPoolExecutor exec(folly::Random::rand32(1) + 1);
folly::fibers::Baton baton;
auto producer = folly::coro::co_invoke([&]() -> CoTask<void> {
co_await folly::coro::sleep(std::chrono::seconds(1));
folly::RequestContext::create();
baton.post();
co_return;
})
.scheduleOn(&exec)
.start();
auto consumer = folly::coro::co_invoke([&]() -> CoTask<void> {
EXPECT_EQ(folly::RequestContext::try_get(), nullptr);
co_await baton;
EXPECT_EQ(folly::RequestContext::try_get(), nullptr);
})
.scheduleOn(&exec)
.start();
producer.wait();
consumer.wait();
}
TEST(TestRequestContext, CoroBaton) {
folly::CPUThreadPoolExecutor exec(folly::Random::rand32(1) + 1);
folly::coro::Baton baton;
auto producer = folly::coro::co_invoke([&]() -> CoTask<void> {
co_await folly::coro::sleep(std::chrono::seconds(1));
folly::RequestContext::create();
baton.post();
co_return;
})
.scheduleOn(&exec)
.start();
auto consumer = folly::coro::co_invoke([&]() -> CoTask<void> {
EXPECT_EQ(folly::RequestContext::try_get(), nullptr);
co_await baton;
EXPECT_EQ(folly::RequestContext::try_get(), nullptr);
})
.scheduleOn(&exec)
.start();
producer.wait();
consumer.wait();
}
TEST(TestRequestContext, Semaphore) {
folly::CPUThreadPoolExecutor exec(folly::Random::rand32(1, 8));
folly::fibers::Semaphore semaphore(folly::Random::rand32(1, 8));
auto worker = [&]() -> CoTask<void> {
for (size_t i = 0; i < folly::Random::rand32(1000, 2000); i++) {
std::optional<folly::ShallowCopyRequestContextScopeGuard> guard;
if (folly::Random::oneIn(2)) {
guard.emplace();
}
auto ctx = folly::RequestContext::try_get();
co_await semaphore.co_wait();
SCOPE_EXIT { semaphore.signal(); };
CO_ASSERT_EQ(ctx, folly::RequestContext::try_get());
}
};
std::vector<folly::SemiFuture<Void>> tasks;
for (size_t i = 0; i < 64; i++) {
tasks.push_back(folly::coro::co_invoke(worker).scheduleOn(&exec).start());
}
for (auto &task : tasks) {
task.wait();
}
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,50 @@
#include <gtest/gtest.h>
#include <scn/scn.h>
#include <scn/tuple_return.h>
namespace hf3fs::test {
namespace {
TEST(TestScnlib, ScanString) {
std::string word;
auto result = scn::scan("Hello world!", "{}", word);
ASSERT_EQ(word, "Hello");
ASSERT_EQ(result.range_as_string(), " world!");
}
TEST(TestScnlib, ScanMultipleValues) {
int i, j;
auto result = scn::scan("123 456 foo", "{} {}", i, j);
ASSERT_EQ(i, 123);
ASSERT_EQ(j, 456);
std::string str;
auto ret = scn::scan(result.range(), "{}", str);
ASSERT_EQ(static_cast<bool>(ret), true);
ASSERT_EQ(str, "foo");
}
TEST(TestScnlib, ScanTuple) {
{
auto [r, i] = scn::scan_tuple<int>("42", "{}");
ASSERT_EQ(static_cast<bool>(r), true);
ASSERT_EQ(i, 42);
}
{
auto [r, i, s] = scn::scan_tuple<int, std::string>("42 foo", "{} {}");
ASSERT_EQ(static_cast<bool>(r), true);
ASSERT_EQ(i, 42);
ASSERT_EQ(s, "foo");
}
}
TEST(TestScnlib, ErrorHandling) {
int i;
auto result = scn::scan("foo", "{}", i);
ASSERT_EQ(static_cast<bool>(result), false);
ASSERT_EQ(result.range_as_string(), "foo");
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,73 @@
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/Collect.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <folly/experimental/coro/Sleep.h>
#include <gtest/gtest.h>
#include <thread>
#include "common/utils/Coroutine.h"
#include "common/utils/Semaphore.h"
#include "common/utils/SemaphoreGuard.h"
namespace hf3fs::test {
namespace {
CoTask<size_t> runTask(Semaphore &sema, size_t numLoops) {
for (size_t i = 0; i < numLoops; i++) {
SemaphoreGuard lock(sema);
co_await lock.coWait();
co_await folly::coro::sleep(std::chrono::microseconds{0});
}
co_return numLoops;
}
std::vector<folly::SemiFuture<size_t>> startTasks(Semaphore &sema,
folly::CPUThreadPoolExecutor &threadPool,
size_t numTasks,
size_t numLoops) {
std::vector<folly::SemiFuture<size_t>> tasks;
for (size_t t = 0; t < numTasks; t++) {
tasks.push_back(runTask(sema, numLoops).scheduleOn(folly::Executor::getKeepAliveToken(threadPool)).start());
}
return tasks;
}
TEST(TestSemaphore, Basic) {
size_t numCpuCores = std::thread::hardware_concurrency();
size_t numTasks = numCpuCores * 2;
size_t numLoops = 100;
Semaphore sema(std::max(size_t(1), numCpuCores / 2));
folly::CPUThreadPoolExecutor threadPool(numCpuCores / 2);
std::vector<folly::SemiFuture<size_t>> tasks = startTasks(sema, threadPool, numTasks, numLoops);
auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(tasks)));
size_t sumLoops = 0;
for (auto res : results) sumLoops += res;
ASSERT_EQ(numTasks * numLoops, sumLoops);
}
TEST(TestSemaphore, ChangeUsableTokens) {
size_t numCpuCores = std::thread::hardware_concurrency();
size_t numTasks = numCpuCores * 2;
size_t numLoops = 100;
Semaphore sema(std::max(size_t(1), numCpuCores / 2));
folly::CPUThreadPoolExecutor threadPool(numCpuCores / 2);
std::vector<folly::SemiFuture<size_t>> tasks = startTasks(sema, threadPool, numTasks, numLoops);
sema.changeUsableTokens(numTasks);
auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(tasks)));
size_t sumLoops = 0;
for (auto res : results) sumLoops += res;
ASSERT_EQ(numTasks * numLoops, sumLoops);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,130 @@
#include <common/utils/SerDeser.h>
#include <gtest/gtest.h>
namespace hf3fs::tests {
namespace {
TEST(Serializer, testPutChar) {
String s;
Serializer ser(s);
ser.putChar('a');
ser.putChar('b');
ser.putChar('c');
ASSERT_EQ(s, "abc");
}
TEST(Serializer, testPutIntAsChar) {
String s;
Serializer ser(s);
ser.putIntAsChar<int16_t>('a');
ser.putIntAsChar<int32_t>('b');
ser.putIntAsChar<int64_t>('c');
ASSERT_EQ(s, "abc");
}
TEST(Serializer, testPutShortString) {
String s;
Serializer ser(s);
ser.putShortString("abc");
ASSERT_EQ(s.size(), 4);
ASSERT_EQ(s[0], 3);
ASSERT_EQ(s.substr(1), "abc");
}
TEST(Deserializer, testGetChar) {
String s("abc");
Deserializer des(s);
String res;
for (auto c = des.getChar(); c.hasValue(); c = des.getChar()) {
res.push_back(c.value());
}
ASSERT_TRUE(des.reachEnd());
ASSERT_TRUE(!des.getChar().hasValue());
ASSERT_EQ(res, "abc");
}
TEST(Deserializer, testGetIntFromChar) {
String s;
Serializer ser(s);
ser.putIntAsChar<int>(100);
ser.putIntAsChar<int16_t>(-50);
ser.putIntAsChar<uint32_t>(50);
Deserializer des(s);
auto c0 = des.getIntFromChar<int>();
ASSERT_TRUE(c0.hasValue());
ASSERT_EQ(c0.value(), 100);
auto c1 = des.getIntFromChar<int16_t>();
ASSERT_TRUE(c1.hasValue());
ASSERT_EQ(c1.value(), -50);
auto c2 = des.getIntFromChar<uint32_t>();
ASSERT_TRUE(c2.hasValue());
ASSERT_EQ(c2.value(), 50);
ASSERT_TRUE(des.reachEnd());
ASSERT_TRUE(!des.getIntFromChar<int8_t>().hasValue());
}
TEST(Deserializer, testGetShortString) {
String s;
Serializer ser(s);
ser.putShortString("abc");
Deserializer des(s);
ASSERT_EQ(des.getShortString().value(), "abc");
ASSERT_TRUE(des.reachEnd());
ASSERT_TRUE(!des.getShortString().hasValue());
}
TEST(Deserializer, testGet) {
struct T {
int x;
char y;
};
T t = {1, 2};
String s;
Serializer ser(s);
ser.put(t);
Deserializer des(s);
auto res = des.get<T>();
ASSERT_TRUE(res.hasValue());
ASSERT_TRUE(des.reachEnd());
ASSERT_EQ(res.value().x, 1);
ASSERT_EQ(res.value().y, 2);
ASSERT_TRUE(!des.get<T>().hasValue());
}
TEST(Deserializer, testGetRaw) {
String s = "abcdefg";
Deserializer des(s);
auto res = des.getRaw(1);
ASSERT_TRUE(res.hasValue());
ASSERT_EQ(res.value(), "a");
res = des.getRaw(2);
ASSERT_TRUE(res.hasValue());
ASSERT_EQ(res.value(), "bc");
res = des.getRaw(5);
ASSERT_TRUE(!res.hasValue());
res = des.getRaw(1);
ASSERT_TRUE(res.hasValue());
ASSERT_EQ(res.value(), "d");
res = des.getRawUntilEnd();
ASSERT_TRUE(res.hasValue());
ASSERT_EQ(res.value(), "efg");
res = des.getRawUntilEnd();
ASSERT_TRUE(res.hasValue());
ASSERT_EQ(res.value(), "");
ASSERT_TRUE(des.reachEnd());
ASSERT_TRUE(!des.getRaw(1).hasValue());
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,41 @@
#include <folly/Random.h>
#include <thread>
#include <vector>
#include "common/utils/RobinHood.h"
#include "common/utils/Shards.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
TEST(TestShards, Normal) {
Shards<std::unordered_map<uint32_t, uint32_t>, 64> shards;
constexpr auto N = 8;
constexpr auto M = 100000;
std::vector<std::thread> threads(N);
for (auto &thread : threads) {
thread = std::thread([&shards] {
for (auto i = 0; i < M; ++i) {
auto r = folly::Random::rand32();
shards.withLock([&](std::unordered_map<uint32_t, uint32_t> &map) { ++map[r]; }, r);
}
});
}
for (auto &thread : threads) {
thread.join();
}
uint64_t sum = 0;
shards.iterate([&sum](std::unordered_map<uint32_t, uint32_t> &map) {
for (auto &[k, v] : map) {
sum += v;
}
});
ASSERT_EQ(sum, M * N);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,121 @@
#include <folly/logging/xlog.h>
#include "common/utils/ConfigBase.h"
#include "common/utils/Size.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
static_assert(config::IsPrimitive<Size>, "size is not primitive");
class Config : public ConfigBase<Config> {
CONFIG_HOT_UPDATED_ITEM(size, 128_KB);
};
TEST(TestSize, Normal) {
Size config;
ASSERT_EQ(size_t(config), 0);
ASSERT_EQ(1024_B, 1_KB);
ASSERT_EQ(4096_B, 4_KB);
ASSERT_EQ(1048576_B, 1_MB);
ASSERT_EQ(1024_KB, 1_MB);
ASSERT_EQ(1024_MB, 1_GB);
ASSERT_EQ(1024_GB, 1_TB);
ASSERT_EQ(1000_B, 1_K);
ASSERT_EQ(1000_K, 1_M);
ASSERT_EQ(1000_M, 1_G);
ASSERT_EQ(1000_G, 1_T);
ASSERT_EQ(Size::toString(0), "0");
ASSERT_EQ(Size::toString(511), "511");
ASSERT_EQ(Size::toString(512), "512");
ASSERT_EQ(Size::toString(1013), "1013");
ASSERT_EQ(Size::toString(1025), "1025");
ASSERT_EQ(Size::toString(1034), "1034");
ASSERT_EQ(Size::toString(1048576), "1MB");
ASSERT_EQ(Size::toString(1000), "1K");
ASSERT_EQ(Size::toString(1024), "1KB");
ASSERT_EQ(Size::toString(1000_K), "1M");
ASSERT_EQ(Size::toString(1024_K), "1024K");
ASSERT_EQ(Size::toString(1000_G), "1T");
ASSERT_EQ(Size::toString(1024_G), "1024G");
ASSERT_EQ(Size::from("1K").value(), 1_K);
ASSERT_EQ(Size::from("2KB").value(), 2_KB);
ASSERT_EQ(Size::from("3M").value(), 3_M);
ASSERT_EQ(Size::from("4MB").value(), 4_MB);
ASSERT_EQ(Size::from("5G").value(), 5_G);
ASSERT_EQ(Size::from("6GB").value(), 6_GB);
ASSERT_EQ(Size::from("7T").value(), 7_T);
ASSERT_EQ(Size::from("8TB").value(), 8_TB);
ASSERT_EQ(Size::around(0), "0");
ASSERT_EQ(Size::around(511), "511");
ASSERT_EQ(Size::around(512), "0.50KB");
ASSERT_EQ(Size::around(1013), "0.99KB");
ASSERT_EQ(Size::around(1025), "1.00KB");
ASSERT_EQ(Size::around(1034), "1.01KB");
ASSERT_EQ(Size::around(1048576), "1.00MB");
}
TEST(TestSize, Parse) {
Size c0 = Size::from("100").value();
ASSERT_EQ(c0.toString(), "100");
Size c1 = Size::from("2KB").value();
ASSERT_EQ(c1.toString(), "2KB");
Size c2 = Size::from("1001MB").value();
ASSERT_EQ(c2.toString(), "1001MB");
Size c3 = Size::from("45").value();
ASSERT_EQ(c3.toString(), "45");
Size c5 = Size::from("08GB").value();
ASSERT_EQ(c5.toString(), "8GB");
Size c6 = Size::from("1TB").value();
ASSERT_EQ(c6.toString(), "1TB");
ASSERT_FALSE(Size::from("10kb"));
ASSERT_FALSE(Size::from(""));
ASSERT_FALSE(Size::from("GB"));
ASSERT_FALSE(Size::from("1.1MB"));
ASSERT_EQ(fmt::format("{}", 100_KB), "100KB");
}
TEST(TestSize, testConfig) {
Config cfg;
ASSERT_EQ(cfg.size(), 128_KB);
cfg.set_size(512_KB);
ASSERT_EQ(cfg.size(), 512_KB);
toml::table result = toml::parse(R"(
size = "5KB"
)");
ASSERT_TRUE(cfg.update(result));
ASSERT_EQ(cfg.size(), 5_KB);
XLOGF(INFO, "toString: {}", cfg.toString());
result = toml::parse(cfg.toString());
Config other;
ASSERT_TRUE(other.update(result));
ASSERT_EQ(cfg.size(), other.size());
ASSERT_TRUE(cfg.update(toml::parse(R"(
size = 4096
)")));
ASSERT_EQ(cfg.size(), 4_KB);
}
TEST(TestSize, Infinity) {
constexpr auto inf = Size::infinity();
ASSERT_EQ(inf, std::numeric_limits<uint64_t>::max());
ASSERT_EQ(inf.toString(), "infinity");
ASSERT_OK(Size::from("infinity"));
ASSERT_EQ(*Size::from("infinity"), Size::infinity());
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,110 @@
#include <gtest/gtest.h>
#include <string>
#include "common/utils/Status.h"
namespace hf3fs::tests {
namespace {
TEST(Status, testCtor) {
Status s1(1);
ASSERT_EQ(s1.code(), 1);
ASSERT_TRUE(s1.message().empty());
ASSERT_TRUE(!s1.hasPayload());
Status s2(1, "test");
ASSERT_EQ(s2.code(), 1);
ASSERT_EQ(s2.message(), "test");
ASSERT_TRUE(!s2.hasPayload());
ASSERT_EQ(fmt::format("{}", s2), "NotImplemented(1) test");
}
TEST(Status, testPayload) {
Status s0(1);
s0.setPayload<int>(5);
ASSERT_TRUE(s0.hasPayload());
ASSERT_EQ(*s0.payload<int>(), 5);
s0.setPayload<std::string>("abc");
ASSERT_TRUE(s0.hasPayload());
ASSERT_EQ(*s0.payload<std::string>(), "abc");
s0.resetPayload();
ASSERT_TRUE(!s0.hasPayload());
s0.emplacePayload<std::string>(3, 'a');
ASSERT_EQ(*s0.payload<std::string>(), "aaa");
}
TEST(Status, testCopy) {
Status s2(1, "test");
s2.setPayload<std::string>("abc");
ASSERT_EQ(s2.code(), 1);
ASSERT_EQ(s2.message(), "test");
ASSERT_TRUE(s2.hasPayload());
ASSERT_EQ(*s2.payload<std::string>(), "abc");
Status s3 = s2;
ASSERT_EQ(s2.code(), 1);
ASSERT_EQ(s2.message(), "test");
ASSERT_TRUE(s2.hasPayload());
ASSERT_EQ(*s2.payload<std::string>(), "abc");
ASSERT_EQ(s3.code(), 1);
ASSERT_EQ(s3.message(), "test");
ASSERT_TRUE(s3.hasPayload());
ASSERT_EQ(*s3.payload<std::string>(), "abc");
auto *ps3 = &s3;
s3 = *ps3;
ASSERT_EQ(s3.code(), 1);
ASSERT_EQ(s3.message(), "test");
ASSERT_TRUE(s3.hasPayload());
ASSERT_EQ(*s3.payload<std::string>(), "abc");
Status s4(0);
s4 = s3;
ASSERT_EQ(s3.code(), 1);
ASSERT_EQ(s3.message(), "test");
ASSERT_TRUE(s3.hasPayload());
ASSERT_EQ(*s3.payload<std::string>(), "abc");
ASSERT_EQ(s4.code(), 1);
ASSERT_EQ(s4.message(), "test");
ASSERT_TRUE(s4.hasPayload());
ASSERT_EQ(*s4.payload<std::string>(), "abc");
}
TEST(Status, testMove) {
Status s2(1, "test");
s2.setPayload<std::string>("abc");
ASSERT_EQ(s2.code(), 1);
ASSERT_EQ(s2.message(), "test");
ASSERT_TRUE(s2.hasPayload());
ASSERT_EQ(*s2.payload<std::string>(), "abc");
Status s3 = std::move(s2);
ASSERT_EQ(s2.code(), StatusCode::kOK);
ASSERT_TRUE(s2.message().empty());
ASSERT_TRUE(!s2.hasPayload());
ASSERT_EQ(s3.code(), 1);
ASSERT_EQ(s3.message(), "test");
ASSERT_TRUE(s3.hasPayload());
ASSERT_EQ(*s3.payload<std::string>(), "abc");
auto *ps3 = &s3;
s3 = std::move(*ps3);
ASSERT_EQ(s3.code(), 1);
ASSERT_EQ(s3.message(), "test");
ASSERT_TRUE(s3.hasPayload());
ASSERT_EQ(*s3.payload<std::string>(), "abc");
Status s4(0);
s4 = std::move(s3);
ASSERT_EQ(s3.code(), StatusCode::kOK);
ASSERT_TRUE(s3.message().empty());
ASSERT_TRUE(!s3.hasPayload());
ASSERT_EQ(s4.code(), 1);
ASSERT_EQ(s4.message(), "test");
ASSERT_TRUE(s4.hasPayload());
ASSERT_EQ(*s4.payload<std::string>(), "abc");
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,14 @@
#include "common/utils/StatusCode.h"
#include "gtest/gtest.h"
namespace hf3fs {
namespace {
TEST(StatusCode, toString) {
ASSERT_EQ(StatusCode::toString(StatusCode::kOK), "OK");
ASSERT_EQ(StatusCode::toString(TransactionCode::kConflict), "Transaction::Conflict");
ASSERT_EQ(StatusCode::toString(-1), "UnknownStatusCode");
}
} // namespace
} // namespace hf3fs

View File

@@ -0,0 +1,28 @@
#include <string>
#include "common/utils/StrongType.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::tests {
namespace {
STRONG_TYPEDEF(int, TypedInt);
STRONG_TYPEDEF(size_t, TypedUint);
STRONG_TYPEDEF(std::string, TypedString);
TEST(StrongTypedefTest, testComparison) {
TypedInt ti0;
ASSERT_TRUE(ti0 < 1);
ASSERT_TRUE(ti0 > -1);
ASSERT_EQ(ti0, 0);
TypedUint tu0;
ASSERT_TRUE(tu0.toUnderType() < 1);
ASSERT_TRUE(tu0.toUnderType() >= 0);
ASSERT_EQ(tu0, 0);
TypedString ts0, ts1("aa");
ASSERT_TRUE(ts0 < ts1);
ASSERT_TRUE(ts0 < "a");
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,57 @@
#include <cstdlib>
#include <folly/logging/xlog.h>
#include <gtest/gtest.h>
#include "common/serde/Serde.h"
#include "common/utils/SysResource.h"
namespace hf3fs {
TEST(TestSysResource, GetHostname) {
auto result = SysResource::hostname();
ASSERT_TRUE(result);
XLOGF(INFO, "hostname is {}", result.value());
}
TEST(TestSysResource, FDLimit) {
auto result = SysResource::increaseProcessFDLimit(8192);
ASSERT_TRUE(result);
}
// todo: fix fs UUID in docker
TEST(TestSysResource, DISABLED_FileSystemUUID) {
auto result = SysResource::fileSystemUUID();
ASSERT_TRUE(result);
auto &map = *result;
ASSERT_TRUE(!map.empty());
for (auto [deviceId, uuid] : map) {
XLOGF(INFO, "device: {}, uuid: {}", deviceId, uuid);
}
}
TEST(TestSysResource, ScanDiskInfo) {
auto result = SysResource::scanDiskInfo();
ASSERT_TRUE(result);
XLOGF(INFO, "info is {}", serde::toJsonString(*result));
}
TEST(TestSysResource, getAllEnvs) {
::setenv("HF3FS_TEST_ONE", "ONE", 1);
::setenv("HF3FS_TEST_TWO", "TWO", 1);
::setenv("TEST_THREE", "THREE", 1);
auto m = SysResource::getAllEnvs("HF3FS_");
ASSERT_TRUE(!m.empty());
for (const auto &[k, v] : m) XLOGF(INFO, "env {} = {}", k, v);
ASSERT_TRUE(m.contains("HF3FS_TEST_ONE"));
ASSERT_EQ(m["HF3FS_TEST_ONE"], "ONE");
ASSERT_TRUE(m.contains("HF3FS_TEST_TWO"));
ASSERT_EQ(m["HF3FS_TEST_TWO"], "TWO");
ASSERT_TRUE(!m.contains("TEST_THREE"));
}
} // namespace hf3fs

View File

@@ -0,0 +1,32 @@
#include <type_traits>
#include "common/utils/Thief.h"
#include "common/utils/TypeTraits.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
class A {
public:
auto &secret() { return secret_; }
private:
int secret_ = 0;
};
template <typename Tag, auto Value>
struct StoreValue : public std::type_identity<thief::steal<Tag, value_identity<Value>>> {};
struct Tag;
template struct StoreValue<Tag, &A::secret_>;
TEST(TestThief, AccessPrivateMember) {
A a;
a.secret() = 20;
ASSERT_EQ(a.*thief::retrieve<Tag>::value, 20);
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,25 @@
#include <gtest/gtest.h>
#include "common/utils/Toml.hpp"
TEST(TestToml, Normal) {
toml::parse_result config = toml::parse(R"(
name = "foo"
[test]
foo = "bar"
val = 123
)");
ASSERT_TRUE(config["name"].is_string());
ASSERT_EQ(config["name"].value_or(std::string_view{}), std::string_view("foo"));
ASSERT_TRUE(config["test"].is_table());
ASSERT_EQ(config["test"]["foo"].value<std::string_view>(), "bar");
ASSERT_EQ(config["test"]["val"].value<int>(), 123);
ASSERT_FALSE(config.contains("not-exists"));
ASSERT_FALSE(config["not-exists"].is_table());
ASSERT_FALSE(config["not-exists"]["not-exists"].is_value());
ASSERT_EQ(config["not-exists"]["not-exists"].value_or(123), 123);
}

View File

@@ -0,0 +1,14 @@
#include "tests/GtestHelpers.h"
#include "toml.hpp"
namespace hf3fs::test {
namespace {
TEST(TestToml11, Normal) {
toml::value v; // not initialized as a table.
v["foo"] = 42; // OK. `v` will be a table.
ASSERT_TRUE(v.is_table());
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,130 @@
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <gtest/gtest.h>
#include "common/utils/Coroutine.h"
#include "common/utils/Tracing.h"
#include "common/utils/TracingEvent.h"
namespace hf3fs::tests {
namespace {
TEST(Tracing, testEventToString) {
ASSERT_EQ(tracing::toString(tracing::kFdbBeginNewTransaction), "Fdb::BeginNewTransaction");
ASSERT_EQ(tracing::toString(tracing::kFdbEndNewTransaction), "Fdb::EndNewTransaction");
ASSERT_EQ(tracing::toString(tracing::kFdbBeginCommitTransaction), "Fdb::BeginCommitTransaction");
ASSERT_EQ(tracing::toString(tracing::kFdbEndCommitTransaction), "Fdb::EndCommitTransaction");
ASSERT_EQ(tracing::toString(tracing::kFdbBeginCancelTransaction), "Fdb::BeginCancelTransaction");
ASSERT_EQ(tracing::toString(tracing::kFdbEndCancelTransaction), "Fdb::EndCancelTransaction");
ASSERT_EQ(tracing::toString(tracing::kMetaLoadInode), "Meta::LoadInode");
ASSERT_EQ(tracing::toString(tracing::kMetaBeginLoadInode), "Meta::BeginLoadInode");
ASSERT_EQ(tracing::toString(tracing::kMetaEndLoadInode), "Meta::EndLoadInode");
ASSERT_EQ(tracing::toString(tracing::kMetaSnapshotLoadInode), "Meta::SnapshotLoadInode");
ASSERT_EQ(tracing::toString(tracing::kMetaBeginSnapshotLoadInode), "Meta::BeginSnapshotLoadInode");
ASSERT_EQ(tracing::toString(tracing::kMetaEndSnapshotLoadInode), "Meta::EndSnapshotLoadInode");
ASSERT_EQ(tracing::toString(tracing::kMetaLoadDirEntry), "Meta::LoadDirEntry");
ASSERT_EQ(tracing::toString(tracing::kMetaBeginLoadDirEntry), "Meta::BeginLoadDirEntry");
ASSERT_EQ(tracing::toString(tracing::kMetaEndLoadDirEntry), "Meta::EndLoadDirEntry");
ASSERT_EQ(tracing::toString(tracing::kMetaSnapshotLoadDirEntry), "Meta::SnapshotLoadDirEntry");
ASSERT_EQ(tracing::toString(tracing::kMetaBeginSnapshotLoadDirEntry), "Meta::BeginSnapshotLoadDirEntry");
ASSERT_EQ(tracing::toString(tracing::kMetaEndSnapshotLoadDirEntry), "Meta::EndSnapshotLoadDirEntry");
}
TEST(Tracing, testScopeGuard) {
tracing::Points points1, points2;
{
SCOPE_SET_TRACING_POINTS(points1);
TRACING_ADD_SCOPE_EVENT(tracing::kFdbNewTransaction);
{
SCOPE_SET_TRACING_POINTS(points2);
TRACING_ADD_SCOPE_EVENT(tracing::kFdbCommitTransaction);
}
TRACING_ADD_EVENT(tracing::kFdbBeginCancelTransaction);
}
TRACING_ADD_EVENT(tracing::kFdbEndCancelTransaction);
auto v1 = points1.extractAll();
ASSERT_EQ(v1.size(), 3);
ASSERT_EQ(v1[0].event(), tracing::kFdbBeginNewTransaction);
ASSERT_EQ(v1[1].event(), tracing::kFdbBeginCancelTransaction);
ASSERT_EQ(v1[2].event(), tracing::kFdbEndNewTransaction);
auto v2 = points2.extractAll();
ASSERT_EQ(v2.size(), 2);
ASSERT_EQ(v2[0].event(), tracing::kFdbBeginCommitTransaction);
ASSERT_EQ(v2[1].event(), tracing::kFdbEndCommitTransaction);
}
TEST(Tracing, testPassThroughCoroutine) {
tracing::Points points;
folly::CPUThreadPoolExecutor executor(1);
{
SCOPE_SET_TRACING_POINTS(points);
folly::coro::blockingWait([&]() -> CoTask<void> {
TRACING_ADD_SCOPE_EVENT(tracing::kFdbNewTransaction);
auto f = []() -> CoTask<void> {
TRACING_ADD_SCOPE_EVENT(tracing::kFdbCommitTransaction);
co_return;
};
co_await f().scheduleOn(folly::getKeepAliveToken(executor));
}());
}
auto v = points.extractAll();
ASSERT_EQ(v.size(), 4);
ASSERT_EQ(v[0].event(), tracing::kFdbBeginNewTransaction);
ASSERT_EQ(v[1].event(), tracing::kFdbBeginCommitTransaction);
ASSERT_EQ(v[2].event(), tracing::kFdbEndCommitTransaction);
ASSERT_EQ(v[3].event(), tracing::kFdbEndNewTransaction);
}
TEST(Tracing, testPointMessage) {
auto t = UtcClock::now();
tracing::Point p0(t, 0);
ASSERT_EQ(p0.timestamp(), t);
ASSERT_EQ(p0.event(), 0);
ASSERT_EQ(p0.message(), "");
std::string_view msg0 = "A very long message which contains more than 16 chars";
p0.setMessage(msg0);
ASSERT_EQ(p0.message(), msg0);
ASSERT_NE(p0.message().data(), msg0.data());
std::string_view msg1 = "A short message";
p0.setMessage(msg1);
ASSERT_EQ(p0.message(), msg1);
}
TEST(Tracing, testPointMove) {
auto t0 = UtcClock::now();
std::string_view msg0 = "A very long message which contains more than 16 chars";
tracing::Point p0(t0, 0, msg0);
ASSERT_EQ(p0.timestamp(), t0);
ASSERT_EQ(p0.event(), 0);
ASSERT_EQ(p0.message(), msg0);
auto t1 = t0 + std::chrono::microseconds(1);
std::string_view msg1 = "Yet another very long message but shorter than msg0";
{
tracing::Point p1(t1, 1, msg1);
tracing::Point p2 = std::move(p1);
ASSERT_EQ(p1.message(), "");
ASSERT_EQ(p2.timestamp(), t1);
ASSERT_EQ(p2.event(), 1);
ASSERT_EQ(p2.message(), msg1);
p0 = std::move(p2);
ASSERT_EQ(p2.message(), "");
}
ASSERT_EQ(p0.timestamp(), t1);
ASSERT_EQ(p0.event(), 1);
ASSERT_EQ(p0.message(), msg1);
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,34 @@
#include <vector>
#include "common/utils/RobinHood.h"
#include "common/utils/TypeTraits.h"
namespace hf3fs {
namespace {
static_assert(is_vector_v<std::vector<int>>, "is vector");
static_assert(!is_vector_v<std::string>, "not vector");
static_assert(is_set_v<std::set<int>>, "is set");
static_assert(!is_set_v<std::string>, "not set");
static_assert(is_map_v<std::map<int, int>>, "is map");
static_assert(!is_map_v<std::vector<int>>, "not map");
static_assert(GenericPair<std::pair<int, int>>, "is generic pair");
static_assert(!GenericPair<std::map<int, int>>, "not generic pair");
static_assert(GenericPair<robin_hood::pair<int, int>>, "is generic pair");
static_assert(Container<robin_hood::unordered_map<int, int>>, "ok");
struct Base {
int x;
};
struct Inherit : Base {
int y;
};
static_assert(std::is_same_v<member_pointer_to_class_t<&Inherit::x>, Base>);
static_assert(std::is_same_v<member_pointer_to_class_t<&Inherit::y>, Inherit>);
} // namespace
} // namespace hf3fs

View File

@@ -0,0 +1,29 @@
#include <stdexcept>
#include <unordered_map>
#include "common/utils/RobinHood.h"
#include "common/utils/UnorderedDense.h"
#include "tests/GtestHelpers.h"
namespace {
template <class Map>
void insertWhileHashIsZero() {
constexpr auto N = 1000;
Map map;
for (auto i = 0; i < N; ++i) {
map[i] = i;
}
}
TEST(TestUnorderedDense, Normal) {
struct Hash {
size_t operator()(int) const { return 0; }
};
insertWhileHashIsZero<ankerl::unordered_dense::map<int, int, Hash>>();
insertWhileHashIsZero<std::unordered_map<int, int, Hash>>();
ASSERT_THROW((insertWhileHashIsZero<robin_hood::unordered_map<int, int, Hash>>()), std::overflow_error);
}
} // namespace

View File

@@ -0,0 +1,17 @@
#include <gtest/gtest.h>
#include "common/utils/UtcTime.h"
namespace hf3fs::tests {
namespace {
TEST(UtcTime, testConvert) {
UtcTime now = UtcClock::now();
int64_t us = now.toMicroseconds();
UtcTime utc = UtcTime::fromMicroseconds(us);
ASSERT_EQ(us, utc.toMicroseconds());
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,124 @@
#include <boost/core/ignore_unused.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/uuid/uuid.hpp>
#include <cstdlib>
#include <double-conversion/utils.h>
#include <fmt/core.h>
#include <gtest/gtest.h>
#include <memory>
#include <string>
#include <sys/wait.h>
#include <unistd.h>
#include <utility>
#include "common/utils/SerDeser.h"
#include "common/utils/Uuid.h"
namespace hf3fs::tests {
namespace {
std::pair<boost::uuids::uuid, boost::uuids::uuid> gen(auto &generator) {
boost::ignore_unused(generator());
int pipefd[2];
EXPECT_NE(pipe(pipefd), -1);
auto pid = fork();
EXPECT_NE(pid, -1);
auto uuid = generator();
fmt::print("{} {}\n", pid == 0 ? "child " : "father", boost::lexical_cast<std::string>(uuid));
if (pid == 0) {
close(pipefd[0]);
write(pipefd[1], uuid.data, sizeof(uuid));
exit(0);
}
close(pipefd[1]);
boost::uuids::uuid another;
read(pipefd[0], another.data, sizeof(another));
int status;
EXPECT_EQ(wait(&status), pid);
return std::make_pair(uuid, another);
}
TEST(Uuid, DISABLED_fork) {
// disable this test by default because it hangs in gitlab CI.
static thread_local boost::uuids::random_generator_mt19937 mt19937;
static thread_local boost::uuids::random_generator random;
fmt::print("random_generator_mt19937\n");
for (size_t i = 0; i < 10; i++) {
auto a = gen(mt19937);
EXPECT_EQ(a.first, a.second); // shouldn't use mt19937
}
fmt::print("random_generator\n");
for (size_t i = 0; i < 10; i++) {
auto b = gen(random);
EXPECT_NE(b.first, b.second);
}
fmt::print("Uuid\n");
for (size_t i = 0; i < 10; i++) {
auto c = gen(Uuid::random);
EXPECT_NE(c.first, c.second);
}
}
TEST(Uuid, testZero) {
uint8_t buffer[Uuid::static_size()] = {0};
Uuid zero = Uuid::zero();
Uuid def = Uuid{};
ASSERT_EQ(memcmp(buffer, zero.data, Uuid::static_size()), 0);
ASSERT_EQ(zero, def);
Uuid zero1 = Uuid::zero();
ASSERT_EQ(zero, zero1);
}
TEST(Uuid, testRandom) {
Uuid a = Uuid::random();
Uuid b = Uuid::random();
ASSERT_NE(a, b);
}
TEST(Uuid, testToString) {
Uuid a = Uuid::random();
std::cout << a.toHexString() << std::endl;
Uuid zero = Uuid::zero();
std::cout << zero.toHexString() << std::endl;
ASSERT_EQ(zero.toHexString(), "00000000-0000-0000-0000-000000000000");
}
TEST(Uuid, Compare) {
for (size_t i = 0; i < 100; i++) {
Uuid a = Uuid::random();
Uuid b = Uuid::random();
ASSERT_EQ((a < b), (b > a));
ASSERT_NE((a < b), (b < a));
ASSERT_TRUE(a > Uuid::zero());
ASSERT_TRUE(b > Uuid::zero());
ASSERT_TRUE(a < Uuid::max());
ASSERT_TRUE(b < Uuid::max());
auto aser = Serializer::serRawArgs(a);
auto bser = Serializer::serRawArgs(b);
ASSERT_EQ((a < b), (aser < bser));
ASSERT_EQ((a > b), (aser > bser));
ASSERT_TRUE(aser > Serializer::serRawArgs(Uuid::zero()));
ASSERT_TRUE(bser > Serializer::serRawArgs(Uuid::zero()));
ASSERT_TRUE(aser < Serializer::serRawArgs(Uuid::max()));
ASSERT_TRUE(bser < Serializer::serRawArgs(Uuid::max()));
}
}
TEST(Uuid, Gen) {
for (size_t i = 0; i < 100; i++) {
fmt::println("{}", Uuid::random());
}
}
} // namespace
} // namespace hf3fs::tests

View File

@@ -0,0 +1,38 @@
#include <gtest/gtest.h>
#include <limits>
#include "common/serde/Serde.h"
#include "common/utils/Varint32.h"
namespace hf3fs::test {
namespace {
TEST(TestVarint32, Normal) {
Varint32 value = 20;
value = 30;
ASSERT_EQ(value, 30);
auto out = serde::serialize(value);
Varint32 de1{};
ASSERT_TRUE(serde::deserialize(de1, out).hasValue());
ASSERT_EQ(de1, value);
Varint64 de2{};
ASSERT_TRUE(serde::deserialize(de2, out).hasValue());
ASSERT_EQ(de2, value);
}
TEST(TestVarint64, Normal) {
for (auto value : {0ul, 1ul << 32, std::numeric_limits<uint64_t>::max()}) {
Varint64 ser = value;
auto out = serde::serialize(ser);
Varint64 der{};
ASSERT_TRUE(serde::deserialize(der, out).hasValue());
ASSERT_EQ(ser, der);
}
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,82 @@
#include <folly/executors/task_queue/UnboundedBlockingQueue.h>
#include "common/utils/WorkStealingBlockingQueue.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::test {
namespace {
template <template <typename, typename> typename QueueTemplate>
auto createQueues(size_t count = 10) {
auto isEnd = [](const int &x) { return x == 0; };
using WSQueue = QueueTemplate<int, decltype(isEnd)>;
auto sharedState = std::make_shared<typename WSQueue::SharedState>(count);
std::vector<std::unique_ptr<WSQueue>> wsQueues;
for (size_t i = 0; i < count; ++i) {
wsQueues.push_back(std::make_unique<WSQueue>(sharedState, i, isEnd));
}
return wsQueues;
}
TEST(WorkStealingBlockingQueue, testWorkStealing) {
auto wsQueues = createQueues<WorkStealingBlockingQueue>();
wsQueues[0]->add(5);
ASSERT_EQ(wsQueues[1]->take(), 5);
ASSERT_TRUE(!wsQueues[0]->try_take_for(std::chrono::milliseconds(100)).hasValue());
wsQueues[2]->add(10);
for (;;) {
auto res = wsQueues[3]->try_take_for(std::chrono::milliseconds(100));
if (res) {
ASSERT_EQ(*res, 10);
break;
}
}
wsQueues[2]->add(0);
auto res = wsQueues[3]->try_take_for(std::chrono::milliseconds(100));
ASSERT_TRUE(!res);
for (;;) {
res = wsQueues[2]->try_take_for(std::chrono::milliseconds(0));
if (res) break;
}
ASSERT_EQ(*res, 0);
}
TEST(WorkStealingBlockingQueue, testRoundRobin) {
auto wsQueues = createQueues<RoundRobinBlockingQueue>(4);
wsQueues[0]->add(5);
ASSERT_EQ(wsQueues[1]->take(), 5);
ASSERT_TRUE(!wsQueues[0]->try_take_for(std::chrono::milliseconds(100)).hasValue());
wsQueues[2]->add(10);
for (;;) {
auto res = wsQueues[3]->try_take_for(std::chrono::milliseconds(100));
if (res) {
ASSERT_EQ(*res, 10);
break;
}
}
wsQueues[2]->add(0);
auto res = wsQueues[3]->try_take_for(std::chrono::milliseconds(100));
ASSERT_TRUE(!res);
for (;;) {
res = wsQueues[2]->try_take_for(std::chrono::milliseconds(0));
if (res) break;
}
ASSERT_EQ(*res, 0);
}
TEST(WorkStealingBlockingQueue, testSharedNothing) {
auto wsQueues = createQueues<SharedNothingBlockingQueue>();
wsQueues[0]->add(5);
ASSERT_TRUE(!wsQueues[1]->try_take_for(std::chrono::milliseconds(100)));
}
} // namespace
} // namespace hf3fs::test

View File

@@ -0,0 +1,25 @@
#include "common/utils/Size.h"
#include "common/utils/ZSTD.h"
#include "tests/GtestHelpers.h"
namespace hf3fs::testing {
namespace {
TEST(TestZstd, Normal) {
std::string src(1_MB, 0xFF);
std::string out(1_MB, '\0');
auto compressed = ZSTD_compress(out.data(), out.size(), src.data(), src.size(), 0);
ASSERT_FALSE(ZSTD_isError(compressed));
ASSERT_LT(compressed, src.size());
out.resize(compressed);
XLOGF(INFO, "zstd compress {} into {}", src.size(), out.size());
std::string des(1_MB, '\0');
auto decompressed = ZSTD_decompress(des.data(), des.size(), out.data(), out.size());
ASSERT_EQ(src.size(), decompressed);
ASSERT_TRUE(src == des);
}
} // namespace
} // namespace hf3fs::testing