DeepSeek/3FS
1
0
Fork 0
mirror of https://github.com/deepseek-ai/3FS synced 2025-06-26 18:16:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
dev
2025-02-27 21:53:53 +08:00
commit 815e55e4c0
1291 changed files with 185445 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
tests/common/monitor/TestMonitor.cc Normal file
View File

@@ -0,0 +1,244 @@
#include <chrono>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <folly/logging/xlog.h>
#include <gtest/gtest.h>
#include <memory>
#include <optional>
#include <ratio>
#include <thread>
#include <vector>
#include "common/monitor/Monitor.h"
#include "common/monitor/Recorder.h"
#include "common/monitor/Sample.h"
namespace hf3fs::monitor {
bool checkRecorderHasTag(const Recorder &rec, const TagSet &tag) { return rec.map_.find(tag) != rec.map_.end(); }
void printSamples(const std::vector<Sample> &samples) {
XLOGF(INFO, "{} Samples:", samples.size());
for (auto &sample : samples) {
if (sample.isNumber()) {
XLOGF(INFO, "\t{}: {}", sample.name, sample.number());
} else if (sample.isDistribution()) {
auto &dist = sample.dist();
XLOGF(INFO,
"\t{}: cnt {} sum {} min {} p50 {} p90 {} p95 {} p99 {} max {}",
sample.name,
dist.cnt,
dist.sum,
dist.min,
dist.p50,
dist.p90,
dist.p95,
dist.p99,
dist.max);
}
}
}
namespace test {
size_t countMatchedSamples(const std::string &counterPrefix, const std::vector<Sample> &samples) {
size_t sampleSize = 0;
for (const auto &s : samples)
if (s.name.starts_with(counterPrefix)) sampleSize++;
return sampleSize;
}
size_t filterSamples(const std::string &counterPrefix, std::vector<Sample> &samples) {
std::vector<Sample> matchedSamples;
for (const auto &s : samples)
if (s.name.starts_with(counterPrefix)) matchedSamples.push_back(s);
matchedSamples.swap(samples);
return samples.size();
}
TEST(TestMonitor, StartAndStop) {
Monitor::Config config;
Monitor::start(config);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
Monitor::stop();
}
TEST(TestMonitor, CountRecorder) {
auto &collector = Monitor::getDefaultInstance().getCollector();
CountRecorder testAdder("test_addr");
CountRecorder tagAdderA("tag_adder", monitor::TagSet{{"tag", "a"}});
CountRecorder tagAdderB("tag_adder", monitor::TagSet{{"tag", "b"}});
constexpr auto N = 8;
constexpr auto M = 1000000;
std::vector<std::thread> threads(N);
for (auto &thread : threads) {
thread = std::thread([&] {
for (auto m = 0; m < M; ++m) {
testAdder.addSample(1);
}
});
}
for (auto &thread : threads) {
thread.join();
}
std::vector<Sample> samples;
testAdder.collect(samples);
ASSERT_EQ(countMatchedSamples(testAdder.name(), samples), 1);
filterSamples(testAdder.name(), samples);
ASSERT_EQ(samples.front().number(), N * M);
// TODO_221013
monitor::TagSet recorder_tag_a;
recorder_tag_a.addTag("tag", "a");
monitor::TagSet recorder_tag_b;
recorder_tag_b.addTag("tag", "b");
testAdder.addSample(1, recorder_tag_a);
testAdder.addSample(1, recorder_tag_b);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(testAdder.name(), samples), 2);
ASSERT_TRUE(checkRecorderHasTag(testAdder, recorder_tag_a));
ASSERT_TRUE(checkRecorderHasTag(testAdder, recorder_tag_b));
testAdder.addSample(1, recorder_tag_b);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(testAdder.name(), samples), 1);
printSamples(samples);
ASSERT_FALSE(checkRecorderHasTag(testAdder, recorder_tag_a));
ASSERT_TRUE(checkRecorderHasTag(testAdder, recorder_tag_b));
testAdder.addSample(1, recorder_tag_a);
testAdder.addSample(1, recorder_tag_b);
samples.clear();
collector.collectAll(0, samples, false);
printSamples(samples);
monitor::TagSet recorder_tag_1{{"anothertag", "1"}};
monitor::TagSet recorder_tag_2{{"anothertag", "2"}};
tagAdderA.addSample(1, recorder_tag_1);
tagAdderA.addSample(1, recorder_tag_2);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(tagAdderA.name(), samples), 2);
tagAdderB.addSample(1, recorder_tag_1);
tagAdderB.addSample(1, recorder_tag_2);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(tagAdderB.name(), samples), 2);
tagAdderA.addSample(1, recorder_tag_1);
tagAdderB.addSample(1, recorder_tag_2);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(tagAdderA.name(), samples), 2);
tagAdderA.addSample(1, recorder_tag_1);
tagAdderA.addSample(1, recorder_tag_2);
tagAdderB.addSample(1, recorder_tag_1);
tagAdderB.addSample(1, recorder_tag_2);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(tagAdderA.name(), samples), 4);
}
TEST(TestMonitor, LatencyRecorder) {
auto &collector = Monitor::getDefaultInstance().getCollector();
LatencyRecorder testLatency("test_latency");
LatencyRecorder tagLatencyA("tag_latency", monitor::TagSet{{"tag", "a"}});
LatencyRecorder tagLatencyB("tag_latency", monitor::TagSet{{"tag", "b"}});
constexpr auto N = 10;
constexpr auto M = 1000000;
std::vector<std::thread> threads(N);
int cnt = 0;
for (auto &thread : threads) {
thread = std::thread([&, duration = cnt++ * M] {
for (auto m = 0; m < M; ++m) {
testLatency.addSample(std::chrono::nanoseconds(duration + m));
}
});
}
for (auto &thread : threads) {
thread.join();
}
std::vector<Sample> samples;
testLatency.collect(samples);
ASSERT_EQ(countMatchedSamples(testLatency.name(), samples), 1);
auto &sample = samples.front();
ASSERT_TRUE(sample.isDistribution());
ASSERT_EQ(sample.dist().cnt, N * M);
ASSERT_NEAR(sample.dist().mean(), N * M / 2.0, 1e1);
ASSERT_NEAR(sample.dist().min, 0, 1e-4);
ASSERT_NEAR(sample.dist().max, N * M - 1.0, 1e-4);
ASSERT_NEAR(sample.dist().p90, N * M * 0.9, 1e4);
ASSERT_NEAR(sample.dist().p99, N * M * 0.99, 1e4);
samples.clear();
testLatency.collect(samples);
ASSERT_EQ(countMatchedSamples(testLatency.name(), samples), 0);
// TODO_221013
monitor::TagSet recorder_tag_a;
recorder_tag_a.addTag("tag", "a");
monitor::TagSet recorder_tag_b;
recorder_tag_b.addTag("tag", "b");
testLatency.addSample(std::chrono::nanoseconds(1), recorder_tag_a);
testLatency.addSample(std::chrono::nanoseconds(1), recorder_tag_b);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(testLatency.name(), samples), 2);
printSamples(samples);
ASSERT_TRUE(checkRecorderHasTag(testLatency, recorder_tag_a));
ASSERT_TRUE(checkRecorderHasTag(testLatency, recorder_tag_b));
testLatency.addSample(std::chrono::nanoseconds(1), recorder_tag_b);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(testLatency.name(), samples), 1);
ASSERT_FALSE(checkRecorderHasTag(testLatency, recorder_tag_a));
ASSERT_TRUE(checkRecorderHasTag(testLatency, recorder_tag_b));
testLatency.addSample(std::chrono::nanoseconds(1), recorder_tag_a);
testLatency.addSample(std::chrono::nanoseconds(1), recorder_tag_b);
samples.clear();
collector.collectAll(0, samples, false);
printSamples(samples);
monitor::TagSet recorder_tag_1{{"anothertag", "1"}};
monitor::TagSet recorder_tag_2{{"anothertag", "2"}};
tagLatencyA.addSample(std::chrono::nanoseconds(1), recorder_tag_1);
tagLatencyA.addSample(std::chrono::nanoseconds(1), recorder_tag_2);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(tagLatencyA.name(), samples), 2);
tagLatencyB.addSample(std::chrono::nanoseconds(1), recorder_tag_1);
tagLatencyB.addSample(std::chrono::nanoseconds(1), recorder_tag_2);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(tagLatencyB.name(), samples), 2);
tagLatencyA.addSample(std::chrono::nanoseconds(1), recorder_tag_1);
tagLatencyB.addSample(std::chrono::nanoseconds(1), recorder_tag_2);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(tagLatencyA.name(), samples), 2);
tagLatencyA.addSample(std::chrono::nanoseconds(1), recorder_tag_1);
tagLatencyA.addSample(std::chrono::nanoseconds(1), recorder_tag_2);
tagLatencyB.addSample(std::chrono::nanoseconds(1), recorder_tag_1);
tagLatencyB.addSample(std::chrono::nanoseconds(1), recorder_tag_2);
samples.clear();
collector.collectAll(0, samples, true);
ASSERT_EQ(countMatchedSamples(tagLatencyA.name(), samples), 4);
}
TEST(TestMonitor, DISABLED_abort_on_duplicate) {
LatencyRecorder recorder1("tag_latency", monitor::TagSet{{"tag", "a"}});
LatencyRecorder recorder2("tag_latency", monitor::TagSet{{"tag", "b"}});
// abort here.
LatencyRecorder recorder3("tag_latency", monitor::TagSet{{"tag", "b"}});
}
} // namespace test
} // namespace hf3fs::monitor

66
tests/common/monitor/TestMonitorCollectorClient.cc Normal file
View File

@@ -0,0 +1,66 @@
#include <chrono>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/GtestHelpers.h>
#include <folly/logging/xlog.h>
#include <gtest/gtest.h>
#include <memory>
#include <optional>
#include <ratio>
#include <thread>
#include <vector>
#include "common/monitor/Monitor.h"
#include "common/monitor/Recorder.h"
namespace hf3fs::monitor {
namespace test {
// const std::string kDatabase = "unit_test";
// All test disabled because they need a running monitor collector server.
TEST(TestMonitorCollector, DISABLED_MonitorCollectorStartAndStop) {
try {
monitor::TagSet recorder_tag_a;
recorder_tag_a.addTag("tag", "this_is_a_tag");
CountRecorder testAdder("test_addr_count", recorder_tag_a);
testAdder.addSample(1);
Monitor::Config config_;
config_.set_reporters_length(1);
config_.reporters(0).set_type("monitor_collector");
config_.reporters(0).monitor_collector().set_remote_ip("1.2.3.4:5678");
Monitor::start(config_);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
Monitor::stop();
} catch (error_t e) {
ASSERT_TRUE(false);
}
}
TEST(TestMonitorCollector, DISABLED_CountRecorderWithMonitorCollector) {
try {
monitor::TagSet recorder_tag_a;
recorder_tag_a.addTag("tag", "a");
monitor::TagSet recorder_tag_b;
recorder_tag_b.addTag("tag", "b");
CountRecorder testAdder("test_addr_count");
testAdder.addSample(1, recorder_tag_a);
testAdder.addSample(1, recorder_tag_a);
testAdder.addSample(1, recorder_tag_b);
Monitor::Config config_;
config_.set_reporters_length(1);
config_.reporters(0).set_type("monitor_collector");
config_.reporters(0).monitor_collector().set_remote_ip("1.2.3.4:5678");
Monitor::start(config_);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
Monitor::stop();
} catch (error_t e) {
ASSERT_TRUE(false);
}
ASSERT_TRUE(true);
}
} // namespace test
} // namespace hf3fs::monitor

62
tests/common/monitor/TestTaosClient.cc Normal file
View File

@@ -0,0 +1,62 @@
#include <chrono>
#include <gtest/gtest.h>
#include "common/monitor/Monitor.h"
namespace hf3fs::monitor::test {
namespace {
/*
TEST(TestTaosClient, Normal) {
constexpr auto kInsertCount = 10000;
const std::string kDatabase = "unittest";
TaosClient::Config config;
config.set_db(kDatabase);
TaosClient client(config);
ASSERT_TRUE(client.init());
ASSERT_TRUE(client.query(fmt::format("drop database if exists {};", kDatabase)));
ASSERT_TRUE(client.query(fmt::format("create database {} precision 'us' update 1 keep 30;", kDatabase)));
auto start = std::chrono::steady_clock::now();
std::vector<Sample> samples;
samples.reserve(kInsertCount);
for (int64_t i = 0; i < kInsertCount; ++i) {
TagSet sample_tag;
samples.emplace_back(Sample{"count", sample_tag, UtcClock::now() + std::chrono::microseconds(i), i});
}
ASSERT_TRUE(client.commit(samples));
auto elapsed = std::chrono::steady_clock::now() - start;
fmt::print("Write {} count samples, takes {}ms\n",
kInsertCount,
std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count());
start = std::chrono::steady_clock::now();
samples.clear();
samples.reserve(kInsertCount);
for (uint64_t i = 0; i < kInsertCount; ++i) {
Distribution dist;
dist.cnt = i + 1;
dist.sum = (i + 1) * 50;
dist.min = 0;
dist.max = 100;
dist.p50 = 50;
dist.p90 = 90;
dist.p95 = 95;
dist.p99 = 99;
TagSet sample_tag;
samples.emplace_back(Sample{"latency", sample_tag, UtcClock::now() + std::chrono::microseconds(i), dist});
}
ASSERT_TRUE(client.commit(samples));
elapsed = std::chrono::steady_clock::now() - start;
fmt::print("Write {} latency samples, takes {}ms\n",
kInsertCount,
std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count());
client.stop();
}
*/
} // namespace
} // namespace hf3fs::monitor::test