Initial commit

src/monitor_collector/CMakeLists.txt

@@ -0,0 +1,2 @@
+target_add_lib(monitor_collector common memory-common MonitorCollectorService-fbs)
+target_add_bin(monitor_collector_main "monitor_collector.cpp" monitor_collector)

src/monitor_collector/monitor_collector.cpp

@@ -0,0 +1,7 @@
+#include "common/app/OnePhaseApplication.h"
+#include "memory/common/OverrideCppNewDelete.h"
+#include "monitor_collector/service/MonitorCollectorServer.h"
+
+int main(int argc, char *argv[]) {
+  return hf3fs::OnePhaseApplication<hf3fs::monitor::MonitorCollectorServer>::instance().run(argc, argv);
+}

src/monitor_collector/service/MonitorCollectorOperator.cc

@@ -0,0 +1,96 @@
+#include "monitor_collector/service/MonitorCollectorOperator.h"
+
+#include <algorithm>
+
+namespace hf3fs::monitor {
+
+static CountRecorder numQueueingSamples{"monitor_collector.num_queueing_samples"};
+
+MonitorCollectorOperator::MonitorCollectorOperator(const MonitorCollectorService::Config &cfg)
+    : cfg_(cfg) {
+  sampleQueue_ = folly::MPMCQueue<std::vector<Sample>>(cfg.queue_capacity());
+
+  for (int i = 0; i < cfg.conn_threads(); i++) {
+    threads_.emplace_back(std::bind_front(&MonitorCollectorOperator::connThreadFunc, this));
+  }
+  threads_.emplace_back(std::bind_front(&MonitorCollectorOperator::monitorThreadFunc, this));
+}
+
+MonitorCollectorOperator::~MonitorCollectorOperator() {
+  for (auto &thread : threads_) {
+    thread.request_stop();
+  }
+  for (auto &thread : threads_) {
+    thread.join();
+  }
+}
+
+CoTryTask<void> MonitorCollectorOperator::write(std::vector<Sample> &&samples) {
+  sampleQueue_.blockingWrite(std::move(samples));
+  numQueueingSamples.addSample(1);
+  cv_.notify_one();
+  co_return Void();
+}
+
+void MonitorCollectorOperator::connThreadFunc(std::stop_token stoken) {
+  std::unique_ptr<Reporter> reporter;
+  auto &reporterConfig = cfg_.reporter();
+  if (reporterConfig.type() == "clickhouse") {
+    reporter = std::make_unique<ClickHouseClient>(reporterConfig.clickhouse());
+  } else if (reporterConfig.type() == "log") {
+    reporter = std::make_unique<LogReporter>(reporterConfig.log());
+  } else if (reporterConfig.type() == "monitor_collector") {
+    reporter = std::make_unique<MonitorCollectorClient>(reporterConfig.monitor_collector());
+  } else {
+    XLOGF(FATAL, "Invalid reporter type: {}", reporterConfig.type());
+  }
+
+  auto result = reporter->init();
+  XLOGF_IF(FATAL, result.hasError(), "Initializing reporter failed. {}", result.error().describe());
+
+  std::vector<Sample> samples;
+
+  while (!stoken.stop_requested()) {
+    std::unique_lock lk(m_);
+    bool has_data = cv_.wait(lk, stoken, [this, &samples]() { return sampleQueue_.read(samples); });
+
+    if (has_data) {
+      std::vector<Sample> gather;
+      for (int i = 1; i < cfg_.batch_commit_size(); i++) {
+        bool continue_gather = sampleQueue_.read(gather);
+        if (!continue_gather) {
+          break;
+        } else {
+          samples.insert(samples.end(), gather.begin(), gather.end());
+        }
+        numQueueingSamples.addSample(-1);
+      }
+
+      samples.erase(std::remove_if(samples.begin(),
+                                   samples.end(),
+                                   [&cfg = this->cfg_](const Sample &s) {
+                                     return cfg.blacklisted_metric_names().count(s.name) > 0;
+                                   }),
+                    samples.end());
+
+      try {
+        reporter->commit(samples);
+      } catch (error_t e) {
+        XLOGF(ERR, "[ERROR]: client commit failed!");
+        XLOGF(ERR, "[ERROR]: {}", e);
+      }
+    }
+  }
+
+  XLOG(INFO, "Database connection thread shutting down.");
+}
+
+void MonitorCollectorOperator::monitorThreadFunc(std::stop_token stoken) {
+  using namespace std::chrono_literals;
+  while (!stoken.stop_requested()) {
+    XLOGF(INFO, "Sample queue capacity: {} / {}.", sampleQueue_.size(), cfg_.queue_capacity());
+    std::this_thread::sleep_for(5s);
+  }
+}
+
+}  // namespace hf3fs::monitor

src/monitor_collector/service/MonitorCollectorOperator.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <condition_variable>
+#include <folly/MPMCQueue.h>
+#include <mutex>
+#include <thread>
+
+#include "common/monitor/Monitor.h"
+#include "monitor_collector/service/MonitorCollectorService.h"
+
+namespace hf3fs::monitor {
+
+class MonitorCollectorOperator {
+ public:
+  MonitorCollectorOperator(const MonitorCollectorService::Config &cfg);
+  ~MonitorCollectorOperator();
+
+  CoTryTask<void> write(std::vector<Sample> &&samples);
+
+ private:
+  void connThreadFunc(std::stop_token stoken);
+  void monitorThreadFunc(std::stop_token stoken);
+
+  const MonitorCollectorService::Config &cfg_;
+  std::vector<std::jthread> threads_;
+  std::mutex m_;
+  std::condition_variable_any cv_;
+  folly::MPMCQueue<std::vector<Sample>> sampleQueue_;
+};
+
+}  // namespace hf3fs::monitor

src/monitor_collector/service/MonitorCollectorServer.cc

@@ -0,0 +1,24 @@
+#include "monitor_collector/service/MonitorCollectorServer.h"
+
+#include "monitor_collector/service/MonitorCollectorOperator.h"
+
+namespace hf3fs::monitor {
+
+MonitorCollectorServer::MonitorCollectorServer(const MonitorCollectorServer::Config &config)
+    : net::Server(config.base()),
+      config_(config) {}
+
+MonitorCollectorServer::~MonitorCollectorServer() { XLOGF(INFO, "Destructor MonitorCollectorServer"); }
+
+Result<Void> MonitorCollectorServer::beforeStart() {
+  monitorCollectorOperator_ = std::make_unique<MonitorCollectorOperator>(config_.monitor_collector());
+  RETURN_ON_ERROR(addSerdeService(std::make_unique<MonitorCollectorService>(*monitorCollectorOperator_), true));
+  return Void{};
+}
+
+Result<Void> MonitorCollectorServer::beforeStop() {
+  monitorCollectorOperator_.reset();
+  return Void{};
+}
+
+}  // namespace hf3fs::monitor

src/monitor_collector/service/MonitorCollectorServer.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "common/net/Server.h"
+#include "monitor_collector/service/MonitorCollectorService.h"
+
+namespace hf3fs::monitor {
+
+class MonitorCollectorServer : public net::Server {
+ public:
+  static constexpr auto kName = "MonitorCollector";
+
+  struct Config : public ConfigBase<Config> {
+    CONFIG_OBJ(base, net::Server::Config, [](net::Server::Config &c) {
+      c.set_groups_length(1);
+      c.groups(0).listener().set_listen_port(10000);
+      c.groups(0).set_network_type(hf3fs::net::Address::TCP);
+      c.groups(0).set_services({"MonitorCollector"});
+    });
+    CONFIG_OBJ(monitor_collector, monitor::MonitorCollectorService::Config);
+  };
+
+  MonitorCollectorServer(const Config &config);
+  ~MonitorCollectorServer() override;
+
+  // set up monitor collector server.
+  Result<Void> beforeStart() final;
+
+  // tear down monitor collector server.
+  Result<Void> beforeStop() final;
+
+ private:
+  std::unique_ptr<MonitorCollectorOperator> monitorCollectorOperator_;
+  const Config &config_;
+};
+
+}  // namespace hf3fs::monitor

src/monitor_collector/service/MonitorCollectorService.cc

@@ -0,0 +1,14 @@
+#include "monitor_collector/service/MonitorCollectorService.h"
+
+#include <chrono>
+
+#include "monitor_collector/service/MonitorCollectorOperator.h"
+
+namespace hf3fs::monitor {
+
+CoTryTask<MonitorCollectorRsp> MonitorCollectorService::write(serde::CallContext &ctx, std::vector<Sample> &samples) {
+  co_await monitorCollectorOperator_.write(std::move(samples));
+  co_return MonitorCollectorRsp{};
+}
+
+}  // namespace hf3fs::monitor

src/monitor_collector/service/MonitorCollectorService.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <condition_variable>
+#include <folly/MPMCQueue.h>
+#include <mutex>
+#include <thread>
+#include <vector>
+
+#include "common/monitor/Monitor.h"
+#include "common/serde/Service.h"
+#include "common/utils/ConfigBase.h"
+#include "fbs/monitor_collector/MonitorCollectorService.h"
+
+namespace hf3fs::monitor {
+class MonitorCollectorOperator;
+class MonitorCollectorService : public serde::ServiceWrapper<MonitorCollectorService, MonitorCollector> {
+ public:
+  class Config : public ConfigBase<Config> {
+    CONFIG_OBJ(reporter, hf3fs::monitor::Monitor::ReporterConfig, [](hf3fs::monitor::Monitor::ReporterConfig &c) {
+      c.set_type("clickhouse");
+    });
+    CONFIG_ITEM(conn_threads, 32);
+    CONFIG_ITEM(queue_capacity, 204800);
+    CONFIG_ITEM(batch_commit_size, 4096);
+    CONFIG_ITEM(blacklisted_metric_names, std::set<std::string>{});
+  };
+
+  MonitorCollectorService(MonitorCollectorOperator &monitorCollectorOperator)
+      : monitorCollectorOperator_(monitorCollectorOperator) {}
+
+  CoTryTask<MonitorCollectorRsp> write(serde::CallContext &ctx, std::vector<Sample> &samples);
+
+ private:
+  MonitorCollectorOperator &monitorCollectorOperator_;
+};
+
+}  // namespace hf3fs::monitor