Initial commit

tests/analytics/CMakeLists.txt

@@ -0,0 +1 @@
+target_add_test(test_analytics analytics meta common storage-fbs meta-fbs mgmtd-fbs)

tests/analytics/TestSerdeObjectReader.cc

@@ -0,0 +1,138 @@
+#include <arrow/io/file.h>
+#include <folly/logging/xlog.h>
+#include <optional>
+#include <parquet/schema.h>
+
+#include "analytics/SerdeObjectReader.h"
+#include "analytics/SerdeObjectWriter.h"
+#include "fbs/core/user/User.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/storage/Common.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::analytics {
+
+template <serde::SerdeType T>
+void readAndCompareSerdeObjectDump(const T &expected) {
+  {
+    auto writer = SerdeObjectWriter<T>::open(fmt::format("{}.parquet", nameof::nameof_short_type<T>()));
+    ASSERT_NE(writer, nullptr);
+    *writer << expected << parquet::EndRowGroup;
+  }
+
+  auto reader = SerdeObjectReader<T>::open(fmt::format("{}.parquet", nameof::nameof_short_type<T>()));
+  ASSERT_NE(reader, nullptr);
+  T fromFile;
+  *reader >> fromFile;
+
+  ASSERT_TRUE(serde::equals(expected, fromFile))
+      << "Expected: " << serde::toJsonString(expected, true, true) << std::endl
+      << "FromFile: " << serde::toJsonString(fromFile, true, true);
+}
+
+TEST(TestSerdeObjectReader, ChunkMeta) {
+  storage::ChunkMeta chunkmeta{
+      .chunkId = storage::ChunkId(1, 1),
+      .updateVer = storage::ChunkVer(1),
+      .commitVer = storage::ChunkVer(2),
+      .checksum =
+          storage::ChecksumInfo{
+              .type = storage::ChecksumType::CRC32C,
+              .value = 123,
+          },
+  };
+
+  readAndCompareSerdeObjectDump(chunkmeta);
+}
+
+TEST(TestSerdeObjectReader, Inode) {
+  meta::Inode inode{
+      meta::InodeId{0xFF},
+      meta::InodeData{
+          .type =
+              meta::File{
+                  meta::Layout{
+                      .tableId = meta::ChainTableId{1},
+                      .tableVersion = meta::ChainTableVersion{1},
+                      .chains = meta::Layout::ChainRange(101 /*baseIndex*/,
+                                                         meta::Layout::ChainRange::Shuffle::STD_SHUFFLE_MT19937,
+                                                         0xFFEE /*seed*/),
+                  },
+              },
+          .nlink = 10,
+          .atime = UtcTime::clock::now(),
+          .ctime = UtcTime::clock::now(),
+          .mtime = UtcTime::clock::now(),
+      },
+  };
+
+  readAndCompareSerdeObjectDump(inode);
+}
+
+TEST(TestSerdeObjectReader, DirEntry) {
+  meta::DirEntry entry{meta::InodeId{0x10},
+                       "direntry",
+                       meta::DirEntryData{meta::InodeId{0xFF},
+                                          meta::InodeType::File,
+                                          meta::Acl(flat::Uid(1), flat::Gid(1), flat::Permission(0644))}};
+
+  readAndCompareSerdeObjectDump(entry);
+  entry.dirAcl = std::nullopt;
+  readAndCompareSerdeObjectDump(entry);
+}
+
+TEST(TestSerdeObjectReader, IOResult) {
+  storage::IOResult ioResult{
+      1024U,
+      storage::ChunkVer{1},
+      storage::ChunkVer{2},
+      storage::ChecksumInfo{storage::ChecksumType::CRC32C, 0xFFEEAABB},
+      storage::ChainVer{123},
+  };
+
+  readAndCompareSerdeObjectDump(ioResult);
+}
+
+TEST(TestSerdeObjectReader, UpdateReq) {
+  storage::UpdateReq updateReq{
+      .payload =
+          {
+              .offset = 123,
+              .length = 456,
+              .chunkSize = 789,
+              .key =
+                  storage::GlobalKey{
+                      .vChainId =
+                          storage::VersionedChainId{
+                              .chainId = storage::ChainId{10001},
+                              .chainVer = storage::ChainVer{2000},
+                          },
+                  },
+              // .rdmabuf = net::RDMARemoteBuf{},
+              .updateVer = storage::ChunkVer{999},
+              .updateType = storage::UpdateType::WRITE,
+              .checksum =
+                  storage::ChecksumInfo{
+                      .type = storage::ChecksumType::CRC32C,
+                      .value = 0xFFEEAABB,
+                  },
+              // .inlinebuf =
+              //     storage::UInt8Vector{
+              //         .data = std::vector<uint8_t>(10, 0xFF),
+              //     },
+          },
+      .tag =
+          storage::MessageTag{
+              ClientId::random(),
+              storage::RequestId{101},
+              storage::UpdateChannel{
+                  .id = storage::ChannelId{1},
+                  .seqnum = storage::ChannelSeqNum{1},
+              },
+          },
+  };
+
+  readAndCompareSerdeObjectDump(updateReq);
+}
+
+}  // namespace hf3fs::analytics

tests/analytics/TestSerdeObjectVisitor.cc

@@ -0,0 +1,115 @@
+#include <folly/logging/xlog.h>
+#include <parquet/schema.h>
+#include <string_view>
+
+#include "analytics/SerdeObjectVisitor.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/storage/Common.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::analytics {
+
+class DebugObjectVisitor : public BaseObjectVisitor<DebugObjectVisitor> {
+ public:
+  template <typename T>
+  void visit(std::string_view k, T &) = delete;
+
+  template <typename T>
+  requires std::is_arithmetic_v<T>
+  void visit(std::string_view k, T &) { XLOGF(DBG3, "arithmetic visit({})", k); }
+
+  template <typename T>
+  requires std::is_enum_v<T>
+  void visit(std::string_view k, T &) { XLOGF(DBG3, "enum visit({})", k); }
+
+  template <typename T>
+  requires std::is_convertible_v<T, std::string_view>
+  void visit(std::string_view k, T &) { XLOGF(DBG3, "string visit({})", k); }
+
+  template <StrongTyped T>
+  void visit(std::string_view k, T &v) {
+    XLOGF(DBG3, "strongtyped visit({})", k);
+    visit<typename T::UnderlyingType>(k, v.toUnderType());
+  }
+
+  template <serde::WithReadableSerdeMethod T>
+  void visit(std::string_view k, T &v) {
+    XLOGF(DBG3, "WithReadableSerdeMethod visit({})", k);
+    auto serialized = serde::SerdeMethod<T>::serdeToReadable(v);
+    visit(k, serialized);
+  }
+
+  template <serde::WithSerdeMethod T>
+  void visit(std::string_view k, T &v) {
+    XLOGF(DBG3, "WithSerdeMethod visit({})", k);
+    auto serialized = serde::SerdeMethod<T>::serdeTo(v);
+    visit(k, serialized);
+  }
+
+  template <serde::WithReadableSerdeMemberMethod T>
+  void visit(std::string_view k, T &v) {
+    XLOGF(DBG3, "WithReadableSerdeMemberMethod visit({})", k);
+    auto serialized = v.serdeToReadable();
+    visit(k, serialized);
+  }
+
+  template <serde::WithSerdeMemberMethod T>
+  void visit(std::string_view k, T &v) {
+    XLOGF(DBG3, "WithSerdeMemberMethod visit({})", k);
+    auto serialized = v.serdeTo();
+    visit(k, serialized);
+  }
+
+  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
+  void visit(std::string_view k, T &val) {
+    XLOGF(DBG3, "serdetype visit({})", k);
+    BaseObjectVisitor<DebugObjectVisitor>::visit(k, val);
+  }
+
+  template <typename T>
+  requires is_variant_v<T>
+  void visit(std::string_view k, T &val) {
+    XLOGF(DBG3, "variant visit({})", k);
+    BaseObjectVisitor<DebugObjectVisitor>::visit(k, val);
+  }
+
+  template <typename T>
+  requires is_vector_v<T> || is_set_v<T>
+  void visit(std::string_view k, T &val) {
+    XLOGF(DBG3, "container visit({})", k);
+    BaseObjectVisitor<DebugObjectVisitor>::visit(k, val);
+  }
+
+  template <typename T>
+  requires is_optional_v<T>
+  void visit(std::string_view k, T &val) {
+    XLOGF(DBG3, "container visit({})", k);
+    BaseObjectVisitor<DebugObjectVisitor>::visit(k, val);
+  }
+};
+
+TEST(TestSerdeObjectVisitor, ChunkId) {
+  storage::ChunkId chunkId;
+  DebugObjectVisitor visitor;
+  visitor.visit("chunkId", chunkId);
+}
+
+TEST(TestSerdeObjectVisitor, ChunkMeta) {
+  storage::ChunkMeta chunkmeta;
+  DebugObjectVisitor visitor;
+  visitor.visit("chunkmeta", chunkmeta);
+}
+
+TEST(TestSerdeObjectVisitor, InodeId) {
+  meta::InodeId inodeId;
+  DebugObjectVisitor visitor;
+  visitor.visit("inodeId", inodeId);
+}
+
+TEST(TestSerdeObjectVisitor, Inode) {
+  DebugObjectVisitor visitor;
+  meta::Inode inode;
+  visitor.visit("inode", inode);
+}
+
+}  // namespace hf3fs::analytics

tests/analytics/TestSerdeObjectWriter.cc

@@ -0,0 +1,40 @@
+#include <arrow/io/file.h>
+#include <folly/logging/xlog.h>
+#include <parquet/schema.h>
+
+#include "analytics/SerdeObjectWriter.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/storage/Common.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::analytics {
+
+TEST(TestSerdeObjectWriter, ChunkMeta) {
+  auto writer = SerdeObjectWriter<storage::ChunkMeta>::open("TestSerdeObjectWriter.ChunkMeta.parquet");
+  ASSERT_NE(writer, nullptr);
+  storage::ChunkMeta chunkmeta;
+  *writer << chunkmeta << parquet::EndRowGroup;
+}
+
+TEST(TestSerdeObjectWriter, Inode) {
+  auto writer = SerdeObjectWriter<meta::Inode>::open("TestSerdeObjectWriter.Inode.parquet");
+  ASSERT_NE(writer, nullptr);
+  meta::Inode inode;
+  *writer << inode << parquet::EndRowGroup;
+}
+
+TEST(TestSerdeObjectWriter, DirEntry) {
+  auto writer = SerdeObjectWriter<meta::DirEntry>::open("TestSerdeObjectWriter.DirEntry.parquet");
+  ASSERT_NE(writer, nullptr);
+  meta::DirEntry entry;
+  *writer << entry << parquet::EndRowGroup;
+}
+
+TEST(TestSerdeObjectWriter, UpdateReq) {
+  auto writer = SerdeObjectWriter<storage::UpdateReq>::open("TestSerdeObjectWriter.UpdateReq.parquet");
+  ASSERT_NE(writer, nullptr);
+  storage::UpdateReq updateReq;
+  *writer << updateReq << parquet::EndRowGroup;
+}
+
+}  // namespace hf3fs::analytics

tests/analytics/TestSerdeSchemaBuilder.cc

@@ -0,0 +1,39 @@
+#include <folly/logging/xlog.h>
+#include <parquet/schema.h>
+
+#include "analytics/SerdeSchemaBuilder.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/storage/Common.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::analytics {
+using namespace parquet;
+
+TEST(TestSerdeSchemaBuilder, ChunkMeta) {
+  SerdeSchemaBuilder<storage::ChunkMeta> builder;
+  auto chunkmetaSchema = builder.getSchema();
+
+  std::ostringstream outstr;
+  schema::PrintSchema(chunkmetaSchema.get(), outstr);
+  XLOGF(INFO, "chunkmeta schema: {}", outstr.str());
+}
+
+TEST(TestSerdeSchemaBuilder, Inode) {
+  SerdeSchemaBuilder<meta::Inode> builder;
+  auto inodeSchema = builder.getSchema();
+
+  std::ostringstream outstr;
+  schema::PrintSchema(inodeSchema.get(), outstr);
+  XLOGF(INFO, "inode schema: {}", outstr.str());
+}
+
+TEST(TestSerdeSchemaBuilder, DirEntry) {
+  SerdeSchemaBuilder<meta::DirEntry> builder;
+  auto dirEntrySchema = builder.getSchema();
+
+  std::ostringstream outstr;
+  schema::PrintSchema(dirEntrySchema.get(), outstr);
+  XLOGF(INFO, "inode schema: {}", outstr.str());
+}
+
+}  // namespace hf3fs::analytics

tests/analytics/TestSerdeStructVisitor.cc

@@ -0,0 +1,139 @@
+#include <folly/logging/xlog.h>
+#include <parquet/schema.h>
+
+#include "analytics/SerdeStructVisitor.h"
+#include "fbs/meta/Schema.h"
+#include "fbs/storage/Common.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::analytics {
+
+class DebugStructVisitor : public BaseStructVisitor<DebugStructVisitor> {
+ public:
+  // default
+  template <typename T>
+  void visit(std::string_view k) = delete;
+
+  template <>
+  void visit<uint16_t>(std::string_view k) {
+    XLOGF(DBG3, "uint16_t visit({})", k);
+  }
+
+  template <>
+  void visit<uint32_t>(std::string_view k) {
+    XLOGF(DBG3, "uint32_t visit({})", k);
+  }
+
+  template <>
+  void visit<uint64_t>(std::string_view k) {
+    XLOGF(DBG3, "uint64_t visit({})", k);
+  }
+
+  template <>
+  void visit<int16_t>(std::string_view k) {
+    XLOGF(DBG3, "int16_t visit({})", k);
+  }
+
+  template <>
+  void visit<int32_t>(std::string_view k) {
+    XLOGF(DBG3, "int32_t visit({})", k);
+  }
+
+  template <>
+  void visit<int64_t>(std::string_view k) {
+    XLOGF(DBG3, "int64_t visit({})", k);
+  }
+
+  template <typename T>
+  requires std::is_enum_v<T>
+  void visit(std::string_view k) { XLOGF(DBG3, "enum visit({})", k); }
+
+  template <serde::ConvertibleToString T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "string visit({})", k);
+  }
+
+  template <StrongTyped T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "strongtyped visit({})", k);
+    BaseStructVisitor<DebugStructVisitor>::visit<T>(k);
+  }
+
+  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "serdetype visit({})", k);
+    BaseStructVisitor<DebugStructVisitor>::visit<T>(k);
+  }
+
+  template <serde::WithReadableSerdeMethod T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "WithReadableSerdeMethod visit({})", k);
+    visit<serde::SerdeToReadableReturnType<T>>(k);
+  }
+
+  template <serde::WithSerdeMethod T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "WithSerdeMethod visit({})", k);
+    visit<serde::SerdeToReturnType<T>>(k);
+  }
+
+  template <serde::WithReadableSerdeMemberMethod T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "WithReadableSerdeMemberMethod visit({})", k);
+    visit<serde::SerdeToReadableMemberMethodReturnType<T>>(k);
+  }
+
+  template <serde::WithSerdeMemberMethod T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "WithSerdeMemberMethod visit({})", k);
+    visit<serde::SerdeToMemberMethodReturnType<T>>(k);
+  }
+
+  template <typename T>
+  requires is_variant_v<T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "variant visit({})", k);
+    BaseStructVisitor<DebugStructVisitor>::visit<T>(k);
+  }
+
+  template <typename T>
+  requires is_vector_v<T> || is_set_v<T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "container visit({})", k);
+    BaseStructVisitor<DebugStructVisitor>::visit<T>(k);
+  }
+
+  template <typename T>
+  requires is_optional_v<T>
+  void visit(std::string_view k) {
+    XLOGF(DBG3, "container visit({})", k);
+    BaseStructVisitor<DebugStructVisitor>::visit<T>(k);
+  }
+};
+
+TEST(TestSerdeStructVisitor, ChunkId) {
+  DebugStructVisitor visitor;
+  visitor.visit<storage::ChunkId>("chunkId");
+}
+
+TEST(TestSerdeStructVisitor, ChunkMeta) {
+  DebugStructVisitor visitor;
+  visitor.visit<storage::ChunkMeta>("chunkmeta");
+}
+
+TEST(TestSerdeStructVisitor, InodeId) {
+  DebugStructVisitor visitor;
+  visitor.visit<meta::InodeId>("inodeId");
+}
+
+TEST(TestSerdeStructVisitor, Inode) {
+  DebugStructVisitor visitor;
+  visitor.visit<meta::Inode>("inode");
+}
+
+TEST(TestSerdeStructVisitor, DirEntry) {
+  DebugStructVisitor visitor;
+  visitor.visit<meta::DirEntry>("dirEntry");
+}
+
+}  // namespace hf3fs::analytics

tests/analytics/TestStructuredTraceLog.cc

@@ -0,0 +1,134 @@
+#include <folly/logging/xlog.h>
+
+#include "analytics/StructuredTraceLog.h"
+#include "common/monitor/ScopedMetricsWriter.h"
+#include "fbs/storage/Common.h"
+#include "meta/event/Event.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::analytics {
+
+TEST(TestStructuredTraceLog, Open) {
+  StructuredTraceLog<storage::StorageEventTrace>::Config config;
+  config.set_enabled(true);
+  StructuredTraceLog<storage::StorageEventTrace> traceLog(config);
+  ASSERT_TRUE(traceLog.open());
+}
+
+TEST(TestStructuredTraceLog, Close) {
+  StructuredTraceLog<storage::StorageEventTrace>::Config config;
+  config.set_enabled(true);
+  StructuredTraceLog<storage::StorageEventTrace> traceLog(config);
+  ASSERT_TRUE(traceLog.open());
+  traceLog.close();
+}
+
+TEST(TestStructuredTraceLog, MetaEventTrace) {
+  StructuredTraceLog<meta::server::MetaEventTrace>::Config config;
+  config.set_enabled(true);
+  config.set_dump_interval(100_ms);
+  StructuredTraceLog<meta::server::MetaEventTrace> traceLog(config);
+  ASSERT_TRUE(traceLog.open());
+
+  for (size_t loop = 0; loop < 100'000; loop++) {
+    traceLog.append(meta::server::MetaEventTrace{
+        .inodeId = meta::InodeId{loop},
+        .entryName = std::to_string(loop),
+        .client = ClientId::zero(),
+    });
+  }
+
+  traceLog.close();
+}
+
+auto createStorageEventTrace(size_t id) {
+  return storage::StorageEventTrace{
+      .updateReq =
+          storage::UpdateReq{
+              .tag =
+                  storage::MessageTag{
+                      ClientId::zero(),
+                      storage::RequestId{id},
+                      storage::UpdateChannel{
+                          .id = storage::ChannelId{id},
+                          .seqnum = storage::ChannelSeqNum{id},
+                      },
+                  },
+          },
+  };
+}
+
+TEST(TestStructuredTraceLog, StorageEventTrace) {
+  StructuredTraceLog<storage::StorageEventTrace>::Config config;
+  config.set_enabled(true);
+  config.set_dump_interval(100_ms);
+  StructuredTraceLog<storage::StorageEventTrace> traceLog(config);
+  ASSERT_TRUE(traceLog.open());
+
+  for (size_t loop = 0; loop < 100'000; loop++) {
+    traceLog.append(createStorageEventTrace(loop));
+  }
+
+  traceLog.close();
+}
+
+TEST(TestStructuredTraceLog, MultiThreadsAppend) {
+  auto concurrency = std::min(std::thread::hardware_concurrency(), 64U);
+  StructuredTraceLog<storage::StorageEventTrace>::Config config;
+  config.set_enabled(true);
+  config.set_max_num_writers(concurrency / 2);
+  config.set_dump_interval(1_s);
+  StructuredTraceLog<storage::StorageEventTrace> traceLog(config);
+  ASSERT_TRUE(traceLog.open());
+
+  std::vector<std::future<void>> tasks;
+
+  for (size_t i = 0; i < concurrency; i++) {
+    tasks.push_back(std::async(std::launch::async, [&traceLog]() {
+      for (size_t loop = 0; loop < 100'000; loop++) {
+        traceLog.append(createStorageEventTrace(loop));
+      }
+    }));
+  }
+
+  for (auto &t : tasks) t.wait();
+  traceLog.close();
+}
+
+TEST(TestStructuredTraceLog, MultiThreadsNewEntry) {
+  auto concurrency = std::min(std::thread::hardware_concurrency(), 64U);
+  StructuredTraceLog<storage::StorageEventTrace>::Config config;
+  config.set_enabled(true);
+  config.set_max_num_writers(concurrency);
+  config.set_dump_interval(1_s);
+  StructuredTraceLog<storage::StorageEventTrace> traceLog(config);
+  ASSERT_TRUE(traceLog.open());
+
+  std::vector<std::future<void>> tasks;
+  monitor::LatencyRecorder latRecorder("TestStructuredTraceLog");
+
+  for (size_t i = 0; i < concurrency; i++) {
+    tasks.push_back(std::async(std::launch::async, [&traceLog, &latRecorder]() {
+      for (size_t loop = 0; loop < 100; loop++) {
+        auto entry = traceLog.newEntry(createStorageEventTrace(loop));
+        entry.reset();
+      }
+      for (size_t loop = 0; loop < 100'000; loop++) {
+        monitor::ScopedLatencyWriter latWriter(latRecorder);
+        auto entry = traceLog.newEntry(createStorageEventTrace(loop));
+        entry.reset();
+      }
+    }));
+  }
+
+  for (auto &t : tasks) t.wait();
+  traceLog.close();
+
+  std::vector<monitor::Sample> samples;
+  latRecorder.collect(samples);
+  for (const auto &s : samples) {
+    XLOGF(WARN, "latency: {}", s.dist());
+  }
+}
+
+}  // namespace hf3fs::analytics