Initial commit

tests/storage/client/TestTargetSelectionStrategy.cc

@@ -0,0 +1,155 @@
+#include <folly/executors/CPUThreadPoolExecutor.h>
+
+#include "client/storage/TargetSelection.h"
+#include "common/utils/Duration.h"
+#include "tests/GtestHelpers.h"
+
+namespace hf3fs::storage::client {
+namespace {
+
+TEST(TargetSelectionStrategy, LoadDistribution) {
+  const uint64_t numReplicas = 3;
+  const uint64_t numNodes = 180;
+  const uint64_t numTargets = numNodes * 16 * 16;
+  const uint64_t numChains = numTargets / 3;
+  const uint64_t numBatches = 100;
+
+  uint64_t nodeSeqNum = 0;
+  uint64_t targetId = 0;
+  std::unordered_map<NodeId, uint64_t> numTargetsOnNode;
+  std::unordered_map<ChainId, SlimChainInfo> chainMap;
+
+  for (uint64_t chainId = 1; chainId <= numChains; chainId++) {
+    auto &chainInfo = chainMap[ChainId(chainId)];
+    chainInfo.chainId = ChainId(chainId);
+    chainInfo.totalNumTargets = numReplicas;
+
+    for (uint64_t replica = 0; replica < numReplicas; replica++) {
+      NodeId nodeId((nodeSeqNum++ % numNodes) + 1);
+      chainInfo.servingTargets.push_back({TargetId(++targetId), nodeId});
+      numTargetsOnNode[nodeId]++;
+      ASSERT_LE(numTargetsOnNode[nodeId], numTargets / numNodes);
+      ASSERT_LE(targetId, numTargets);
+    }
+  }
+
+  auto computeIODist = [](const std::unordered_map<NodeId, uint64_t> &numIOsOnNode) {
+    std::vector<uint64_t> numIOs;
+    uint64_t sumIOs = 0;
+
+    for (const auto &item : numIOsOnNode) {
+      numIOs.push_back(item.second);
+      sumIOs += item.second;
+    }
+
+    std::sort(numIOs.begin(), numIOs.end());
+    return std::vector{
+        numIOs.front(),
+        numIOs.back(),
+        numIOs[numIOs.size() / 2],
+        sumIOs / numIOsOnNode.size(),
+        numIOsOnNode.size(),
+    };
+  };
+
+  for (uint64_t numIOsPerBatch : {100, 200, 400, 1000}) {
+    for (uint64_t mode = 1; mode < TargetSelectionMode::EndOfMode; mode++) {
+      std::unordered_map<NodeId, uint64_t> totalIOsOnNode;
+      std::vector<uint64_t> avgBatchDist(5);
+
+      for (uint64_t k = 0; k < numBatches; k++) {
+        TargetSelectionOptions options;
+        options.set_mode(static_cast<TargetSelectionMode>(mode));
+
+        auto strategy = TargetSelectionStrategy::create(options);
+        if (k == 0) strategy->reset();
+
+        std::unordered_map<NodeId, uint64_t> batchIOsOnNode;
+
+        for (uint64_t i = 0; i < numIOsPerBatch; i++) {
+          ChainId chainId(folly::Random::rand32(1, numChains + 1));
+          ASSERT_TRUE(!chainMap[chainId].servingTargets.empty()) << "chainId " << chainId;
+          auto selectedTarget = strategy->selectTarget(chainMap[chainId]);
+          ASSERT_TRUE(std::find(chainMap[chainId].servingTargets.cbegin(),
+                                chainMap[chainId].servingTargets.cend(),
+                                *selectedTarget) != chainMap[chainId].servingTargets.end());
+          totalIOsOnNode[selectedTarget->nodeId]++;
+          batchIOsOnNode[selectedTarget->nodeId]++;
+        }
+
+        auto dist = computeIODist(batchIOsOnNode);
+        for (uint64_t i = 0; i < dist.size(); i++) {
+          avgBatchDist[i] += dist[i];
+        }
+      }
+
+      auto overallDist = computeIODist(totalIOsOnNode);
+
+      fmt::print("\n=== numIOsPerBatch {}, TargetSelectionMode {} ===\n", numIOsPerBatch, mode);
+      fmt::print("   overall :  min {:<5d} max {:<5d} median {:<5d} avg {:<5d} #nodes {:<5d}\n",
+                 overallDist[0],
+                 overallDist[1],
+                 overallDist[2],
+                 overallDist[3],
+                 overallDist[4]);
+      fmt::print(" per batch :  min {:<5d} max {:<5d} median {:<5d} avg {:<5d} #nodes {:<5d}\n",
+                 avgBatchDist[0] / numBatches,
+                 avgBatchDist[1] / numBatches,
+                 avgBatchDist[2] / numBatches,
+                 avgBatchDist[3] / numBatches,
+                 avgBatchDist[4] / numBatches);
+    }
+  }
+}
+
+TEST(TargetSelectionStrategy, MultiThreads) {
+  constexpr uint64_t numReplicas = 2;
+  constexpr uint64_t numNodes = 180;
+  constexpr uint64_t numTargets = numNodes * 16 * 16;
+  constexpr uint64_t numChains = numTargets / numReplicas;
+  constexpr uint64_t numIOsPerBatch = 1024;
+  constexpr uint32_t numThreads = 32;
+
+  uint64_t nodeSeqNum = 0;
+  uint64_t targetId = 0;
+  std::unordered_map<NodeId, uint64_t> numTargetsOnNode;
+  std::unordered_map<ChainId, SlimChainInfo> chainMap;
+
+  for (uint64_t chainId = 1; chainId <= numChains; chainId++) {
+    auto &chainInfo = chainMap[ChainId(chainId)];
+    chainInfo.chainId = ChainId(chainId);
+    chainInfo.totalNumTargets = numReplicas;
+
+    for (uint64_t replica = 0; replica < numReplicas; replica++) {
+      NodeId nodeId((nodeSeqNum++ % numNodes) + 1);
+      chainInfo.servingTargets.push_back({TargetId(++targetId), nodeId});
+      numTargetsOnNode[nodeId]++;
+    }
+  }
+
+  folly::CPUThreadPoolExecutor executor(numThreads);
+  auto startTime = RelativeTime::now();
+  std::atomic<uint64_t> finished;
+  for (auto i = 0u; i < numThreads; ++i) {
+    executor.add([&] {
+      while (startTime + 1_s >= RelativeTime::now()) {
+        TargetSelectionOptions options;
+        options.set_mode(TargetSelectionMode::RoundRobin);
+        auto strategy = TargetSelectionStrategy::create(options);
+
+        for (auto j = 0u; j < numIOsPerBatch; ++j) {
+          ChainId chainId{folly::Random::rand32(1, numChains + 1)};
+          ASSERT_TRUE(!chainMap[chainId].servingTargets.empty()) << "chainId " << chainId;
+          auto selectedTarget = strategy->selectTarget(chainMap[chainId]);
+          ASSERT_OK(selectedTarget);
+        }
+        finished += numIOsPerBatch;
+      }
+    });
+  }
+  executor.join();
+  XLOGF(WARNING, "select target {} times", finished.load());
+}
+
+}  // namespace
+}  // namespace hf3fs::storage::client