From c15454e15344f4e92510dd9540ac24acfdf7b43c Mon Sep 17 00:00:00 2001 From: connerohnesorge Date: Tue, 27 May 2025 10:10:28 -0500 Subject: [PATCH] added the nixos module tests and config tests for nixos --- nixos-module-test.nix | 226 ++++++++++++++++++++++++++++++++++++++++++ nixos-test-config.nix | 102 +++++++++++++++++++ 2 files changed, 328 insertions(+) create mode 100644 nixos-module-test.nix create mode 100644 nixos-test-config.nix diff --git a/nixos-module-test.nix b/nixos-module-test.nix new file mode 100644 index 0000000..d524200 --- /dev/null +++ b/nixos-module-test.nix @@ -0,0 +1,226 @@ +# NixOS VM test for 3FS services +# Run with: nix build .#checks.x86_64-linux.3fs-integration-test + +{ pkgs, lib, ... }: + +let + # Test configuration for 3FS cluster + testConfig = { + meta = { + port = 6000; + dataDir = "/var/lib/3fs/meta"; + }; + + storage = { + port = 7000; + targets = [ + "/var/lib/3fs/storage/target1" + "/var/lib/3fs/storage/target2" + ]; + }; + + mgmtd = { + port = 8000; + dataDir = "/var/lib/3fs/mgmtd"; + }; + + foundationdb = { + clusterFile = "/etc/foundationdb/fdb.cluster"; + clusterString = "test:test@127.0.0.1:4500"; + }; + }; + +in { + name = "3fs-integration-test"; + + nodes = { + # Master node running all services + master = { config, pkgs, ... }: { + imports = [ self.nixosModules.default ]; + + # Enable 3FS services + services."3fs" = { + enable = true; + + meta = { + enable = true; + config = { + inherit (testConfig.meta) port dataDir; + }; + }; + + storage = { + enable = true; + targets = testConfig.storage.targets; + config = { + inherit (testConfig.storage) port; + }; + }; + + mgmtd = { + enable = true; + config = { + inherit (testConfig.mgmtd) port dataDir; + }; + }; + + monitor = { + enable = true; + config = { + port = 9000; + }; + }; + + foundationdb = { + clusterFile = testConfig.foundationdb.clusterFile; + }; + }; + + # Configure FoundationDB + services.foundationdb = { + enable = true; + clusterFile = testConfig.foundationdb.clusterFile; + listenAddress = "127.0.0.1:4500"; + dataDir = "/var/lib/foundationdb"; + logDir = "/var/log/foundationdb"; + + # Initialize the database + initialScript = '' + configure single ssd + ''; + }; + + # Open firewall ports + networking.firewall.enable = false; + + # Additional test utilities + environment.systemPackages = with pkgs; [ + netcat + curl + jq + ]; + }; + + # Client node with FUSE mount + client = { config, pkgs, ... }: { + imports = [ self.nixosModules.default ]; + + services."3fs" = { + enable = true; + + fuse = { + enable = true; + mountPoint = "/mnt/3fs"; + config = { + metaServers = [ "master:6000" ]; + mgmtdServers = [ "master:8000" ]; + }; + }; + + foundationdb = { + clusterFile = testConfig.foundationdb.clusterFile; + }; + }; + + # Configure FoundationDB client + services.foundationdb = { + enable = true; + clusterFile = testConfig.foundationdb.clusterFile; + pidFile = "/run/foundationdb.pid"; + }; + + # Write cluster file pointing to master + environment.etc."foundationdb/fdb.cluster" = { + text = testConfig.foundationdb.clusterString; + mode = "0644"; + }; + + networking.firewall.enable = false; + + environment.systemPackages = with pkgs; [ + fio + iozone + sysbench + ]; + }; + }; + + testScript = '' + start_all() + + # Wait for FoundationDB to be ready + master.wait_for_unit("foundationdb.service") + master.wait_for_open_port(4500) + master.succeed("fdbcli --exec 'status' -C ${testConfig.foundationdb.clusterFile}") + + # Wait for 3FS services to start + master.wait_for_unit("3fs-mgmtd.service") + master.wait_for_unit("3fs-meta.service") + master.wait_for_unit("3fs-storage.service") + master.wait_for_unit("3fs-monitor.service") + + # Check that services are listening on correct ports + master.wait_for_open_port(${toString testConfig.meta.port}) + master.wait_for_open_port(${toString testConfig.storage.port}) + master.wait_for_open_port(${toString testConfig.mgmtd.port}) + master.wait_for_open_port(9000) # monitor + + # Check service status + master.succeed("systemctl is-active 3fs-mgmtd.service") + master.succeed("systemctl is-active 3fs-meta.service") + master.succeed("systemctl is-active 3fs-storage.service") + master.succeed("systemctl is-active 3fs-monitor.service") + + # Initialize 3FS cluster + master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config ${testConfig.foundationdb.clusterFile}") + + # Register storage nodes + master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}") + + # Create storage targets + for target in ${lib.concatStringsSep " " (map (t: "\"${t}\"") testConfig.storage.targets)}; do + master.succeed("${pkgs."3fs"}/bin/admin create-target --path $target") + done + + # Wait for client to connect + client.wait_for_unit("3fs-fuse.service") + client.wait_until_succeeds("mountpoint -q /mnt/3fs") + + # Basic filesystem operations test + client.succeed("echo 'Hello 3FS!' > /mnt/3fs/test.txt") + client.succeed("cat /mnt/3fs/test.txt | grep 'Hello 3FS!'") + + # Create directory structure + client.succeed("mkdir -p /mnt/3fs/test/deep/directory") + client.succeed("touch /mnt/3fs/test/deep/directory/file.txt") + client.succeed("ls -la /mnt/3fs/test/deep/directory/") + + # Test file operations + client.succeed("dd if=/dev/urandom of=/mnt/3fs/random.dat bs=1M count=10") + client.succeed("cp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat") + client.succeed("cmp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat") + + # Test permissions + client.succeed("chmod 755 /mnt/3fs/test") + client.succeed("test -d /mnt/3fs/test") + + # Clean up + client.succeed("rm -rf /mnt/3fs/test*") + client.succeed("rm -f /mnt/3fs/random*.dat") + + # Verify cleanup + client.succeed("test -z \"$(ls -A /mnt/3fs)\"") + + # Check metrics + master.succeed("${pkgs."3fs"}/bin/admin list-nodes") + master.succeed("${pkgs."3fs"}/bin/admin list-targets") + + # Test service restart + master.succeed("systemctl restart 3fs-storage.service") + master.wait_for_unit("3fs-storage.service") + + # Ensure filesystem still works after restart + client.succeed("echo 'Still working!' > /mnt/3fs/restart-test.txt") + client.succeed("cat /mnt/3fs/restart-test.txt") + ''; +} \ No newline at end of file diff --git a/nixos-test-config.nix b/nixos-test-config.nix new file mode 100644 index 0000000..34680e2 --- /dev/null +++ b/nixos-test-config.nix @@ -0,0 +1,102 @@ +# Example NixOS configuration for testing 3FS +{ config, pkgs, ... }: + +{ + imports = [ + # Import the 3FS flake module + # In a real system, you would use: + # (builtins.getFlake "github:deepseek-ai/3fs").nixosModules.default + ]; + + # Enable 3FS services + services."3fs" = { + enable = true; + + # Configure user and group + user = "threefs"; + group = "threefs"; + + # Configure directories + configDir = "/etc/3fs"; + dataDir = "/var/lib/3fs"; + + # Enable metadata service + meta = { + enable = true; + config = { + # Additional meta service configuration + }; + }; + + # Enable storage service + storage = { + enable = true; + targets = [ + "/var/lib/3fs/storage/target1" + "/var/lib/3fs/storage/target2" + ]; + config = { + # Additional storage service configuration + }; + }; + + # Enable management daemon + mgmtd = { + enable = true; + config = { + # Additional mgmtd configuration + }; + }; + + # Enable monitor collector + monitor = { + enable = true; + config = { + # Additional monitor configuration + }; + }; + + # Enable FUSE client + fuse = { + enable = true; + mountPoint = "/mnt/3fs"; + config = { + # Additional FUSE configuration + }; + }; + + # FoundationDB configuration + foundationdb = { + clusterFile = "/etc/foundationdb/fdb.cluster"; + }; + }; + + # Ensure FoundationDB is also installed and configured + services.foundationdb = { + enable = true; + clusterFile = "/etc/foundationdb/fdb.cluster"; + }; + + # Open necessary ports for 3FS services + networking.firewall = { + allowedTCPPorts = [ + # Meta service ports + 6000 6001 + # Storage service ports + 7000 7001 + # Management daemon ports + 8000 8001 + # Monitor collector ports + 9000 9001 + ]; + }; + + # Example of how to override package + nixpkgs.overlays = [ + (final: prev: { + "3fs" = prev."3fs".overrideAttrs (oldAttrs: { + # Custom overrides if needed + }); + }) + ]; +} \ No newline at end of file