added the nixos module tests and config tests for nixos

This commit is contained in:
connerohnesorge 2025-05-27 10:10:28 -05:00
parent 310118ef08
commit c15454e153
2 changed files with 328 additions and 0 deletions

226
nixos-module-test.nix Normal file
View File

@ -0,0 +1,226 @@
# NixOS VM test for 3FS services
# Run with: nix build .#checks.x86_64-linux.3fs-integration-test
{ pkgs, lib, ... }:
let
# Test configuration for 3FS cluster
testConfig = {
meta = {
port = 6000;
dataDir = "/var/lib/3fs/meta";
};
storage = {
port = 7000;
targets = [
"/var/lib/3fs/storage/target1"
"/var/lib/3fs/storage/target2"
];
};
mgmtd = {
port = 8000;
dataDir = "/var/lib/3fs/mgmtd";
};
foundationdb = {
clusterFile = "/etc/foundationdb/fdb.cluster";
clusterString = "test:test@127.0.0.1:4500";
};
};
in {
name = "3fs-integration-test";
nodes = {
# Master node running all services
master = { config, pkgs, ... }: {
imports = [ self.nixosModules.default ];
# Enable 3FS services
services."3fs" = {
enable = true;
meta = {
enable = true;
config = {
inherit (testConfig.meta) port dataDir;
};
};
storage = {
enable = true;
targets = testConfig.storage.targets;
config = {
inherit (testConfig.storage) port;
};
};
mgmtd = {
enable = true;
config = {
inherit (testConfig.mgmtd) port dataDir;
};
};
monitor = {
enable = true;
config = {
port = 9000;
};
};
foundationdb = {
clusterFile = testConfig.foundationdb.clusterFile;
};
};
# Configure FoundationDB
services.foundationdb = {
enable = true;
clusterFile = testConfig.foundationdb.clusterFile;
listenAddress = "127.0.0.1:4500";
dataDir = "/var/lib/foundationdb";
logDir = "/var/log/foundationdb";
# Initialize the database
initialScript = ''
configure single ssd
'';
};
# Open firewall ports
networking.firewall.enable = false;
# Additional test utilities
environment.systemPackages = with pkgs; [
netcat
curl
jq
];
};
# Client node with FUSE mount
client = { config, pkgs, ... }: {
imports = [ self.nixosModules.default ];
services."3fs" = {
enable = true;
fuse = {
enable = true;
mountPoint = "/mnt/3fs";
config = {
metaServers = [ "master:6000" ];
mgmtdServers = [ "master:8000" ];
};
};
foundationdb = {
clusterFile = testConfig.foundationdb.clusterFile;
};
};
# Configure FoundationDB client
services.foundationdb = {
enable = true;
clusterFile = testConfig.foundationdb.clusterFile;
pidFile = "/run/foundationdb.pid";
};
# Write cluster file pointing to master
environment.etc."foundationdb/fdb.cluster" = {
text = testConfig.foundationdb.clusterString;
mode = "0644";
};
networking.firewall.enable = false;
environment.systemPackages = with pkgs; [
fio
iozone
sysbench
];
};
};
testScript = ''
start_all()
# Wait for FoundationDB to be ready
master.wait_for_unit("foundationdb.service")
master.wait_for_open_port(4500)
master.succeed("fdbcli --exec 'status' -C ${testConfig.foundationdb.clusterFile}")
# Wait for 3FS services to start
master.wait_for_unit("3fs-mgmtd.service")
master.wait_for_unit("3fs-meta.service")
master.wait_for_unit("3fs-storage.service")
master.wait_for_unit("3fs-monitor.service")
# Check that services are listening on correct ports
master.wait_for_open_port(${toString testConfig.meta.port})
master.wait_for_open_port(${toString testConfig.storage.port})
master.wait_for_open_port(${toString testConfig.mgmtd.port})
master.wait_for_open_port(9000) # monitor
# Check service status
master.succeed("systemctl is-active 3fs-mgmtd.service")
master.succeed("systemctl is-active 3fs-meta.service")
master.succeed("systemctl is-active 3fs-storage.service")
master.succeed("systemctl is-active 3fs-monitor.service")
# Initialize 3FS cluster
master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config ${testConfig.foundationdb.clusterFile}")
# Register storage nodes
master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}")
# Create storage targets
for target in ${lib.concatStringsSep " " (map (t: "\"${t}\"") testConfig.storage.targets)}; do
master.succeed("${pkgs."3fs"}/bin/admin create-target --path $target")
done
# Wait for client to connect
client.wait_for_unit("3fs-fuse.service")
client.wait_until_succeeds("mountpoint -q /mnt/3fs")
# Basic filesystem operations test
client.succeed("echo 'Hello 3FS!' > /mnt/3fs/test.txt")
client.succeed("cat /mnt/3fs/test.txt | grep 'Hello 3FS!'")
# Create directory structure
client.succeed("mkdir -p /mnt/3fs/test/deep/directory")
client.succeed("touch /mnt/3fs/test/deep/directory/file.txt")
client.succeed("ls -la /mnt/3fs/test/deep/directory/")
# Test file operations
client.succeed("dd if=/dev/urandom of=/mnt/3fs/random.dat bs=1M count=10")
client.succeed("cp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat")
client.succeed("cmp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat")
# Test permissions
client.succeed("chmod 755 /mnt/3fs/test")
client.succeed("test -d /mnt/3fs/test")
# Clean up
client.succeed("rm -rf /mnt/3fs/test*")
client.succeed("rm -f /mnt/3fs/random*.dat")
# Verify cleanup
client.succeed("test -z \"$(ls -A /mnt/3fs)\"")
# Check metrics
master.succeed("${pkgs."3fs"}/bin/admin list-nodes")
master.succeed("${pkgs."3fs"}/bin/admin list-targets")
# Test service restart
master.succeed("systemctl restart 3fs-storage.service")
master.wait_for_unit("3fs-storage.service")
# Ensure filesystem still works after restart
client.succeed("echo 'Still working!' > /mnt/3fs/restart-test.txt")
client.succeed("cat /mnt/3fs/restart-test.txt")
'';
}

102
nixos-test-config.nix Normal file
View File

@ -0,0 +1,102 @@
# Example NixOS configuration for testing 3FS
{ config, pkgs, ... }:
{
imports = [
# Import the 3FS flake module
# In a real system, you would use:
# (builtins.getFlake "github:deepseek-ai/3fs").nixosModules.default
];
# Enable 3FS services
services."3fs" = {
enable = true;
# Configure user and group
user = "threefs";
group = "threefs";
# Configure directories
configDir = "/etc/3fs";
dataDir = "/var/lib/3fs";
# Enable metadata service
meta = {
enable = true;
config = {
# Additional meta service configuration
};
};
# Enable storage service
storage = {
enable = true;
targets = [
"/var/lib/3fs/storage/target1"
"/var/lib/3fs/storage/target2"
];
config = {
# Additional storage service configuration
};
};
# Enable management daemon
mgmtd = {
enable = true;
config = {
# Additional mgmtd configuration
};
};
# Enable monitor collector
monitor = {
enable = true;
config = {
# Additional monitor configuration
};
};
# Enable FUSE client
fuse = {
enable = true;
mountPoint = "/mnt/3fs";
config = {
# Additional FUSE configuration
};
};
# FoundationDB configuration
foundationdb = {
clusterFile = "/etc/foundationdb/fdb.cluster";
};
};
# Ensure FoundationDB is also installed and configured
services.foundationdb = {
enable = true;
clusterFile = "/etc/foundationdb/fdb.cluster";
};
# Open necessary ports for 3FS services
networking.firewall = {
allowedTCPPorts = [
# Meta service ports
6000 6001
# Storage service ports
7000 7001
# Management daemon ports
8000 8001
# Monitor collector ports
9000 9001
];
};
# Example of how to override package
nixpkgs.overlays = [
(final: prev: {
"3fs" = prev."3fs".overrideAttrs (oldAttrs: {
# Custom overrides if needed
});
})
];
}