Merge 16468b0110 into 2db69ced80

README-nix.md

@@ -0,0 +1,235 @@
+# 3FS Nix Package and NixOS Module
+
+This directory contains a Nix flake for building and deploying 3FS (Fire-Flyer File System) on NixOS systems.
+
+## Features
+
+- **Complete Package**: Builds 3FS with all dependencies including RDMA support, FoundationDB integration, and FUSE 3
+- **NixOS Module**: Provides systemd services for all 3FS components (meta, storage, mgmtd, monitor, fuse)
+- **Development Shell**: Includes all build tools and dependencies for 3FS development
+- **Integration Tests**: NixOS VM tests to validate the complete system
+
+## Quick Start
+
+### Building the Package
+
+1. First build (will fail with hash mismatch):
+```bash
+nix build .#3fs
+```
+
+2. Update the hash in `package.nix` with the correct one from the error message
+
+3. Rebuild:
+```bash
+nix build .#3fs
+```
+
+### Using in NixOS
+
+Add to your `flake.nix`:
+
+```nix
+{
+  inputs = {
+    threefs.url = "github:deepseek-ai/3fs";
+  };
+  
+  outputs = { self, nixpkgs, threefs, ... }: {
+    nixosConfigurations.myserver = nixpkgs.lib.nixosSystem {
+      modules = [
+        threefs.nixosModules.default
+        {
+          services."3fs" = {
+            enable = true;
+            meta.enable = true;
+            storage.enable = true;
+            mgmtd.enable = true;
+          };
+        }
+      ];
+    };
+  };
+}
+```
+
+### Development Environment
+
+Enter the development shell:
+```bash
+nix develop
+```
+
+This provides:
+- Clang 14 with C++20 support
+- CMake and build tools
+- All 3FS dependencies
+- Rust toolchain
+- Debug tools (gdb, valgrind, perf)
+
+## NixOS Module Options
+
+### Basic Configuration
+
+```nix
+services."3fs" = {
+  enable = true;
+  
+  # User and group for services
+  user = "threefs";
+  group = "threefs";
+  
+  # Directories
+  configDir = "/etc/3fs";
+  dataDir = "/var/lib/3fs";
+  
+  # FoundationDB cluster file
+  foundationdb.clusterFile = "/etc/foundationdb/fdb.cluster";
+};
+```
+
+### Service Configuration
+
+#### Metadata Service
+```nix
+services."3fs".meta = {
+  enable = true;
+  config = {
+    # Additional meta configuration
+  };
+};
+```
+
+#### Storage Service
+```nix
+services."3fs".storage = {
+  enable = true;
+  targets = [
+    "/var/lib/3fs/storage/target1"
+    "/var/lib/3fs/storage/target2"
+  ];
+  config = {
+    # Additional storage configuration
+  };
+};
+```
+
+#### Management Daemon
+```nix
+services."3fs".mgmtd = {
+  enable = true;
+  config = {
+    # Additional mgmtd configuration
+  };
+};
+```
+
+#### Monitor Collector
+```nix
+services."3fs".monitor = {
+  enable = true;
+  config = {
+    # Additional monitor configuration
+  };
+};
+```
+
+#### FUSE Client
+```nix
+services."3fs".fuse = {
+  enable = true;
+  mountPoint = "/mnt/3fs";
+  config = {
+    # Additional FUSE configuration
+  };
+};
+```
+
+## Testing
+
+### Run Package Tests
+```bash
+./test-build.sh
+```
+
+### Run Integration Tests
+```bash
+nix build .#checks.x86_64-linux.integration-test
+```
+
+### Manual Testing
+
+1. Start FoundationDB:
+```bash
+sudo systemctl start foundationdb
+```
+
+2. Initialize 3FS cluster:
+```bash
+3fs-admin init-cluster
+```
+
+3. Start 3FS services:
+```bash
+sudo systemctl start 3fs-mgmtd
+sudo systemctl start 3fs-meta
+sudo systemctl start 3fs-storage
+```
+
+4. Mount filesystem:
+```bash
+sudo systemctl start 3fs-fuse
+```
+
+## Package Contents
+
+The package includes:
+
+- **Binaries**:
+  - `admin` - Administrative CLI tool
+  - `meta_main` - Metadata service
+  - `storage_main` - Storage service
+  - `mgmtd_main` - Management daemon
+  - `monitor_collector_main` - Metrics collector
+  - `hf3fs_fuse_main` - FUSE filesystem client
+
+- **Libraries**: All required shared libraries in `/lib`
+- **Configuration**: Default configuration files in `/etc/3fs`
+- **Headers**: Development headers in `/include/3fs`
+
+## Troubleshooting
+
+### Build Issues
+
+1. **Hash mismatch**: Update the SHA256 hash in `package.nix`
+2. **Missing dependencies**: Check that all system libraries are available
+3. **Compilation errors**: Ensure you're using Clang 14 or newer
+
+### Runtime Issues
+
+1. **Service fails to start**: Check logs with `journalctl -u 3fs-<service>`
+2. **FoundationDB connection**: Ensure FDB cluster file is correct
+3. **FUSE mount fails**: Check that FUSE 3 is installed and kernel module is loaded
+
+## Architecture
+
+The Nix package handles:
+
+1. **Complex Dependencies**: Automatically builds custom jemalloc, handles RDMA libraries
+2. **Third-party Libraries**: Uses system libraries where possible, builds others from source
+3. **Rust Components**: Properly integrates Cargo builds for Rust components
+4. **Service Management**: Complete systemd integration with proper dependencies
+5. **Configuration**: Manages TOML configuration files for all components
+
+## Contributing
+
+When modifying the package:
+
+1. Update version in `package.nix`
+2. Test build locally
+3. Run integration tests
+4. Update documentation
+
+## License
+
+MIT License (same as 3FS project)

flake.lock

@@ -0,0 +1,61 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1748190013,
+        "narHash": "sha256-R5HJFflOfsP5FBtk+zE8FpL8uqE7n62jqOsADvVshhE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "62b852f6c6742134ade1abdd2a21685fd617a291",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,368 @@
+{
+  description = "Fire-Flyer File System (3FS) - High-performance distributed file system for AI workloads";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, flake-utils }:
+    let
+      overlay = final: prev: {
+        # Build clickhouse-cpp from source if not available in nixpkgs
+        clickhouse-cpp = prev.clickhouse-cpp or (prev.stdenv.mkDerivation rec {
+          pname = "clickhouse-cpp";
+          version = "2.5.1";
+          
+          src = prev.fetchFromGitHub {
+            owner = "ClickHouse";
+            repo = "clickhouse-cpp";
+            rev = "v${version}";
+            sha256 = "sha256-6kqcANO4S9Z1ee4kBPKGCnsPEGDaWPCx2hUi4APPWHU=";
+          };
+          
+          nativeBuildInputs = [ prev.cmake ];
+          buildInputs = [ prev.zlib prev.openssl prev.lz4 ];
+          
+          cmakeFlags = [
+            "-DBUILD_SHARED_LIBS=ON"
+            "-DWITH_OPENSSL=ON"
+          ];
+        });
+        
+        # Build scnlib from source if not available in nixpkgs
+        scnlib = prev.scnlib or (prev.stdenv.mkDerivation rec {
+          pname = "scnlib";
+          version = "2.0.2";
+          
+          src = prev.fetchFromGitHub {
+            owner = "eliaskosunen";
+            repo = "scnlib";
+            rev = "v${version}";
+            sha256 = "sha256-YWlJiHAKKJd7jWv8Z0GmKqIfXI3HwVqA7AgZiHN2W8I=";
+          };
+          
+          nativeBuildInputs = [ prev.cmake ];
+          
+          cmakeFlags = [
+            "-DSCN_TESTS=OFF"
+            "-DSCN_EXAMPLES=OFF"
+            "-DSCN_BENCHMARKS=OFF"
+          ];
+        });
+        
+        "3fs" = prev.callPackage ./package.nix {
+          inherit (final) clickhouse-cpp scnlib;
+          libibverbs = prev.rdma-core;
+        };
+      };
+      
+      nixosModule = { config, lib, pkgs, ... }:
+        with lib;
+        let
+          cfg = config.services."3fs";
+          
+          # Helper function to create service configuration
+          mkServiceConfig = component: {
+            description = "3FS ${component} service";
+            wantedBy = [ "multi-user.target" ];
+            after = [ "network-online.target" ];
+            requires = [ "network-online.target" ];
+            
+            serviceConfig = {
+              Type = "simple";
+              ExecStart = "${pkgs."3fs"}/bin/${component}_main --launcher_cfg ${cfg.configDir}/${component}_main_launcher.toml --app-cfg ${cfg.configDir}/${component}_main_app.toml";
+              Restart = "on-failure";
+              RestartSec = 5;
+              LimitNOFILE = 1000000;
+              User = cfg.user;
+              Group = cfg.group;
+            } // (if component == "storage" then {
+              LimitMEMLOCK = "infinity";
+              TimeoutStopSec = "5m";
+            } else {});
+            
+            environment = {
+              LD_LIBRARY_PATH = "${pkgs."3fs"}/lib:${pkgs.foundationdb}/lib";
+            };
+          };
+        in
+        {
+          options.services."3fs" = {
+            enable = mkEnableOption "3FS distributed file system";
+            
+            user = mkOption {
+              type = types.str;
+              default = "threefs";
+              description = "User under which 3FS services run";
+            };
+            
+            group = mkOption {
+              type = types.str;
+              default = "threefs";
+              description = "Group under which 3FS services run";
+            };
+            
+            configDir = mkOption {
+              type = types.path;
+              default = "/etc/3fs";
+              description = "Directory containing 3FS configuration files";
+            };
+            
+            dataDir = mkOption {
+              type = types.path;
+              default = "/var/lib/3fs";
+              description = "Directory for 3FS data storage";
+            };
+            
+            meta = {
+              enable = mkEnableOption "3FS metadata service";
+              
+              config = mkOption {
+                type = types.attrs;
+                default = {};
+                description = "Additional configuration for meta service";
+              };
+            };
+            
+            storage = {
+              enable = mkEnableOption "3FS storage service";
+              
+              targets = mkOption {
+                type = types.listOf types.str;
+                default = [ "/var/lib/3fs/storage" ];
+                description = "List of storage target directories";
+              };
+              
+              config = mkOption {
+                type = types.attrs;
+                default = {};
+                description = "Additional configuration for storage service";
+              };
+            };
+            
+            mgmtd = {
+              enable = mkEnableOption "3FS management daemon";
+              
+              config = mkOption {
+                type = types.attrs;
+                default = {};
+                description = "Additional configuration for management daemon";
+              };
+            };
+            
+            monitor = {
+              enable = mkEnableOption "3FS monitor collector";
+              
+              config = mkOption {
+                type = types.attrs;
+                default = {};
+                description = "Additional configuration for monitor collector";
+              };
+            };
+            
+            fuse = {
+              enable = mkEnableOption "3FS FUSE client";
+              
+              mountPoint = mkOption {
+                type = types.path;
+                default = "/mnt/3fs";
+                description = "Mount point for 3FS FUSE filesystem";
+              };
+              
+              config = mkOption {
+                type = types.attrs;
+                default = {};
+                description = "Additional configuration for FUSE client";
+              };
+            };
+            
+            foundationdb = {
+              clusterFile = mkOption {
+                type = types.path;
+                default = "/etc/foundationdb/fdb.cluster";
+                description = "Path to FoundationDB cluster file";
+              };
+            };
+          };
+          
+          config = mkIf cfg.enable {
+            # Create system user and group
+            users.users.${cfg.user} = {
+              isSystemUser = true;
+              group = cfg.group;
+              home = cfg.dataDir;
+              createHome = true;
+              description = "3FS system user";
+            };
+            
+            users.groups.${cfg.group} = {};
+            
+            # Create necessary directories
+            systemd.tmpfiles.rules = [
+              "d '${cfg.configDir}' 0755 ${cfg.user} ${cfg.group} -"
+              "d '${cfg.dataDir}' 0755 ${cfg.user} ${cfg.group} -"
+              "d '${cfg.dataDir}/meta' 0755 ${cfg.user} ${cfg.group} -"
+              "d '${cfg.dataDir}/mgmtd' 0755 ${cfg.user} ${cfg.group} -"
+              "d '${cfg.dataDir}/monitor' 0755 ${cfg.user} ${cfg.group} -"
+            ] ++ (map (target: "d '${target}' 0755 ${cfg.user} ${cfg.group} -") cfg.storage.targets);
+            
+            # Install default configuration files
+            environment.etc = {
+              "3fs/meta_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/meta_main_launcher.toml";
+              "3fs/meta_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/meta_main_app.toml";
+              "3fs/storage_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/storage_main_launcher.toml";
+              "3fs/storage_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/storage_main_app.toml";
+              "3fs/mgmtd_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/mgmtd_main_launcher.toml";
+              "3fs/mgmtd_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/mgmtd_main_app.toml";
+              "3fs/monitor_collector_main.toml".source = "${pkgs."3fs"}/etc/3fs/monitor_collector_main.toml";
+              "3fs/hf3fs_fuse_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/hf3fs_fuse_main_launcher.toml";
+              "3fs/hf3fs_fuse_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/hf3fs_fuse_main_app.toml";
+            };
+            
+            # Define systemd services
+            systemd.services = {
+              "3fs-meta" = mkIf cfg.meta.enable (mkServiceConfig "meta");
+              "3fs-storage" = mkIf cfg.storage.enable (mkServiceConfig "storage");
+              "3fs-mgmtd" = mkIf cfg.mgmtd.enable (mkServiceConfig "mgmtd");
+              "3fs-monitor" = mkIf cfg.monitor.enable (mkServiceConfig "monitor_collector");
+              
+              "3fs-fuse" = mkIf cfg.fuse.enable {
+                description = "3FS FUSE client";
+                wantedBy = [ "multi-user.target" ];
+                after = [ "network-online.target" "3fs-meta.service" ];
+                requires = [ "network-online.target" ];
+                
+                serviceConfig = {
+                  Type = "simple";
+                  ExecStart = "${pkgs."3fs"}/bin/hf3fs_fuse_main --launcher_cfg ${cfg.configDir}/hf3fs_fuse_main_launcher.toml --app-cfg ${cfg.configDir}/hf3fs_fuse_main_app.toml ${cfg.fuse.mountPoint}";
+                  ExecStop = "${pkgs.fuse3}/bin/fusermount3 -u ${cfg.fuse.mountPoint}";
+                  Restart = "on-failure";
+                  RestartSec = 5;
+                  LimitNOFILE = 1000000;
+                  User = "root";  # FUSE requires root
+                  Group = "root";
+                };
+                
+                preStart = ''
+                  mkdir -p ${cfg.fuse.mountPoint}
+                '';
+              };
+            };
+            
+            # Add 3fs package to system packages
+            environment.systemPackages = [ pkgs."3fs" ];
+          };
+        };
+    in
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [ overlay ];
+        };
+        
+        isLinux = pkgs.stdenv.isLinux;
+      in
+      {
+        packages = pkgs.lib.optionalAttrs isLinux {
+          default = pkgs."3fs";
+          "3fs" = pkgs."3fs";
+        };
+
+        # Don't export overlays per-system
+
+        devShells.default = pkgs.mkShell {
+          inputsFrom = pkgs.lib.optionals isLinux [ pkgs."3fs" ];
+          buildInputs = with pkgs; [
+            # Build dependencies that are cross-platform
+            cmake
+            pkg-config
+            boost
+            protobuf
+            grpc
+            gtest
+            glog
+            lz4
+            zlib
+            openssl
+            zstd
+            jemalloc
+            libevent
+            thrift
+            bison
+            flex
+            git
+            which
+            fmt
+            folly
+            rocksdb
+            leveldb
+            arrow-cpp
+            mimalloc
+            tomlplusplus
+            rustc
+            cargo
+            rustfmt
+            clippy
+            rustPlatform.bindgenHook
+            clickhouse-cpp
+            scnlib
+            
+            # Additional dev tools
+            clang-tools_14
+            cmake-format
+            cmake-language-server
+            gdb
+            rust-analyzer
+            cargo-watch
+          ] ++ pkgs.lib.optionals isLinux [
+            # Linux-only dependencies
+            foundationdb
+            fuse3
+            rdma-core
+            numactl
+            liburing
+            valgrind
+            perf-tools
+          ];
+          
+          shellHook = ''
+            echo "3FS development environment"
+            ${if isLinux then ''
+              echo "Build with: nix build .#3fs"
+            '' else ''
+              echo "Note: 3FS can only be built on Linux systems"
+            ''}
+            echo "Enter shell with: nix develop"
+            echo ""
+            echo "To test locally:"
+            echo "  1. Start FoundationDB"
+            echo "  2. Configure and start 3FS services"
+            echo "  3. Mount FUSE filesystem"
+          '';
+        };
+        checks = pkgs.lib.optionalAttrs isLinux {
+          # Package build test
+          package = pkgs."3fs";
+        } // pkgs.lib.optionalAttrs isLinux {
+          # Integration test using NixOS VM (Linux only)
+          integration-test = pkgs.nixosTest (import ./nixos-module-test.nix {
+            inherit pkgs self;
+            lib = pkgs.lib;
+          });
+        };
+      }
+    ) // {
+      nixosModules.default = nixosModule;
+      nixosModules."3fs" = nixosModule;
+      
+      overlays.default = overlay;
+      
+      # Hydra job for CI - only for Linux systems
+      hydraJobs = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ] (system: {
+        packages = self.packages.${system};
+        tests = self.checks.${system} or {};
+      });
+    };
+}

nixos-module-test.nix

@@ -0,0 +1,210 @@
+# NixOS VM test for 3FS services
+# Run with: nix build .#checks.x86_64-linux.3fs-integration-test
+
+{ pkgs, lib, self, ... }:
+
+let
+  # Test configuration for 3FS cluster
+  testConfig = {
+    meta = {
+      port = 6000;
+      dataDir = "/var/lib/3fs/meta";
+    };
+    
+    storage = {
+      port = 7000;
+      targets = [
+        "/var/lib/3fs/storage/target1"
+        "/var/lib/3fs/storage/target2"
+      ];
+    };
+    
+    mgmtd = {
+      port = 8000;
+      dataDir = "/var/lib/3fs/mgmtd";
+    };
+  };
+  
+in {
+  name = "3fs-integration-test";
+  
+  nodes = {
+    # Master node running all services
+    master = { config, pkgs, ... }: {
+      imports = [ self.nixosModules.default ];
+      
+      # Enable 3FS services
+      services."3fs" = {
+        enable = true;
+        
+        meta = {
+          enable = true;
+          config = {
+            inherit (testConfig.meta) port dataDir;
+          };
+        };
+        
+        storage = {
+          enable = true;
+          targets = testConfig.storage.targets;
+          config = {
+            inherit (testConfig.storage) port;
+          };
+        };
+        
+        mgmtd = {
+          enable = true;
+          config = {
+            inherit (testConfig.mgmtd) port dataDir;
+          };
+        };
+        
+        monitor = {
+          enable = true;
+          config = {
+            port = 9000;
+          };
+        };
+        
+        # FoundationDB configuration is handled by the system service
+      };
+      
+      # Configure FoundationDB
+      services.foundationdb = {
+        enable = true;
+        package = pkgs.foundationdb;
+        listenAddress = "127.0.0.1:4500";
+        dataDir = "/var/lib/foundationdb";
+        logDir = "/var/log/foundationdb";
+      };
+      
+      # Open firewall ports
+      networking.firewall.enable = false;
+      
+      # Additional test utilities
+      environment.systemPackages = with pkgs; [
+        netcat
+        curl
+        jq
+      ];
+    };
+    
+    # Client node with FUSE mount
+    client = { config, pkgs, ... }: {
+      imports = [ self.nixosModules.default ];
+      
+      services."3fs" = {
+        enable = true;
+        
+        fuse = {
+          enable = true;
+          mountPoint = "/mnt/3fs";
+          config = {
+            metaServers = [ "master:6000" ];
+            mgmtdServers = [ "master:8000" ];
+          };
+        };
+        
+        # FoundationDB configuration is handled by the system service
+      };
+      
+      # Configure FoundationDB client
+      services.foundationdb = {
+        enable = true;
+        package = pkgs.foundationdb;
+      };
+      
+      # Write cluster file pointing to master
+      environment.etc."foundationdb/fdb.cluster" = {
+        text = "test:test@master:4500";
+        mode = "0644";
+      };
+      
+      networking.firewall.enable = false;
+      
+      environment.systemPackages = with pkgs; [
+        fio
+        sysbench
+      ];
+    };
+  };
+  
+  testScript = ''
+    start_all()
+    
+    # Wait for FoundationDB to be ready
+    master.wait_for_unit("foundationdb.service")
+    master.wait_for_open_port(4500)
+    master.succeed("fdbcli --exec 'status' -C /etc/foundationdb/fdb.cluster")
+    
+    # Wait for 3FS services to start
+    master.wait_for_unit("3fs-mgmtd.service")
+    master.wait_for_unit("3fs-meta.service")
+    master.wait_for_unit("3fs-storage.service")
+    master.wait_for_unit("3fs-monitor.service")
+    
+    # Check that services are listening on correct ports
+    master.wait_for_open_port(${toString testConfig.meta.port})
+    master.wait_for_open_port(${toString testConfig.storage.port})
+    master.wait_for_open_port(${toString testConfig.mgmtd.port})
+    master.wait_for_open_port(9000)  # monitor
+    
+    # Check service status
+    master.succeed("systemctl is-active 3fs-mgmtd.service")
+    master.succeed("systemctl is-active 3fs-meta.service")
+    master.succeed("systemctl is-active 3fs-storage.service")
+    master.succeed("systemctl is-active 3fs-monitor.service")
+    
+    # Initialize 3FS cluster
+    master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config /etc/foundationdb/fdb.cluster")
+    
+    # Register storage nodes
+    master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}")
+    
+    # Create storage targets
+    for target in ${lib.concatStringsSep " " (map (t: "\"${t}\"") testConfig.storage.targets)}; do
+      master.succeed("${pkgs."3fs"}/bin/admin create-target --path $target")
+    done
+    
+    # Wait for client to connect
+    client.wait_for_unit("3fs-fuse.service")
+    client.wait_until_succeeds("mountpoint -q /mnt/3fs")
+    
+    # Basic filesystem operations test
+    client.succeed("echo 'Hello 3FS!' > /mnt/3fs/test.txt")
+    client.succeed("cat /mnt/3fs/test.txt | grep 'Hello 3FS!'")
+    
+    # Create directory structure
+    client.succeed("mkdir -p /mnt/3fs/test/deep/directory")
+    client.succeed("touch /mnt/3fs/test/deep/directory/file.txt")
+    client.succeed("ls -la /mnt/3fs/test/deep/directory/")
+    
+    # Test file operations
+    client.succeed("dd if=/dev/urandom of=/mnt/3fs/random.dat bs=1M count=10")
+    client.succeed("cp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat")
+    client.succeed("cmp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat")
+    
+    # Test permissions
+    client.succeed("chmod 755 /mnt/3fs/test")
+    client.succeed("test -d /mnt/3fs/test")
+    
+    # Clean up
+    client.succeed("rm -rf /mnt/3fs/test*")
+    client.succeed("rm -f /mnt/3fs/random*.dat")
+    
+    # Verify cleanup
+    client.succeed("test -z \"$(ls -A /mnt/3fs)\"")
+    
+    # Check metrics
+    master.succeed("${pkgs."3fs"}/bin/admin list-nodes")
+    master.succeed("${pkgs."3fs"}/bin/admin list-targets")
+    
+    # Test service restart
+    master.succeed("systemctl restart 3fs-storage.service")
+    master.wait_for_unit("3fs-storage.service")
+    
+    # Ensure filesystem still works after restart
+    client.succeed("echo 'Still working!' > /mnt/3fs/restart-test.txt")
+    client.succeed("cat /mnt/3fs/restart-test.txt")
+  '';
+}

nixos-test-config.nix

@@ -0,0 +1,102 @@
+# Example NixOS configuration for testing 3FS
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    # Import the 3FS flake module
+    # In a real system, you would use:
+    # (builtins.getFlake "github:deepseek-ai/3fs").nixosModules.default
+  ];
+
+  # Enable 3FS services
+  services."3fs" = {
+    enable = true;
+    
+    # Configure user and group
+    user = "threefs";
+    group = "threefs";
+    
+    # Configure directories
+    configDir = "/etc/3fs";
+    dataDir = "/var/lib/3fs";
+    
+    # Enable metadata service
+    meta = {
+      enable = true;
+      config = {
+        # Additional meta service configuration
+      };
+    };
+    
+    # Enable storage service
+    storage = {
+      enable = true;
+      targets = [
+        "/var/lib/3fs/storage/target1"
+        "/var/lib/3fs/storage/target2"
+      ];
+      config = {
+        # Additional storage service configuration
+      };
+    };
+    
+    # Enable management daemon
+    mgmtd = {
+      enable = true;
+      config = {
+        # Additional mgmtd configuration
+      };
+    };
+    
+    # Enable monitor collector
+    monitor = {
+      enable = true;
+      config = {
+        # Additional monitor configuration
+      };
+    };
+    
+    # Enable FUSE client
+    fuse = {
+      enable = true;
+      mountPoint = "/mnt/3fs";
+      config = {
+        # Additional FUSE configuration
+      };
+    };
+    
+    # FoundationDB configuration
+    foundationdb = {
+      clusterFile = "/etc/foundationdb/fdb.cluster";
+    };
+  };
+  
+  # Ensure FoundationDB is also installed and configured
+  services.foundationdb = {
+    enable = true;
+    clusterFile = "/etc/foundationdb/fdb.cluster";
+  };
+  
+  # Open necessary ports for 3FS services
+  networking.firewall = {
+    allowedTCPPorts = [
+      # Meta service ports
+      6000 6001
+      # Storage service ports  
+      7000 7001
+      # Management daemon ports
+      8000 8001
+      # Monitor collector ports
+      9000 9001
+    ];
+  };
+  
+  # Example of how to override package
+  nixpkgs.overlays = [
+    (final: prev: {
+      "3fs" = prev."3fs".overrideAttrs (oldAttrs: {
+        # Custom overrides if needed
+      });
+    })
+  ];
+}

package.nix

@@ -0,0 +1,378 @@
+{ stdenv
+, lib
+, fetchFromGitHub
+, runCommand
+, writeText
+, makeWrapper
+, cmake
+, pkg-config
+, clang_14
+, lld_14
+, rustc
+, cargo
+, rustPlatform
+, autoconf
+, automake
+, libtool
+, python3
+, boost
+, libuv
+, lz4
+, xz
+, double-conversion
+, libdwarf
+, libunwind
+, libaio
+, gflags
+, glog
+, gtest
+, gperftools
+, openssl
+, fuse3
+, foundationdb
+, rdma-core
+, libibverbs
+, zstd
+, jemalloc
+, libevent
+, numactl
+, liburing
+, thrift
+, bison
+, flex
+, git
+, which
+, fmt
+, folly
+, rocksdb
+, leveldb
+, arrow-cpp
+, mimalloc
+, clickhouse-cpp
+, tomlplusplus
+, scnlib
+}:
+
+let
+  # Build jemalloc with specific configuration for 3FS
+  jemalloc-custom = jemalloc.overrideAttrs (oldAttrs: {
+    configureFlags = (oldAttrs.configureFlags or []) ++ [
+      "--disable-cxx"
+      "--enable-prof"
+      "--disable-initial-exec-tls"
+    ];
+  });
+  
+  # Custom FoundationDB build if needed
+  fdb-version = "7.1.5";
+  
+in stdenv.mkDerivation rec {
+  pname = "3fs";
+  version = "0.1.5";
+
+  src = fetchFromGitHub {
+    owner = "deepseek-ai";
+    repo = "3fs";
+    rev = "91bfcf39a9e4b5ded959f7b5c2cb0cf858ebbff5";
+    sha256 = "sha256-0000000000000000000000000000000000000000000=";  # To be replaced
+    fetchSubmodules = true;
+  };
+
+  nativeBuildInputs = [
+    cmake
+    pkg-config
+    clang_14
+    lld_14
+    rustc
+    cargo
+    rustPlatform.cargoSetupHook
+    autoconf
+    automake
+    libtool
+    python3
+    git
+    which
+    bison
+    flex
+  ];
+
+  buildInputs = [
+    # Core dependencies
+    boost
+    libuv
+    lz4
+    xz
+    double-conversion
+    libdwarf
+    libunwind
+    libaio
+    gflags
+    glog
+    gtest
+    gperftools
+    openssl
+    
+    # FUSE 3
+    fuse3
+    
+    # FoundationDB
+    foundationdb
+    
+    # RDMA/InfiniBand
+    rdma-core
+    libibverbs
+    
+    # Additional libraries
+    zstd
+    jemalloc-custom
+    libevent
+    numactl
+    liburing
+    thrift
+    fmt
+    folly
+    rocksdb
+    leveldb
+    mimalloc
+    tomlplusplus
+    
+    # Python bindings
+    python3.pkgs.pybind11
+  ];
+
+  patches = [
+    # Add any necessary patches here
+  ];
+
+  postPatch = ''
+    # Apply the repository patches
+    patchShebangs ./patches/apply.sh
+    ./patches/apply.sh || true
+    
+    # Fix CMake to use system libraries
+    substituteInPlace CMakeLists.txt \
+      --replace 'add_link_options(-fuse-ld=lld)' '# add_link_options(-fuse-ld=lld)' \
+      --replace 'set(CMAKE_CXX_FLAGS "$' 'set(CMAKE_CXX_FLAGS "$'
+    
+    # Disable bundled third-party libraries
+    for lib in fmt zstd googletest folly leveldb rocksdb scnlib pybind11 toml11 mimalloc clickhouse-cpp liburing-cmake; do
+      substituteInPlace CMakeLists.txt \
+        --replace "add_subdirectory(\"third_party/$lib\"" "#add_subdirectory(\"third_party/$lib\""
+    done
+    
+    # Fix jemalloc build
+    substituteInPlace cmake/Jemalloc.cmake \
+      --replace 'ExternalProject_add(' 'return() #ExternalProject_add(' \
+      --replace "\''${JEMALLOC_DIR}/lib/libjemalloc.so.2" "\''${jemalloc-custom}/lib/libjemalloc.so"
+    
+    # Fix Apache Arrow build
+    substituteInPlace cmake/ApacheArrow.cmake \
+      --replace 'ExternalProject_Add(' 'return() #ExternalProject_Add('
+    
+    # Set up Rust dependencies
+    cd src/client/trash_cleaner
+    cargoDepsCopy="$NIX_BUILD_TOP/cargo-vendor-dir"
+    if [ -d "$cargoDepsCopy" ]; then
+      chmod -R +w "$cargoDepsCopy"
+    fi
+    cd $NIX_BUILD_TOP/source
+    
+    # Fix storage chunk_engine Rust build
+    cd src/storage/chunk_engine
+    cargoDepsCopy="$NIX_BUILD_TOP/cargo-vendor-dir"
+    if [ -d "$cargoDepsCopy" ]; then
+      chmod -R +w "$cargoDepsCopy"
+    fi
+    cd $NIX_BUILD_TOP/source
+  '';
+
+  preConfigure = ''
+    # Create necessary directories
+    mkdir -p $TMP/third_party
+    
+    # Setup environment
+    export HOME=$TMP
+    export CARGO_HOME=$TMP/.cargo
+    
+    # Set up library paths
+    export BOOST_ROOT=${boost}
+    export BOOST_INCLUDEDIR=${boost}/include
+    export BOOST_LIBRARYDIR=${boost}/lib
+    
+    # FoundationDB paths
+    export FDB_LIBRARY_DIR=${foundationdb}/lib
+    export FDB_INCLUDE_DIR=${foundationdb}/include
+    
+    # Jemalloc paths
+    export JEMALLOC_OVERRIDE=${jemalloc-custom}/lib/libjemalloc.so
+    export JEMALLOC_DIR=${jemalloc-custom}
+    
+    # Compiler flags
+    export CXXFLAGS="-I${boost}/include -I${gtest}/include -I${liburing}/include -I${fmt}/include -I${folly}/include"
+    export LDFLAGS="-L${boost}/lib -L${jemalloc-custom}/lib -L${liburing}/lib -L${fmt}/lib -L${folly}/lib"
+  '';
+
+  cmakeFlags = [
+    "-DCMAKE_CXX_COMPILER=${clang_14}/bin/clang++"
+    "-DCMAKE_C_COMPILER=${clang_14}/bin/clang"
+    "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
+    "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
+    "-DENABLE_FUSE_APPLICATION=ON"
+    "-DOVERRIDE_CXX_NEW_DELETE=OFF"
+    "-DSAVE_ALLOCATE_SIZE=OFF"
+    "-DBoost_USE_STATIC_LIBS=ON"
+    "-DFDB_VERSION=${fdb-version}"
+    
+    # System library paths
+    "-DCMAKE_PREFIX_PATH=${boost};${gtest};${liburing};${fmt};${folly}"
+    "-DBoost_DIR=${boost}/lib/cmake/Boost"
+    "-DBoost_INCLUDE_DIR=${boost}/include"
+    "-Dfmt_DIR=${fmt}/lib/cmake/fmt"
+    "-DFolly_DIR=${folly}/lib/cmake/folly"
+    "-DGTest_DIR=${gtest}/lib/cmake/GTest"
+    "-DZSTD_LIBRARY=${zstd}/lib/libzstd.so"
+    "-DZSTD_INCLUDE_DIR=${zstd}/include"
+    "-Djemalloc_INCLUDE_DIR=${jemalloc-custom}/include"
+    "-Djemalloc_LIBRARY=${jemalloc-custom}/lib/libjemalloc.so"
+    "-DJEMALLOC_DIR=${jemalloc-custom}"
+    "-DARROW_INCLUDE_DIR=${arrow-cpp}/include"
+    "-DARROW_LIB_DIR=${arrow-cpp}/lib"
+    "-Darrow_DIR=${arrow-cpp}/lib/cmake/arrow"
+    
+    # Initially disable tests to simplify build
+    "-DBUILD_TESTING=OFF"
+  ];
+
+  # Custom build phase to handle complex build process
+  buildPhase = ''
+    runHook preBuild
+    
+    # Build with decreasing parallelism if needed
+    cmake --build . -j $NIX_BUILD_CORES || \
+    cmake --build . -j 4 || \
+    cmake --build . -j 1
+    
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+    
+    # Create output directories
+    mkdir -p $out/bin $out/lib $out/etc/3fs
+    
+    # Install binaries
+    for bin in \
+      src/fuse/hf3fs_fuse \
+      src/client/cli/admin/admin \
+      src/meta/meta \
+      src/mgmtd/mgmtd \
+      src/storage/storage \
+      src/monitor_collector/monitor_collector \
+      src/tools/admin
+    do
+      if [ -f "$bin" ] && [ -x "$bin" ]; then
+        install -D -m755 "$bin" "$out/bin/$(basename $bin)"
+      fi
+    done
+    
+    # Rename main binaries to match systemd service expectations
+    [ -f "$out/bin/meta" ] && mv "$out/bin/meta" "$out/bin/meta_main"
+    [ -f "$out/bin/mgmtd" ] && mv "$out/bin/mgmtd" "$out/bin/mgmtd_main"
+    [ -f "$out/bin/storage" ] && mv "$out/bin/storage" "$out/bin/storage_main"
+    [ -f "$out/bin/monitor_collector" ] && mv "$out/bin/monitor_collector" "$out/bin/monitor_collector_main"
+    [ -f "$out/bin/hf3fs_fuse" ] && cp "$out/bin/hf3fs_fuse" "$out/bin/hf3fs_fuse_main"
+    
+    # Install libraries
+    find . -name "*.so" -o -name "*.so.*" | while read lib; do
+      if [ -f "$lib" ]; then
+        install -D -m644 "$lib" "$out/lib/$(basename $lib)"
+      fi
+    done
+    
+    # Install static libraries (optional)
+    find . -name "*.a" | while read lib; do
+      if [ -f "$lib" ] && [[ ! "$lib" =~ third_party ]]; then
+        install -D -m644 "$lib" "$out/lib/$(basename $lib)"
+      fi
+    done
+    
+    # Install Python modules
+    if [ -d "src/lib/py" ]; then
+      mkdir -p $out/lib/python${python3.pythonVersion}/site-packages
+      cp -r src/lib/py/* $out/lib/python${python3.pythonVersion}/site-packages/
+    fi
+    
+    # Install configuration files
+    cp -r configs/* $out/etc/3fs/
+    
+    # Install headers (for development)
+    mkdir -p $out/include/3fs
+    find src -name "*.h" -o -name "*.hpp" | while read header; do
+      rel_path=$(echo "$header" | sed 's|^src/||')
+      install -D -m644 "$header" "$out/include/3fs/$rel_path"
+    done
+    
+    runHook postInstall
+  '';
+
+  postFixup = ''
+    # Fix RPATH for all binaries
+    for exe in $out/bin/*; do
+      if [ -f "$exe" ] && [ -x "$exe" ]; then
+        patchelf --set-rpath "$out/lib:${lib.makeLibraryPath buildInputs}" "$exe" || true
+      fi
+    done
+    
+    # Fix RPATH for all libraries
+    for lib in $out/lib/*.so*; do
+      if [ -f "$lib" ]; then
+        patchelf --set-rpath "$out/lib:${lib.makeLibraryPath buildInputs}" "$lib" || true
+      fi
+    done
+    
+    # Create wrapper scripts if needed
+    for bin in $out/bin/*_main; do
+      if [ -f "$bin" ]; then
+        makeWrapper "$bin" "$bin.wrapped" \
+          --prefix LD_LIBRARY_PATH : "$out/lib:${lib.makeLibraryPath buildInputs}" \
+          --set RUST_BACKTRACE 1
+        mv "$bin.wrapped" "$bin"
+      fi
+    done
+  '';
+
+  # Enable parallel building
+  enableParallelBuilding = true;
+  NIX_BUILD_CORES = 8;
+
+  # Set up proper library paths
+  setupHook = writeText "setup-hook" ''
+    export LD_LIBRARY_PATH="$1/lib''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH"
+  '';
+
+  passthru = {
+    inherit jemalloc-custom;
+    tests = {
+      # Add package tests here
+      version = runCommand "3fs-version-test" {} ''
+        ${pname}/bin/admin --version
+        touch $out
+      '';
+    };
+  };
+
+  meta = with lib; {
+    description = "Fire-Flyer File System - High-performance distributed file system for AI workloads";
+    longDescription = ''
+      3FS is a high-performance distributed file system designed to address the challenges
+      of AI training and inference workloads. It leverages modern SSDs and RDMA networks
+      to provide a shared storage layer that simplifies development of distributed applications.
+    '';
+    homepage = "https://github.com/deepseek-ai/3fs";
+    license = licenses.mit;
+    maintainers = with maintainers; [ ];
+    platforms = platforms.linux;
+    mainProgram = "admin";
+  };
+}