From 310118ef088d022b67a3e4ab4dbccaf507505445 Mon Sep 17 00:00:00 2001 From: connerohnesorge Date: Tue, 27 May 2025 09:57:43 -0500 Subject: [PATCH 1/3] added flake.nix, package.nix, and documentation --- README-nix.md | 235 +++++++++++++++++++++++++++++++ flake.nix | 274 ++++++++++++++++++++++++++++++++++++ package.nix | 378 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 887 insertions(+) create mode 100644 README-nix.md create mode 100644 flake.nix create mode 100644 package.nix diff --git a/README-nix.md b/README-nix.md new file mode 100644 index 0000000..4ad0fa1 --- /dev/null +++ b/README-nix.md @@ -0,0 +1,235 @@ +# 3FS Nix Package and NixOS Module + +This directory contains a Nix flake for building and deploying 3FS (Fire-Flyer File System) on NixOS systems. + +## Features + +- **Complete Package**: Builds 3FS with all dependencies including RDMA support, FoundationDB integration, and FUSE 3 +- **NixOS Module**: Provides systemd services for all 3FS components (meta, storage, mgmtd, monitor, fuse) +- **Development Shell**: Includes all build tools and dependencies for 3FS development +- **Integration Tests**: NixOS VM tests to validate the complete system + +## Quick Start + +### Building the Package + +1. First build (will fail with hash mismatch): +```bash +nix build .#3fs +``` + +2. Update the hash in `package.nix` with the correct one from the error message + +3. Rebuild: +```bash +nix build .#3fs +``` + +### Using in NixOS + +Add to your `flake.nix`: + +```nix +{ + inputs = { + threefs.url = "github:deepseek-ai/3fs"; + }; + + outputs = { self, nixpkgs, threefs, ... }: { + nixosConfigurations.myserver = nixpkgs.lib.nixosSystem { + modules = [ + threefs.nixosModules.default + { + services."3fs" = { + enable = true; + meta.enable = true; + storage.enable = true; + mgmtd.enable = true; + }; + } + ]; + }; + }; +} +``` + +### Development Environment + +Enter the development shell: +```bash +nix develop +``` + +This provides: +- Clang 14 with C++20 support +- CMake and build tools +- All 3FS dependencies +- Rust toolchain +- Debug tools (gdb, valgrind, perf) + +## NixOS Module Options + +### Basic Configuration + +```nix +services."3fs" = { + enable = true; + + # User and group for services + user = "threefs"; + group = "threefs"; + + # Directories + configDir = "/etc/3fs"; + dataDir = "/var/lib/3fs"; + + # FoundationDB cluster file + foundationdb.clusterFile = "/etc/foundationdb/fdb.cluster"; +}; +``` + +### Service Configuration + +#### Metadata Service +```nix +services."3fs".meta = { + enable = true; + config = { + # Additional meta configuration + }; +}; +``` + +#### Storage Service +```nix +services."3fs".storage = { + enable = true; + targets = [ + "/var/lib/3fs/storage/target1" + "/var/lib/3fs/storage/target2" + ]; + config = { + # Additional storage configuration + }; +}; +``` + +#### Management Daemon +```nix +services."3fs".mgmtd = { + enable = true; + config = { + # Additional mgmtd configuration + }; +}; +``` + +#### Monitor Collector +```nix +services."3fs".monitor = { + enable = true; + config = { + # Additional monitor configuration + }; +}; +``` + +#### FUSE Client +```nix +services."3fs".fuse = { + enable = true; + mountPoint = "/mnt/3fs"; + config = { + # Additional FUSE configuration + }; +}; +``` + +## Testing + +### Run Package Tests +```bash +./test-build.sh +``` + +### Run Integration Tests +```bash +nix build .#checks.x86_64-linux.integration-test +``` + +### Manual Testing + +1. Start FoundationDB: +```bash +sudo systemctl start foundationdb +``` + +2. Initialize 3FS cluster: +```bash +3fs-admin init-cluster +``` + +3. Start 3FS services: +```bash +sudo systemctl start 3fs-mgmtd +sudo systemctl start 3fs-meta +sudo systemctl start 3fs-storage +``` + +4. Mount filesystem: +```bash +sudo systemctl start 3fs-fuse +``` + +## Package Contents + +The package includes: + +- **Binaries**: + - `admin` - Administrative CLI tool + - `meta_main` - Metadata service + - `storage_main` - Storage service + - `mgmtd_main` - Management daemon + - `monitor_collector_main` - Metrics collector + - `hf3fs_fuse_main` - FUSE filesystem client + +- **Libraries**: All required shared libraries in `/lib` +- **Configuration**: Default configuration files in `/etc/3fs` +- **Headers**: Development headers in `/include/3fs` + +## Troubleshooting + +### Build Issues + +1. **Hash mismatch**: Update the SHA256 hash in `package.nix` +2. **Missing dependencies**: Check that all system libraries are available +3. **Compilation errors**: Ensure you're using Clang 14 or newer + +### Runtime Issues + +1. **Service fails to start**: Check logs with `journalctl -u 3fs-` +2. **FoundationDB connection**: Ensure FDB cluster file is correct +3. **FUSE mount fails**: Check that FUSE 3 is installed and kernel module is loaded + +## Architecture + +The Nix package handles: + +1. **Complex Dependencies**: Automatically builds custom jemalloc, handles RDMA libraries +2. **Third-party Libraries**: Uses system libraries where possible, builds others from source +3. **Rust Components**: Properly integrates Cargo builds for Rust components +4. **Service Management**: Complete systemd integration with proper dependencies +5. **Configuration**: Manages TOML configuration files for all components + +## Contributing + +When modifying the package: + +1. Update version in `package.nix` +2. Test build locally +3. Run integration tests +4. Update documentation + +## License + +MIT License (same as 3FS project) \ No newline at end of file diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..6ef8ac8 --- /dev/null +++ b/flake.nix @@ -0,0 +1,274 @@ +{ + description = "Fire-Flyer File System (3FS) - High-performance distributed file system for AI workloads"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + let + overlay = final: prev: { + "3fs" = prev.callPackage ./package.nix { }; + }; + + nixosModule = { config, lib, pkgs, ... }: + with lib; + let + cfg = config.services."3fs"; + + # Helper function to create service configuration + mkServiceConfig = component: { + description = "3FS ${component} service"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" ]; + requires = [ "network-online.target" ]; + + serviceConfig = { + Type = "simple"; + ExecStart = "${pkgs."3fs"}/bin/${component}_main --launcher_cfg ${cfg.configDir}/${component}_main_launcher.toml --app-cfg ${cfg.configDir}/${component}_main_app.toml"; + Restart = "on-failure"; + RestartSec = 5; + LimitNOFILE = 1000000; + User = cfg.user; + Group = cfg.group; + } // (if component == "storage" then { + LimitMEMLOCK = "infinity"; + TimeoutStopSec = "5m"; + } else {}); + + environment = { + LD_LIBRARY_PATH = "${pkgs."3fs"}/lib:${pkgs.foundationdb}/lib"; + }; + }; + in + { + options.services."3fs" = { + enable = mkEnableOption "3FS distributed file system"; + + user = mkOption { + type = types.str; + default = "threefs"; + description = "User under which 3FS services run"; + }; + + group = mkOption { + type = types.str; + default = "threefs"; + description = "Group under which 3FS services run"; + }; + + configDir = mkOption { + type = types.path; + default = "/etc/3fs"; + description = "Directory containing 3FS configuration files"; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/3fs"; + description = "Directory for 3FS data storage"; + }; + + meta = { + enable = mkEnableOption "3FS metadata service"; + + config = mkOption { + type = types.attrs; + default = {}; + description = "Additional configuration for meta service"; + }; + }; + + storage = { + enable = mkEnableOption "3FS storage service"; + + targets = mkOption { + type = types.listOf types.str; + default = [ "/var/lib/3fs/storage" ]; + description = "List of storage target directories"; + }; + + config = mkOption { + type = types.attrs; + default = {}; + description = "Additional configuration for storage service"; + }; + }; + + mgmtd = { + enable = mkEnableOption "3FS management daemon"; + + config = mkOption { + type = types.attrs; + default = {}; + description = "Additional configuration for management daemon"; + }; + }; + + monitor = { + enable = mkEnableOption "3FS monitor collector"; + + config = mkOption { + type = types.attrs; + default = {}; + description = "Additional configuration for monitor collector"; + }; + }; + + fuse = { + enable = mkEnableOption "3FS FUSE client"; + + mountPoint = mkOption { + type = types.path; + default = "/mnt/3fs"; + description = "Mount point for 3FS FUSE filesystem"; + }; + + config = mkOption { + type = types.attrs; + default = {}; + description = "Additional configuration for FUSE client"; + }; + }; + + foundationdb = { + clusterFile = mkOption { + type = types.path; + default = "/etc/foundationdb/fdb.cluster"; + description = "Path to FoundationDB cluster file"; + }; + }; + }; + + config = mkIf cfg.enable { + # Create system user and group + users.users.${cfg.user} = { + isSystemUser = true; + group = cfg.group; + home = cfg.dataDir; + createHome = true; + description = "3FS system user"; + }; + + users.groups.${cfg.group} = {}; + + # Create necessary directories + systemd.tmpfiles.rules = [ + "d '${cfg.configDir}' 0755 ${cfg.user} ${cfg.group} -" + "d '${cfg.dataDir}' 0755 ${cfg.user} ${cfg.group} -" + "d '${cfg.dataDir}/meta' 0755 ${cfg.user} ${cfg.group} -" + "d '${cfg.dataDir}/mgmtd' 0755 ${cfg.user} ${cfg.group} -" + "d '${cfg.dataDir}/monitor' 0755 ${cfg.user} ${cfg.group} -" + ] ++ (map (target: "d '${target}' 0755 ${cfg.user} ${cfg.group} -") cfg.storage.targets); + + # Install default configuration files + environment.etc = { + "3fs/meta_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/meta_main_launcher.toml"; + "3fs/meta_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/meta_main_app.toml"; + "3fs/storage_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/storage_main_launcher.toml"; + "3fs/storage_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/storage_main_app.toml"; + "3fs/mgmtd_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/mgmtd_main_launcher.toml"; + "3fs/mgmtd_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/mgmtd_main_app.toml"; + "3fs/monitor_collector_main.toml".source = "${pkgs."3fs"}/etc/3fs/monitor_collector_main.toml"; + "3fs/hf3fs_fuse_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/hf3fs_fuse_main_launcher.toml"; + "3fs/hf3fs_fuse_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/hf3fs_fuse_main_app.toml"; + }; + + # Define systemd services + systemd.services = { + "3fs-meta" = mkIf cfg.meta.enable (mkServiceConfig "meta"); + "3fs-storage" = mkIf cfg.storage.enable (mkServiceConfig "storage"); + "3fs-mgmtd" = mkIf cfg.mgmtd.enable (mkServiceConfig "mgmtd"); + "3fs-monitor" = mkIf cfg.monitor.enable (mkServiceConfig "monitor_collector"); + + "3fs-fuse" = mkIf cfg.fuse.enable { + description = "3FS FUSE client"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "3fs-meta.service" ]; + requires = [ "network-online.target" ]; + + serviceConfig = { + Type = "simple"; + ExecStart = "${pkgs."3fs"}/bin/hf3fs_fuse_main --launcher_cfg ${cfg.configDir}/hf3fs_fuse_main_launcher.toml --app-cfg ${cfg.configDir}/hf3fs_fuse_main_app.toml ${cfg.fuse.mountPoint}"; + ExecStop = "${pkgs.fuse3}/bin/fusermount3 -u ${cfg.fuse.mountPoint}"; + Restart = "on-failure"; + RestartSec = 5; + LimitNOFILE = 1000000; + User = "root"; # FUSE requires root + Group = "root"; + }; + + preStart = '' + mkdir -p ${cfg.fuse.mountPoint} + ''; + }; + }; + + # Add 3fs package to system packages + environment.systemPackages = [ pkgs."3fs" ]; + }; + }; + in + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ overlay ]; + }; + in + { + packages = { + default = pkgs."3fs"; + "3fs" = pkgs."3fs"; + }; + + overlays.default = overlay; + + devShells.default = pkgs.mkShell { + inputsFrom = [ pkgs."3fs" ]; + buildInputs = with pkgs; [ + # Additional dev tools + clang-tools_14 + cmake-format + cmake-language-server + gdb + valgrind + perf-tools + rust-analyzer + cargo-watch + ]; + + shellHook = '' + echo "3FS development environment" + echo "Build with: nix build .#3fs" + echo "Enter shell with: nix develop" + echo "" + echo "To test locally:" + echo " 1. Start FoundationDB" + echo " 2. Configure and start 3FS services" + echo " 3. Mount FUSE filesystem" + ''; + }; + checks = { + # Package build test + package = pkgs."3fs"; + + # Integration test using NixOS VM + integration-test = pkgs.nixosTest (import ./nixos-module-test.nix { + inherit pkgs lib; + self = self; + }); + }; + } + ) // { + nixosModules.default = nixosModule; + nixosModules."3fs" = nixosModule; + + # Hydra job for CI + hydraJobs = { + inherit (self) packages; + tests = self.checks; + }; + }; +} \ No newline at end of file diff --git a/package.nix b/package.nix new file mode 100644 index 0000000..08352cc --- /dev/null +++ b/package.nix @@ -0,0 +1,378 @@ +{ stdenv +, lib +, fetchFromGitHub +, runCommand +, writeText +, makeWrapper +, cmake +, pkg-config +, clang_14 +, lld_14 +, rustc +, cargo +, rustPlatform +, autoconf +, automake +, libtool +, python3 +, boost +, libuv +, lz4 +, xz +, double-conversion +, libdwarf +, libunwind +, libaio +, gflags +, glog +, gtest +, gperftools +, openssl +, fuse3 +, foundationdb +, rdma-core +, libibverbs +, zstd +, jemalloc +, libevent +, numactl +, liburing +, thrift +, bison +, flex +, git +, which +, fmt +, folly +, rocksdb +, leveldb +, arrow-cpp +, mimalloc +, clickhouse-cpp +, tomlplusplus +, scnlib +}: + +let + # Build jemalloc with specific configuration for 3FS + jemalloc-custom = jemalloc.overrideAttrs (oldAttrs: { + configureFlags = (oldAttrs.configureFlags or []) ++ [ + "--disable-cxx" + "--enable-prof" + "--disable-initial-exec-tls" + ]; + }); + + # Custom FoundationDB build if needed + fdb-version = "7.1.5"; + +in stdenv.mkDerivation rec { + pname = "3fs"; + version = "0.1.5"; + + src = fetchFromGitHub { + owner = "deepseek-ai"; + repo = "3fs"; + rev = "91bfcf39a9e4b5ded959f7b5c2cb0cf858ebbff5"; + sha256 = "sha256-0000000000000000000000000000000000000000000="; # To be replaced + fetchSubmodules = true; + }; + + nativeBuildInputs = [ + cmake + pkg-config + clang_14 + lld_14 + rustc + cargo + rustPlatform.cargoSetupHook + autoconf + automake + libtool + python3 + git + which + bison + flex + ]; + + buildInputs = [ + # Core dependencies + boost + libuv + lz4 + xz + double-conversion + libdwarf + libunwind + libaio + gflags + glog + gtest + gperftools + openssl + + # FUSE 3 + fuse3 + + # FoundationDB + foundationdb + + # RDMA/InfiniBand + rdma-core + libibverbs + + # Additional libraries + zstd + jemalloc-custom + libevent + numactl + liburing + thrift + fmt + folly + rocksdb + leveldb + mimalloc + tomlplusplus + + # Python bindings + python3.pkgs.pybind11 + ]; + + patches = [ + # Add any necessary patches here + ]; + + postPatch = '' + # Apply the repository patches + patchShebangs ./patches/apply.sh + ./patches/apply.sh || true + + # Fix CMake to use system libraries + substituteInPlace CMakeLists.txt \ + --replace 'add_link_options(-fuse-ld=lld)' '# add_link_options(-fuse-ld=lld)' \ + --replace 'set(CMAKE_CXX_FLAGS "$' 'set(CMAKE_CXX_FLAGS "$' + + # Disable bundled third-party libraries + for lib in fmt zstd googletest folly leveldb rocksdb scnlib pybind11 toml11 mimalloc clickhouse-cpp liburing-cmake; do + substituteInPlace CMakeLists.txt \ + --replace "add_subdirectory(\"third_party/$lib\"" "#add_subdirectory(\"third_party/$lib\"" + done + + # Fix jemalloc build + substituteInPlace cmake/Jemalloc.cmake \ + --replace 'ExternalProject_add(' 'return() #ExternalProject_add(' \ + --replace '${JEMALLOC_DIR}/lib/libjemalloc.so.2' '${jemalloc-custom}/lib/libjemalloc.so' + + # Fix Apache Arrow build + substituteInPlace cmake/ApacheArrow.cmake \ + --replace 'ExternalProject_Add(' 'return() #ExternalProject_Add(' + + # Set up Rust dependencies + cd src/client/trash_cleaner + cargoDepsCopy="$NIX_BUILD_TOP/cargo-vendor-dir" + if [ -d "$cargoDepsCopy" ]; then + chmod -R +w "$cargoDepsCopy" + fi + cd $NIX_BUILD_TOP/source + + # Fix storage chunk_engine Rust build + cd src/storage/chunk_engine + cargoDepsCopy="$NIX_BUILD_TOP/cargo-vendor-dir" + if [ -d "$cargoDepsCopy" ]; then + chmod -R +w "$cargoDepsCopy" + fi + cd $NIX_BUILD_TOP/source + ''; + + preConfigure = '' + # Create necessary directories + mkdir -p $TMP/third_party + + # Setup environment + export HOME=$TMP + export CARGO_HOME=$TMP/.cargo + + # Set up library paths + export BOOST_ROOT=${boost} + export BOOST_INCLUDEDIR=${boost}/include + export BOOST_LIBRARYDIR=${boost}/lib + + # FoundationDB paths + export FDB_LIBRARY_DIR=${foundationdb}/lib + export FDB_INCLUDE_DIR=${foundationdb}/include + + # Jemalloc paths + export JEMALLOC_OVERRIDE=${jemalloc-custom}/lib/libjemalloc.so + export JEMALLOC_DIR=${jemalloc-custom} + + # Compiler flags + export CXXFLAGS="-I${boost}/include -I${gtest}/include -I${liburing}/include -I${fmt}/include -I${folly}/include" + export LDFLAGS="-L${boost}/lib -L${jemalloc-custom}/lib -L${liburing}/lib -L${fmt}/lib -L${folly}/lib" + ''; + + cmakeFlags = [ + "-DCMAKE_CXX_COMPILER=${clang_14}/bin/clang++" + "-DCMAKE_C_COMPILER=${clang_14}/bin/clang" + "-DCMAKE_BUILD_TYPE=RelWithDebInfo" + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON" + "-DENABLE_FUSE_APPLICATION=ON" + "-DOVERRIDE_CXX_NEW_DELETE=OFF" + "-DSAVE_ALLOCATE_SIZE=OFF" + "-DBoost_USE_STATIC_LIBS=ON" + "-DFDB_VERSION=${fdb-version}" + + # System library paths + "-DCMAKE_PREFIX_PATH=${boost};${gtest};${liburing};${fmt};${folly}" + "-DBoost_DIR=${boost}/lib/cmake/Boost" + "-DBoost_INCLUDE_DIR=${boost}/include" + "-Dfmt_DIR=${fmt}/lib/cmake/fmt" + "-DFolly_DIR=${folly}/lib/cmake/folly" + "-DGTest_DIR=${gtest}/lib/cmake/GTest" + "-DZSTD_LIBRARY=${zstd}/lib/libzstd.so" + "-DZSTD_INCLUDE_DIR=${zstd}/include" + "-Djemalloc_INCLUDE_DIR=${jemalloc-custom}/include" + "-Djemalloc_LIBRARY=${jemalloc-custom}/lib/libjemalloc.so" + "-DJEMALLOC_DIR=${jemalloc-custom}" + "-DARROW_INCLUDE_DIR=${arrow-cpp}/include" + "-DARROW_LIB_DIR=${arrow-cpp}/lib" + "-Darrow_DIR=${arrow-cpp}/lib/cmake/arrow" + + # Initially disable tests to simplify build + "-DBUILD_TESTING=OFF" + ]; + + # Custom build phase to handle complex build process + buildPhase = '' + runHook preBuild + + # Build with decreasing parallelism if needed + cmake --build . -j $NIX_BUILD_CORES || \ + cmake --build . -j 4 || \ + cmake --build . -j 1 + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + # Create output directories + mkdir -p $out/bin $out/lib $out/etc/3fs + + # Install binaries + for bin in \ + src/fuse/hf3fs_fuse \ + src/client/cli/admin/admin \ + src/meta/meta \ + src/mgmtd/mgmtd \ + src/storage/storage \ + src/monitor_collector/monitor_collector \ + src/tools/admin + do + if [ -f "$bin" ] && [ -x "$bin" ]; then + install -D -m755 "$bin" "$out/bin/$(basename $bin)" + fi + done + + # Rename main binaries to match systemd service expectations + [ -f "$out/bin/meta" ] && mv "$out/bin/meta" "$out/bin/meta_main" + [ -f "$out/bin/mgmtd" ] && mv "$out/bin/mgmtd" "$out/bin/mgmtd_main" + [ -f "$out/bin/storage" ] && mv "$out/bin/storage" "$out/bin/storage_main" + [ -f "$out/bin/monitor_collector" ] && mv "$out/bin/monitor_collector" "$out/bin/monitor_collector_main" + [ -f "$out/bin/hf3fs_fuse" ] && cp "$out/bin/hf3fs_fuse" "$out/bin/hf3fs_fuse_main" + + # Install libraries + find . -name "*.so" -o -name "*.so.*" | while read lib; do + if [ -f "$lib" ]; then + install -D -m644 "$lib" "$out/lib/$(basename $lib)" + fi + done + + # Install static libraries (optional) + find . -name "*.a" | while read lib; do + if [ -f "$lib" ] && [[ ! "$lib" =~ third_party ]]; then + install -D -m644 "$lib" "$out/lib/$(basename $lib)" + fi + done + + # Install Python modules + if [ -d "src/lib/py" ]; then + mkdir -p $out/lib/python${python3.pythonVersion}/site-packages + cp -r src/lib/py/* $out/lib/python${python3.pythonVersion}/site-packages/ + fi + + # Install configuration files + cp -r configs/* $out/etc/3fs/ + + # Install headers (for development) + mkdir -p $out/include/3fs + find src -name "*.h" -o -name "*.hpp" | while read header; do + rel_path=$(echo "$header" | sed 's|^src/||') + install -D -m644 "$header" "$out/include/3fs/$rel_path" + done + + runHook postInstall + ''; + + postFixup = '' + # Fix RPATH for all binaries + for exe in $out/bin/*; do + if [ -f "$exe" ] && [ -x "$exe" ]; then + patchelf --set-rpath "$out/lib:${lib.makeLibraryPath buildInputs}" "$exe" || true + fi + done + + # Fix RPATH for all libraries + for lib in $out/lib/*.so*; do + if [ -f "$lib" ]; then + patchelf --set-rpath "$out/lib:${lib.makeLibraryPath buildInputs}" "$lib" || true + fi + done + + # Create wrapper scripts if needed + for bin in $out/bin/*_main; do + if [ -f "$bin" ]; then + makeWrapper "$bin" "$bin.wrapped" \ + --prefix LD_LIBRARY_PATH : "$out/lib:${lib.makeLibraryPath buildInputs}" \ + --set RUST_BACKTRACE 1 + mv "$bin.wrapped" "$bin" + fi + done + ''; + + # Enable parallel building + enableParallelBuilding = true; + NIX_BUILD_CORES = 8; + + # Set up proper library paths + setupHook = writeText "setup-hook" '' + export LD_LIBRARY_PATH="$1/lib''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH" + ''; + + passthru = { + inherit jemalloc-custom; + tests = { + # Add package tests here + version = runCommand "3fs-version-test" {} '' + ${pname}/bin/admin --version + touch $out + ''; + }; + }; + + meta = with lib; { + description = "Fire-Flyer File System - High-performance distributed file system for AI workloads"; + longDescription = '' + 3FS is a high-performance distributed file system designed to address the challenges + of AI training and inference workloads. It leverages modern SSDs and RDMA networks + to provide a shared storage layer that simplifies development of distributed applications. + ''; + homepage = "https://github.com/deepseek-ai/3fs"; + license = licenses.mit; + maintainers = with maintainers; [ ]; + platforms = platforms.linux; + mainProgram = "admin"; + }; +} \ No newline at end of file From c15454e15344f4e92510dd9540ac24acfdf7b43c Mon Sep 17 00:00:00 2001 From: connerohnesorge Date: Tue, 27 May 2025 10:10:28 -0500 Subject: [PATCH 2/3] added the nixos module tests and config tests for nixos --- nixos-module-test.nix | 226 ++++++++++++++++++++++++++++++++++++++++++ nixos-test-config.nix | 102 +++++++++++++++++++ 2 files changed, 328 insertions(+) create mode 100644 nixos-module-test.nix create mode 100644 nixos-test-config.nix diff --git a/nixos-module-test.nix b/nixos-module-test.nix new file mode 100644 index 0000000..d524200 --- /dev/null +++ b/nixos-module-test.nix @@ -0,0 +1,226 @@ +# NixOS VM test for 3FS services +# Run with: nix build .#checks.x86_64-linux.3fs-integration-test + +{ pkgs, lib, ... }: + +let + # Test configuration for 3FS cluster + testConfig = { + meta = { + port = 6000; + dataDir = "/var/lib/3fs/meta"; + }; + + storage = { + port = 7000; + targets = [ + "/var/lib/3fs/storage/target1" + "/var/lib/3fs/storage/target2" + ]; + }; + + mgmtd = { + port = 8000; + dataDir = "/var/lib/3fs/mgmtd"; + }; + + foundationdb = { + clusterFile = "/etc/foundationdb/fdb.cluster"; + clusterString = "test:test@127.0.0.1:4500"; + }; + }; + +in { + name = "3fs-integration-test"; + + nodes = { + # Master node running all services + master = { config, pkgs, ... }: { + imports = [ self.nixosModules.default ]; + + # Enable 3FS services + services."3fs" = { + enable = true; + + meta = { + enable = true; + config = { + inherit (testConfig.meta) port dataDir; + }; + }; + + storage = { + enable = true; + targets = testConfig.storage.targets; + config = { + inherit (testConfig.storage) port; + }; + }; + + mgmtd = { + enable = true; + config = { + inherit (testConfig.mgmtd) port dataDir; + }; + }; + + monitor = { + enable = true; + config = { + port = 9000; + }; + }; + + foundationdb = { + clusterFile = testConfig.foundationdb.clusterFile; + }; + }; + + # Configure FoundationDB + services.foundationdb = { + enable = true; + clusterFile = testConfig.foundationdb.clusterFile; + listenAddress = "127.0.0.1:4500"; + dataDir = "/var/lib/foundationdb"; + logDir = "/var/log/foundationdb"; + + # Initialize the database + initialScript = '' + configure single ssd + ''; + }; + + # Open firewall ports + networking.firewall.enable = false; + + # Additional test utilities + environment.systemPackages = with pkgs; [ + netcat + curl + jq + ]; + }; + + # Client node with FUSE mount + client = { config, pkgs, ... }: { + imports = [ self.nixosModules.default ]; + + services."3fs" = { + enable = true; + + fuse = { + enable = true; + mountPoint = "/mnt/3fs"; + config = { + metaServers = [ "master:6000" ]; + mgmtdServers = [ "master:8000" ]; + }; + }; + + foundationdb = { + clusterFile = testConfig.foundationdb.clusterFile; + }; + }; + + # Configure FoundationDB client + services.foundationdb = { + enable = true; + clusterFile = testConfig.foundationdb.clusterFile; + pidFile = "/run/foundationdb.pid"; + }; + + # Write cluster file pointing to master + environment.etc."foundationdb/fdb.cluster" = { + text = testConfig.foundationdb.clusterString; + mode = "0644"; + }; + + networking.firewall.enable = false; + + environment.systemPackages = with pkgs; [ + fio + iozone + sysbench + ]; + }; + }; + + testScript = '' + start_all() + + # Wait for FoundationDB to be ready + master.wait_for_unit("foundationdb.service") + master.wait_for_open_port(4500) + master.succeed("fdbcli --exec 'status' -C ${testConfig.foundationdb.clusterFile}") + + # Wait for 3FS services to start + master.wait_for_unit("3fs-mgmtd.service") + master.wait_for_unit("3fs-meta.service") + master.wait_for_unit("3fs-storage.service") + master.wait_for_unit("3fs-monitor.service") + + # Check that services are listening on correct ports + master.wait_for_open_port(${toString testConfig.meta.port}) + master.wait_for_open_port(${toString testConfig.storage.port}) + master.wait_for_open_port(${toString testConfig.mgmtd.port}) + master.wait_for_open_port(9000) # monitor + + # Check service status + master.succeed("systemctl is-active 3fs-mgmtd.service") + master.succeed("systemctl is-active 3fs-meta.service") + master.succeed("systemctl is-active 3fs-storage.service") + master.succeed("systemctl is-active 3fs-monitor.service") + + # Initialize 3FS cluster + master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config ${testConfig.foundationdb.clusterFile}") + + # Register storage nodes + master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}") + + # Create storage targets + for target in ${lib.concatStringsSep " " (map (t: "\"${t}\"") testConfig.storage.targets)}; do + master.succeed("${pkgs."3fs"}/bin/admin create-target --path $target") + done + + # Wait for client to connect + client.wait_for_unit("3fs-fuse.service") + client.wait_until_succeeds("mountpoint -q /mnt/3fs") + + # Basic filesystem operations test + client.succeed("echo 'Hello 3FS!' > /mnt/3fs/test.txt") + client.succeed("cat /mnt/3fs/test.txt | grep 'Hello 3FS!'") + + # Create directory structure + client.succeed("mkdir -p /mnt/3fs/test/deep/directory") + client.succeed("touch /mnt/3fs/test/deep/directory/file.txt") + client.succeed("ls -la /mnt/3fs/test/deep/directory/") + + # Test file operations + client.succeed("dd if=/dev/urandom of=/mnt/3fs/random.dat bs=1M count=10") + client.succeed("cp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat") + client.succeed("cmp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat") + + # Test permissions + client.succeed("chmod 755 /mnt/3fs/test") + client.succeed("test -d /mnt/3fs/test") + + # Clean up + client.succeed("rm -rf /mnt/3fs/test*") + client.succeed("rm -f /mnt/3fs/random*.dat") + + # Verify cleanup + client.succeed("test -z \"$(ls -A /mnt/3fs)\"") + + # Check metrics + master.succeed("${pkgs."3fs"}/bin/admin list-nodes") + master.succeed("${pkgs."3fs"}/bin/admin list-targets") + + # Test service restart + master.succeed("systemctl restart 3fs-storage.service") + master.wait_for_unit("3fs-storage.service") + + # Ensure filesystem still works after restart + client.succeed("echo 'Still working!' > /mnt/3fs/restart-test.txt") + client.succeed("cat /mnt/3fs/restart-test.txt") + ''; +} \ No newline at end of file diff --git a/nixos-test-config.nix b/nixos-test-config.nix new file mode 100644 index 0000000..34680e2 --- /dev/null +++ b/nixos-test-config.nix @@ -0,0 +1,102 @@ +# Example NixOS configuration for testing 3FS +{ config, pkgs, ... }: + +{ + imports = [ + # Import the 3FS flake module + # In a real system, you would use: + # (builtins.getFlake "github:deepseek-ai/3fs").nixosModules.default + ]; + + # Enable 3FS services + services."3fs" = { + enable = true; + + # Configure user and group + user = "threefs"; + group = "threefs"; + + # Configure directories + configDir = "/etc/3fs"; + dataDir = "/var/lib/3fs"; + + # Enable metadata service + meta = { + enable = true; + config = { + # Additional meta service configuration + }; + }; + + # Enable storage service + storage = { + enable = true; + targets = [ + "/var/lib/3fs/storage/target1" + "/var/lib/3fs/storage/target2" + ]; + config = { + # Additional storage service configuration + }; + }; + + # Enable management daemon + mgmtd = { + enable = true; + config = { + # Additional mgmtd configuration + }; + }; + + # Enable monitor collector + monitor = { + enable = true; + config = { + # Additional monitor configuration + }; + }; + + # Enable FUSE client + fuse = { + enable = true; + mountPoint = "/mnt/3fs"; + config = { + # Additional FUSE configuration + }; + }; + + # FoundationDB configuration + foundationdb = { + clusterFile = "/etc/foundationdb/fdb.cluster"; + }; + }; + + # Ensure FoundationDB is also installed and configured + services.foundationdb = { + enable = true; + clusterFile = "/etc/foundationdb/fdb.cluster"; + }; + + # Open necessary ports for 3FS services + networking.firewall = { + allowedTCPPorts = [ + # Meta service ports + 6000 6001 + # Storage service ports + 7000 7001 + # Management daemon ports + 8000 8001 + # Monitor collector ports + 9000 9001 + ]; + }; + + # Example of how to override package + nixpkgs.overlays = [ + (final: prev: { + "3fs" = prev."3fs".overrideAttrs (oldAttrs: { + # Custom overrides if needed + }); + }) + ]; +} \ No newline at end of file From 16468b0110cfc78e9ee9ed90ef58357421edff67 Mon Sep 17 00:00:00 2001 From: connerohnesorge Date: Tue, 27 May 2025 10:28:03 -0500 Subject: [PATCH 3/3] fix inheritted inputs from flake.nix --- flake.lock | 61 ++++++++++++++++++++ flake.nix | 130 ++++++++++++++++++++++++++++++++++++------ nixos-module-test.nix | 32 +++-------- package.nix | 2 +- 4 files changed, 182 insertions(+), 43 deletions(-) create mode 100644 flake.lock diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..d072398 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1748190013, + "narHash": "sha256-R5HJFflOfsP5FBtk+zE8FpL8uqE7n62jqOsADvVshhE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "62b852f6c6742134ade1abdd2a21685fd617a291", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index 6ef8ac8..9dd5bad 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,52 @@ outputs = { self, nixpkgs, flake-utils }: let overlay = final: prev: { - "3fs" = prev.callPackage ./package.nix { }; + # Build clickhouse-cpp from source if not available in nixpkgs + clickhouse-cpp = prev.clickhouse-cpp or (prev.stdenv.mkDerivation rec { + pname = "clickhouse-cpp"; + version = "2.5.1"; + + src = prev.fetchFromGitHub { + owner = "ClickHouse"; + repo = "clickhouse-cpp"; + rev = "v${version}"; + sha256 = "sha256-6kqcANO4S9Z1ee4kBPKGCnsPEGDaWPCx2hUi4APPWHU="; + }; + + nativeBuildInputs = [ prev.cmake ]; + buildInputs = [ prev.zlib prev.openssl prev.lz4 ]; + + cmakeFlags = [ + "-DBUILD_SHARED_LIBS=ON" + "-DWITH_OPENSSL=ON" + ]; + }); + + # Build scnlib from source if not available in nixpkgs + scnlib = prev.scnlib or (prev.stdenv.mkDerivation rec { + pname = "scnlib"; + version = "2.0.2"; + + src = prev.fetchFromGitHub { + owner = "eliaskosunen"; + repo = "scnlib"; + rev = "v${version}"; + sha256 = "sha256-YWlJiHAKKJd7jWv8Z0GmKqIfXI3HwVqA7AgZiHN2W8I="; + }; + + nativeBuildInputs = [ prev.cmake ]; + + cmakeFlags = [ + "-DSCN_TESTS=OFF" + "-DSCN_EXAMPLES=OFF" + "-DSCN_BENCHMARKS=OFF" + ]; + }); + + "3fs" = prev.callPackage ./package.nix { + inherit (final) clickhouse-cpp scnlib; + libibverbs = prev.rdma-core; + }; }; nixosModule = { config, lib, pkgs, ... }: @@ -216,32 +261,79 @@ inherit system; overlays = [ overlay ]; }; + + isLinux = pkgs.stdenv.isLinux; in { - packages = { + packages = pkgs.lib.optionalAttrs isLinux { default = pkgs."3fs"; "3fs" = pkgs."3fs"; }; - overlays.default = overlay; + # Don't export overlays per-system devShells.default = pkgs.mkShell { - inputsFrom = [ pkgs."3fs" ]; + inputsFrom = pkgs.lib.optionals isLinux [ pkgs."3fs" ]; buildInputs = with pkgs; [ + # Build dependencies that are cross-platform + cmake + pkg-config + boost + protobuf + grpc + gtest + glog + lz4 + zlib + openssl + zstd + jemalloc + libevent + thrift + bison + flex + git + which + fmt + folly + rocksdb + leveldb + arrow-cpp + mimalloc + tomlplusplus + rustc + cargo + rustfmt + clippy + rustPlatform.bindgenHook + clickhouse-cpp + scnlib + # Additional dev tools clang-tools_14 cmake-format cmake-language-server gdb - valgrind - perf-tools rust-analyzer cargo-watch + ] ++ pkgs.lib.optionals isLinux [ + # Linux-only dependencies + foundationdb + fuse3 + rdma-core + numactl + liburing + valgrind + perf-tools ]; shellHook = '' echo "3FS development environment" - echo "Build with: nix build .#3fs" + ${if isLinux then '' + echo "Build with: nix build .#3fs" + '' else '' + echo "Note: 3FS can only be built on Linux systems" + ''} echo "Enter shell with: nix develop" echo "" echo "To test locally:" @@ -250,14 +342,14 @@ echo " 3. Mount FUSE filesystem" ''; }; - checks = { + checks = pkgs.lib.optionalAttrs isLinux { # Package build test package = pkgs."3fs"; - - # Integration test using NixOS VM + } // pkgs.lib.optionalAttrs isLinux { + # Integration test using NixOS VM (Linux only) integration-test = pkgs.nixosTest (import ./nixos-module-test.nix { - inherit pkgs lib; - self = self; + inherit pkgs self; + lib = pkgs.lib; }); }; } @@ -265,10 +357,12 @@ nixosModules.default = nixosModule; nixosModules."3fs" = nixosModule; - # Hydra job for CI - hydraJobs = { - inherit (self) packages; - tests = self.checks; - }; + overlays.default = overlay; + + # Hydra job for CI - only for Linux systems + hydraJobs = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ] (system: { + packages = self.packages.${system}; + tests = self.checks.${system} or {}; + }); }; -} \ No newline at end of file +} diff --git a/nixos-module-test.nix b/nixos-module-test.nix index d524200..9113e6e 100644 --- a/nixos-module-test.nix +++ b/nixos-module-test.nix @@ -1,7 +1,7 @@ # NixOS VM test for 3FS services # Run with: nix build .#checks.x86_64-linux.3fs-integration-test -{ pkgs, lib, ... }: +{ pkgs, lib, self, ... }: let # Test configuration for 3FS cluster @@ -23,11 +23,6 @@ let port = 8000; dataDir = "/var/lib/3fs/mgmtd"; }; - - foundationdb = { - clusterFile = "/etc/foundationdb/fdb.cluster"; - clusterString = "test:test@127.0.0.1:4500"; - }; }; in { @@ -71,23 +66,16 @@ in { }; }; - foundationdb = { - clusterFile = testConfig.foundationdb.clusterFile; - }; + # FoundationDB configuration is handled by the system service }; # Configure FoundationDB services.foundationdb = { enable = true; - clusterFile = testConfig.foundationdb.clusterFile; + package = pkgs.foundationdb; listenAddress = "127.0.0.1:4500"; dataDir = "/var/lib/foundationdb"; logDir = "/var/log/foundationdb"; - - # Initialize the database - initialScript = '' - configure single ssd - ''; }; # Open firewall ports @@ -117,21 +105,18 @@ in { }; }; - foundationdb = { - clusterFile = testConfig.foundationdb.clusterFile; - }; + # FoundationDB configuration is handled by the system service }; # Configure FoundationDB client services.foundationdb = { enable = true; - clusterFile = testConfig.foundationdb.clusterFile; - pidFile = "/run/foundationdb.pid"; + package = pkgs.foundationdb; }; # Write cluster file pointing to master environment.etc."foundationdb/fdb.cluster" = { - text = testConfig.foundationdb.clusterString; + text = "test:test@master:4500"; mode = "0644"; }; @@ -139,7 +124,6 @@ in { environment.systemPackages = with pkgs; [ fio - iozone sysbench ]; }; @@ -151,7 +135,7 @@ in { # Wait for FoundationDB to be ready master.wait_for_unit("foundationdb.service") master.wait_for_open_port(4500) - master.succeed("fdbcli --exec 'status' -C ${testConfig.foundationdb.clusterFile}") + master.succeed("fdbcli --exec 'status' -C /etc/foundationdb/fdb.cluster") # Wait for 3FS services to start master.wait_for_unit("3fs-mgmtd.service") @@ -172,7 +156,7 @@ in { master.succeed("systemctl is-active 3fs-monitor.service") # Initialize 3FS cluster - master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config ${testConfig.foundationdb.clusterFile}") + master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config /etc/foundationdb/fdb.cluster") # Register storage nodes master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}") diff --git a/package.nix b/package.nix index 08352cc..df5dd08 100644 --- a/package.nix +++ b/package.nix @@ -163,7 +163,7 @@ in stdenv.mkDerivation rec { # Fix jemalloc build substituteInPlace cmake/Jemalloc.cmake \ --replace 'ExternalProject_add(' 'return() #ExternalProject_add(' \ - --replace '${JEMALLOC_DIR}/lib/libjemalloc.so.2' '${jemalloc-custom}/lib/libjemalloc.so' + --replace "\''${JEMALLOC_DIR}/lib/libjemalloc.so.2" "\''${jemalloc-custom}/lib/libjemalloc.so" # Fix Apache Arrow build substituteInPlace cmake/ApacheArrow.cmake \