mirror of
https://github.com/deepseek-ai/3FS
synced 2025-06-26 18:16:45 +00:00
Merge 16468b0110 into 2db69ced80
This commit is contained in:
conneroisu
GitHub
commit
666225ccc5
235
README-nix.md
Normal file
235
README-nix.md
Normal file
@ -0,0 +1,235 @@
|
||||
# 3FS Nix Package and NixOS Module
|
||||
|
||||
This directory contains a Nix flake for building and deploying 3FS (Fire-Flyer File System) on NixOS systems.
|
||||
|
||||
## Features
|
||||
|
||||
- **Complete Package**: Builds 3FS with all dependencies including RDMA support, FoundationDB integration, and FUSE 3
|
||||
- **NixOS Module**: Provides systemd services for all 3FS components (meta, storage, mgmtd, monitor, fuse)
|
||||
- **Development Shell**: Includes all build tools and dependencies for 3FS development
|
||||
- **Integration Tests**: NixOS VM tests to validate the complete system
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Building the Package
|
||||
|
||||
1. First build (will fail with hash mismatch):
|
||||
```bash
|
||||
nix build .#3fs
|
||||
```
|
||||
|
||||
2. Update the hash in `package.nix` with the correct one from the error message
|
||||
|
||||
3. Rebuild:
|
||||
```bash
|
||||
nix build .#3fs
|
||||
```
|
||||
|
||||
### Using in NixOS
|
||||
|
||||
Add to your `flake.nix`:
|
||||
|
||||
```nix
|
||||
{
|
||||
inputs = {
|
||||
threefs.url = "github:deepseek-ai/3fs";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, threefs, ... }: {
|
||||
nixosConfigurations.myserver = nixpkgs.lib.nixosSystem {
|
||||
modules = [
|
||||
threefs.nixosModules.default
|
||||
{
|
||||
services."3fs" = {
|
||||
enable = true;
|
||||
meta.enable = true;
|
||||
storage.enable = true;
|
||||
mgmtd.enable = true;
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### Development Environment
|
||||
|
||||
Enter the development shell:
|
||||
```bash
|
||||
nix develop
|
||||
```
|
||||
|
||||
This provides:
|
||||
- Clang 14 with C++20 support
|
||||
- CMake and build tools
|
||||
- All 3FS dependencies
|
||||
- Rust toolchain
|
||||
- Debug tools (gdb, valgrind, perf)
|
||||
|
||||
## NixOS Module Options
|
||||
|
||||
### Basic Configuration
|
||||
|
||||
```nix
|
||||
services."3fs" = {
|
||||
enable = true;
|
||||
|
||||
# User and group for services
|
||||
user = "threefs";
|
||||
group = "threefs";
|
||||
|
||||
# Directories
|
||||
configDir = "/etc/3fs";
|
||||
dataDir = "/var/lib/3fs";
|
||||
|
||||
# FoundationDB cluster file
|
||||
foundationdb.clusterFile = "/etc/foundationdb/fdb.cluster";
|
||||
};
|
||||
```
|
||||
|
||||
### Service Configuration
|
||||
|
||||
#### Metadata Service
|
||||
```nix
|
||||
services."3fs".meta = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional meta configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
#### Storage Service
|
||||
```nix
|
||||
services."3fs".storage = {
|
||||
enable = true;
|
||||
targets = [
|
||||
"/var/lib/3fs/storage/target1"
|
||||
"/var/lib/3fs/storage/target2"
|
||||
];
|
||||
config = {
|
||||
# Additional storage configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
#### Management Daemon
|
||||
```nix
|
||||
services."3fs".mgmtd = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional mgmtd configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
#### Monitor Collector
|
||||
```nix
|
||||
services."3fs".monitor = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional monitor configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
#### FUSE Client
|
||||
```nix
|
||||
services."3fs".fuse = {
|
||||
enable = true;
|
||||
mountPoint = "/mnt/3fs";
|
||||
config = {
|
||||
# Additional FUSE configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
### Run Package Tests
|
||||
```bash
|
||||
./test-build.sh
|
||||
```
|
||||
|
||||
### Run Integration Tests
|
||||
```bash
|
||||
nix build .#checks.x86_64-linux.integration-test
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
|
||||
1. Start FoundationDB:
|
||||
```bash
|
||||
sudo systemctl start foundationdb
|
||||
```
|
||||
|
||||
2. Initialize 3FS cluster:
|
||||
```bash
|
||||
3fs-admin init-cluster
|
||||
```
|
||||
|
||||
3. Start 3FS services:
|
||||
```bash
|
||||
sudo systemctl start 3fs-mgmtd
|
||||
sudo systemctl start 3fs-meta
|
||||
sudo systemctl start 3fs-storage
|
||||
```
|
||||
|
||||
4. Mount filesystem:
|
||||
```bash
|
||||
sudo systemctl start 3fs-fuse
|
||||
```
|
||||
|
||||
## Package Contents
|
||||
|
||||
The package includes:
|
||||
|
||||
- **Binaries**:
|
||||
- `admin` - Administrative CLI tool
|
||||
- `meta_main` - Metadata service
|
||||
- `storage_main` - Storage service
|
||||
- `mgmtd_main` - Management daemon
|
||||
- `monitor_collector_main` - Metrics collector
|
||||
- `hf3fs_fuse_main` - FUSE filesystem client
|
||||
|
||||
- **Libraries**: All required shared libraries in `/lib`
|
||||
- **Configuration**: Default configuration files in `/etc/3fs`
|
||||
- **Headers**: Development headers in `/include/3fs`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Build Issues
|
||||
|
||||
1. **Hash mismatch**: Update the SHA256 hash in `package.nix`
|
||||
2. **Missing dependencies**: Check that all system libraries are available
|
||||
3. **Compilation errors**: Ensure you're using Clang 14 or newer
|
||||
|
||||
### Runtime Issues
|
||||
|
||||
1. **Service fails to start**: Check logs with `journalctl -u 3fs-<service>`
|
||||
2. **FoundationDB connection**: Ensure FDB cluster file is correct
|
||||
3. **FUSE mount fails**: Check that FUSE 3 is installed and kernel module is loaded
|
||||
|
||||
## Architecture
|
||||
|
||||
The Nix package handles:
|
||||
|
||||
1. **Complex Dependencies**: Automatically builds custom jemalloc, handles RDMA libraries
|
||||
2. **Third-party Libraries**: Uses system libraries where possible, builds others from source
|
||||
3. **Rust Components**: Properly integrates Cargo builds for Rust components
|
||||
4. **Service Management**: Complete systemd integration with proper dependencies
|
||||
5. **Configuration**: Manages TOML configuration files for all components
|
||||
|
||||
## Contributing
|
||||
|
||||
When modifying the package:
|
||||
|
||||
1. Update version in `package.nix`
|
||||
2. Test build locally
|
||||
3. Run integration tests
|
||||
4. Update documentation
|
||||
|
||||
## License
|
||||
|
||||
MIT License (same as 3FS project)
|
||||
61
flake.lock
Normal file
61
flake.lock
Normal file
@ -0,0 +1,61 @@
|
||||
{
|
||||
"nodes": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1748190013,
|
||||
"narHash": "sha256-R5HJFflOfsP5FBtk+zE8FpL8uqE7n62jqOsADvVshhE=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "62b852f6c6742134ade1abdd2a21685fd617a291",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
368
flake.nix
Normal file
368
flake.nix
Normal file
@ -0,0 +1,368 @@
|
||||
{
|
||||
description = "Fire-Flyer File System (3FS) - High-performance distributed file system for AI workloads";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, flake-utils }:
|
||||
let
|
||||
overlay = final: prev: {
|
||||
# Build clickhouse-cpp from source if not available in nixpkgs
|
||||
clickhouse-cpp = prev.clickhouse-cpp or (prev.stdenv.mkDerivation rec {
|
||||
pname = "clickhouse-cpp";
|
||||
version = "2.5.1";
|
||||
|
||||
src = prev.fetchFromGitHub {
|
||||
owner = "ClickHouse";
|
||||
repo = "clickhouse-cpp";
|
||||
rev = "v${version}";
|
||||
sha256 = "sha256-6kqcANO4S9Z1ee4kBPKGCnsPEGDaWPCx2hUi4APPWHU=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ prev.cmake ];
|
||||
buildInputs = [ prev.zlib prev.openssl prev.lz4 ];
|
||||
|
||||
cmakeFlags = [
|
||||
"-DBUILD_SHARED_LIBS=ON"
|
||||
"-DWITH_OPENSSL=ON"
|
||||
];
|
||||
});
|
||||
|
||||
# Build scnlib from source if not available in nixpkgs
|
||||
scnlib = prev.scnlib or (prev.stdenv.mkDerivation rec {
|
||||
pname = "scnlib";
|
||||
version = "2.0.2";
|
||||
|
||||
src = prev.fetchFromGitHub {
|
||||
owner = "eliaskosunen";
|
||||
repo = "scnlib";
|
||||
rev = "v${version}";
|
||||
sha256 = "sha256-YWlJiHAKKJd7jWv8Z0GmKqIfXI3HwVqA7AgZiHN2W8I=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ prev.cmake ];
|
||||
|
||||
cmakeFlags = [
|
||||
"-DSCN_TESTS=OFF"
|
||||
"-DSCN_EXAMPLES=OFF"
|
||||
"-DSCN_BENCHMARKS=OFF"
|
||||
];
|
||||
});
|
||||
|
||||
"3fs" = prev.callPackage ./package.nix {
|
||||
inherit (final) clickhouse-cpp scnlib;
|
||||
libibverbs = prev.rdma-core;
|
||||
};
|
||||
};
|
||||
|
||||
nixosModule = { config, lib, pkgs, ... }:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.services."3fs";
|
||||
|
||||
# Helper function to create service configuration
|
||||
mkServiceConfig = component: {
|
||||
description = "3FS ${component} service";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
requires = [ "network-online.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${pkgs."3fs"}/bin/${component}_main --launcher_cfg ${cfg.configDir}/${component}_main_launcher.toml --app-cfg ${cfg.configDir}/${component}_main_app.toml";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
LimitNOFILE = 1000000;
|
||||
User = cfg.user;
|
||||
Group = cfg.group;
|
||||
} // (if component == "storage" then {
|
||||
LimitMEMLOCK = "infinity";
|
||||
TimeoutStopSec = "5m";
|
||||
} else {});
|
||||
|
||||
environment = {
|
||||
LD_LIBRARY_PATH = "${pkgs."3fs"}/lib:${pkgs.foundationdb}/lib";
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services."3fs" = {
|
||||
enable = mkEnableOption "3FS distributed file system";
|
||||
|
||||
user = mkOption {
|
||||
type = types.str;
|
||||
default = "threefs";
|
||||
description = "User under which 3FS services run";
|
||||
};
|
||||
|
||||
group = mkOption {
|
||||
type = types.str;
|
||||
default = "threefs";
|
||||
description = "Group under which 3FS services run";
|
||||
};
|
||||
|
||||
configDir = mkOption {
|
||||
type = types.path;
|
||||
default = "/etc/3fs";
|
||||
description = "Directory containing 3FS configuration files";
|
||||
};
|
||||
|
||||
dataDir = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/lib/3fs";
|
||||
description = "Directory for 3FS data storage";
|
||||
};
|
||||
|
||||
meta = {
|
||||
enable = mkEnableOption "3FS metadata service";
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for meta service";
|
||||
};
|
||||
};
|
||||
|
||||
storage = {
|
||||
enable = mkEnableOption "3FS storage service";
|
||||
|
||||
targets = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ "/var/lib/3fs/storage" ];
|
||||
description = "List of storage target directories";
|
||||
};
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for storage service";
|
||||
};
|
||||
};
|
||||
|
||||
mgmtd = {
|
||||
enable = mkEnableOption "3FS management daemon";
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for management daemon";
|
||||
};
|
||||
};
|
||||
|
||||
monitor = {
|
||||
enable = mkEnableOption "3FS monitor collector";
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for monitor collector";
|
||||
};
|
||||
};
|
||||
|
||||
fuse = {
|
||||
enable = mkEnableOption "3FS FUSE client";
|
||||
|
||||
mountPoint = mkOption {
|
||||
type = types.path;
|
||||
default = "/mnt/3fs";
|
||||
description = "Mount point for 3FS FUSE filesystem";
|
||||
};
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for FUSE client";
|
||||
};
|
||||
};
|
||||
|
||||
foundationdb = {
|
||||
clusterFile = mkOption {
|
||||
type = types.path;
|
||||
default = "/etc/foundationdb/fdb.cluster";
|
||||
description = "Path to FoundationDB cluster file";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
# Create system user and group
|
||||
users.users.${cfg.user} = {
|
||||
isSystemUser = true;
|
||||
group = cfg.group;
|
||||
home = cfg.dataDir;
|
||||
createHome = true;
|
||||
description = "3FS system user";
|
||||
};
|
||||
|
||||
users.groups.${cfg.group} = {};
|
||||
|
||||
# Create necessary directories
|
||||
systemd.tmpfiles.rules = [
|
||||
"d '${cfg.configDir}' 0755 ${cfg.user} ${cfg.group} -"
|
||||
"d '${cfg.dataDir}' 0755 ${cfg.user} ${cfg.group} -"
|
||||
"d '${cfg.dataDir}/meta' 0755 ${cfg.user} ${cfg.group} -"
|
||||
"d '${cfg.dataDir}/mgmtd' 0755 ${cfg.user} ${cfg.group} -"
|
||||
"d '${cfg.dataDir}/monitor' 0755 ${cfg.user} ${cfg.group} -"
|
||||
] ++ (map (target: "d '${target}' 0755 ${cfg.user} ${cfg.group} -") cfg.storage.targets);
|
||||
|
||||
# Install default configuration files
|
||||
environment.etc = {
|
||||
"3fs/meta_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/meta_main_launcher.toml";
|
||||
"3fs/meta_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/meta_main_app.toml";
|
||||
"3fs/storage_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/storage_main_launcher.toml";
|
||||
"3fs/storage_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/storage_main_app.toml";
|
||||
"3fs/mgmtd_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/mgmtd_main_launcher.toml";
|
||||
"3fs/mgmtd_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/mgmtd_main_app.toml";
|
||||
"3fs/monitor_collector_main.toml".source = "${pkgs."3fs"}/etc/3fs/monitor_collector_main.toml";
|
||||
"3fs/hf3fs_fuse_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/hf3fs_fuse_main_launcher.toml";
|
||||
"3fs/hf3fs_fuse_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/hf3fs_fuse_main_app.toml";
|
||||
};
|
||||
|
||||
# Define systemd services
|
||||
systemd.services = {
|
||||
"3fs-meta" = mkIf cfg.meta.enable (mkServiceConfig "meta");
|
||||
"3fs-storage" = mkIf cfg.storage.enable (mkServiceConfig "storage");
|
||||
"3fs-mgmtd" = mkIf cfg.mgmtd.enable (mkServiceConfig "mgmtd");
|
||||
"3fs-monitor" = mkIf cfg.monitor.enable (mkServiceConfig "monitor_collector");
|
||||
|
||||
"3fs-fuse" = mkIf cfg.fuse.enable {
|
||||
description = "3FS FUSE client";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" "3fs-meta.service" ];
|
||||
requires = [ "network-online.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${pkgs."3fs"}/bin/hf3fs_fuse_main --launcher_cfg ${cfg.configDir}/hf3fs_fuse_main_launcher.toml --app-cfg ${cfg.configDir}/hf3fs_fuse_main_app.toml ${cfg.fuse.mountPoint}";
|
||||
ExecStop = "${pkgs.fuse3}/bin/fusermount3 -u ${cfg.fuse.mountPoint}";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
LimitNOFILE = 1000000;
|
||||
User = "root"; # FUSE requires root
|
||||
Group = "root";
|
||||
};
|
||||
|
||||
preStart = ''
|
||||
mkdir -p ${cfg.fuse.mountPoint}
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# Add 3fs package to system packages
|
||||
environment.systemPackages = [ pkgs."3fs" ];
|
||||
};
|
||||
};
|
||||
in
|
||||
flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
pkgs = import nixpkgs {
|
||||
inherit system;
|
||||
overlays = [ overlay ];
|
||||
};
|
||||
|
||||
isLinux = pkgs.stdenv.isLinux;
|
||||
in
|
||||
{
|
||||
packages = pkgs.lib.optionalAttrs isLinux {
|
||||
default = pkgs."3fs";
|
||||
"3fs" = pkgs."3fs";
|
||||
};
|
||||
|
||||
# Don't export overlays per-system
|
||||
|
||||
devShells.default = pkgs.mkShell {
|
||||
inputsFrom = pkgs.lib.optionals isLinux [ pkgs."3fs" ];
|
||||
buildInputs = with pkgs; [
|
||||
# Build dependencies that are cross-platform
|
||||
cmake
|
||||
pkg-config
|
||||
boost
|
||||
protobuf
|
||||
grpc
|
||||
gtest
|
||||
glog
|
||||
lz4
|
||||
zlib
|
||||
openssl
|
||||
zstd
|
||||
jemalloc
|
||||
libevent
|
||||
thrift
|
||||
bison
|
||||
flex
|
||||
git
|
||||
which
|
||||
fmt
|
||||
folly
|
||||
rocksdb
|
||||
leveldb
|
||||
arrow-cpp
|
||||
mimalloc
|
||||
tomlplusplus
|
||||
rustc
|
||||
cargo
|
||||
rustfmt
|
||||
clippy
|
||||
rustPlatform.bindgenHook
|
||||
clickhouse-cpp
|
||||
scnlib
|
||||
|
||||
# Additional dev tools
|
||||
clang-tools_14
|
||||
cmake-format
|
||||
cmake-language-server
|
||||
gdb
|
||||
rust-analyzer
|
||||
cargo-watch
|
||||
] ++ pkgs.lib.optionals isLinux [
|
||||
# Linux-only dependencies
|
||||
foundationdb
|
||||
fuse3
|
||||
rdma-core
|
||||
numactl
|
||||
liburing
|
||||
valgrind
|
||||
perf-tools
|
||||
];
|
||||
|
||||
shellHook = ''
|
||||
echo "3FS development environment"
|
||||
${if isLinux then ''
|
||||
echo "Build with: nix build .#3fs"
|
||||
'' else ''
|
||||
echo "Note: 3FS can only be built on Linux systems"
|
||||
''}
|
||||
echo "Enter shell with: nix develop"
|
||||
echo ""
|
||||
echo "To test locally:"
|
||||
echo " 1. Start FoundationDB"
|
||||
echo " 2. Configure and start 3FS services"
|
||||
echo " 3. Mount FUSE filesystem"
|
||||
'';
|
||||
};
|
||||
checks = pkgs.lib.optionalAttrs isLinux {
|
||||
# Package build test
|
||||
package = pkgs."3fs";
|
||||
} // pkgs.lib.optionalAttrs isLinux {
|
||||
# Integration test using NixOS VM (Linux only)
|
||||
integration-test = pkgs.nixosTest (import ./nixos-module-test.nix {
|
||||
inherit pkgs self;
|
||||
lib = pkgs.lib;
|
||||
});
|
||||
};
|
||||
}
|
||||
) // {
|
||||
nixosModules.default = nixosModule;
|
||||
nixosModules."3fs" = nixosModule;
|
||||
|
||||
overlays.default = overlay;
|
||||
|
||||
# Hydra job for CI - only for Linux systems
|
||||
hydraJobs = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ] (system: {
|
||||
packages = self.packages.${system};
|
||||
tests = self.checks.${system} or {};
|
||||
});
|
||||
};
|
||||
}
|
||||
210
nixos-module-test.nix
Normal file
210
nixos-module-test.nix
Normal file
@ -0,0 +1,210 @@
|
||||
# NixOS VM test for 3FS services
|
||||
# Run with: nix build .#checks.x86_64-linux.3fs-integration-test
|
||||
|
||||
{ pkgs, lib, self, ... }:
|
||||
|
||||
let
|
||||
# Test configuration for 3FS cluster
|
||||
testConfig = {
|
||||
meta = {
|
||||
port = 6000;
|
||||
dataDir = "/var/lib/3fs/meta";
|
||||
};
|
||||
|
||||
storage = {
|
||||
port = 7000;
|
||||
targets = [
|
||||
"/var/lib/3fs/storage/target1"
|
||||
"/var/lib/3fs/storage/target2"
|
||||
];
|
||||
};
|
||||
|
||||
mgmtd = {
|
||||
port = 8000;
|
||||
dataDir = "/var/lib/3fs/mgmtd";
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
name = "3fs-integration-test";
|
||||
|
||||
nodes = {
|
||||
# Master node running all services
|
||||
master = { config, pkgs, ... }: {
|
||||
imports = [ self.nixosModules.default ];
|
||||
|
||||
# Enable 3FS services
|
||||
services."3fs" = {
|
||||
enable = true;
|
||||
|
||||
meta = {
|
||||
enable = true;
|
||||
config = {
|
||||
inherit (testConfig.meta) port dataDir;
|
||||
};
|
||||
};
|
||||
|
||||
storage = {
|
||||
enable = true;
|
||||
targets = testConfig.storage.targets;
|
||||
config = {
|
||||
inherit (testConfig.storage) port;
|
||||
};
|
||||
};
|
||||
|
||||
mgmtd = {
|
||||
enable = true;
|
||||
config = {
|
||||
inherit (testConfig.mgmtd) port dataDir;
|
||||
};
|
||||
};
|
||||
|
||||
monitor = {
|
||||
enable = true;
|
||||
config = {
|
||||
port = 9000;
|
||||
};
|
||||
};
|
||||
|
||||
# FoundationDB configuration is handled by the system service
|
||||
};
|
||||
|
||||
# Configure FoundationDB
|
||||
services.foundationdb = {
|
||||
enable = true;
|
||||
package = pkgs.foundationdb;
|
||||
listenAddress = "127.0.0.1:4500";
|
||||
dataDir = "/var/lib/foundationdb";
|
||||
logDir = "/var/log/foundationdb";
|
||||
};
|
||||
|
||||
# Open firewall ports
|
||||
networking.firewall.enable = false;
|
||||
|
||||
# Additional test utilities
|
||||
environment.systemPackages = with pkgs; [
|
||||
netcat
|
||||
curl
|
||||
jq
|
||||
];
|
||||
};
|
||||
|
||||
# Client node with FUSE mount
|
||||
client = { config, pkgs, ... }: {
|
||||
imports = [ self.nixosModules.default ];
|
||||
|
||||
services."3fs" = {
|
||||
enable = true;
|
||||
|
||||
fuse = {
|
||||
enable = true;
|
||||
mountPoint = "/mnt/3fs";
|
||||
config = {
|
||||
metaServers = [ "master:6000" ];
|
||||
mgmtdServers = [ "master:8000" ];
|
||||
};
|
||||
};
|
||||
|
||||
# FoundationDB configuration is handled by the system service
|
||||
};
|
||||
|
||||
# Configure FoundationDB client
|
||||
services.foundationdb = {
|
||||
enable = true;
|
||||
package = pkgs.foundationdb;
|
||||
};
|
||||
|
||||
# Write cluster file pointing to master
|
||||
environment.etc."foundationdb/fdb.cluster" = {
|
||||
text = "test:test@master:4500";
|
||||
mode = "0644";
|
||||
};
|
||||
|
||||
networking.firewall.enable = false;
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
fio
|
||||
sysbench
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
# Wait for FoundationDB to be ready
|
||||
master.wait_for_unit("foundationdb.service")
|
||||
master.wait_for_open_port(4500)
|
||||
master.succeed("fdbcli --exec 'status' -C /etc/foundationdb/fdb.cluster")
|
||||
|
||||
# Wait for 3FS services to start
|
||||
master.wait_for_unit("3fs-mgmtd.service")
|
||||
master.wait_for_unit("3fs-meta.service")
|
||||
master.wait_for_unit("3fs-storage.service")
|
||||
master.wait_for_unit("3fs-monitor.service")
|
||||
|
||||
# Check that services are listening on correct ports
|
||||
master.wait_for_open_port(${toString testConfig.meta.port})
|
||||
master.wait_for_open_port(${toString testConfig.storage.port})
|
||||
master.wait_for_open_port(${toString testConfig.mgmtd.port})
|
||||
master.wait_for_open_port(9000) # monitor
|
||||
|
||||
# Check service status
|
||||
master.succeed("systemctl is-active 3fs-mgmtd.service")
|
||||
master.succeed("systemctl is-active 3fs-meta.service")
|
||||
master.succeed("systemctl is-active 3fs-storage.service")
|
||||
master.succeed("systemctl is-active 3fs-monitor.service")
|
||||
|
||||
# Initialize 3FS cluster
|
||||
master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config /etc/foundationdb/fdb.cluster")
|
||||
|
||||
# Register storage nodes
|
||||
master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}")
|
||||
|
||||
# Create storage targets
|
||||
for target in ${lib.concatStringsSep " " (map (t: "\"${t}\"") testConfig.storage.targets)}; do
|
||||
master.succeed("${pkgs."3fs"}/bin/admin create-target --path $target")
|
||||
done
|
||||
|
||||
# Wait for client to connect
|
||||
client.wait_for_unit("3fs-fuse.service")
|
||||
client.wait_until_succeeds("mountpoint -q /mnt/3fs")
|
||||
|
||||
# Basic filesystem operations test
|
||||
client.succeed("echo 'Hello 3FS!' > /mnt/3fs/test.txt")
|
||||
client.succeed("cat /mnt/3fs/test.txt | grep 'Hello 3FS!'")
|
||||
|
||||
# Create directory structure
|
||||
client.succeed("mkdir -p /mnt/3fs/test/deep/directory")
|
||||
client.succeed("touch /mnt/3fs/test/deep/directory/file.txt")
|
||||
client.succeed("ls -la /mnt/3fs/test/deep/directory/")
|
||||
|
||||
# Test file operations
|
||||
client.succeed("dd if=/dev/urandom of=/mnt/3fs/random.dat bs=1M count=10")
|
||||
client.succeed("cp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat")
|
||||
client.succeed("cmp /mnt/3fs/random.dat /mnt/3fs/random_copy.dat")
|
||||
|
||||
# Test permissions
|
||||
client.succeed("chmod 755 /mnt/3fs/test")
|
||||
client.succeed("test -d /mnt/3fs/test")
|
||||
|
||||
# Clean up
|
||||
client.succeed("rm -rf /mnt/3fs/test*")
|
||||
client.succeed("rm -f /mnt/3fs/random*.dat")
|
||||
|
||||
# Verify cleanup
|
||||
client.succeed("test -z \"$(ls -A /mnt/3fs)\"")
|
||||
|
||||
# Check metrics
|
||||
master.succeed("${pkgs."3fs"}/bin/admin list-nodes")
|
||||
master.succeed("${pkgs."3fs"}/bin/admin list-targets")
|
||||
|
||||
# Test service restart
|
||||
master.succeed("systemctl restart 3fs-storage.service")
|
||||
master.wait_for_unit("3fs-storage.service")
|
||||
|
||||
# Ensure filesystem still works after restart
|
||||
client.succeed("echo 'Still working!' > /mnt/3fs/restart-test.txt")
|
||||
client.succeed("cat /mnt/3fs/restart-test.txt")
|
||||
'';
|
||||
}
|
||||
102
nixos-test-config.nix
Normal file
102
nixos-test-config.nix
Normal file
@ -0,0 +1,102 @@
|
||||
# Example NixOS configuration for testing 3FS
|
||||
{ config, pkgs, ... }:
|
||||
|
||||
{
|
||||
imports = [
|
||||
# Import the 3FS flake module
|
||||
# In a real system, you would use:
|
||||
# (builtins.getFlake "github:deepseek-ai/3fs").nixosModules.default
|
||||
];
|
||||
|
||||
# Enable 3FS services
|
||||
services."3fs" = {
|
||||
enable = true;
|
||||
|
||||
# Configure user and group
|
||||
user = "threefs";
|
||||
group = "threefs";
|
||||
|
||||
# Configure directories
|
||||
configDir = "/etc/3fs";
|
||||
dataDir = "/var/lib/3fs";
|
||||
|
||||
# Enable metadata service
|
||||
meta = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional meta service configuration
|
||||
};
|
||||
};
|
||||
|
||||
# Enable storage service
|
||||
storage = {
|
||||
enable = true;
|
||||
targets = [
|
||||
"/var/lib/3fs/storage/target1"
|
||||
"/var/lib/3fs/storage/target2"
|
||||
];
|
||||
config = {
|
||||
# Additional storage service configuration
|
||||
};
|
||||
};
|
||||
|
||||
# Enable management daemon
|
||||
mgmtd = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional mgmtd configuration
|
||||
};
|
||||
};
|
||||
|
||||
# Enable monitor collector
|
||||
monitor = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional monitor configuration
|
||||
};
|
||||
};
|
||||
|
||||
# Enable FUSE client
|
||||
fuse = {
|
||||
enable = true;
|
||||
mountPoint = "/mnt/3fs";
|
||||
config = {
|
||||
# Additional FUSE configuration
|
||||
};
|
||||
};
|
||||
|
||||
# FoundationDB configuration
|
||||
foundationdb = {
|
||||
clusterFile = "/etc/foundationdb/fdb.cluster";
|
||||
};
|
||||
};
|
||||
|
||||
# Ensure FoundationDB is also installed and configured
|
||||
services.foundationdb = {
|
||||
enable = true;
|
||||
clusterFile = "/etc/foundationdb/fdb.cluster";
|
||||
};
|
||||
|
||||
# Open necessary ports for 3FS services
|
||||
networking.firewall = {
|
||||
allowedTCPPorts = [
|
||||
# Meta service ports
|
||||
6000 6001
|
||||
# Storage service ports
|
||||
7000 7001
|
||||
# Management daemon ports
|
||||
8000 8001
|
||||
# Monitor collector ports
|
||||
9000 9001
|
||||
];
|
||||
};
|
||||
|
||||
# Example of how to override package
|
||||
nixpkgs.overlays = [
|
||||
(final: prev: {
|
||||
"3fs" = prev."3fs".overrideAttrs (oldAttrs: {
|
||||
# Custom overrides if needed
|
||||
});
|
||||
})
|
||||
];
|
||||
}
|
||||
378
package.nix
Normal file
378
package.nix
Normal file
@ -0,0 +1,378 @@
|
||||
{ stdenv
|
||||
, lib
|
||||
, fetchFromGitHub
|
||||
, runCommand
|
||||
, writeText
|
||||
, makeWrapper
|
||||
, cmake
|
||||
, pkg-config
|
||||
, clang_14
|
||||
, lld_14
|
||||
, rustc
|
||||
, cargo
|
||||
, rustPlatform
|
||||
, autoconf
|
||||
, automake
|
||||
, libtool
|
||||
, python3
|
||||
, boost
|
||||
, libuv
|
||||
, lz4
|
||||
, xz
|
||||
, double-conversion
|
||||
, libdwarf
|
||||
, libunwind
|
||||
, libaio
|
||||
, gflags
|
||||
, glog
|
||||
, gtest
|
||||
, gperftools
|
||||
, openssl
|
||||
, fuse3
|
||||
, foundationdb
|
||||
, rdma-core
|
||||
, libibverbs
|
||||
, zstd
|
||||
, jemalloc
|
||||
, libevent
|
||||
, numactl
|
||||
, liburing
|
||||
, thrift
|
||||
, bison
|
||||
, flex
|
||||
, git
|
||||
, which
|
||||
, fmt
|
||||
, folly
|
||||
, rocksdb
|
||||
, leveldb
|
||||
, arrow-cpp
|
||||
, mimalloc
|
||||
, clickhouse-cpp
|
||||
, tomlplusplus
|
||||
, scnlib
|
||||
}:
|
||||
|
||||
let
|
||||
# Build jemalloc with specific configuration for 3FS
|
||||
jemalloc-custom = jemalloc.overrideAttrs (oldAttrs: {
|
||||
configureFlags = (oldAttrs.configureFlags or []) ++ [
|
||||
"--disable-cxx"
|
||||
"--enable-prof"
|
||||
"--disable-initial-exec-tls"
|
||||
];
|
||||
});
|
||||
|
||||
# Custom FoundationDB build if needed
|
||||
fdb-version = "7.1.5";
|
||||
|
||||
in stdenv.mkDerivation rec {
|
||||
pname = "3fs";
|
||||
version = "0.1.5";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "deepseek-ai";
|
||||
repo = "3fs";
|
||||
rev = "91bfcf39a9e4b5ded959f7b5c2cb0cf858ebbff5";
|
||||
sha256 = "sha256-0000000000000000000000000000000000000000000="; # To be replaced
|
||||
fetchSubmodules = true;
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
cmake
|
||||
pkg-config
|
||||
clang_14
|
||||
lld_14
|
||||
rustc
|
||||
cargo
|
||||
rustPlatform.cargoSetupHook
|
||||
autoconf
|
||||
automake
|
||||
libtool
|
||||
python3
|
||||
git
|
||||
which
|
||||
bison
|
||||
flex
|
||||
];
|
||||
|
||||
buildInputs = [
|
||||
# Core dependencies
|
||||
boost
|
||||
libuv
|
||||
lz4
|
||||
xz
|
||||
double-conversion
|
||||
libdwarf
|
||||
libunwind
|
||||
libaio
|
||||
gflags
|
||||
glog
|
||||
gtest
|
||||
gperftools
|
||||
openssl
|
||||
|
||||
# FUSE 3
|
||||
fuse3
|
||||
|
||||
# FoundationDB
|
||||
foundationdb
|
||||
|
||||
# RDMA/InfiniBand
|
||||
rdma-core
|
||||
libibverbs
|
||||
|
||||
# Additional libraries
|
||||
zstd
|
||||
jemalloc-custom
|
||||
libevent
|
||||
numactl
|
||||
liburing
|
||||
thrift
|
||||
fmt
|
||||
folly
|
||||
rocksdb
|
||||
leveldb
|
||||
mimalloc
|
||||
tomlplusplus
|
||||
|
||||
# Python bindings
|
||||
python3.pkgs.pybind11
|
||||
];
|
||||
|
||||
patches = [
|
||||
# Add any necessary patches here
|
||||
];
|
||||
|
||||
postPatch = ''
|
||||
# Apply the repository patches
|
||||
patchShebangs ./patches/apply.sh
|
||||
./patches/apply.sh || true
|
||||
|
||||
# Fix CMake to use system libraries
|
||||
substituteInPlace CMakeLists.txt \
|
||||
--replace 'add_link_options(-fuse-ld=lld)' '# add_link_options(-fuse-ld=lld)' \
|
||||
--replace 'set(CMAKE_CXX_FLAGS "$' 'set(CMAKE_CXX_FLAGS "$'
|
||||
|
||||
# Disable bundled third-party libraries
|
||||
for lib in fmt zstd googletest folly leveldb rocksdb scnlib pybind11 toml11 mimalloc clickhouse-cpp liburing-cmake; do
|
||||
substituteInPlace CMakeLists.txt \
|
||||
--replace "add_subdirectory(\"third_party/$lib\"" "#add_subdirectory(\"third_party/$lib\""
|
||||
done
|
||||
|
||||
# Fix jemalloc build
|
||||
substituteInPlace cmake/Jemalloc.cmake \
|
||||
--replace 'ExternalProject_add(' 'return() #ExternalProject_add(' \
|
||||
--replace "\''${JEMALLOC_DIR}/lib/libjemalloc.so.2" "\''${jemalloc-custom}/lib/libjemalloc.so"
|
||||
|
||||
# Fix Apache Arrow build
|
||||
substituteInPlace cmake/ApacheArrow.cmake \
|
||||
--replace 'ExternalProject_Add(' 'return() #ExternalProject_Add('
|
||||
|
||||
# Set up Rust dependencies
|
||||
cd src/client/trash_cleaner
|
||||
cargoDepsCopy="$NIX_BUILD_TOP/cargo-vendor-dir"
|
||||
if [ -d "$cargoDepsCopy" ]; then
|
||||
chmod -R +w "$cargoDepsCopy"
|
||||
fi
|
||||
cd $NIX_BUILD_TOP/source
|
||||
|
||||
# Fix storage chunk_engine Rust build
|
||||
cd src/storage/chunk_engine
|
||||
cargoDepsCopy="$NIX_BUILD_TOP/cargo-vendor-dir"
|
||||
if [ -d "$cargoDepsCopy" ]; then
|
||||
chmod -R +w "$cargoDepsCopy"
|
||||
fi
|
||||
cd $NIX_BUILD_TOP/source
|
||||
'';
|
||||
|
||||
preConfigure = ''
|
||||
# Create necessary directories
|
||||
mkdir -p $TMP/third_party
|
||||
|
||||
# Setup environment
|
||||
export HOME=$TMP
|
||||
export CARGO_HOME=$TMP/.cargo
|
||||
|
||||
# Set up library paths
|
||||
export BOOST_ROOT=${boost}
|
||||
export BOOST_INCLUDEDIR=${boost}/include
|
||||
export BOOST_LIBRARYDIR=${boost}/lib
|
||||
|
||||
# FoundationDB paths
|
||||
export FDB_LIBRARY_DIR=${foundationdb}/lib
|
||||
export FDB_INCLUDE_DIR=${foundationdb}/include
|
||||
|
||||
# Jemalloc paths
|
||||
export JEMALLOC_OVERRIDE=${jemalloc-custom}/lib/libjemalloc.so
|
||||
export JEMALLOC_DIR=${jemalloc-custom}
|
||||
|
||||
# Compiler flags
|
||||
export CXXFLAGS="-I${boost}/include -I${gtest}/include -I${liburing}/include -I${fmt}/include -I${folly}/include"
|
||||
export LDFLAGS="-L${boost}/lib -L${jemalloc-custom}/lib -L${liburing}/lib -L${fmt}/lib -L${folly}/lib"
|
||||
'';
|
||||
|
||||
cmakeFlags = [
|
||||
"-DCMAKE_CXX_COMPILER=${clang_14}/bin/clang++"
|
||||
"-DCMAKE_C_COMPILER=${clang_14}/bin/clang"
|
||||
"-DCMAKE_BUILD_TYPE=RelWithDebInfo"
|
||||
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
|
||||
"-DENABLE_FUSE_APPLICATION=ON"
|
||||
"-DOVERRIDE_CXX_NEW_DELETE=OFF"
|
||||
"-DSAVE_ALLOCATE_SIZE=OFF"
|
||||
"-DBoost_USE_STATIC_LIBS=ON"
|
||||
"-DFDB_VERSION=${fdb-version}"
|
||||
|
||||
# System library paths
|
||||
"-DCMAKE_PREFIX_PATH=${boost};${gtest};${liburing};${fmt};${folly}"
|
||||
"-DBoost_DIR=${boost}/lib/cmake/Boost"
|
||||
"-DBoost_INCLUDE_DIR=${boost}/include"
|
||||
"-Dfmt_DIR=${fmt}/lib/cmake/fmt"
|
||||
"-DFolly_DIR=${folly}/lib/cmake/folly"
|
||||
"-DGTest_DIR=${gtest}/lib/cmake/GTest"
|
||||
"-DZSTD_LIBRARY=${zstd}/lib/libzstd.so"
|
||||
"-DZSTD_INCLUDE_DIR=${zstd}/include"
|
||||
"-Djemalloc_INCLUDE_DIR=${jemalloc-custom}/include"
|
||||
"-Djemalloc_LIBRARY=${jemalloc-custom}/lib/libjemalloc.so"
|
||||
"-DJEMALLOC_DIR=${jemalloc-custom}"
|
||||
"-DARROW_INCLUDE_DIR=${arrow-cpp}/include"
|
||||
"-DARROW_LIB_DIR=${arrow-cpp}/lib"
|
||||
"-Darrow_DIR=${arrow-cpp}/lib/cmake/arrow"
|
||||
|
||||
# Initially disable tests to simplify build
|
||||
"-DBUILD_TESTING=OFF"
|
||||
];
|
||||
|
||||
# Custom build phase to handle complex build process
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
|
||||
# Build with decreasing parallelism if needed
|
||||
cmake --build . -j $NIX_BUILD_CORES || \
|
||||
cmake --build . -j 4 || \
|
||||
cmake --build . -j 1
|
||||
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
# Create output directories
|
||||
mkdir -p $out/bin $out/lib $out/etc/3fs
|
||||
|
||||
# Install binaries
|
||||
for bin in \
|
||||
src/fuse/hf3fs_fuse \
|
||||
src/client/cli/admin/admin \
|
||||
src/meta/meta \
|
||||
src/mgmtd/mgmtd \
|
||||
src/storage/storage \
|
||||
src/monitor_collector/monitor_collector \
|
||||
src/tools/admin
|
||||
do
|
||||
if [ -f "$bin" ] && [ -x "$bin" ]; then
|
||||
install -D -m755 "$bin" "$out/bin/$(basename $bin)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Rename main binaries to match systemd service expectations
|
||||
[ -f "$out/bin/meta" ] && mv "$out/bin/meta" "$out/bin/meta_main"
|
||||
[ -f "$out/bin/mgmtd" ] && mv "$out/bin/mgmtd" "$out/bin/mgmtd_main"
|
||||
[ -f "$out/bin/storage" ] && mv "$out/bin/storage" "$out/bin/storage_main"
|
||||
[ -f "$out/bin/monitor_collector" ] && mv "$out/bin/monitor_collector" "$out/bin/monitor_collector_main"
|
||||
[ -f "$out/bin/hf3fs_fuse" ] && cp "$out/bin/hf3fs_fuse" "$out/bin/hf3fs_fuse_main"
|
||||
|
||||
# Install libraries
|
||||
find . -name "*.so" -o -name "*.so.*" | while read lib; do
|
||||
if [ -f "$lib" ]; then
|
||||
install -D -m644 "$lib" "$out/lib/$(basename $lib)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Install static libraries (optional)
|
||||
find . -name "*.a" | while read lib; do
|
||||
if [ -f "$lib" ] && [[ ! "$lib" =~ third_party ]]; then
|
||||
install -D -m644 "$lib" "$out/lib/$(basename $lib)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Install Python modules
|
||||
if [ -d "src/lib/py" ]; then
|
||||
mkdir -p $out/lib/python${python3.pythonVersion}/site-packages
|
||||
cp -r src/lib/py/* $out/lib/python${python3.pythonVersion}/site-packages/
|
||||
fi
|
||||
|
||||
# Install configuration files
|
||||
cp -r configs/* $out/etc/3fs/
|
||||
|
||||
# Install headers (for development)
|
||||
mkdir -p $out/include/3fs
|
||||
find src -name "*.h" -o -name "*.hpp" | while read header; do
|
||||
rel_path=$(echo "$header" | sed 's|^src/||')
|
||||
install -D -m644 "$header" "$out/include/3fs/$rel_path"
|
||||
done
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
# Fix RPATH for all binaries
|
||||
for exe in $out/bin/*; do
|
||||
if [ -f "$exe" ] && [ -x "$exe" ]; then
|
||||
patchelf --set-rpath "$out/lib:${lib.makeLibraryPath buildInputs}" "$exe" || true
|
||||
fi
|
||||
done
|
||||
|
||||
# Fix RPATH for all libraries
|
||||
for lib in $out/lib/*.so*; do
|
||||
if [ -f "$lib" ]; then
|
||||
patchelf --set-rpath "$out/lib:${lib.makeLibraryPath buildInputs}" "$lib" || true
|
||||
fi
|
||||
done
|
||||
|
||||
# Create wrapper scripts if needed
|
||||
for bin in $out/bin/*_main; do
|
||||
if [ -f "$bin" ]; then
|
||||
makeWrapper "$bin" "$bin.wrapped" \
|
||||
--prefix LD_LIBRARY_PATH : "$out/lib:${lib.makeLibraryPath buildInputs}" \
|
||||
--set RUST_BACKTRACE 1
|
||||
mv "$bin.wrapped" "$bin"
|
||||
fi
|
||||
done
|
||||
'';
|
||||
|
||||
# Enable parallel building
|
||||
enableParallelBuilding = true;
|
||||
NIX_BUILD_CORES = 8;
|
||||
|
||||
# Set up proper library paths
|
||||
setupHook = writeText "setup-hook" ''
|
||||
export LD_LIBRARY_PATH="$1/lib''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH"
|
||||
'';
|
||||
|
||||
passthru = {
|
||||
inherit jemalloc-custom;
|
||||
tests = {
|
||||
# Add package tests here
|
||||
version = runCommand "3fs-version-test" {} ''
|
||||
${pname}/bin/admin --version
|
||||
touch $out
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
meta = with lib; {
|
||||
description = "Fire-Flyer File System - High-performance distributed file system for AI workloads";
|
||||
longDescription = ''
|
||||
3FS is a high-performance distributed file system designed to address the challenges
|
||||
of AI training and inference workloads. It leverages modern SSDs and RDMA networks
|
||||
to provide a shared storage layer that simplifies development of distributed applications.
|
||||
'';
|
||||
homepage = "https://github.com/deepseek-ai/3fs";
|
||||
license = licenses.mit;
|
||||
maintainers = with maintainers; [ ];
|
||||
platforms = platforms.linux;
|
||||
mainProgram = "admin";
|
||||
};
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user