mirror of
https://github.com/deepseek-ai/3FS
synced 2025-06-26 18:16:45 +00:00
added flake.nix, package.nix, and documentation
This commit is contained in:
connerohnesorge
parent
91bfcf3606
commit
310118ef08
235
README-nix.md
Normal file
235
README-nix.md
Normal file
@ -0,0 +1,235 @@
|
||||
# 3FS Nix Package and NixOS Module
|
||||
|
||||
This directory contains a Nix flake for building and deploying 3FS (Fire-Flyer File System) on NixOS systems.
|
||||
|
||||
## Features
|
||||
|
||||
- **Complete Package**: Builds 3FS with all dependencies including RDMA support, FoundationDB integration, and FUSE 3
|
||||
- **NixOS Module**: Provides systemd services for all 3FS components (meta, storage, mgmtd, monitor, fuse)
|
||||
- **Development Shell**: Includes all build tools and dependencies for 3FS development
|
||||
- **Integration Tests**: NixOS VM tests to validate the complete system
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Building the Package
|
||||
|
||||
1. First build (will fail with hash mismatch):
|
||||
```bash
|
||||
nix build .#3fs
|
||||
```
|
||||
|
||||
2. Update the hash in `package.nix` with the correct one from the error message
|
||||
|
||||
3. Rebuild:
|
||||
```bash
|
||||
nix build .#3fs
|
||||
```
|
||||
|
||||
### Using in NixOS
|
||||
|
||||
Add to your `flake.nix`:
|
||||
|
||||
```nix
|
||||
{
|
||||
inputs = {
|
||||
threefs.url = "github:deepseek-ai/3fs";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, threefs, ... }: {
|
||||
nixosConfigurations.myserver = nixpkgs.lib.nixosSystem {
|
||||
modules = [
|
||||
threefs.nixosModules.default
|
||||
{
|
||||
services."3fs" = {
|
||||
enable = true;
|
||||
meta.enable = true;
|
||||
storage.enable = true;
|
||||
mgmtd.enable = true;
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### Development Environment
|
||||
|
||||
Enter the development shell:
|
||||
```bash
|
||||
nix develop
|
||||
```
|
||||
|
||||
This provides:
|
||||
- Clang 14 with C++20 support
|
||||
- CMake and build tools
|
||||
- All 3FS dependencies
|
||||
- Rust toolchain
|
||||
- Debug tools (gdb, valgrind, perf)
|
||||
|
||||
## NixOS Module Options
|
||||
|
||||
### Basic Configuration
|
||||
|
||||
```nix
|
||||
services."3fs" = {
|
||||
enable = true;
|
||||
|
||||
# User and group for services
|
||||
user = "threefs";
|
||||
group = "threefs";
|
||||
|
||||
# Directories
|
||||
configDir = "/etc/3fs";
|
||||
dataDir = "/var/lib/3fs";
|
||||
|
||||
# FoundationDB cluster file
|
||||
foundationdb.clusterFile = "/etc/foundationdb/fdb.cluster";
|
||||
};
|
||||
```
|
||||
|
||||
### Service Configuration
|
||||
|
||||
#### Metadata Service
|
||||
```nix
|
||||
services."3fs".meta = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional meta configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
#### Storage Service
|
||||
```nix
|
||||
services."3fs".storage = {
|
||||
enable = true;
|
||||
targets = [
|
||||
"/var/lib/3fs/storage/target1"
|
||||
"/var/lib/3fs/storage/target2"
|
||||
];
|
||||
config = {
|
||||
# Additional storage configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
#### Management Daemon
|
||||
```nix
|
||||
services."3fs".mgmtd = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional mgmtd configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
#### Monitor Collector
|
||||
```nix
|
||||
services."3fs".monitor = {
|
||||
enable = true;
|
||||
config = {
|
||||
# Additional monitor configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
#### FUSE Client
|
||||
```nix
|
||||
services."3fs".fuse = {
|
||||
enable = true;
|
||||
mountPoint = "/mnt/3fs";
|
||||
config = {
|
||||
# Additional FUSE configuration
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
### Run Package Tests
|
||||
```bash
|
||||
./test-build.sh
|
||||
```
|
||||
|
||||
### Run Integration Tests
|
||||
```bash
|
||||
nix build .#checks.x86_64-linux.integration-test
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
|
||||
1. Start FoundationDB:
|
||||
```bash
|
||||
sudo systemctl start foundationdb
|
||||
```
|
||||
|
||||
2. Initialize 3FS cluster:
|
||||
```bash
|
||||
3fs-admin init-cluster
|
||||
```
|
||||
|
||||
3. Start 3FS services:
|
||||
```bash
|
||||
sudo systemctl start 3fs-mgmtd
|
||||
sudo systemctl start 3fs-meta
|
||||
sudo systemctl start 3fs-storage
|
||||
```
|
||||
|
||||
4. Mount filesystem:
|
||||
```bash
|
||||
sudo systemctl start 3fs-fuse
|
||||
```
|
||||
|
||||
## Package Contents
|
||||
|
||||
The package includes:
|
||||
|
||||
- **Binaries**:
|
||||
- `admin` - Administrative CLI tool
|
||||
- `meta_main` - Metadata service
|
||||
- `storage_main` - Storage service
|
||||
- `mgmtd_main` - Management daemon
|
||||
- `monitor_collector_main` - Metrics collector
|
||||
- `hf3fs_fuse_main` - FUSE filesystem client
|
||||
|
||||
- **Libraries**: All required shared libraries in `/lib`
|
||||
- **Configuration**: Default configuration files in `/etc/3fs`
|
||||
- **Headers**: Development headers in `/include/3fs`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Build Issues
|
||||
|
||||
1. **Hash mismatch**: Update the SHA256 hash in `package.nix`
|
||||
2. **Missing dependencies**: Check that all system libraries are available
|
||||
3. **Compilation errors**: Ensure you're using Clang 14 or newer
|
||||
|
||||
### Runtime Issues
|
||||
|
||||
1. **Service fails to start**: Check logs with `journalctl -u 3fs-<service>`
|
||||
2. **FoundationDB connection**: Ensure FDB cluster file is correct
|
||||
3. **FUSE mount fails**: Check that FUSE 3 is installed and kernel module is loaded
|
||||
|
||||
## Architecture
|
||||
|
||||
The Nix package handles:
|
||||
|
||||
1. **Complex Dependencies**: Automatically builds custom jemalloc, handles RDMA libraries
|
||||
2. **Third-party Libraries**: Uses system libraries where possible, builds others from source
|
||||
3. **Rust Components**: Properly integrates Cargo builds for Rust components
|
||||
4. **Service Management**: Complete systemd integration with proper dependencies
|
||||
5. **Configuration**: Manages TOML configuration files for all components
|
||||
|
||||
## Contributing
|
||||
|
||||
When modifying the package:
|
||||
|
||||
1. Update version in `package.nix`
|
||||
2. Test build locally
|
||||
3. Run integration tests
|
||||
4. Update documentation
|
||||
|
||||
## License
|
||||
|
||||
MIT License (same as 3FS project)
|
||||
274
flake.nix
Normal file
274
flake.nix
Normal file
@ -0,0 +1,274 @@
|
||||
{
|
||||
description = "Fire-Flyer File System (3FS) - High-performance distributed file system for AI workloads";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, flake-utils }:
|
||||
let
|
||||
overlay = final: prev: {
|
||||
"3fs" = prev.callPackage ./package.nix { };
|
||||
};
|
||||
|
||||
nixosModule = { config, lib, pkgs, ... }:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.services."3fs";
|
||||
|
||||
# Helper function to create service configuration
|
||||
mkServiceConfig = component: {
|
||||
description = "3FS ${component} service";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" ];
|
||||
requires = [ "network-online.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${pkgs."3fs"}/bin/${component}_main --launcher_cfg ${cfg.configDir}/${component}_main_launcher.toml --app-cfg ${cfg.configDir}/${component}_main_app.toml";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
LimitNOFILE = 1000000;
|
||||
User = cfg.user;
|
||||
Group = cfg.group;
|
||||
} // (if component == "storage" then {
|
||||
LimitMEMLOCK = "infinity";
|
||||
TimeoutStopSec = "5m";
|
||||
} else {});
|
||||
|
||||
environment = {
|
||||
LD_LIBRARY_PATH = "${pkgs."3fs"}/lib:${pkgs.foundationdb}/lib";
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services."3fs" = {
|
||||
enable = mkEnableOption "3FS distributed file system";
|
||||
|
||||
user = mkOption {
|
||||
type = types.str;
|
||||
default = "threefs";
|
||||
description = "User under which 3FS services run";
|
||||
};
|
||||
|
||||
group = mkOption {
|
||||
type = types.str;
|
||||
default = "threefs";
|
||||
description = "Group under which 3FS services run";
|
||||
};
|
||||
|
||||
configDir = mkOption {
|
||||
type = types.path;
|
||||
default = "/etc/3fs";
|
||||
description = "Directory containing 3FS configuration files";
|
||||
};
|
||||
|
||||
dataDir = mkOption {
|
||||
type = types.path;
|
||||
default = "/var/lib/3fs";
|
||||
description = "Directory for 3FS data storage";
|
||||
};
|
||||
|
||||
meta = {
|
||||
enable = mkEnableOption "3FS metadata service";
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for meta service";
|
||||
};
|
||||
};
|
||||
|
||||
storage = {
|
||||
enable = mkEnableOption "3FS storage service";
|
||||
|
||||
targets = mkOption {
|
||||
type = types.listOf types.str;
|
||||
default = [ "/var/lib/3fs/storage" ];
|
||||
description = "List of storage target directories";
|
||||
};
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for storage service";
|
||||
};
|
||||
};
|
||||
|
||||
mgmtd = {
|
||||
enable = mkEnableOption "3FS management daemon";
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for management daemon";
|
||||
};
|
||||
};
|
||||
|
||||
monitor = {
|
||||
enable = mkEnableOption "3FS monitor collector";
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for monitor collector";
|
||||
};
|
||||
};
|
||||
|
||||
fuse = {
|
||||
enable = mkEnableOption "3FS FUSE client";
|
||||
|
||||
mountPoint = mkOption {
|
||||
type = types.path;
|
||||
default = "/mnt/3fs";
|
||||
description = "Mount point for 3FS FUSE filesystem";
|
||||
};
|
||||
|
||||
config = mkOption {
|
||||
type = types.attrs;
|
||||
default = {};
|
||||
description = "Additional configuration for FUSE client";
|
||||
};
|
||||
};
|
||||
|
||||
foundationdb = {
|
||||
clusterFile = mkOption {
|
||||
type = types.path;
|
||||
default = "/etc/foundationdb/fdb.cluster";
|
||||
description = "Path to FoundationDB cluster file";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
# Create system user and group
|
||||
users.users.${cfg.user} = {
|
||||
isSystemUser = true;
|
||||
group = cfg.group;
|
||||
home = cfg.dataDir;
|
||||
createHome = true;
|
||||
description = "3FS system user";
|
||||
};
|
||||
|
||||
users.groups.${cfg.group} = {};
|
||||
|
||||
# Create necessary directories
|
||||
systemd.tmpfiles.rules = [
|
||||
"d '${cfg.configDir}' 0755 ${cfg.user} ${cfg.group} -"
|
||||
"d '${cfg.dataDir}' 0755 ${cfg.user} ${cfg.group} -"
|
||||
"d '${cfg.dataDir}/meta' 0755 ${cfg.user} ${cfg.group} -"
|
||||
"d '${cfg.dataDir}/mgmtd' 0755 ${cfg.user} ${cfg.group} -"
|
||||
"d '${cfg.dataDir}/monitor' 0755 ${cfg.user} ${cfg.group} -"
|
||||
] ++ (map (target: "d '${target}' 0755 ${cfg.user} ${cfg.group} -") cfg.storage.targets);
|
||||
|
||||
# Install default configuration files
|
||||
environment.etc = {
|
||||
"3fs/meta_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/meta_main_launcher.toml";
|
||||
"3fs/meta_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/meta_main_app.toml";
|
||||
"3fs/storage_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/storage_main_launcher.toml";
|
||||
"3fs/storage_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/storage_main_app.toml";
|
||||
"3fs/mgmtd_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/mgmtd_main_launcher.toml";
|
||||
"3fs/mgmtd_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/mgmtd_main_app.toml";
|
||||
"3fs/monitor_collector_main.toml".source = "${pkgs."3fs"}/etc/3fs/monitor_collector_main.toml";
|
||||
"3fs/hf3fs_fuse_main_launcher.toml".source = "${pkgs."3fs"}/etc/3fs/hf3fs_fuse_main_launcher.toml";
|
||||
"3fs/hf3fs_fuse_main_app.toml".source = "${pkgs."3fs"}/etc/3fs/hf3fs_fuse_main_app.toml";
|
||||
};
|
||||
|
||||
# Define systemd services
|
||||
systemd.services = {
|
||||
"3fs-meta" = mkIf cfg.meta.enable (mkServiceConfig "meta");
|
||||
"3fs-storage" = mkIf cfg.storage.enable (mkServiceConfig "storage");
|
||||
"3fs-mgmtd" = mkIf cfg.mgmtd.enable (mkServiceConfig "mgmtd");
|
||||
"3fs-monitor" = mkIf cfg.monitor.enable (mkServiceConfig "monitor_collector");
|
||||
|
||||
"3fs-fuse" = mkIf cfg.fuse.enable {
|
||||
description = "3FS FUSE client";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-online.target" "3fs-meta.service" ];
|
||||
requires = [ "network-online.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "simple";
|
||||
ExecStart = "${pkgs."3fs"}/bin/hf3fs_fuse_main --launcher_cfg ${cfg.configDir}/hf3fs_fuse_main_launcher.toml --app-cfg ${cfg.configDir}/hf3fs_fuse_main_app.toml ${cfg.fuse.mountPoint}";
|
||||
ExecStop = "${pkgs.fuse3}/bin/fusermount3 -u ${cfg.fuse.mountPoint}";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 5;
|
||||
LimitNOFILE = 1000000;
|
||||
User = "root"; # FUSE requires root
|
||||
Group = "root";
|
||||
};
|
||||
|
||||
preStart = ''
|
||||
mkdir -p ${cfg.fuse.mountPoint}
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# Add 3fs package to system packages
|
||||
environment.systemPackages = [ pkgs."3fs" ];
|
||||
};
|
||||
};
|
||||
in
|
||||
flake-utils.lib.eachDefaultSystem (system:
|
||||
let
|
||||
pkgs = import nixpkgs {
|
||||
inherit system;
|
||||
overlays = [ overlay ];
|
||||
};
|
||||
in
|
||||
{
|
||||
packages = {
|
||||
default = pkgs."3fs";
|
||||
"3fs" = pkgs."3fs";
|
||||
};
|
||||
|
||||
overlays.default = overlay;
|
||||
|
||||
devShells.default = pkgs.mkShell {
|
||||
inputsFrom = [ pkgs."3fs" ];
|
||||
buildInputs = with pkgs; [
|
||||
# Additional dev tools
|
||||
clang-tools_14
|
||||
cmake-format
|
||||
cmake-language-server
|
||||
gdb
|
||||
valgrind
|
||||
perf-tools
|
||||
rust-analyzer
|
||||
cargo-watch
|
||||
];
|
||||
|
||||
shellHook = ''
|
||||
echo "3FS development environment"
|
||||
echo "Build with: nix build .#3fs"
|
||||
echo "Enter shell with: nix develop"
|
||||
echo ""
|
||||
echo "To test locally:"
|
||||
echo " 1. Start FoundationDB"
|
||||
echo " 2. Configure and start 3FS services"
|
||||
echo " 3. Mount FUSE filesystem"
|
||||
'';
|
||||
};
|
||||
checks = {
|
||||
# Package build test
|
||||
package = pkgs."3fs";
|
||||
|
||||
# Integration test using NixOS VM
|
||||
integration-test = pkgs.nixosTest (import ./nixos-module-test.nix {
|
||||
inherit pkgs lib;
|
||||
self = self;
|
||||
});
|
||||
};
|
||||
}
|
||||
) // {
|
||||
nixosModules.default = nixosModule;
|
||||
nixosModules."3fs" = nixosModule;
|
||||
|
||||
# Hydra job for CI
|
||||
hydraJobs = {
|
||||
inherit (self) packages;
|
||||
tests = self.checks;
|
||||
};
|
||||
};
|
||||
}
|
||||
378
package.nix
Normal file
378
package.nix
Normal file
@ -0,0 +1,378 @@
|
||||
{ stdenv
|
||||
, lib
|
||||
, fetchFromGitHub
|
||||
, runCommand
|
||||
, writeText
|
||||
, makeWrapper
|
||||
, cmake
|
||||
, pkg-config
|
||||
, clang_14
|
||||
, lld_14
|
||||
, rustc
|
||||
, cargo
|
||||
, rustPlatform
|
||||
, autoconf
|
||||
, automake
|
||||
, libtool
|
||||
, python3
|
||||
, boost
|
||||
, libuv
|
||||
, lz4
|
||||
, xz
|
||||
, double-conversion
|
||||
, libdwarf
|
||||
, libunwind
|
||||
, libaio
|
||||
, gflags
|
||||
, glog
|
||||
, gtest
|
||||
, gperftools
|
||||
, openssl
|
||||
, fuse3
|
||||
, foundationdb
|
||||
, rdma-core
|
||||
, libibverbs
|
||||
, zstd
|
||||
, jemalloc
|
||||
, libevent
|
||||
, numactl
|
||||
, liburing
|
||||
, thrift
|
||||
, bison
|
||||
, flex
|
||||
, git
|
||||
, which
|
||||
, fmt
|
||||
, folly
|
||||
, rocksdb
|
||||
, leveldb
|
||||
, arrow-cpp
|
||||
, mimalloc
|
||||
, clickhouse-cpp
|
||||
, tomlplusplus
|
||||
, scnlib
|
||||
}:
|
||||
|
||||
let
|
||||
# Build jemalloc with specific configuration for 3FS
|
||||
jemalloc-custom = jemalloc.overrideAttrs (oldAttrs: {
|
||||
configureFlags = (oldAttrs.configureFlags or []) ++ [
|
||||
"--disable-cxx"
|
||||
"--enable-prof"
|
||||
"--disable-initial-exec-tls"
|
||||
];
|
||||
});
|
||||
|
||||
# Custom FoundationDB build if needed
|
||||
fdb-version = "7.1.5";
|
||||
|
||||
in stdenv.mkDerivation rec {
|
||||
pname = "3fs";
|
||||
version = "0.1.5";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "deepseek-ai";
|
||||
repo = "3fs";
|
||||
rev = "91bfcf39a9e4b5ded959f7b5c2cb0cf858ebbff5";
|
||||
sha256 = "sha256-0000000000000000000000000000000000000000000="; # To be replaced
|
||||
fetchSubmodules = true;
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
cmake
|
||||
pkg-config
|
||||
clang_14
|
||||
lld_14
|
||||
rustc
|
||||
cargo
|
||||
rustPlatform.cargoSetupHook
|
||||
autoconf
|
||||
automake
|
||||
libtool
|
||||
python3
|
||||
git
|
||||
which
|
||||
bison
|
||||
flex
|
||||
];
|
||||
|
||||
buildInputs = [
|
||||
# Core dependencies
|
||||
boost
|
||||
libuv
|
||||
lz4
|
||||
xz
|
||||
double-conversion
|
||||
libdwarf
|
||||
libunwind
|
||||
libaio
|
||||
gflags
|
||||
glog
|
||||
gtest
|
||||
gperftools
|
||||
openssl
|
||||
|
||||
# FUSE 3
|
||||
fuse3
|
||||
|
||||
# FoundationDB
|
||||
foundationdb
|
||||
|
||||
# RDMA/InfiniBand
|
||||
rdma-core
|
||||
libibverbs
|
||||
|
||||
# Additional libraries
|
||||
zstd
|
||||
jemalloc-custom
|
||||
libevent
|
||||
numactl
|
||||
liburing
|
||||
thrift
|
||||
fmt
|
||||
folly
|
||||
rocksdb
|
||||
leveldb
|
||||
mimalloc
|
||||
tomlplusplus
|
||||
|
||||
# Python bindings
|
||||
python3.pkgs.pybind11
|
||||
];
|
||||
|
||||
patches = [
|
||||
# Add any necessary patches here
|
||||
];
|
||||
|
||||
postPatch = ''
|
||||
# Apply the repository patches
|
||||
patchShebangs ./patches/apply.sh
|
||||
./patches/apply.sh || true
|
||||
|
||||
# Fix CMake to use system libraries
|
||||
substituteInPlace CMakeLists.txt \
|
||||
--replace 'add_link_options(-fuse-ld=lld)' '# add_link_options(-fuse-ld=lld)' \
|
||||
--replace 'set(CMAKE_CXX_FLAGS "$' 'set(CMAKE_CXX_FLAGS "$'
|
||||
|
||||
# Disable bundled third-party libraries
|
||||
for lib in fmt zstd googletest folly leveldb rocksdb scnlib pybind11 toml11 mimalloc clickhouse-cpp liburing-cmake; do
|
||||
substituteInPlace CMakeLists.txt \
|
||||
--replace "add_subdirectory(\"third_party/$lib\"" "#add_subdirectory(\"third_party/$lib\""
|
||||
done
|
||||
|
||||
# Fix jemalloc build
|
||||
substituteInPlace cmake/Jemalloc.cmake \
|
||||
--replace 'ExternalProject_add(' 'return() #ExternalProject_add(' \
|
||||
--replace '${JEMALLOC_DIR}/lib/libjemalloc.so.2' '${jemalloc-custom}/lib/libjemalloc.so'
|
||||
|
||||
# Fix Apache Arrow build
|
||||
substituteInPlace cmake/ApacheArrow.cmake \
|
||||
--replace 'ExternalProject_Add(' 'return() #ExternalProject_Add('
|
||||
|
||||
# Set up Rust dependencies
|
||||
cd src/client/trash_cleaner
|
||||
cargoDepsCopy="$NIX_BUILD_TOP/cargo-vendor-dir"
|
||||
if [ -d "$cargoDepsCopy" ]; then
|
||||
chmod -R +w "$cargoDepsCopy"
|
||||
fi
|
||||
cd $NIX_BUILD_TOP/source
|
||||
|
||||
# Fix storage chunk_engine Rust build
|
||||
cd src/storage/chunk_engine
|
||||
cargoDepsCopy="$NIX_BUILD_TOP/cargo-vendor-dir"
|
||||
if [ -d "$cargoDepsCopy" ]; then
|
||||
chmod -R +w "$cargoDepsCopy"
|
||||
fi
|
||||
cd $NIX_BUILD_TOP/source
|
||||
'';
|
||||
|
||||
preConfigure = ''
|
||||
# Create necessary directories
|
||||
mkdir -p $TMP/third_party
|
||||
|
||||
# Setup environment
|
||||
export HOME=$TMP
|
||||
export CARGO_HOME=$TMP/.cargo
|
||||
|
||||
# Set up library paths
|
||||
export BOOST_ROOT=${boost}
|
||||
export BOOST_INCLUDEDIR=${boost}/include
|
||||
export BOOST_LIBRARYDIR=${boost}/lib
|
||||
|
||||
# FoundationDB paths
|
||||
export FDB_LIBRARY_DIR=${foundationdb}/lib
|
||||
export FDB_INCLUDE_DIR=${foundationdb}/include
|
||||
|
||||
# Jemalloc paths
|
||||
export JEMALLOC_OVERRIDE=${jemalloc-custom}/lib/libjemalloc.so
|
||||
export JEMALLOC_DIR=${jemalloc-custom}
|
||||
|
||||
# Compiler flags
|
||||
export CXXFLAGS="-I${boost}/include -I${gtest}/include -I${liburing}/include -I${fmt}/include -I${folly}/include"
|
||||
export LDFLAGS="-L${boost}/lib -L${jemalloc-custom}/lib -L${liburing}/lib -L${fmt}/lib -L${folly}/lib"
|
||||
'';
|
||||
|
||||
cmakeFlags = [
|
||||
"-DCMAKE_CXX_COMPILER=${clang_14}/bin/clang++"
|
||||
"-DCMAKE_C_COMPILER=${clang_14}/bin/clang"
|
||||
"-DCMAKE_BUILD_TYPE=RelWithDebInfo"
|
||||
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
|
||||
"-DENABLE_FUSE_APPLICATION=ON"
|
||||
"-DOVERRIDE_CXX_NEW_DELETE=OFF"
|
||||
"-DSAVE_ALLOCATE_SIZE=OFF"
|
||||
"-DBoost_USE_STATIC_LIBS=ON"
|
||||
"-DFDB_VERSION=${fdb-version}"
|
||||
|
||||
# System library paths
|
||||
"-DCMAKE_PREFIX_PATH=${boost};${gtest};${liburing};${fmt};${folly}"
|
||||
"-DBoost_DIR=${boost}/lib/cmake/Boost"
|
||||
"-DBoost_INCLUDE_DIR=${boost}/include"
|
||||
"-Dfmt_DIR=${fmt}/lib/cmake/fmt"
|
||||
"-DFolly_DIR=${folly}/lib/cmake/folly"
|
||||
"-DGTest_DIR=${gtest}/lib/cmake/GTest"
|
||||
"-DZSTD_LIBRARY=${zstd}/lib/libzstd.so"
|
||||
"-DZSTD_INCLUDE_DIR=${zstd}/include"
|
||||
"-Djemalloc_INCLUDE_DIR=${jemalloc-custom}/include"
|
||||
"-Djemalloc_LIBRARY=${jemalloc-custom}/lib/libjemalloc.so"
|
||||
"-DJEMALLOC_DIR=${jemalloc-custom}"
|
||||
"-DARROW_INCLUDE_DIR=${arrow-cpp}/include"
|
||||
"-DARROW_LIB_DIR=${arrow-cpp}/lib"
|
||||
"-Darrow_DIR=${arrow-cpp}/lib/cmake/arrow"
|
||||
|
||||
# Initially disable tests to simplify build
|
||||
"-DBUILD_TESTING=OFF"
|
||||
];
|
||||
|
||||
# Custom build phase to handle complex build process
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
|
||||
# Build with decreasing parallelism if needed
|
||||
cmake --build . -j $NIX_BUILD_CORES || \
|
||||
cmake --build . -j 4 || \
|
||||
cmake --build . -j 1
|
||||
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
# Create output directories
|
||||
mkdir -p $out/bin $out/lib $out/etc/3fs
|
||||
|
||||
# Install binaries
|
||||
for bin in \
|
||||
src/fuse/hf3fs_fuse \
|
||||
src/client/cli/admin/admin \
|
||||
src/meta/meta \
|
||||
src/mgmtd/mgmtd \
|
||||
src/storage/storage \
|
||||
src/monitor_collector/monitor_collector \
|
||||
src/tools/admin
|
||||
do
|
||||
if [ -f "$bin" ] && [ -x "$bin" ]; then
|
||||
install -D -m755 "$bin" "$out/bin/$(basename $bin)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Rename main binaries to match systemd service expectations
|
||||
[ -f "$out/bin/meta" ] && mv "$out/bin/meta" "$out/bin/meta_main"
|
||||
[ -f "$out/bin/mgmtd" ] && mv "$out/bin/mgmtd" "$out/bin/mgmtd_main"
|
||||
[ -f "$out/bin/storage" ] && mv "$out/bin/storage" "$out/bin/storage_main"
|
||||
[ -f "$out/bin/monitor_collector" ] && mv "$out/bin/monitor_collector" "$out/bin/monitor_collector_main"
|
||||
[ -f "$out/bin/hf3fs_fuse" ] && cp "$out/bin/hf3fs_fuse" "$out/bin/hf3fs_fuse_main"
|
||||
|
||||
# Install libraries
|
||||
find . -name "*.so" -o -name "*.so.*" | while read lib; do
|
||||
if [ -f "$lib" ]; then
|
||||
install -D -m644 "$lib" "$out/lib/$(basename $lib)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Install static libraries (optional)
|
||||
find . -name "*.a" | while read lib; do
|
||||
if [ -f "$lib" ] && [[ ! "$lib" =~ third_party ]]; then
|
||||
install -D -m644 "$lib" "$out/lib/$(basename $lib)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Install Python modules
|
||||
if [ -d "src/lib/py" ]; then
|
||||
mkdir -p $out/lib/python${python3.pythonVersion}/site-packages
|
||||
cp -r src/lib/py/* $out/lib/python${python3.pythonVersion}/site-packages/
|
||||
fi
|
||||
|
||||
# Install configuration files
|
||||
cp -r configs/* $out/etc/3fs/
|
||||
|
||||
# Install headers (for development)
|
||||
mkdir -p $out/include/3fs
|
||||
find src -name "*.h" -o -name "*.hpp" | while read header; do
|
||||
rel_path=$(echo "$header" | sed 's|^src/||')
|
||||
install -D -m644 "$header" "$out/include/3fs/$rel_path"
|
||||
done
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
# Fix RPATH for all binaries
|
||||
for exe in $out/bin/*; do
|
||||
if [ -f "$exe" ] && [ -x "$exe" ]; then
|
||||
patchelf --set-rpath "$out/lib:${lib.makeLibraryPath buildInputs}" "$exe" || true
|
||||
fi
|
||||
done
|
||||
|
||||
# Fix RPATH for all libraries
|
||||
for lib in $out/lib/*.so*; do
|
||||
if [ -f "$lib" ]; then
|
||||
patchelf --set-rpath "$out/lib:${lib.makeLibraryPath buildInputs}" "$lib" || true
|
||||
fi
|
||||
done
|
||||
|
||||
# Create wrapper scripts if needed
|
||||
for bin in $out/bin/*_main; do
|
||||
if [ -f "$bin" ]; then
|
||||
makeWrapper "$bin" "$bin.wrapped" \
|
||||
--prefix LD_LIBRARY_PATH : "$out/lib:${lib.makeLibraryPath buildInputs}" \
|
||||
--set RUST_BACKTRACE 1
|
||||
mv "$bin.wrapped" "$bin"
|
||||
fi
|
||||
done
|
||||
'';
|
||||
|
||||
# Enable parallel building
|
||||
enableParallelBuilding = true;
|
||||
NIX_BUILD_CORES = 8;
|
||||
|
||||
# Set up proper library paths
|
||||
setupHook = writeText "setup-hook" ''
|
||||
export LD_LIBRARY_PATH="$1/lib''${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH"
|
||||
'';
|
||||
|
||||
passthru = {
|
||||
inherit jemalloc-custom;
|
||||
tests = {
|
||||
# Add package tests here
|
||||
version = runCommand "3fs-version-test" {} ''
|
||||
${pname}/bin/admin --version
|
||||
touch $out
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
meta = with lib; {
|
||||
description = "Fire-Flyer File System - High-performance distributed file system for AI workloads";
|
||||
longDescription = ''
|
||||
3FS is a high-performance distributed file system designed to address the challenges
|
||||
of AI training and inference workloads. It leverages modern SSDs and RDMA networks
|
||||
to provide a shared storage layer that simplifies development of distributed applications.
|
||||
'';
|
||||
homepage = "https://github.com/deepseek-ai/3fs";
|
||||
license = licenses.mit;
|
||||
maintainers = with maintainers; [ ];
|
||||
platforms = platforms.linux;
|
||||
mainProgram = "admin";
|
||||
};
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user