mirror of
https://github.com/deepseek-ai/3FS
synced 2025-06-26 18:16:45 +00:00
fix inheritted inputs from flake.nix
This commit is contained in:
61
flake.lock
generated
Normal file
61
flake.lock
generated
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"flake-utils": {
|
||||||
|
"inputs": {
|
||||||
|
"systems": "systems"
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1731533236,
|
||||||
|
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1748190013,
|
||||||
|
"narHash": "sha256-R5HJFflOfsP5FBtk+zE8FpL8uqE7n62jqOsADvVshhE=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "62b852f6c6742134ade1abdd2a21685fd617a291",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixos-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-utils": "flake-utils",
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"systems": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1681028828,
|
||||||
|
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-systems",
|
||||||
|
"repo": "default",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
128
flake.nix
128
flake.nix
@@ -9,7 +9,52 @@
|
|||||||
outputs = { self, nixpkgs, flake-utils }:
|
outputs = { self, nixpkgs, flake-utils }:
|
||||||
let
|
let
|
||||||
overlay = final: prev: {
|
overlay = final: prev: {
|
||||||
"3fs" = prev.callPackage ./package.nix { };
|
# Build clickhouse-cpp from source if not available in nixpkgs
|
||||||
|
clickhouse-cpp = prev.clickhouse-cpp or (prev.stdenv.mkDerivation rec {
|
||||||
|
pname = "clickhouse-cpp";
|
||||||
|
version = "2.5.1";
|
||||||
|
|
||||||
|
src = prev.fetchFromGitHub {
|
||||||
|
owner = "ClickHouse";
|
||||||
|
repo = "clickhouse-cpp";
|
||||||
|
rev = "v${version}";
|
||||||
|
sha256 = "sha256-6kqcANO4S9Z1ee4kBPKGCnsPEGDaWPCx2hUi4APPWHU=";
|
||||||
|
};
|
||||||
|
|
||||||
|
nativeBuildInputs = [ prev.cmake ];
|
||||||
|
buildInputs = [ prev.zlib prev.openssl prev.lz4 ];
|
||||||
|
|
||||||
|
cmakeFlags = [
|
||||||
|
"-DBUILD_SHARED_LIBS=ON"
|
||||||
|
"-DWITH_OPENSSL=ON"
|
||||||
|
];
|
||||||
|
});
|
||||||
|
|
||||||
|
# Build scnlib from source if not available in nixpkgs
|
||||||
|
scnlib = prev.scnlib or (prev.stdenv.mkDerivation rec {
|
||||||
|
pname = "scnlib";
|
||||||
|
version = "2.0.2";
|
||||||
|
|
||||||
|
src = prev.fetchFromGitHub {
|
||||||
|
owner = "eliaskosunen";
|
||||||
|
repo = "scnlib";
|
||||||
|
rev = "v${version}";
|
||||||
|
sha256 = "sha256-YWlJiHAKKJd7jWv8Z0GmKqIfXI3HwVqA7AgZiHN2W8I=";
|
||||||
|
};
|
||||||
|
|
||||||
|
nativeBuildInputs = [ prev.cmake ];
|
||||||
|
|
||||||
|
cmakeFlags = [
|
||||||
|
"-DSCN_TESTS=OFF"
|
||||||
|
"-DSCN_EXAMPLES=OFF"
|
||||||
|
"-DSCN_BENCHMARKS=OFF"
|
||||||
|
];
|
||||||
|
});
|
||||||
|
|
||||||
|
"3fs" = prev.callPackage ./package.nix {
|
||||||
|
inherit (final) clickhouse-cpp scnlib;
|
||||||
|
libibverbs = prev.rdma-core;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
nixosModule = { config, lib, pkgs, ... }:
|
nixosModule = { config, lib, pkgs, ... }:
|
||||||
@@ -216,32 +261,79 @@
|
|||||||
inherit system;
|
inherit system;
|
||||||
overlays = [ overlay ];
|
overlays = [ overlay ];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
isLinux = pkgs.stdenv.isLinux;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
packages = {
|
packages = pkgs.lib.optionalAttrs isLinux {
|
||||||
default = pkgs."3fs";
|
default = pkgs."3fs";
|
||||||
"3fs" = pkgs."3fs";
|
"3fs" = pkgs."3fs";
|
||||||
};
|
};
|
||||||
|
|
||||||
overlays.default = overlay;
|
# Don't export overlays per-system
|
||||||
|
|
||||||
devShells.default = pkgs.mkShell {
|
devShells.default = pkgs.mkShell {
|
||||||
inputsFrom = [ pkgs."3fs" ];
|
inputsFrom = pkgs.lib.optionals isLinux [ pkgs."3fs" ];
|
||||||
buildInputs = with pkgs; [
|
buildInputs = with pkgs; [
|
||||||
|
# Build dependencies that are cross-platform
|
||||||
|
cmake
|
||||||
|
pkg-config
|
||||||
|
boost
|
||||||
|
protobuf
|
||||||
|
grpc
|
||||||
|
gtest
|
||||||
|
glog
|
||||||
|
lz4
|
||||||
|
zlib
|
||||||
|
openssl
|
||||||
|
zstd
|
||||||
|
jemalloc
|
||||||
|
libevent
|
||||||
|
thrift
|
||||||
|
bison
|
||||||
|
flex
|
||||||
|
git
|
||||||
|
which
|
||||||
|
fmt
|
||||||
|
folly
|
||||||
|
rocksdb
|
||||||
|
leveldb
|
||||||
|
arrow-cpp
|
||||||
|
mimalloc
|
||||||
|
tomlplusplus
|
||||||
|
rustc
|
||||||
|
cargo
|
||||||
|
rustfmt
|
||||||
|
clippy
|
||||||
|
rustPlatform.bindgenHook
|
||||||
|
clickhouse-cpp
|
||||||
|
scnlib
|
||||||
|
|
||||||
# Additional dev tools
|
# Additional dev tools
|
||||||
clang-tools_14
|
clang-tools_14
|
||||||
cmake-format
|
cmake-format
|
||||||
cmake-language-server
|
cmake-language-server
|
||||||
gdb
|
gdb
|
||||||
valgrind
|
|
||||||
perf-tools
|
|
||||||
rust-analyzer
|
rust-analyzer
|
||||||
cargo-watch
|
cargo-watch
|
||||||
|
] ++ pkgs.lib.optionals isLinux [
|
||||||
|
# Linux-only dependencies
|
||||||
|
foundationdb
|
||||||
|
fuse3
|
||||||
|
rdma-core
|
||||||
|
numactl
|
||||||
|
liburing
|
||||||
|
valgrind
|
||||||
|
perf-tools
|
||||||
];
|
];
|
||||||
|
|
||||||
shellHook = ''
|
shellHook = ''
|
||||||
echo "3FS development environment"
|
echo "3FS development environment"
|
||||||
echo "Build with: nix build .#3fs"
|
${if isLinux then ''
|
||||||
|
echo "Build with: nix build .#3fs"
|
||||||
|
'' else ''
|
||||||
|
echo "Note: 3FS can only be built on Linux systems"
|
||||||
|
''}
|
||||||
echo "Enter shell with: nix develop"
|
echo "Enter shell with: nix develop"
|
||||||
echo ""
|
echo ""
|
||||||
echo "To test locally:"
|
echo "To test locally:"
|
||||||
@@ -250,14 +342,14 @@
|
|||||||
echo " 3. Mount FUSE filesystem"
|
echo " 3. Mount FUSE filesystem"
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
checks = {
|
checks = pkgs.lib.optionalAttrs isLinux {
|
||||||
# Package build test
|
# Package build test
|
||||||
package = pkgs."3fs";
|
package = pkgs."3fs";
|
||||||
|
} // pkgs.lib.optionalAttrs isLinux {
|
||||||
# Integration test using NixOS VM
|
# Integration test using NixOS VM (Linux only)
|
||||||
integration-test = pkgs.nixosTest (import ./nixos-module-test.nix {
|
integration-test = pkgs.nixosTest (import ./nixos-module-test.nix {
|
||||||
inherit pkgs lib;
|
inherit pkgs self;
|
||||||
self = self;
|
lib = pkgs.lib;
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -265,10 +357,12 @@
|
|||||||
nixosModules.default = nixosModule;
|
nixosModules.default = nixosModule;
|
||||||
nixosModules."3fs" = nixosModule;
|
nixosModules."3fs" = nixosModule;
|
||||||
|
|
||||||
# Hydra job for CI
|
overlays.default = overlay;
|
||||||
hydraJobs = {
|
|
||||||
inherit (self) packages;
|
# Hydra job for CI - only for Linux systems
|
||||||
tests = self.checks;
|
hydraJobs = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ] (system: {
|
||||||
};
|
packages = self.packages.${system};
|
||||||
|
tests = self.checks.${system} or {};
|
||||||
|
});
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
# NixOS VM test for 3FS services
|
# NixOS VM test for 3FS services
|
||||||
# Run with: nix build .#checks.x86_64-linux.3fs-integration-test
|
# Run with: nix build .#checks.x86_64-linux.3fs-integration-test
|
||||||
|
|
||||||
{ pkgs, lib, ... }:
|
{ pkgs, lib, self, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
# Test configuration for 3FS cluster
|
# Test configuration for 3FS cluster
|
||||||
@@ -23,11 +23,6 @@ let
|
|||||||
port = 8000;
|
port = 8000;
|
||||||
dataDir = "/var/lib/3fs/mgmtd";
|
dataDir = "/var/lib/3fs/mgmtd";
|
||||||
};
|
};
|
||||||
|
|
||||||
foundationdb = {
|
|
||||||
clusterFile = "/etc/foundationdb/fdb.cluster";
|
|
||||||
clusterString = "test:test@127.0.0.1:4500";
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
in {
|
in {
|
||||||
@@ -71,23 +66,16 @@ in {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
foundationdb = {
|
# FoundationDB configuration is handled by the system service
|
||||||
clusterFile = testConfig.foundationdb.clusterFile;
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# Configure FoundationDB
|
# Configure FoundationDB
|
||||||
services.foundationdb = {
|
services.foundationdb = {
|
||||||
enable = true;
|
enable = true;
|
||||||
clusterFile = testConfig.foundationdb.clusterFile;
|
package = pkgs.foundationdb;
|
||||||
listenAddress = "127.0.0.1:4500";
|
listenAddress = "127.0.0.1:4500";
|
||||||
dataDir = "/var/lib/foundationdb";
|
dataDir = "/var/lib/foundationdb";
|
||||||
logDir = "/var/log/foundationdb";
|
logDir = "/var/log/foundationdb";
|
||||||
|
|
||||||
# Initialize the database
|
|
||||||
initialScript = ''
|
|
||||||
configure single ssd
|
|
||||||
'';
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# Open firewall ports
|
# Open firewall ports
|
||||||
@@ -117,21 +105,18 @@ in {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
foundationdb = {
|
# FoundationDB configuration is handled by the system service
|
||||||
clusterFile = testConfig.foundationdb.clusterFile;
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# Configure FoundationDB client
|
# Configure FoundationDB client
|
||||||
services.foundationdb = {
|
services.foundationdb = {
|
||||||
enable = true;
|
enable = true;
|
||||||
clusterFile = testConfig.foundationdb.clusterFile;
|
package = pkgs.foundationdb;
|
||||||
pidFile = "/run/foundationdb.pid";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# Write cluster file pointing to master
|
# Write cluster file pointing to master
|
||||||
environment.etc."foundationdb/fdb.cluster" = {
|
environment.etc."foundationdb/fdb.cluster" = {
|
||||||
text = testConfig.foundationdb.clusterString;
|
text = "test:test@master:4500";
|
||||||
mode = "0644";
|
mode = "0644";
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -139,7 +124,6 @@ in {
|
|||||||
|
|
||||||
environment.systemPackages = with pkgs; [
|
environment.systemPackages = with pkgs; [
|
||||||
fio
|
fio
|
||||||
iozone
|
|
||||||
sysbench
|
sysbench
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
@@ -151,7 +135,7 @@ in {
|
|||||||
# Wait for FoundationDB to be ready
|
# Wait for FoundationDB to be ready
|
||||||
master.wait_for_unit("foundationdb.service")
|
master.wait_for_unit("foundationdb.service")
|
||||||
master.wait_for_open_port(4500)
|
master.wait_for_open_port(4500)
|
||||||
master.succeed("fdbcli --exec 'status' -C ${testConfig.foundationdb.clusterFile}")
|
master.succeed("fdbcli --exec 'status' -C /etc/foundationdb/fdb.cluster")
|
||||||
|
|
||||||
# Wait for 3FS services to start
|
# Wait for 3FS services to start
|
||||||
master.wait_for_unit("3fs-mgmtd.service")
|
master.wait_for_unit("3fs-mgmtd.service")
|
||||||
@@ -172,7 +156,7 @@ in {
|
|||||||
master.succeed("systemctl is-active 3fs-monitor.service")
|
master.succeed("systemctl is-active 3fs-monitor.service")
|
||||||
|
|
||||||
# Initialize 3FS cluster
|
# Initialize 3FS cluster
|
||||||
master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config ${testConfig.foundationdb.clusterFile}")
|
master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config /etc/foundationdb/fdb.cluster")
|
||||||
|
|
||||||
# Register storage nodes
|
# Register storage nodes
|
||||||
master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}")
|
master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}")
|
||||||
|
|||||||
@@ -163,7 +163,7 @@ in stdenv.mkDerivation rec {
|
|||||||
# Fix jemalloc build
|
# Fix jemalloc build
|
||||||
substituteInPlace cmake/Jemalloc.cmake \
|
substituteInPlace cmake/Jemalloc.cmake \
|
||||||
--replace 'ExternalProject_add(' 'return() #ExternalProject_add(' \
|
--replace 'ExternalProject_add(' 'return() #ExternalProject_add(' \
|
||||||
--replace '${JEMALLOC_DIR}/lib/libjemalloc.so.2' '${jemalloc-custom}/lib/libjemalloc.so'
|
--replace "\''${JEMALLOC_DIR}/lib/libjemalloc.so.2" "\''${jemalloc-custom}/lib/libjemalloc.so"
|
||||||
|
|
||||||
# Fix Apache Arrow build
|
# Fix Apache Arrow build
|
||||||
substituteInPlace cmake/ApacheArrow.cmake \
|
substituteInPlace cmake/ApacheArrow.cmake \
|
||||||
|
|||||||
Reference in New Issue
Block a user