mirror of
https://github.com/deepseek-ai/3FS
synced 2025-06-26 18:16:45 +00:00
fix inheritted inputs from flake.nix
This commit is contained in:
connerohnesorge
parent
c15454e153
commit
16468b0110
61
flake.lock
Normal file
61
flake.lock
Normal file
@ -0,0 +1,61 @@
|
||||
{
|
||||
"nodes": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1748190013,
|
||||
"narHash": "sha256-R5HJFflOfsP5FBtk+zE8FpL8uqE7n62jqOsADvVshhE=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "62b852f6c6742134ade1abdd2a21685fd617a291",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
130
flake.nix
130
flake.nix
@ -9,7 +9,52 @@
|
||||
outputs = { self, nixpkgs, flake-utils }:
|
||||
let
|
||||
overlay = final: prev: {
|
||||
"3fs" = prev.callPackage ./package.nix { };
|
||||
# Build clickhouse-cpp from source if not available in nixpkgs
|
||||
clickhouse-cpp = prev.clickhouse-cpp or (prev.stdenv.mkDerivation rec {
|
||||
pname = "clickhouse-cpp";
|
||||
version = "2.5.1";
|
||||
|
||||
src = prev.fetchFromGitHub {
|
||||
owner = "ClickHouse";
|
||||
repo = "clickhouse-cpp";
|
||||
rev = "v${version}";
|
||||
sha256 = "sha256-6kqcANO4S9Z1ee4kBPKGCnsPEGDaWPCx2hUi4APPWHU=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ prev.cmake ];
|
||||
buildInputs = [ prev.zlib prev.openssl prev.lz4 ];
|
||||
|
||||
cmakeFlags = [
|
||||
"-DBUILD_SHARED_LIBS=ON"
|
||||
"-DWITH_OPENSSL=ON"
|
||||
];
|
||||
});
|
||||
|
||||
# Build scnlib from source if not available in nixpkgs
|
||||
scnlib = prev.scnlib or (prev.stdenv.mkDerivation rec {
|
||||
pname = "scnlib";
|
||||
version = "2.0.2";
|
||||
|
||||
src = prev.fetchFromGitHub {
|
||||
owner = "eliaskosunen";
|
||||
repo = "scnlib";
|
||||
rev = "v${version}";
|
||||
sha256 = "sha256-YWlJiHAKKJd7jWv8Z0GmKqIfXI3HwVqA7AgZiHN2W8I=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ prev.cmake ];
|
||||
|
||||
cmakeFlags = [
|
||||
"-DSCN_TESTS=OFF"
|
||||
"-DSCN_EXAMPLES=OFF"
|
||||
"-DSCN_BENCHMARKS=OFF"
|
||||
];
|
||||
});
|
||||
|
||||
"3fs" = prev.callPackage ./package.nix {
|
||||
inherit (final) clickhouse-cpp scnlib;
|
||||
libibverbs = prev.rdma-core;
|
||||
};
|
||||
};
|
||||
|
||||
nixosModule = { config, lib, pkgs, ... }:
|
||||
@ -216,32 +261,79 @@
|
||||
inherit system;
|
||||
overlays = [ overlay ];
|
||||
};
|
||||
|
||||
isLinux = pkgs.stdenv.isLinux;
|
||||
in
|
||||
{
|
||||
packages = {
|
||||
packages = pkgs.lib.optionalAttrs isLinux {
|
||||
default = pkgs."3fs";
|
||||
"3fs" = pkgs."3fs";
|
||||
};
|
||||
|
||||
overlays.default = overlay;
|
||||
# Don't export overlays per-system
|
||||
|
||||
devShells.default = pkgs.mkShell {
|
||||
inputsFrom = [ pkgs."3fs" ];
|
||||
inputsFrom = pkgs.lib.optionals isLinux [ pkgs."3fs" ];
|
||||
buildInputs = with pkgs; [
|
||||
# Build dependencies that are cross-platform
|
||||
cmake
|
||||
pkg-config
|
||||
boost
|
||||
protobuf
|
||||
grpc
|
||||
gtest
|
||||
glog
|
||||
lz4
|
||||
zlib
|
||||
openssl
|
||||
zstd
|
||||
jemalloc
|
||||
libevent
|
||||
thrift
|
||||
bison
|
||||
flex
|
||||
git
|
||||
which
|
||||
fmt
|
||||
folly
|
||||
rocksdb
|
||||
leveldb
|
||||
arrow-cpp
|
||||
mimalloc
|
||||
tomlplusplus
|
||||
rustc
|
||||
cargo
|
||||
rustfmt
|
||||
clippy
|
||||
rustPlatform.bindgenHook
|
||||
clickhouse-cpp
|
||||
scnlib
|
||||
|
||||
# Additional dev tools
|
||||
clang-tools_14
|
||||
cmake-format
|
||||
cmake-language-server
|
||||
gdb
|
||||
valgrind
|
||||
perf-tools
|
||||
rust-analyzer
|
||||
cargo-watch
|
||||
] ++ pkgs.lib.optionals isLinux [
|
||||
# Linux-only dependencies
|
||||
foundationdb
|
||||
fuse3
|
||||
rdma-core
|
||||
numactl
|
||||
liburing
|
||||
valgrind
|
||||
perf-tools
|
||||
];
|
||||
|
||||
shellHook = ''
|
||||
echo "3FS development environment"
|
||||
echo "Build with: nix build .#3fs"
|
||||
${if isLinux then ''
|
||||
echo "Build with: nix build .#3fs"
|
||||
'' else ''
|
||||
echo "Note: 3FS can only be built on Linux systems"
|
||||
''}
|
||||
echo "Enter shell with: nix develop"
|
||||
echo ""
|
||||
echo "To test locally:"
|
||||
@ -250,14 +342,14 @@
|
||||
echo " 3. Mount FUSE filesystem"
|
||||
'';
|
||||
};
|
||||
checks = {
|
||||
checks = pkgs.lib.optionalAttrs isLinux {
|
||||
# Package build test
|
||||
package = pkgs."3fs";
|
||||
|
||||
# Integration test using NixOS VM
|
||||
} // pkgs.lib.optionalAttrs isLinux {
|
||||
# Integration test using NixOS VM (Linux only)
|
||||
integration-test = pkgs.nixosTest (import ./nixos-module-test.nix {
|
||||
inherit pkgs lib;
|
||||
self = self;
|
||||
inherit pkgs self;
|
||||
lib = pkgs.lib;
|
||||
});
|
||||
};
|
||||
}
|
||||
@ -265,10 +357,12 @@
|
||||
nixosModules.default = nixosModule;
|
||||
nixosModules."3fs" = nixosModule;
|
||||
|
||||
# Hydra job for CI
|
||||
hydraJobs = {
|
||||
inherit (self) packages;
|
||||
tests = self.checks;
|
||||
};
|
||||
overlays.default = overlay;
|
||||
|
||||
# Hydra job for CI - only for Linux systems
|
||||
hydraJobs = nixpkgs.lib.genAttrs [ "x86_64-linux" "aarch64-linux" ] (system: {
|
||||
packages = self.packages.${system};
|
||||
tests = self.checks.${system} or {};
|
||||
});
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# NixOS VM test for 3FS services
|
||||
# Run with: nix build .#checks.x86_64-linux.3fs-integration-test
|
||||
|
||||
{ pkgs, lib, ... }:
|
||||
{ pkgs, lib, self, ... }:
|
||||
|
||||
let
|
||||
# Test configuration for 3FS cluster
|
||||
@ -23,11 +23,6 @@ let
|
||||
port = 8000;
|
||||
dataDir = "/var/lib/3fs/mgmtd";
|
||||
};
|
||||
|
||||
foundationdb = {
|
||||
clusterFile = "/etc/foundationdb/fdb.cluster";
|
||||
clusterString = "test:test@127.0.0.1:4500";
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
@ -71,23 +66,16 @@ in {
|
||||
};
|
||||
};
|
||||
|
||||
foundationdb = {
|
||||
clusterFile = testConfig.foundationdb.clusterFile;
|
||||
};
|
||||
# FoundationDB configuration is handled by the system service
|
||||
};
|
||||
|
||||
# Configure FoundationDB
|
||||
services.foundationdb = {
|
||||
enable = true;
|
||||
clusterFile = testConfig.foundationdb.clusterFile;
|
||||
package = pkgs.foundationdb;
|
||||
listenAddress = "127.0.0.1:4500";
|
||||
dataDir = "/var/lib/foundationdb";
|
||||
logDir = "/var/log/foundationdb";
|
||||
|
||||
# Initialize the database
|
||||
initialScript = ''
|
||||
configure single ssd
|
||||
'';
|
||||
};
|
||||
|
||||
# Open firewall ports
|
||||
@ -117,21 +105,18 @@ in {
|
||||
};
|
||||
};
|
||||
|
||||
foundationdb = {
|
||||
clusterFile = testConfig.foundationdb.clusterFile;
|
||||
};
|
||||
# FoundationDB configuration is handled by the system service
|
||||
};
|
||||
|
||||
# Configure FoundationDB client
|
||||
services.foundationdb = {
|
||||
enable = true;
|
||||
clusterFile = testConfig.foundationdb.clusterFile;
|
||||
pidFile = "/run/foundationdb.pid";
|
||||
package = pkgs.foundationdb;
|
||||
};
|
||||
|
||||
# Write cluster file pointing to master
|
||||
environment.etc."foundationdb/fdb.cluster" = {
|
||||
text = testConfig.foundationdb.clusterString;
|
||||
text = "test:test@master:4500";
|
||||
mode = "0644";
|
||||
};
|
||||
|
||||
@ -139,7 +124,6 @@ in {
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
fio
|
||||
iozone
|
||||
sysbench
|
||||
];
|
||||
};
|
||||
@ -151,7 +135,7 @@ in {
|
||||
# Wait for FoundationDB to be ready
|
||||
master.wait_for_unit("foundationdb.service")
|
||||
master.wait_for_open_port(4500)
|
||||
master.succeed("fdbcli --exec 'status' -C ${testConfig.foundationdb.clusterFile}")
|
||||
master.succeed("fdbcli --exec 'status' -C /etc/foundationdb/fdb.cluster")
|
||||
|
||||
# Wait for 3FS services to start
|
||||
master.wait_for_unit("3fs-mgmtd.service")
|
||||
@ -172,7 +156,7 @@ in {
|
||||
master.succeed("systemctl is-active 3fs-monitor.service")
|
||||
|
||||
# Initialize 3FS cluster
|
||||
master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config ${testConfig.foundationdb.clusterFile}")
|
||||
master.succeed("${pkgs."3fs"}/bin/admin init-cluster --config /etc/foundationdb/fdb.cluster")
|
||||
|
||||
# Register storage nodes
|
||||
master.succeed("${pkgs."3fs"}/bin/admin register-node --type storage --address master:${toString testConfig.storage.port}")
|
||||
|
||||
@ -163,7 +163,7 @@ in stdenv.mkDerivation rec {
|
||||
# Fix jemalloc build
|
||||
substituteInPlace cmake/Jemalloc.cmake \
|
||||
--replace 'ExternalProject_add(' 'return() #ExternalProject_add(' \
|
||||
--replace '${JEMALLOC_DIR}/lib/libjemalloc.so.2' '${jemalloc-custom}/lib/libjemalloc.so'
|
||||
--replace "\''${JEMALLOC_DIR}/lib/libjemalloc.so.2" "\''${jemalloc-custom}/lib/libjemalloc.so"
|
||||
|
||||
# Fix Apache Arrow build
|
||||
substituteInPlace cmake/ApacheArrow.cmake \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user