diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..2443e26 --- /dev/null +++ b/build.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -e + +# Default build config +OS_TYPE="ubuntu2204" + +# Parse arguments +if [[ "$1" == "-h" || "$1" == "--help" ]]; then + cat < + +### Hardware Requirements +| Node | OS | IP Address | Memory | Storage | Networking | RDMA port| +|-----------|--------------|--------------|---------|------------------|------------|----------| +| Meta | Ubuntu 20.04 | 192.168.1.1 | 128GB | - | RoCE | mlx5_2 | +| Storage1 | Ubuntu 20.04 | 192.168.1.2 | 128GB | 2×7TB NVMe SSD | RoCE | mlx5_2 | +| Storage2 | Ubuntu 20.04 | 192.168.1.3 | 128GB | 2×7TB NVMe SSD | RoCE | mlx5_2 | + +> **RDMA Configuration** +> 1. Assign IP addresses to RDMA NICs. Multiple RDMA NICs (InfiniBand or RoCE) are supported on each node. +> 2. Check RDMA connectivity between nodes using `ib_write_bw`. + + +## Build Docker Images + +1. Clone repository and checkout source code: + ```bash + git clone https://github.com/your-org/3fs.git + cd 3fs + git submodule update --init --recursive + ./patches/apply.sh + ``` + +2. Build base environment: + ```bash + ./build.sh docker-ubuntu2004 + ``` + +3. Build component images: + ```bash + docker build -t hf3fs-monitor -f dockerfile/monitor.ubuntu2004.Dockerfile . + docker build -t hf3fs-mgmtd -f dockerfile/mgmtd.ubuntu2004.Dockerfile . + docker build -t hf3fs-meta -f dockerfile/meta.ubuntu2004.Dockerfile . + docker build -t hf3fs-storage -f dockerfile/storage.ubuntu2004.Dockerfile . + docker build -t hf3fs-fuse -f dockerfile/fuse.ubuntu2004.Dockerfile . + ``` + +## Deployment Steps + +### ClickHouse Service +```bash +# Pull official image +docker pull clickhouse/clickhouse-server:25.3.1.2703 + +# Start container +docker run -d \ + -p19000:9000 \ + -e CLICKHOUSE_PASSWORD=3fs \ + --name clickhouse-server \ + --ulimit nofile=262144:262144 \ + clickhouse/clickhouse-server:25.3.1.2703 + +# Copy 3fs-monitor.sql to clickhouse-server docker container +docker cp 3fs-monitor.sql clickhouse-server:/root + +# Login clickhouse-server docker container and import the SQL file into ClickHouse: +docker exec -it clickhouse-server /bin/bash +clickhouse-client -n < /root/3fs-monitor.sql +``` + +### FoundationDB Service +```bash +# Pull official image +docker pull foundationdb/foundationdb:7.3.63 + +# Start container +docker run -d \ + --privileged \ + --network host \ + -e FDB_NETWORKING_MODE=host \ + --name foundationdb-server \ + foundationdb/foundationdb:7.3.63 + +# Configure FoundationDB +docker exec -it foundationdb-server /bin/sh -c "echo 'export PUBLIC_IP=192.168.1.1' > /var/fdb/.fdbenv" +docker restart foundationdb-server + +# Initialize database +docker exec foundationdb-server /usr/bin/fdbcli -C /var/fdb/fdb.cluster --exec 'configure new single ssd' + +# Check status +docker exec foundationdb-server /usr/bin/fdbcli -C /var/fdb/fdb.cluster --exec 'status' +``` + +### Monitor Service +```bash +docker run --name hf3fs-monitor \ + --privileged \ + --network host \ + -d --restart always \ + --env CLICKHOUSE_DB=3fs \ + --env CLICKHOUSE_HOST=192.168.1.1 \ + --env CLICKHOUSE_PASSWD=3fs \ + --env CLICKHOUSE_PORT=19000 \ + --env CLICKHOUSE_USER=default \ + --env DEVICE_FILTER=mlx5_2 \ + hf3fs-monitor:latest +``` + +### Management Service +```bash +docker run --name hf3fs-mgmtd \ + --privileged \ + --network host \ + -d --restart always \ + --env CLUSTER_ID=stage \ + --env FDB_CLUSTER=docker:docker@192.168.1.1:4500 \ + --env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \ + --env MGMTD_NODE_ID=1 \ + --env DEVICE_FILTER=mlx5_2 \ + --env REMOTE_IP=192.168.1.1:10000 \ + hf3fs-mgmtd:latest +``` + +### Metadata Service +```bash +docker run --name hf3fs-meta \ + --privileged \ + -d --restart always \ + --network host \ + --env CLUSTER_ID=stage \ + --env FDB_CLUSTER=docker:docker@192.168.1.1:4500 \ + --env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \ + --env META_NODE_ID=100 \ + --env DEVICE_FILTER=mlx5_2 \ + --env REMOTE_IP=192.168.1.1:10000 \ + hf3fs-meta:latest +``` + +### Storage Services + +#### Storage Node Preparation +```bash +# Format and mount SSDs +mkdir -p /storage/data{0..1} +for i in {0..1}; do + mkfs.xfs -L data${i} -s size=4096 /dev/nvme${i}n1 + mount -o noatime,nodiratime -L data${i} /storage/data${i} + mkdir -p /storage/data${i}/3fs +done +``` + +#### Storage1 Deployment +```bash +docker run --name hf3fs-storage \ + --privileged \ + -d --restart always \ + --network host \ + -v /storage:/storage \ + --env CLUSTER_ID=stage \ + --env FDB_CLUSTER=docker:docker@192.168.1.1:4500 \ + --env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \ + --env STORAGE_NODE_ID=10001 \ + --env TARGET_PATHS='/storage/data0/3fs,/storage/data1/3fs' \ + --env DEVICE_FILTER=mlx5_2 \ + --env REMOTE_IP=192.168.1.1:10000 \ + hf3fs-storage:latest +``` + +#### Storage2 Deployment +```bash +docker run --name hf3fs-storage \ + --privileged \ + -d --restart always \ + --network host \ + -v /storage:/storage \ + --env CLUSTER_ID=stage \ + --env FDB_CLUSTER=docker:docker@192.168.1.1:4500 \ + --env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \ + --env STORAGE_NODE_ID=10002 \ + --env TARGET_PATHS='/storage/data0/3fs,/storage/data1/3fs' \ + --env DEVICE_FILTER=mlx5_2 \ + --env REMOTE_IP=192.168.1.1:10000 \ + hf3fs-storage:latest +``` + +## Post-Deployment Configuration + +1. Generate cluster configuration: +```bash +pip install -r deploy/data_placement/requirements.txt +python3 deploy/data_placement/src/model/data_placement.py \ + -ql -relax -type CR --num_nodes 2 --replication_factor 2 --min_targets_per_disk 6 + +python3 deploy/data_placement/src/setup/gen_chain_table.py \ + --chain_table_type CR --node_id_begin 10001 --node_id_end 10002 \ + --num_disks_per_node 2 --num_targets_per_disk 6 \ + --target_id_prefix 1 --chain_id_prefix 9 \ + --incidence_matrix_path output/DataPlacementModel-v_2-b_6-r_6-k_2-λ_6-lb_1-ub_0/incidence_matrix.pickle +``` + +2. Transfer configuration files: +```bash +docker cp output/create_target_cmd.txt hf3fs-mgmtd:/opt/3fs/etc/ +docker cp output/generated_chains.csv hf3fs-mgmtd:/opt/3fs/etc/ +docker cp output/generated_chain_table.csv hf3fs-mgmtd:/opt/3fs/etc/ +``` + +3. Configure administrative access: + +```bash +docker exec -it hf3fs-mgmtd /bin/bash +``` +The admin token is printed to the console, save it to /opt/3fs/etc/token.txt. +```bash +/opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "user-add --root --admin 0 root" +``` + +4. Initialize storage targets: +```bash +/opt/3fs/bin/admin_cli --cfg /opt/3fs/etc/admin_cli.toml \ + --config.user_info.token $(<"/opt/3fs/etc/token.txt") \ + < /opt/3fs/etc/create_target_cmd.txt +``` + +5. Upload chain configuration: +```bash +/opt/3fs/bin/admin_cli --cfg /opt/3fs/etc/admin_cli.toml \ + --config.user_info.token $(<"/opt/3fs/etc/token.txt") \ + "upload-chains /opt/3fs/etc/generated_chains.csv" + +/opt/3fs/bin/admin_cli --cfg /opt/3fs/etc/admin_cli.toml \ + --config.user_info.token $(<"/opt/3fs/etc/token.txt") \ + "upload-chain-table --desc stage 1 /opt/3fs/etc/generated_chain_table.csv" +``` + +6. List chains and chain tables to check if they have been correctly uploaded +```bash +/opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-chains" +/opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-chain-tables" +``` + +## FUSE Client Setup on Meta Node +```bash +docker run --name hf3fs-fuse \ + --privileged \ + -d --restart always \ + --network host \ + --env CLUSTER_ID=stage \ + --env FDB_CLUSTER=docker:docker@192.168.1.1:4500 \ + --env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \ + --env REMOTE_IP=192.168.1.1:10000 \ + --env DEVICE_FILTER=mlx5_2 \ + --env TOKEN=${TOKEN} \ + hf3fs-fuse:latest + +# Login hf3fs-fuse Docker container +docker exec -it hf3fs-fuse /bin/bash + +# Verify mount +mount | grep '/3fs/stage' +``` diff --git a/deploy/scripts/_3fs_common.sh b/deploy/scripts/_3fs_common.sh new file mode 100755 index 0000000..5001069 --- /dev/null +++ b/deploy/scripts/_3fs_common.sh @@ -0,0 +1,20 @@ +#!/bin/bash +CONFIG_DONE_FLAG="/opt/3fs/etc/.done" + +function config_cluster_id() { + # env: CLUSTER_ID + sed -i "s/^cluster_id.*/cluster_id = \"${CLUSTER_ID:-default}\"/" /opt/3fs/etc/* +} + + +function config_admin_cli() { + # env: FDB_CLUSTER, MGMTD_SERVER_ADDRESSES, DEVICE_FILTER, REMOTE_IP + # admin_cli.toml + echo ${FDB_CLUSTER} >/etc/foundationdb/fdb.cluster + sed -i "s|^clusterFile.*|clusterFile = '/etc/foundationdb/fdb.cluster'|" /opt/3fs/etc/admin_cli.toml + # device_filter + if [[ -n "${DEVICE_FILTER}" ]]; then + sed -i "s|device_filter = \[\]|device_filter = [\"${DEVICE_FILTER//,/\",\"}\"]|g" /opt/3fs/etc/admin_cli.toml + fi + sed -i "s|mgmtd_server_addresses = \[\]|mgmtd_server_addresses = [\"${MGMTD_SERVER_ADDRESSES//,/\",\"}\"]|g" /opt/3fs/etc/admin_cli.toml +} diff --git a/deploy/scripts/start_fuse.sh b/deploy/scripts/start_fuse.sh new file mode 100755 index 0000000..826b9fb --- /dev/null +++ b/deploy/scripts/start_fuse.sh @@ -0,0 +1,43 @@ +#!/bin/bash +source "$(dirname "$0")/_3fs_common.sh" + +function run_fuse() { + for var in FDB_CLUSTER MGMTD_SERVER_ADDRESSES TOKEN REMOTE_IP CLUSTER_ID; do + if [[ -z "${!var}" ]]; then + echo "ERROR: Environment variable $var is not set" + exit 1 + fi + done + + if [[ ! -f "$CONFIG_DONE_FLAG" ]]; then + config_cluster_id + # env: FDB_CLUSTER, MGMTD_SERVER_ADDRESSES, TOKEN, DEVICE_FILTER, REMOTE_IP + config_admin_cli + mkdir -p /3fs/stage + # TOKEN + echo ${TOKEN} >/opt/3fs/etc/token.txt + # hf3fs_fuse_main_launcher.toml + sed -i "s|mgmtd_server_addresses = \[\]|mgmtd_server_addresses = [\"${MGMTD_SERVER_ADDRESSES//,/\",\"}\"]|g" /opt/3fs/etc/hf3fs_fuse_main_launcher.toml + sed -i "s|mountpoint = ''|mountpoint = '/3fs/stage'|g" /opt/3fs/etc/hf3fs_fuse_main_launcher.toml + sed -i "s|token_file = ''|token_file = '/opt/3fs/etc/token.txt'|g" /opt/3fs/etc/hf3fs_fuse_main_launcher.toml + # hf3fs_fuse_main.toml + sed -i "s|remote_ip = ''|remote_ip = \"${REMOTE_IP}\"|g" /opt/3fs/etc/hf3fs_fuse_main.toml + sed -i "s|mgmtd_server_addresses = \[\]|mgmtd_server_addresses = [\"${MGMTD_SERVER_ADDRESSES//,/\",\"}\"]|g" /opt/3fs/etc/hf3fs_fuse_main.toml + # device_filter + if [[ -n "${DEVICE_FILTER}" ]]; then + sed -i "s|device_filter = \[\]|device_filter = [\"${DEVICE_FILTER//,/\",\"}\"]|g" /opt/3fs/etc/hf3fs_fuse_main_launcher.toml + fi + # init fuse + /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "set-config --type FUSE --file /opt/3fs/etc/hf3fs_fuse_main.toml" + + touch "$CONFIG_DONE_FLAG" + fi + # run fuse + /opt/3fs/bin/hf3fs_fuse_main --launcher_cfg /opt/3fs/etc/hf3fs_fuse_main_launcher.toml + + # Prevent the main process from exiting, thereby avoiding container termination. + tail -f /dev/null +} + + +run_fuse diff --git a/deploy/scripts/start_meta.sh b/deploy/scripts/start_meta.sh new file mode 100755 index 0000000..163c7c7 --- /dev/null +++ b/deploy/scripts/start_meta.sh @@ -0,0 +1,39 @@ +#!/bin/bash +source "$(dirname "$0")/_3fs_common.sh" + +function run_meta() { + for var in FDB_CLUSTER MGMTD_SERVER_ADDRESSES META_NODE_ID REMOTE_IP CLUSTER_ID; do + if [[ -z "${!var}" ]]; then + echo "ERROR: Environment variable $var is not set" + exit 1 + fi + done + + if [[ ! -f "$CONFIG_DONE_FLAG" ]]; then + config_cluster_id + # env: FDB_CLUSTER, MGMTD_SERVER_ADDRESSES, META_NODE_ID, DEVICE_FILTER, REMOTE_IP, CLUSTER_ID + config_admin_cli + # meta_main_launcher.toml + sed -i "s|mgmtd_server_addresses = \[\]|mgmtd_server_addresses = [\"${MGMTD_SERVER_ADDRESSES//,/\",\"}\"]|g" /opt/3fs/etc/meta_main_launcher.toml + # meta_main_app.toml + sed -i "s/^node_id.*/node_id = ${META_NODE_ID}/" /opt/3fs/etc/meta_main_app.toml + # meta_main.toml + sed -i "s|mgmtd_server_addresses = \[\]|mgmtd_server_addresses = [\"${MGMTD_SERVER_ADDRESSES//,/\",\"}\"]|g" /opt/3fs/etc/meta_main.toml + sed -i "s|remote_ip = .*|remote_ip = '${REMOTE_IP}'|g" /opt/3fs/etc/meta_main.toml + # device_filter + if [[ -n "${DEVICE_FILTER}" ]]; then + sed -i "s|device_filter = \[\]|device_filter = [\"${DEVICE_FILTER//,/\",\"}\"]|g" /opt/3fs/etc/meta_main_launcher.toml + fi + # init meta + /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "set-config --type META --file /opt/3fs/etc/meta_main.toml" + + touch "$CONFIG_DONE_FLAG" + fi + # run meta + /opt/3fs/bin/meta_main --launcher_cfg /opt/3fs/etc/meta_main_launcher.toml --app-cfg /opt/3fs/etc/meta_main_app.toml + # Prevent the main process from exiting, thereby avoiding container termination. + tail -f /dev/null +} + + +run_meta diff --git a/deploy/scripts/start_mgmtd.sh b/deploy/scripts/start_mgmtd.sh new file mode 100755 index 0000000..fe5cf92 --- /dev/null +++ b/deploy/scripts/start_mgmtd.sh @@ -0,0 +1,47 @@ +#!/bin/bash +source "$(dirname "$0")/_3fs_common.sh" + +function run_mgmtd() { + for var in FDB_CLUSTER MGMTD_SERVER_ADDRESSES MGMTD_NODE_ID REMOTE_IP CLUSTER_ID; do + if [[ -z "${!var}" ]]; then + echo "ERROR: Environment variable $var is not set" + exit 1 + fi + done + + if [[ ! -f "$CONFIG_DONE_FLAG" ]]; then + config_cluster_id + # env: FDB_CLUSTER, MGMTD_SERVER_ADDRESSES, MGMTD_NODE_ID, DEVICE_FILTER, REMOTE_IP, CLUSTER_ID + config_admin_cli + echo ${FDB_CLUSTER} >/etc/foundationdb/fdb.cluster + # mgmtd_main_launcher.toml + sed -i "/\[fdb\]/,/^\[/{s|^clusterFile.*|clusterFile = '/etc/foundationdb/fdb.cluster'|}" /opt/3fs/etc/mgmtd_main_launcher.toml + # mgmtd_main_app.toml + sed -i "s/^node_id.*/node_id = ${MGMTD_NODE_ID}/" /opt/3fs/etc/mgmtd_main_app.toml + # mgmtd_main.toml + sed -i "s|remote_ip = .*|remote_ip = '${REMOTE_IP}'|g" /opt/3fs/etc/mgmtd_main.toml + # device_filter + if [[ -n "${DEVICE_FILTER}" ]]; then + sed -i "s|device_filter = \[\]|device_filter = [\"${DEVICE_FILTER//,/\",\"}\"]|g" /opt/3fs/etc/mgmtd_main_launcher.toml + fi + + if [[ -z "${STRIP_SIZE}" ]]; then + STRIP_SIZE=6 + fi + + /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "init-cluster --mgmtd /opt/3fs/etc/mgmtd_main.toml 1 1048576 $STRIP_SIZE" + if [ $? -ne 0 ]; then + echo "ERROR: init-cluster failed" + exit 1 + fi + + touch "$CONFIG_DONE_FLAG" + fi + # run mgmtd + /opt/3fs/bin/mgmtd_main --launcher_cfg /opt/3fs/etc/mgmtd_main_launcher.toml --app-cfg /opt/3fs/etc/mgmtd_main_app.toml + # Prevent the main process from exiting, thereby avoiding container termination. + tail -f /dev/null +} + + +run_mgmtd diff --git a/deploy/scripts/start_monitor.sh b/deploy/scripts/start_monitor.sh new file mode 100755 index 0000000..801fc2c --- /dev/null +++ b/deploy/scripts/start_monitor.sh @@ -0,0 +1,36 @@ +#!/bin/bash +source "$(dirname "$0")/_3fs_common.sh" + +function run_monitor() { + for var in CLICKHOUSE_DB CLICKHOUSE_HOST CLICKHOUSE_PASSWD CLICKHOUSE_PORT CLICKHOUSE_USER; do + if [[ -z "${!var}" ]]; then + echo "ERROR: Environment variable $var is not set" + exit 1 + fi + done + + if [[ ! -f "$CONFIG_DONE_FLAG" ]]; then + # env: CLICKHOUSE_DB, CLICKHOUSE_HOST, CLICKHOUSE_PASSWD, CLICKHOUSE_PORT, CLICKHOUSE_USER, DEVICE_FILTER + # monitor_collector_main.toml + sed -i "/^\[server.monitor_collector.reporter.clickhouse\]/,/^\s*$/{ + s/db = '.*/db = '${CLICKHOUSE_DB}'/; + s/host = '.*/host = '${CLICKHOUSE_HOST}'/; + s/passwd = '.*/passwd = '${CLICKHOUSE_PASSWD}'/; + s/port = '.*/port = '${CLICKHOUSE_PORT}'/; + s/user = '.*/user = '${CLICKHOUSE_USER}'/; + }" /opt/3fs/etc/monitor_collector_main.toml + # device_filter if set + if [[ -n "${DEVICE_FILTER}" ]]; then + sed -i "s|device_filter = \[\]|device_filter = [\"${DEVICE_FILTER//,/\",\"}\"]|g" /opt/3fs/etc/monitor_collector_main.toml + fi + + touch "$CONFIG_DONE_FLAG" + fi + # run monitor + /opt/3fs/bin/monitor_collector_main --cfg /opt/3fs/etc/monitor_collector_main.toml + # Prevent the main process from exiting, thereby avoiding container termination. + tail -f /dev/null +} + + +run_monitor diff --git a/deploy/scripts/start_storage.sh b/deploy/scripts/start_storage.sh new file mode 100755 index 0000000..2da22b1 --- /dev/null +++ b/deploy/scripts/start_storage.sh @@ -0,0 +1,43 @@ +#!/bin/bash +source "$(dirname "$0")/_3fs_common.sh" + +function run_storage() { + for var in FDB_CLUSTER MGMTD_SERVER_ADDRESSES STORAGE_NODE_ID REMOTE_IP CLUSTER_ID; do + if [[ -z "${!var}" ]]; then + echo "ERROR: Environment variable $var is not set" + exit 1 + fi + done + + if [[ ! -f "$CONFIG_DONE_FLAG" ]]; then + config_cluster_id + # env: FDB_CLUSTER, MGMTD_SERVER_ADDRESSES, STORAGE_NODE_ID, TARGET_PATHS, DEVICE_FILTER, REMOTE_IP, CLUSTER_ID + config_admin_cli + # storage_main_launcher.toml + sed -i "s|mgmtd_server_addresses = \[\]|mgmtd_server_addresses = [\"${MGMTD_SERVER_ADDRESSES//,/\",\"}\"]|g" /opt/3fs/etc/storage_main_launcher.toml + # storage_main_app.toml + sed -i "s/^node_id.*/node_id = ${STORAGE_NODE_ID}/" /opt/3fs/etc/storage_main_app.toml + # storage_main.toml + sed -i "s|mgmtd_server_addresses = \[\]|mgmtd_server_addresses = [\"${MGMTD_SERVER_ADDRESSES//,/\",\"}\"]|g" /opt/3fs/etc/storage_main.toml + # /opt/3fs/etc/storage_main.toml + if [[ -n "${TARGET_PATHS}" ]]; then + sed -i "s|^target_paths = .*|target_paths = [\"${TARGET_PATHS//,/\",\"}\"]|g" /opt/3fs/etc/storage_main.toml + fi + sed -i "s|remote_ip = \".*\"|remote_ip = \"${REMOTE_IP}\"|g" /opt/3fs/etc/storage_main.toml + # device_filter + if [[ -n "${DEVICE_FILTER}" ]]; then + sed -i "s|device_filter = \[\]|device_filter = [\"${DEVICE_FILTER//,/\",\"}\"]|g" /opt/3fs/etc/storage_main_launcher.toml + fi + # init storage + /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "set-config --type STORAGE --file /opt/3fs/etc/storage_main.toml" + + touch "$CONFIG_DONE_FLAG" + fi + # run storage + /opt/3fs/bin/storage_main --launcher_cfg /opt/3fs/etc/storage_main_launcher.toml --app-cfg /opt/3fs/etc/storage_main_app.toml + # Prevent the main process from exiting, thereby avoiding container termination. + tail -f /dev/null +} + + +run_storage diff --git a/dockerfile/dev.ubuntu2004.dockerfile b/dockerfile/dev.ubuntu2004.dockerfile new file mode 100644 index 0000000..d85e047 --- /dev/null +++ b/dockerfile/dev.ubuntu2004.dockerfile @@ -0,0 +1,64 @@ +FROM ubuntu:20.04 + +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update &&\ + apt-get install -y --no-install-recommends \ + git wget ca-certificates software-properties-common \ + build-essential meson cmake \ + google-perftools \ + libaio-dev \ + libboost1.71-all-dev \ + libdouble-conversion-dev \ + libdwarf-dev \ + libgflags-dev \ + libgmock-dev \ + libgoogle-glog-dev \ + libgoogle-perftools-dev \ + libgtest-dev \ + liblz4-dev \ + liblzma-dev \ + libunwind-dev \ + libuv1-dev \ + libssl-dev \ + gnupg &&\ + apt-get clean &&\ + rm -rf /var/lib/apt/lists/* + +RUN wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key |tee /etc/apt/trusted.gpg.d/llvm.asc &&\ + add-apt-repository -y "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" &&\ + apt-get update && apt-get install -y clang-format-14 clang-14 clang-tidy-14 lld-14 libclang-rt-14-dev gcc-10 g++-10 &&\ + apt-get clean &&\ + rm -rf /var/lib/apt/lists/* + +ARG FDB_VERSION=7.3.63 +RUN FDB_ARCH_SUFFIX=$(dpkg --print-architecture) && \ + case "${FDB_ARCH_SUFFIX}" in \ + amd64) ;; \ + arm64) FDB_ARCH_SUFFIX="aarch64" ;; \ + *) echo "Unsupported architecture: ${FDB_ARCH_SUFFIX}"; exit 1 ;; \ + esac && \ + FDB_CLIENT_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + FDB_SERVER_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + wget -q "${FDB_CLIENT_URL}" && \ + # wget -q "${FDB_SERVER_URL}" && \ + dpkg -i foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb && \ + # dpkg -i foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb && \ + rm foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + +ARG LIBFUSE_VERSION=3.16.2 +ARG LIBFUSE_DOWNLOAD_URL=https://github.com/libfuse/libfuse/releases/download/fuse-${LIBFUSE_VERSION}/fuse-${LIBFUSE_VERSION}.tar.gz +RUN wget -O- ${LIBFUSE_DOWNLOAD_URL} |\ + tar -xzvf - -C /tmp &&\ + cd /tmp/fuse-${LIBFUSE_VERSION} &&\ + mkdir build && cd build &&\ + meson setup .. && meson configure -D default_library=both &&\ + ninja && ninja install &&\ + rm -f -r /tmp/fuse-${LIBFUSE_VERSION}* + +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + + diff --git a/dockerfile/fuse.ubuntu2004.Dockerfile b/dockerfile/fuse.ubuntu2004.Dockerfile new file mode 100644 index 0000000..f3a51f1 --- /dev/null +++ b/dockerfile/fuse.ubuntu2004.Dockerfile @@ -0,0 +1,66 @@ +FROM ubuntu2004-3fs-builder:latest as builder + +WORKDIR /3fs +COPY . . +#RUN cmake -S . -B build_dir -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +# && cmake --build build_dir -j `nproc` + +FROM ubuntu:20.04 +RUN sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates systemd vim net-tools gdb build-essential meson\ + libboost1.71-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libdwarf-dev \ + libdouble-conversion-dev \ + libaio-dev \ + libgoogle-perftools-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +ARG FDB_VERSION=7.3.63 +RUN FDB_ARCH_SUFFIX=$(dpkg --print-architecture) && \ + case "${FDB_ARCH_SUFFIX}" in \ + amd64) ;; \ + arm64) FDB_ARCH_SUFFIX="aarch64" ;; \ + *) echo "Unsupported architecture: ${FDB_ARCH_SUFFIX}"; exit 1 ;; \ + esac && \ + FDB_CLIENT_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + FDB_SERVER_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + wget -q "${FDB_CLIENT_URL}" && \ + wget -q "${FDB_SERVER_URL}" && \ + dpkg -i foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # dpkg -i --force-all foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + +RUN wget https://raw.githubusercontent.com/Mellanox/container_scripts/refs/heads/master/ibdev2netdev -O /usr/sbin/ibdev2netdev && \ +chmod +x /usr/sbin/ibdev2netdev + +ARG LIBFUSE_VERSION=3.16.2 +ARG LIBFUSE_DOWNLOAD_URL=https://github.com/libfuse/libfuse/releases/download/fuse-${LIBFUSE_VERSION}/fuse-${LIBFUSE_VERSION}.tar.gz +RUN wget -O- ${LIBFUSE_DOWNLOAD_URL} |\ + tar -xzvf - -C /tmp &&\ + cd /tmp/fuse-${LIBFUSE_VERSION} &&\ + mkdir build && cd build &&\ + meson setup .. && meson configure -D default_library=both &&\ + ninja && ninja install &&\ + rm -f -r /tmp/fuse-${LIBFUSE_VERSION}* + +RUN mkdir -p /opt/3fs/bin && mkdir -p /opt/3fs/etc && mkdir -p /opt/3fs/scripts && mkdir -p /var/log/3fs && mkdir -p /3fs/stage +COPY --from=builder /3fs/build_dir/bin/hf3fs_fuse_main /opt/3fs/bin/ +COPY --from=builder /3fs/configs/hf3fs_fuse_main*.toml /opt/3fs/etc/ +COPY --from=builder /3fs/deploy/systemd/hf3fs_fuse_main.service /usr/lib/systemd/system/ + +COPY --from=builder /3fs/deploy/scripts/_3fs_common.sh /opt/3fs/scripts/ +COPY --from=builder /3fs/deploy/scripts/start_fuse.sh /opt/3fs/scripts/ + +COPY --from=builder /3fs/build_dir/bin/admin_cli /opt/3fs/bin/ +COPY --from=builder /3fs/configs/admin_cli.toml /opt/3fs/etc/ + +WORKDIR /opt/3fs/bin + +CMD ["/opt/3fs/scripts/start_fuse.sh"] + diff --git a/dockerfile/meta.ubuntu2004.Dockerfile b/dockerfile/meta.ubuntu2004.Dockerfile new file mode 100644 index 0000000..2139ad6 --- /dev/null +++ b/dockerfile/meta.ubuntu2004.Dockerfile @@ -0,0 +1,57 @@ +FROM ubuntu2004-3fs-builder:latest as builder + +WORKDIR /3fs +COPY . . +#RUN cmake -S . -B build_dir -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +# && cmake --build build_dir -j `nproc` + +FROM ubuntu:20.04 +RUN sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates systemd vim net-tools gdb\ + libboost1.71-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libdwarf-dev \ + libdouble-conversion-dev \ + libaio-dev \ + libgoogle-perftools-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +ARG FDB_VERSION=7.3.63 +RUN FDB_ARCH_SUFFIX=$(dpkg --print-architecture) && \ + case "${FDB_ARCH_SUFFIX}" in \ + amd64) ;; \ + arm64) FDB_ARCH_SUFFIX="aarch64" ;; \ + *) echo "Unsupported architecture: ${FDB_ARCH_SUFFIX}"; exit 1 ;; \ + esac && \ + FDB_CLIENT_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + FDB_SERVER_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + wget -q "${FDB_CLIENT_URL}" && \ + wget -q "${FDB_SERVER_URL}" && \ + dpkg -i foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # dpkg -i --force-all foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + +RUN wget https://raw.githubusercontent.com/Mellanox/container_scripts/refs/heads/master/ibdev2netdev -O /usr/sbin/ibdev2netdev && \ +chmod +x /usr/sbin/ibdev2netdev + +RUN mkdir -p /opt/3fs/bin && mkdir -p /opt/3fs/etc && mkdir -p /opt/3fs/scripts && mkdir -p /var/log/3fs +COPY --from=builder /3fs/build_dir/bin/meta_main /opt/3fs/bin/ +COPY --from=builder /3fs/configs/meta_main*.toml /opt/3fs/etc/ +COPY --from=builder /3fs/deploy/systemd/meta_main.service /usr/lib/systemd/system/ +COPY --from=builder /3fs/build_dir/third_party/jemalloc/lib/libjemalloc.so.2 /usr/lib/ + +COPY --from=builder /3fs/deploy/scripts/_3fs_common.sh /opt/3fs/scripts/ +COPY --from=builder /3fs/deploy/scripts/start_meta.sh /opt/3fs/scripts/ + +COPY --from=builder /3fs/build_dir/bin/admin_cli /opt/3fs/bin/ +COPY --from=builder /3fs/configs/admin_cli.toml /opt/3fs/etc/ + +WORKDIR /opt/3fs/bin + +CMD ["/opt/3fs/scripts/start_meta.sh"] + diff --git a/dockerfile/mgmtd.ubuntu2004.Dockerfile b/dockerfile/mgmtd.ubuntu2004.Dockerfile new file mode 100644 index 0000000..363f970 --- /dev/null +++ b/dockerfile/mgmtd.ubuntu2004.Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu2004-3fs-builder:latest as builder + +WORKDIR /3fs +COPY . . +#RUN cmake -S . -B build_dir -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +# && cmake --build build_dir -j `nproc` + +FROM ubuntu:20.04 +RUN sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates systemd vim net-tools gdb\ + libboost1.71-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libdwarf-dev \ + libdouble-conversion-dev \ + libaio-dev \ + libgoogle-perftools-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +ARG FDB_VERSION=7.3.63 +RUN FDB_ARCH_SUFFIX=$(dpkg --print-architecture) && \ + case "${FDB_ARCH_SUFFIX}" in \ + amd64) ;; \ + arm64) FDB_ARCH_SUFFIX="aarch64" ;; \ + *) echo "Unsupported architecture: ${FDB_ARCH_SUFFIX}"; exit 1 ;; \ + esac && \ + FDB_CLIENT_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + FDB_SERVER_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + wget -q "${FDB_CLIENT_URL}" && \ + wget -q "${FDB_SERVER_URL}" && \ + dpkg -i foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # dpkg -i --force-all foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + +RUN wget https://raw.githubusercontent.com/Mellanox/container_scripts/refs/heads/master/ibdev2netdev -O /usr/sbin/ibdev2netdev && \ +chmod +x /usr/sbin/ibdev2netdev + +RUN mkdir -p /opt/3fs/bin && mkdir -p /opt/3fs/etc && mkdir -p /opt/3fs/scripts && mkdir -p /var/log/3fs +COPY --from=builder /3fs/build_dir/bin/mgmtd_main /opt/3fs/bin/ +COPY --from=builder /3fs/configs/mgmtd_main*.toml /opt/3fs/etc/ +COPY --from=builder /3fs/deploy/systemd/mgmtd_main.service /usr/lib/systemd/system/ + +COPY --from=builder /3fs/deploy/scripts/_3fs_common.sh /opt/3fs/scripts/ +COPY --from=builder /3fs/deploy/scripts/start_mgmtd.sh /opt/3fs/scripts/ + +COPY --from=builder /3fs/build_dir/bin/admin_cli /opt/3fs/bin/ +COPY --from=builder /3fs/configs/admin_cli.toml /opt/3fs/etc/ + +WORKDIR /opt/3fs/bin + +CMD ["/opt/3fs/scripts/start_mgmtd.sh"] + diff --git a/dockerfile/monitor.ubuntu2004.Dockerfile b/dockerfile/monitor.ubuntu2004.Dockerfile new file mode 100644 index 0000000..d564c24 --- /dev/null +++ b/dockerfile/monitor.ubuntu2004.Dockerfile @@ -0,0 +1,58 @@ +FROM ubuntu2004-3fs-builder:latest as builder + +WORKDIR /3fs +COPY . . +#RUN cmake -S . -B build_dir -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +# && cmake --build build_dir -j `nproc` + +FROM ubuntu:20.04 +RUN sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates systemd vim net-tools gdb\ + libboost1.71-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libdwarf-dev \ + libdouble-conversion-dev \ + libaio-dev \ + libgoogle-perftools-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +ARG FDB_VERSION=7.3.63 +RUN FDB_ARCH_SUFFIX=$(dpkg --print-architecture) && \ + case "${FDB_ARCH_SUFFIX}" in \ + amd64) ;; \ + arm64) FDB_ARCH_SUFFIX="aarch64" ;; \ + *) echo "Unsupported architecture: ${FDB_ARCH_SUFFIX}"; exit 1 ;; \ + esac && \ + FDB_CLIENT_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + FDB_SERVER_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + wget -q "${FDB_CLIENT_URL}" && \ + wget -q "${FDB_SERVER_URL}" && \ + dpkg -i foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # dpkg -i --force-all foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + +RUN wget https://raw.githubusercontent.com/Mellanox/container_scripts/refs/heads/master/ibdev2netdev -O /usr/sbin/ibdev2netdev && \ +chmod +x /usr/sbin/ibdev2netdev + +RUN mkdir -p /opt/3fs/bin && mkdir -p /opt/3fs/etc && mkdir -p /opt/3fs/scripts && mkdir -p /var/log/3fs +COPY --from=builder /3fs/build_dir/bin/monitor_collector_main /opt/3fs/bin/ +COPY --from=builder /3fs/configs/monitor_collector_main.toml /opt/3fs/etc/ +COPY --from=builder /3fs/deploy/sql/3fs-monitor.sql /opt/3fs/etc/ +COPY --from=builder /3fs/deploy/systemd/monitor_collector_main.service /usr/lib/systemd/system/ + +COPY --from=builder /3fs/deploy/scripts/_3fs_common.sh /opt/3fs/scripts/ +COPY --from=builder /3fs/deploy/scripts/start_monitor.sh /opt/3fs/scripts/ + + +COPY --from=builder /3fs/build_dir/bin/admin_cli /opt/3fs/bin/ +COPY --from=builder /3fs/configs/admin_cli.toml /opt/3fs/etc/ + +WORKDIR /opt/3fs/bin + +CMD ["/opt/3fs/scripts/start_monitor.sh"] + diff --git a/dockerfile/storage.ubuntu2004.Dockerfile b/dockerfile/storage.ubuntu2004.Dockerfile new file mode 100644 index 0000000..fe0dbb3 --- /dev/null +++ b/dockerfile/storage.ubuntu2004.Dockerfile @@ -0,0 +1,57 @@ +FROM ubuntu2004-3fs-builder:latest as builder + +WORKDIR /3fs +COPY . . +#RUN cmake -S . -B build_dir -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +# && cmake --build build_dir -j `nproc` + +FROM ubuntu:20.04 +RUN sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates systemd vim net-tools gdb\ + libboost1.71-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libdwarf-dev \ + libdouble-conversion-dev \ + libaio-dev \ + libgoogle-perftools-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +ARG FDB_VERSION=7.3.63 +RUN FDB_ARCH_SUFFIX=$(dpkg --print-architecture) && \ + case "${FDB_ARCH_SUFFIX}" in \ + amd64) ;; \ + arm64) FDB_ARCH_SUFFIX="aarch64" ;; \ + *) echo "Unsupported architecture: ${FDB_ARCH_SUFFIX}"; exit 1 ;; \ + esac && \ + FDB_CLIENT_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + FDB_SERVER_URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb" && \ + wget -q "${FDB_CLIENT_URL}" && \ + wget -q "${FDB_SERVER_URL}" && \ + dpkg -i foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # dpkg -i --force-all foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + # rm foundationdb-server_${FDB_VERSION}-1_${FDB_ARCH_SUFFIX}.deb + +RUN wget https://raw.githubusercontent.com/Mellanox/container_scripts/refs/heads/master/ibdev2netdev -O /usr/sbin/ibdev2netdev && \ +chmod +x /usr/sbin/ibdev2netdev + +RUN mkdir -p /opt/3fs/bin && mkdir -p /opt/3fs/etc && mkdir -p /opt/3fs/scripts && mkdir -p /var/log/3fs +COPY --from=builder /3fs/build_dir/bin/storage_main /opt/3fs/bin/ +COPY --from=builder /3fs/configs/storage_main*.toml /opt/3fs/etc/ +COPY --from=builder /3fs/deploy/systemd/storage_main.service /usr/lib/systemd/system/ +COPY --from=builder /3fs/build_dir/third_party/jemalloc/lib/libjemalloc.so.2 /usr/lib/ + +COPY --from=builder /3fs/deploy/scripts/_3fs_common.sh /opt/3fs/scripts/ +COPY --from=builder /3fs/deploy/scripts/start_storage.sh /opt/3fs/scripts/ + +COPY --from=builder /3fs/build_dir/bin/admin_cli /opt/3fs/bin/ +COPY --from=builder /3fs/configs/admin_cli.toml /opt/3fs/etc/ + +WORKDIR /opt/3fs/bin + +CMD ["/opt/3fs/scripts/start_storage.sh"] +