From 550fc63ebd08de7c4108eef1aa302897db86aee8 Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Wed, 10 Apr 2024 13:13:39 +0100 Subject: [PATCH 1/4] feat: parallelize x86/arm docker builds --- .github/workflows/docker-build.yaml | 339 ++++++++++++++++++++++++++-- 1 file changed, 317 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 7f043c248..de163ef1a 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -1,6 +1,5 @@ name: Create and publish Docker images with specific build args -# Configures this workflow to run every time a change is pushed to the branch called `release`. on: workflow_dispatch: push: @@ -10,30 +9,39 @@ on: tags: - v* -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + FULL_IMAGE_NAME: ghcr.io/${{ github.repository }} -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: - build-and-push-image: + build-main-image: runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read packages: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - name: Checkout repository uses: actions/checkout@v4 - # Required for multi architecture build + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # Required for multi architecture build + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry uses: docker/login-action@v3 with: @@ -42,10 +50,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata for Docker images (default latest tag) - id: meta-latest + id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: ${{ env.FULL_IMAGE_NAME }} tags: | type=ref,event=branch type=ref,event=tag @@ -55,31 +63,318 @@ jobs: flavor: | latest=${{ github.ref == 'refs/heads/main' }} - - name: Build and push Docker image (latest) + - name: Build Docker image (latest) uses: docker/build-push-action@v5 + id: build with: context: . push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta-latest.outputs.tags }} - labels: ${{ steps.meta-latest.outputs.labels }} + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha + cache-to: type=gha,mode=max - - name: Build and push Docker image with CUDA + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-main-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + build-cuda-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (default latest tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + suffix=-cuda,onlatest=true + + - name: Build Docker image (cuda) uses: docker/build-push-action@v5 - if: github.ref == 'refs/heads/main' + id: build with: context: . push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cuda + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha + cache-to: type=gha,mode=max build-args: USE_CUDA=true - - name: Build and push Docker image with Ollama + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-cuda-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + build-ollama-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm64 + + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (ollama tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + suffix=-ollama,onlatest=true + + - name: Build Docker image (ollama) uses: docker/build-push-action@v5 - if: github.ref == 'refs/heads/main' + id: build with: context: . push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:ollama + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.FULL_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha + cache-to: type=gha,mode=max build-args: USE_OLLAMA=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-ollama-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge-main-images: + runs-on: ubuntu-latest + needs: [ build-main-image ] + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-main-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (default latest tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} + + + merge-cuda-images: + runs-on: ubuntu-latest + needs: [ build-cuda-image ] + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-cuda-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (default latest tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + suffix=-cuda,onlatest=true + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} + + merge-ollama-images: + runs-on: ubuntu-latest + needs: [ build-ollama-image ] + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-ollama-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker images (default ollama tag) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FULL_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha,prefix=git- + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + flavor: | + latest=${{ github.ref == 'refs/heads/main' }} + suffix=-ollama,onlatest=true + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.FULL_IMAGE_NAME }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.FULL_IMAGE_NAME }}:${{ steps.meta.outputs.version }} From eddaa7fa89e05fe1b70045a39ea1c4dbd4b0b05b Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Wed, 10 Apr 2024 13:29:10 +0100 Subject: [PATCH 2/4] cross compile node --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4a23d54fb..6b44afd3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG USE_CUDA_VER=cu121 ARG USE_EMBEDDING_MODEL=all-MiniLM-L6-v2 ######## WebUI frontend ######## -FROM node:21-alpine3.19 as build +FROM --platform=$BUILDPLATFORM node:21-alpine3.19 as build WORKDIR /app From cd91d8a987518ac1a050f24489ee9fc7c4e3c9a7 Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Wed, 10 Apr 2024 13:39:46 +0100 Subject: [PATCH 3/4] run apt install first for better potential layer caching --- Dockerfile | 64 +++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6b44afd3f..6cf8ded28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,43 +67,43 @@ ENV RAG_EMBEDDING_MODEL="$USE_EMBEDDING_MODEL_DOCKER" \ #### Other models ########################################################## WORKDIR /app/backend + +RUN if [ "$USE_OLLAMA" = "true" ]; then \ + apt-get update && \ + # Install pandoc and netcat + apt-get install -y --no-install-recommends pandoc netcat-openbsd && \ + # for RAG OCR + apt-get install -y --no-install-recommends ffmpeg libsm6 libxext6 && \ + # install helper tools + apt-get install -y --no-install-recommends curl && \ + # install ollama + curl -fsSL https://ollama.com/install.sh | sh && \ + # cleanup + rm -rf /var/lib/apt/lists/*; \ + else \ + apt-get update && \ + # Install pandoc and netcat + apt-get install -y --no-install-recommends pandoc netcat-openbsd && \ + # for RAG OCR + apt-get install -y --no-install-recommends ffmpeg libsm6 libxext6 && \ + # cleanup + rm -rf /var/lib/apt/lists/*; \ + fi + # install python dependencies COPY ./backend/requirements.txt ./requirements.txt RUN if [ "$USE_CUDA" = "true" ]; then \ - # If you use CUDA the whisper and embedding modell will be downloaded on first use - pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir && \ - pip3 install -r requirements.txt --no-cache-dir && \ - python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])" && \ - python -c "import os; from chromadb.utils import embedding_functions; sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=os.environ['RAG_EMBEDDING_MODEL'], device='cpu')"; \ + # If you use CUDA the whisper and embedding model will be downloaded on first use + pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir && \ + pip3 install -r requirements.txt --no-cache-dir && \ + python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])" && \ + python -c "import os; from chromadb.utils import embedding_functions; sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=os.environ['RAG_EMBEDDING_MODEL'], device='cpu')"; \ else \ - pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir && \ - pip3 install -r requirements.txt --no-cache-dir && \ - python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])" && \ - python -c "import os; from chromadb.utils import embedding_functions; sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=os.environ['RAG_EMBEDDING_MODEL'], device='cpu')"; \ - fi - - -RUN if [ "$USE_OLLAMA" = "true" ]; then \ - apt-get update && \ - # Install pandoc and netcat - apt-get install -y --no-install-recommends pandoc netcat-openbsd && \ - # for RAG OCR - apt-get install -y --no-install-recommends ffmpeg libsm6 libxext6 && \ - # install helper tools - apt-get install -y --no-install-recommends curl && \ - # install ollama - curl -fsSL https://ollama.com/install.sh | sh && \ - # cleanup - rm -rf /var/lib/apt/lists/*; \ - else \ - apt-get update && \ - # Install pandoc and netcat - apt-get install -y --no-install-recommends pandoc netcat-openbsd && \ - # for RAG OCR - apt-get install -y --no-install-recommends ffmpeg libsm6 libxext6 && \ - # cleanup - rm -rf /var/lib/apt/lists/*; \ + pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu --no-cache-dir && \ + pip3 install -r requirements.txt --no-cache-dir && \ + python -c "import os; from faster_whisper import WhisperModel; WhisperModel(os.environ['WHISPER_MODEL'], device='cpu', compute_type='int8', download_root=os.environ['WHISPER_MODEL_DIR'])" && \ + python -c "import os; from chromadb.utils import embedding_functions; sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=os.environ['RAG_EMBEDDING_MODEL'], device='cpu')"; \ fi From 7050d5371883f0781bac275302962639e2727dcd Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Wed, 10 Apr 2024 20:41:36 +0100 Subject: [PATCH 4/4] build ollama and cuda tags --- .github/workflows/docker-build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index de163ef1a..b703e43da 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -134,6 +134,7 @@ jobs: type=sha,prefix=git- type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} + type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=cuda flavor: | latest=${{ github.ref == 'refs/heads/main' }} suffix=-cuda,onlatest=true @@ -210,6 +211,7 @@ jobs: type=sha,prefix=git- type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} + type=raw,enable=${{ github.ref == 'refs/heads/main' }},prefix=,suffix=,value=ollama flavor: | latest=${{ github.ref == 'refs/heads/main' }} suffix=-ollama,onlatest=true