Merge pull request #379 from Fyve-Labs/fix/install-at-build

Fix - install dependencies at docker image build

Dockerfile

@@ -4,6 +4,8 @@ FROM python:3.11-slim-bookworm AS base
 ARG MINIMUM_BUILD
 ARG USE_CUDA
 ARG USE_CUDA_VER
+ARG PIPELINES_URLS
+ARG PIPELINES_REQUIREMENTS_PATH
 
 ## Basis ##
 ENV ENV=prod \
@@ -39,11 +41,25 @@ RUN if [ "$MINIMUM_BUILD" = "true" ]; then \
         uv pip install --system -r requirements.txt --no-cache-dir; \
     fi
 
+
+# Layer on for other components
+FROM base AS app
+
+ENV PIPELINES_URLS=${PIPELINES_URLS} \
+    PIPELINES_REQUIREMENTS_PATH=${PIPELINES_REQUIREMENTS_PATH}
+
 # Copy the application code
 COPY . .
 
+# Run a docker command if either PIPELINES_URLS or PIPELINES_REQUIREMENTS_PATH is not empty
+RUN if [ -n "$PIPELINES_URLS" ] || [ -n "$PIPELINES_REQUIREMENTS_PATH" ]; then \
+    echo "Running docker command with PIPELINES_URLS or PIPELINES_REQUIREMENTS_PATH"; \
+    ./start.sh --mode setup; \
+    fi
+
 # Expose the port
 ENV HOST="0.0.0.0"
 ENV PORT="9099"
 
+# if we already installed the requirements on build, we can skip this step on run
 ENTRYPOINT [ "bash", "start.sh" ]

README.md

@@ -101,6 +101,38 @@ Get started with Pipelines in a few easy steps:
 
 Once the server is running, set the OpenAI URL on your client to the Pipelines URL. This unlocks the full capabilities of Pipelines, integrating any Python library and creating custom workflows tailored to your needs.
 
+### Advanced Docker Builds
+If you create your own pipelines, you can install them when the Docker image is built.  For example,
+create a bash script with the snippet below to collect files from a path, add them as install URLs, 
+and build the Docker image with the new pipelines automatically installed.
+
+NOTE: The pipelines module will still attempt to install any package dependencies found at in your
+file headers at start time, but they will not be downloaded again.
+
+```sh
+# build in the specific pipelines
+PIPELINE_DIR="pipelines-custom"
+# assuming the above directory is in your source repo and not skipped by `.dockerignore`, it will get copied to the image
+PIPELINE_PREFIX="file:///app"
+
+# retrieve all the sub files
+export PIPELINES_URLS=
+for file in "$PIPELINE_DIR"/*; do
+    if [[ -f "$file" ]]; then
+        if [[ "$file" == *.py ]]; then
+            if [ -z "$PIPELINES_URLS" ]; then
+                PIPELINES_URLS="$PIPELINE_PREFIX/$file"
+            else
+                PIPELINES_URLS="$PIPELINES_URLS;$PIPELINE_PREFIX/$file"
+            fi
+        fi
+    fi
+done
+echo "New Custom Install Pipes: $PIPELINES_URLS"
+
+docker build --build-arg PIPELINES_URLS=$PIPELINES_URLS --build-arg MINIMUM_BUILD=true -f Dockerfile .
+```
+
 ## 📂 Directory Structure and Examples
 
 The `/pipelines` directory is the core of your setup. Add new modules, customize existing ones, and manage your workflows here. All the pipelines in the `/pipelines` directory will be **automatically loaded** when the server launches.

start.sh

@@ -26,9 +26,6 @@ reset_pipelines_dir() {
   fi
 }
 
-# Example usage of the function
-reset_pipelines_dir
-
 # Function to install requirements if requirements.txt is provided
 install_requirements() {
   if [[ -f "$1" ]]; then
@@ -105,29 +102,54 @@ install_frontmatter_requirements() {
 }
 
 
-
-# Check if PIPELINES_URLS environment variable is set and non-empty
-if [[ -n "$PIPELINES_URLS" ]]; then
-  if [ ! -d "$PIPELINES_DIR" ]; then
-    mkdir -p "$PIPELINES_DIR"
-  fi
-
-  # Split PIPELINES_URLS by ';' and iterate over each path
-  IFS=';' read -ra ADDR <<< "$PIPELINES_URLS"
-  for path in "${ADDR[@]}"; do
-    download_pipelines "$path" "$PIPELINES_DIR"
-  done
-
-  for file in "$pipelines_dir"/*; do
-    if [[ -f "$file" ]]; then
-      install_frontmatter_requirements "$file"
-    fi
-  done
-else
-  echo "PIPELINES_URLS not specified. Skipping pipelines download and installation."
+# Parse command line arguments for mode
+MODE="full"   # select a runmode ("setup", "run", "full" (setup + run))
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    --mode) MODE="$2"; shift ;;
+    *) echo "Unknown parameter passed: $1"; exit 1 ;;
+  esac
+  shift
+done
+if [[ "$MODE" != "setup" && "$MODE" != "run" && "$MODE" != "full" ]]; then
+  echo "Invalid script mode: $MODE"
+  echo "  Example usage: './start.sh --mode [setup|run|full]' "
+  exit 1
 fi
 
+# Function to handle different modes, added 1/29/24
+if [[ "$MODE" == "setup" || "$MODE" == "full" ]]; then
+  echo "Download + install Executed in mode: $MODE"
 
+  reset_pipelines_dir
+  if [[ -n "$PIPELINES_REQUIREMENTS_PATH" ]]; then
+    install_requirements "$PIPELINES_REQUIREMENTS_PATH"
+  else
+    echo "PIPELINES_REQUIREMENTS_PATH not specified. Skipping installation of requirements."
+  fi
+
+  if [[ -n "$PIPELINES_URLS" ]]; then
+    if [ ! -d "$PIPELINES_DIR" ]; then
+      mkdir -p "$PIPELINES_DIR"
+    fi
+
+    IFS=';' read -ra ADDR <<< "$PIPELINES_URLS"
+    for path in "${ADDR[@]}"; do
+      download_pipelines "$path" "$PIPELINES_DIR"
+    done
+
+    for file in "$PIPELINES_DIR"/*; do
+      if [[ -f "$file" ]]; then
+        install_frontmatter_requirements "$file"
+      fi
+    done
+  else
+    echo "PIPELINES_URLS not specified. Skipping pipelines download and installation."
+  fi
+fi
+
+if [[ "$MODE" == "run" || "$MODE" == "full" ]]; then
+  echo "Running via Mode: $MODE"
+  uvicorn main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*'
+fi
 
-# Start the server
-uvicorn main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*'