Add Docker Model Runner backend

.env.example

@@ -5,6 +5,9 @@ OLLAMA_BASE_URL='http://localhost:11434'
 OPENAI_API_BASE_URL=''
 OPENAI_API_KEY=''
 
+# Docker Model Runner API base URL
+DMR_BASE_URL='http://localhost:12434'
+
 # AUTOMATIC1111_BASE_URL="http://localhost:7860"
 
 # For production, you should only need one host as

README.md

@@ -25,7 +25,7 @@ For more information, be sure to check out our [Open WebUI Documentation](https:
 
 - 🚀 **Effortless Setup**: Install seamlessly using Docker or Kubernetes (kubectl, kustomize or helm) for a hassle-free experience with support for both `:ollama` and `:cuda` tagged images.
 
-- 🤝 **Ollama/OpenAI API Integration**: Effortlessly integrate OpenAI-compatible APIs for versatile conversations alongside Ollama models. Customize the OpenAI API URL to link with **LMStudio, GroqCloud, Mistral, OpenRouter, and more**.
+- 🤝 **Ollama/OpenAI API Integration**: Effortlessly integrate OpenAI-compatible APIs for versatile conversations alongside Ollama models. Customize the OpenAI API URL to link with **LMStudio, GroqCloud, Mistral, OpenRouter, Docker Model Runner, and more**.
 
 - 🛡️ **Granular Permissions and User Groups**: By allowing administrators to create detailed user roles and permissions, we ensure a secure user environment. This granularity not only enhances security but also allows for customized user experiences, fostering a sense of ownership and responsibility amongst users.
 
@@ -165,6 +165,14 @@ This will start the Open WebUI server, which you can access at [http://localhost
   docker run -d -p 3000:8080 -e OPENAI_API_KEY=your_secret_key -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main
   ```
 
+### Using Docker Model Runner
+
+Enable Docker Model Runner in Docker Desktop and set the `DMR_BASE_URL` environment variable to the exposed API endpoint:
+
+```bash
+docker run -d -p 3000:8080 -e DMR_BASE_URL=http://localhost:12434 -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main
+```
+
 ### Installing Open WebUI with Bundled Ollama Support
 
 This installation method uses a single container image that bundles Open WebUI with Ollama, allowing for a streamlined setup via a single command. Choose the appropriate command based on your hardware setup:

backend/open_webui/config.py

@@ -895,6 +895,36 @@ except Exception:
     pass
 OPENAI_API_BASE_URL = "https://api.openai.com/v1"
 
+####################################
+# DOCKER MODEL RUNNER
+####################################
+
+ENABLE_DMR_API = PersistentConfig(
+    "ENABLE_DMR_API",
+    "dmr.enable",
+    os.environ.get("ENABLE_DMR_API", "True").lower() == "true",
+)
+
+DMR_API_KEYS = [k.strip() for k in os.environ.get("DMR_API_KEYS", "").split(";")]
+DMR_API_KEYS = PersistentConfig("DMR_API_KEYS", "dmr.api_keys", DMR_API_KEYS)
+
+DMR_BASE_URL = os.environ.get("DMR_BASE_URL", "")
+if DMR_BASE_URL:
+    DMR_BASE_URL = DMR_BASE_URL[:-1] if DMR_BASE_URL.endswith("/") else DMR_BASE_URL
+
+DMR_BASE_URLS = os.environ.get("DMR_BASE_URLS", "")
+DMR_BASE_URLS = DMR_BASE_URLS if DMR_BASE_URLS != "" else DMR_BASE_URL or "http://localhost:12434"
+DMR_BASE_URLS = [url.strip() for url in DMR_BASE_URLS.split(";")]
+DMR_BASE_URLS = PersistentConfig(
+    "DMR_BASE_URLS", "dmr.base_urls", DMR_BASE_URLS
+)
+
+DMR_API_CONFIGS = PersistentConfig(
+    "DMR_API_CONFIGS",
+    "dmr.api_configs",
+    {},
+)
+
 ####################################
 # TOOL_SERVERS
 ####################################

backend/open_webui/main.py

@@ -63,6 +63,7 @@ from open_webui.routers import (
     images,
     ollama,
     openai,
+    docker_model_runner,
     retrieval,
     pipelines,
     tasks,
@@ -112,6 +113,11 @@ from open_webui.config import (
     OPENAI_API_BASE_URLS,
     OPENAI_API_KEYS,
     OPENAI_API_CONFIGS,
+    # Docker Model Runner
+    ENABLE_DMR_API,
+    DMR_BASE_URLS,
+    DMR_API_KEYS,
+    DMR_API_CONFIGS,
     # Direct Connections
     ENABLE_DIRECT_CONNECTIONS,
     # Thread pool size for FastAPI/AnyIO
@@ -589,6 +595,19 @@ app.state.config.OPENAI_API_CONFIGS = OPENAI_API_CONFIGS
 
 app.state.OPENAI_MODELS = {}
 
+########################################
+#
+# DOCKER MODEL RUNNER
+#
+########################################
+
+app.state.config.ENABLE_DMR_API = ENABLE_DMR_API
+app.state.config.DMR_BASE_URLS = DMR_BASE_URLS
+app.state.config.DMR_API_KEYS = DMR_API_KEYS
+app.state.config.DMR_API_CONFIGS = DMR_API_CONFIGS
+
+app.state.DMR_MODELS = {}
+
 ########################################
 #
 # TOOL SERVERS
@@ -1111,6 +1130,7 @@ app.mount("/ws", socket_app)
 
 app.include_router(ollama.router, prefix="/ollama", tags=["ollama"])
 app.include_router(openai.router, prefix="/openai", tags=["openai"])
+app.include_router(docker_model_runner.router, prefix="/dmr", tags=["dmr"])
 
 
 app.include_router(pipelines.router, prefix="/api/v1/pipelines", tags=["pipelines"])

backend/open_webui/routers/docker_model_runner.py

@@ -0,0 +1,110 @@
+from contextlib import contextmanager
+from typing import Optional
+
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel
+
+from open_webui.models.users import UserModel
+from open_webui.routers import openai
+from open_webui.routers.openai import ConnectionVerificationForm
+from open_webui.utils.auth import get_admin_user, get_verified_user
+
+router = APIRouter()
+
+@contextmanager
+def _dmr_context(request: Request):
+    orig_urls = request.app.state.config.OPENAI_API_BASE_URLS
+    orig_keys = request.app.state.config.OPENAI_API_KEYS
+    orig_configs = request.app.state.config.OPENAI_API_CONFIGS
+    orig_models = request.app.state.OPENAI_MODELS
+    request.app.state.config.OPENAI_API_BASE_URLS = request.app.state.config.DMR_BASE_URLS
+    request.app.state.config.OPENAI_API_KEYS = request.app.state.config.DMR_API_KEYS
+    request.app.state.config.OPENAI_API_CONFIGS = request.app.state.config.DMR_API_CONFIGS
+    request.app.state.OPENAI_MODELS = request.app.state.DMR_MODELS
+    try:
+        yield
+    finally:
+        request.app.state.config.OPENAI_API_BASE_URLS = orig_urls
+        request.app.state.config.OPENAI_API_KEYS = orig_keys
+        request.app.state.config.OPENAI_API_CONFIGS = orig_configs
+        request.app.state.OPENAI_MODELS = orig_models
+
+
+@router.get("/config")
+async def get_config(request: Request, user=Depends(get_admin_user)):
+    return {
+        "ENABLE_DMR_API": request.app.state.config.ENABLE_DMR_API,
+        "DMR_BASE_URLS": request.app.state.config.DMR_BASE_URLS,
+        "DMR_API_KEYS": request.app.state.config.DMR_API_KEYS,
+        "DMR_API_CONFIGS": request.app.state.config.DMR_API_CONFIGS,
+    }
+
+
+class DMRConfigForm(BaseModel):
+    ENABLE_DMR_API: Optional[bool] = None
+    DMR_BASE_URLS: list[str]
+    DMR_API_KEYS: list[str] = []
+    DMR_API_CONFIGS: dict = {}
+
+
+@router.post("/config/update")
+async def update_config(request: Request, form_data: DMRConfigForm, user=Depends(get_admin_user)):
+    request.app.state.config.ENABLE_DMR_API = form_data.ENABLE_DMR_API
+    request.app.state.config.DMR_BASE_URLS = form_data.DMR_BASE_URLS
+    request.app.state.config.DMR_API_KEYS = form_data.DMR_API_KEYS
+    request.app.state.config.DMR_API_CONFIGS = form_data.DMR_API_CONFIGS
+
+    if len(request.app.state.config.DMR_API_KEYS) != len(request.app.state.config.DMR_BASE_URLS):
+        if len(request.app.state.config.DMR_API_KEYS) > len(request.app.state.config.DMR_BASE_URLS):
+            request.app.state.config.DMR_API_KEYS = request.app.state.config.DMR_API_KEYS[: len(request.app.state.config.DMR_BASE_URLS)]
+        else:
+            request.app.state.config.DMR_API_KEYS += [""] * (
+                len(request.app.state.config.DMR_BASE_URLS) - len(request.app.state.config.DMR_API_KEYS)
+            )
+
+    keys = list(map(str, range(len(request.app.state.config.DMR_BASE_URLS))))
+    request.app.state.config.DMR_API_CONFIGS = {
+        k: v for k, v in request.app.state.config.DMR_API_CONFIGS.items() if k in keys
+    }
+
+    return {
+        "ENABLE_DMR_API": request.app.state.config.ENABLE_DMR_API,
+        "DMR_BASE_URLS": request.app.state.config.DMR_BASE_URLS,
+        "DMR_API_KEYS": request.app.state.config.DMR_API_KEYS,
+        "DMR_API_CONFIGS": request.app.state.config.DMR_API_CONFIGS,
+    }
+
+
+@router.post("/verify")
+async def verify_connection(form_data: ConnectionVerificationForm, user=Depends(get_admin_user)):
+    return await openai.verify_connection(form_data, user)
+
+
+@router.get("/models")
+@router.get("/models/{url_idx}")
+async def get_models(request: Request, url_idx: Optional[int] = None, user=Depends(get_verified_user)):
+    with _dmr_context(request):
+        return await openai.get_models(request, url_idx=url_idx, user=user)
+
+
+@router.post("/chat/completions")
+async def generate_chat_completion(request: Request, form_data: dict, user=Depends(get_verified_user)):
+    with _dmr_context(request):
+        return await openai.generate_chat_completion(request, form_data, user=user)
+
+
+@router.post("/completions")
+async def completions(request: Request, form_data: dict, user=Depends(get_verified_user)):
+    with _dmr_context(request):
+        return await openai.completions(request, form_data, user=user)
+
+
+@router.post("/embeddings")
+async def embeddings(request: Request, form_data: dict, user=Depends(get_verified_user)):
+    with _dmr_context(request):
+        return await openai.embeddings(request, form_data, user=user)
+
+
+async def get_all_models(request: Request, user: UserModel = None):
+    with _dmr_context(request):
+        return await openai.get_all_models.__wrapped__(request, user)

backend/open_webui/utils/models.py

@@ -6,7 +6,7 @@ import sys
 from aiocache import cached
 from fastapi import Request
 
-from open_webui.routers import openai, ollama
+from open_webui.routers import openai, ollama, docker_model_runner
 from open_webui.functions import get_function_models
 
 
@@ -56,6 +56,11 @@ async def fetch_openai_models(request: Request, user: UserModel = None):
     return openai_response["data"]
 
 
+async def fetch_docker_models(request: Request, user: UserModel = None):
+    dmr_response = await docker_model_runner.get_all_models(request, user=user)
+    return dmr_response["data"]
+
+
 async def get_all_base_models(request: Request, user: UserModel = None):
     openai_task = (
         fetch_openai_models(request, user)
@@ -67,13 +72,18 @@ async def get_all_base_models(request: Request, user: UserModel = None):
         if request.app.state.config.ENABLE_OLLAMA_API
         else asyncio.sleep(0, result=[])
     )
+    dmr_task = (
+        fetch_docker_models(request, user)
+        if request.app.state.config.ENABLE_DMR_API
+        else asyncio.sleep(0, result=[])
+    )
     function_task = get_function_models(request)
 
-    openai_models, ollama_models, function_models = await asyncio.gather(
-        openai_task, ollama_task, function_task
+    openai_models, ollama_models, function_models, dmr_models = await asyncio.gather(
+        openai_task, ollama_task, function_task, dmr_task
     )
 
-    return function_models + openai_models + ollama_models
+    return function_models + openai_models + ollama_models + dmr_models
 
 
 async def get_all_models(request, user: UserModel = None):