diff --git a/.env.example b/.env.example index 35ea12a88..8d08d0709 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,9 @@ OLLAMA_BASE_URL='http://localhost:11434' OPENAI_API_BASE_URL='' OPENAI_API_KEY='' +# Docker Model Runner API base URL +DMR_BASE_URL='http://localhost:12434' + # AUTOMATIC1111_BASE_URL="http://localhost:7860" # For production, you should only need one host as diff --git a/README.md b/README.md index 9d6a66e41..c6e095138 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ For more information, be sure to check out our [Open WebUI Documentation](https: - 🚀 **Effortless Setup**: Install seamlessly using Docker or Kubernetes (kubectl, kustomize or helm) for a hassle-free experience with support for both `:ollama` and `:cuda` tagged images. -- 🤝 **Ollama/OpenAI API Integration**: Effortlessly integrate OpenAI-compatible APIs for versatile conversations alongside Ollama models. Customize the OpenAI API URL to link with **LMStudio, GroqCloud, Mistral, OpenRouter, and more**. +- 🤝 **Ollama/OpenAI API Integration**: Effortlessly integrate OpenAI-compatible APIs for versatile conversations alongside Ollama models. Customize the OpenAI API URL to link with **LMStudio, GroqCloud, Mistral, OpenRouter, Docker Model Runner, and more**. - 🛡️ **Granular Permissions and User Groups**: By allowing administrators to create detailed user roles and permissions, we ensure a secure user environment. This granularity not only enhances security but also allows for customized user experiences, fostering a sense of ownership and responsibility amongst users. @@ -165,6 +165,14 @@ This will start the Open WebUI server, which you can access at [http://localhost docker run -d -p 3000:8080 -e OPENAI_API_KEY=your_secret_key -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main ``` +### Using Docker Model Runner + +Enable Docker Model Runner in Docker Desktop and set the `DMR_BASE_URL` environment variable to the exposed API endpoint: + +```bash +docker run -d -p 3000:8080 -e DMR_BASE_URL=http://localhost:12434 -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main +``` + ### Installing Open WebUI with Bundled Ollama Support This installation method uses a single container image that bundles Open WebUI with Ollama, allowing for a streamlined setup via a single command. Choose the appropriate command based on your hardware setup: diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index b48ba4f2e..d72885f2d 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -895,6 +895,36 @@ except Exception: pass OPENAI_API_BASE_URL = "https://api.openai.com/v1" +#################################### +# DOCKER MODEL RUNNER +#################################### + +ENABLE_DMR_API = PersistentConfig( + "ENABLE_DMR_API", + "dmr.enable", + os.environ.get("ENABLE_DMR_API", "True").lower() == "true", +) + +DMR_API_KEYS = [k.strip() for k in os.environ.get("DMR_API_KEYS", "").split(";")] +DMR_API_KEYS = PersistentConfig("DMR_API_KEYS", "dmr.api_keys", DMR_API_KEYS) + +DMR_BASE_URL = os.environ.get("DMR_BASE_URL", "") +if DMR_BASE_URL: + DMR_BASE_URL = DMR_BASE_URL[:-1] if DMR_BASE_URL.endswith("/") else DMR_BASE_URL + +DMR_BASE_URLS = os.environ.get("DMR_BASE_URLS", "") +DMR_BASE_URLS = DMR_BASE_URLS if DMR_BASE_URLS != "" else DMR_BASE_URL or "http://localhost:12434" +DMR_BASE_URLS = [url.strip() for url in DMR_BASE_URLS.split(";")] +DMR_BASE_URLS = PersistentConfig( + "DMR_BASE_URLS", "dmr.base_urls", DMR_BASE_URLS +) + +DMR_API_CONFIGS = PersistentConfig( + "DMR_API_CONFIGS", + "dmr.api_configs", + {}, +) + #################################### # TOOL_SERVERS #################################### diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index b6f26a827..8f11d812e 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -63,6 +63,7 @@ from open_webui.routers import ( images, ollama, openai, + docker_model_runner, retrieval, pipelines, tasks, @@ -112,6 +113,11 @@ from open_webui.config import ( OPENAI_API_BASE_URLS, OPENAI_API_KEYS, OPENAI_API_CONFIGS, + # Docker Model Runner + ENABLE_DMR_API, + DMR_BASE_URLS, + DMR_API_KEYS, + DMR_API_CONFIGS, # Direct Connections ENABLE_DIRECT_CONNECTIONS, # Thread pool size for FastAPI/AnyIO @@ -589,6 +595,19 @@ app.state.config.OPENAI_API_CONFIGS = OPENAI_API_CONFIGS app.state.OPENAI_MODELS = {} +######################################## +# +# DOCKER MODEL RUNNER +# +######################################## + +app.state.config.ENABLE_DMR_API = ENABLE_DMR_API +app.state.config.DMR_BASE_URLS = DMR_BASE_URLS +app.state.config.DMR_API_KEYS = DMR_API_KEYS +app.state.config.DMR_API_CONFIGS = DMR_API_CONFIGS + +app.state.DMR_MODELS = {} + ######################################## # # TOOL SERVERS @@ -1111,6 +1130,7 @@ app.mount("/ws", socket_app) app.include_router(ollama.router, prefix="/ollama", tags=["ollama"]) app.include_router(openai.router, prefix="/openai", tags=["openai"]) +app.include_router(docker_model_runner.router, prefix="/dmr", tags=["dmr"]) app.include_router(pipelines.router, prefix="/api/v1/pipelines", tags=["pipelines"]) diff --git a/backend/open_webui/routers/docker_model_runner.py b/backend/open_webui/routers/docker_model_runner.py new file mode 100644 index 000000000..7623cfb88 --- /dev/null +++ b/backend/open_webui/routers/docker_model_runner.py @@ -0,0 +1,110 @@ +from contextlib import contextmanager +from typing import Optional + +from fastapi import APIRouter, Depends, Request +from pydantic import BaseModel + +from open_webui.models.users import UserModel +from open_webui.routers import openai +from open_webui.routers.openai import ConnectionVerificationForm +from open_webui.utils.auth import get_admin_user, get_verified_user + +router = APIRouter() + +@contextmanager +def _dmr_context(request: Request): + orig_urls = request.app.state.config.OPENAI_API_BASE_URLS + orig_keys = request.app.state.config.OPENAI_API_KEYS + orig_configs = request.app.state.config.OPENAI_API_CONFIGS + orig_models = request.app.state.OPENAI_MODELS + request.app.state.config.OPENAI_API_BASE_URLS = request.app.state.config.DMR_BASE_URLS + request.app.state.config.OPENAI_API_KEYS = request.app.state.config.DMR_API_KEYS + request.app.state.config.OPENAI_API_CONFIGS = request.app.state.config.DMR_API_CONFIGS + request.app.state.OPENAI_MODELS = request.app.state.DMR_MODELS + try: + yield + finally: + request.app.state.config.OPENAI_API_BASE_URLS = orig_urls + request.app.state.config.OPENAI_API_KEYS = orig_keys + request.app.state.config.OPENAI_API_CONFIGS = orig_configs + request.app.state.OPENAI_MODELS = orig_models + + +@router.get("/config") +async def get_config(request: Request, user=Depends(get_admin_user)): + return { + "ENABLE_DMR_API": request.app.state.config.ENABLE_DMR_API, + "DMR_BASE_URLS": request.app.state.config.DMR_BASE_URLS, + "DMR_API_KEYS": request.app.state.config.DMR_API_KEYS, + "DMR_API_CONFIGS": request.app.state.config.DMR_API_CONFIGS, + } + + +class DMRConfigForm(BaseModel): + ENABLE_DMR_API: Optional[bool] = None + DMR_BASE_URLS: list[str] + DMR_API_KEYS: list[str] = [] + DMR_API_CONFIGS: dict = {} + + +@router.post("/config/update") +async def update_config(request: Request, form_data: DMRConfigForm, user=Depends(get_admin_user)): + request.app.state.config.ENABLE_DMR_API = form_data.ENABLE_DMR_API + request.app.state.config.DMR_BASE_URLS = form_data.DMR_BASE_URLS + request.app.state.config.DMR_API_KEYS = form_data.DMR_API_KEYS + request.app.state.config.DMR_API_CONFIGS = form_data.DMR_API_CONFIGS + + if len(request.app.state.config.DMR_API_KEYS) != len(request.app.state.config.DMR_BASE_URLS): + if len(request.app.state.config.DMR_API_KEYS) > len(request.app.state.config.DMR_BASE_URLS): + request.app.state.config.DMR_API_KEYS = request.app.state.config.DMR_API_KEYS[: len(request.app.state.config.DMR_BASE_URLS)] + else: + request.app.state.config.DMR_API_KEYS += [""] * ( + len(request.app.state.config.DMR_BASE_URLS) - len(request.app.state.config.DMR_API_KEYS) + ) + + keys = list(map(str, range(len(request.app.state.config.DMR_BASE_URLS)))) + request.app.state.config.DMR_API_CONFIGS = { + k: v for k, v in request.app.state.config.DMR_API_CONFIGS.items() if k in keys + } + + return { + "ENABLE_DMR_API": request.app.state.config.ENABLE_DMR_API, + "DMR_BASE_URLS": request.app.state.config.DMR_BASE_URLS, + "DMR_API_KEYS": request.app.state.config.DMR_API_KEYS, + "DMR_API_CONFIGS": request.app.state.config.DMR_API_CONFIGS, + } + + +@router.post("/verify") +async def verify_connection(form_data: ConnectionVerificationForm, user=Depends(get_admin_user)): + return await openai.verify_connection(form_data, user) + + +@router.get("/models") +@router.get("/models/{url_idx}") +async def get_models(request: Request, url_idx: Optional[int] = None, user=Depends(get_verified_user)): + with _dmr_context(request): + return await openai.get_models(request, url_idx=url_idx, user=user) + + +@router.post("/chat/completions") +async def generate_chat_completion(request: Request, form_data: dict, user=Depends(get_verified_user)): + with _dmr_context(request): + return await openai.generate_chat_completion(request, form_data, user=user) + + +@router.post("/completions") +async def completions(request: Request, form_data: dict, user=Depends(get_verified_user)): + with _dmr_context(request): + return await openai.completions(request, form_data, user=user) + + +@router.post("/embeddings") +async def embeddings(request: Request, form_data: dict, user=Depends(get_verified_user)): + with _dmr_context(request): + return await openai.embeddings(request, form_data, user=user) + + +async def get_all_models(request: Request, user: UserModel = None): + with _dmr_context(request): + return await openai.get_all_models.__wrapped__(request, user) diff --git a/backend/open_webui/utils/models.py b/backend/open_webui/utils/models.py index f637449ba..c551377cd 100644 --- a/backend/open_webui/utils/models.py +++ b/backend/open_webui/utils/models.py @@ -6,7 +6,7 @@ import sys from aiocache import cached from fastapi import Request -from open_webui.routers import openai, ollama +from open_webui.routers import openai, ollama, docker_model_runner from open_webui.functions import get_function_models @@ -56,6 +56,11 @@ async def fetch_openai_models(request: Request, user: UserModel = None): return openai_response["data"] +async def fetch_docker_models(request: Request, user: UserModel = None): + dmr_response = await docker_model_runner.get_all_models(request, user=user) + return dmr_response["data"] + + async def get_all_base_models(request: Request, user: UserModel = None): openai_task = ( fetch_openai_models(request, user) @@ -67,13 +72,18 @@ async def get_all_base_models(request: Request, user: UserModel = None): if request.app.state.config.ENABLE_OLLAMA_API else asyncio.sleep(0, result=[]) ) + dmr_task = ( + fetch_docker_models(request, user) + if request.app.state.config.ENABLE_DMR_API + else asyncio.sleep(0, result=[]) + ) function_task = get_function_models(request) - openai_models, ollama_models, function_models = await asyncio.gather( - openai_task, ollama_task, function_task + openai_models, ollama_models, function_models, dmr_models = await asyncio.gather( + openai_task, ollama_task, function_task, dmr_task ) - return function_models + openai_models + ollama_models + return function_models + openai_models + ollama_models + dmr_models async def get_all_models(request, user: UserModel = None):