From de0084c8df777390704c21670dde216085ef7101 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Thu, 22 Feb 2024 03:22:23 -0800 Subject: [PATCH 01/19] feat: litellm integration --- backend/main.py | 4 ++++ backend/requirements.txt | 2 ++ 2 files changed, 6 insertions(+) diff --git a/backend/main.py b/backend/main.py index d1fb0c205..2ec741b09 100644 --- a/backend/main.py +++ b/backend/main.py @@ -8,6 +8,8 @@ from fastapi.middleware.cors import CORSMiddleware from starlette.exceptions import HTTPException as StarletteHTTPException +from litellm.proxy.proxy_server import app as litellm_app + from apps.ollama.main import app as ollama_app from apps.openai.main import app as openai_app from apps.audio.main import app as audio_app @@ -54,6 +56,8 @@ async def check_url(request: Request, call_next): app.mount("/api/v1", webui_app) +app.mount("/litellm/api", litellm_app) + app.mount("/ollama/api", ollama_app) app.mount("/openai/api", openai_app) diff --git a/backend/requirements.txt b/backend/requirements.txt index 56e1d36eb..75d368d8c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -16,6 +16,8 @@ aiohttp peewee bcrypt +litellm[proxy] + langchain langchain-community chromadb From 9b6dca3d7f59d853acbeb7a00f57c925ac111cc4 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Thu, 22 Feb 2024 04:12:26 -0800 Subject: [PATCH 02/19] feat: litellm frontend integration --- src/lib/apis/litellm/index.ts | 42 +++++++ src/lib/apis/ollama/index.ts | 8 +- src/lib/apis/openai/index.ts | 12 +- src/lib/components/chat/ModelSelector.svelte | 4 +- src/lib/components/chat/SettingsModal.svelte | 9 ++ src/lib/constants.ts | 2 + src/routes/(app)/+layout.svelte | 8 ++ src/routes/(app)/+page.svelte | 111 ++++++++++--------- 8 files changed, 134 insertions(+), 62 deletions(-) create mode 100644 src/lib/apis/litellm/index.ts diff --git a/src/lib/apis/litellm/index.ts b/src/lib/apis/litellm/index.ts new file mode 100644 index 000000000..87d2654e5 --- /dev/null +++ b/src/lib/apis/litellm/index.ts @@ -0,0 +1,42 @@ +import { LITELLM_API_BASE_URL } from '$lib/constants'; + +export const getLiteLLMModels = async (token: string = '') => { + let error = null; + + const res = await fetch(`${LITELLM_API_BASE_URL}/v1/models`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = `OpenAI: ${err?.error?.message ?? 'Network Problem'}`; + return []; + }); + + if (error) { + throw error; + } + + const models = Array.isArray(res) ? res : res?.data ?? null; + + return models + ? models + .map((model) => ({ + id: model.id, + name: model.name ?? model.id, + external: true, + source: 'litellm' + })) + .sort((a, b) => { + return a.name.localeCompare(b.name); + }) + : models; +}; diff --git a/src/lib/apis/ollama/index.ts b/src/lib/apis/ollama/index.ts index 954956aca..5fc8a5fef 100644 --- a/src/lib/apis/ollama/index.ts +++ b/src/lib/apis/ollama/index.ts @@ -128,9 +128,11 @@ export const getOllamaModels = async (token: string = '') => { throw error; } - return (res?.models ?? []).sort((a, b) => { - return a.name.localeCompare(b.name); - }); + return (res?.models ?? []) + .map((model) => ({ id: model.model, name: model.name ?? model.model, ...model })) + .sort((a, b) => { + return a.name.localeCompare(b.name); + }); }; // TODO: migrate to backend diff --git a/src/lib/apis/openai/index.ts b/src/lib/apis/openai/index.ts index f9173f706..414455813 100644 --- a/src/lib/apis/openai/index.ts +++ b/src/lib/apis/openai/index.ts @@ -163,7 +163,7 @@ export const getOpenAIModels = async (token: string = '') => { return models ? models - .map((model) => ({ name: model.id, external: true })) + .map((model) => ({ id: model.id, name: model.name ?? model.id, external: true })) .sort((a, b) => { return a.name.localeCompare(b.name); }) @@ -200,17 +200,21 @@ export const getOpenAIModelsDirect = async ( const models = Array.isArray(res) ? res : res?.data ?? null; return models - .map((model) => ({ name: model.id, external: true })) + .map((model) => ({ id: model.id, name: model.name ?? model.id, external: true })) .filter((model) => (base_url.includes('openai') ? model.name.includes('gpt') : true)) .sort((a, b) => { return a.name.localeCompare(b.name); }); }; -export const generateOpenAIChatCompletion = async (token: string = '', body: object) => { +export const generateOpenAIChatCompletion = async ( + token: string = '', + body: object, + url: string = OPENAI_API_BASE_URL +) => { let error = null; - const res = await fetch(`${OPENAI_API_BASE_URL}/chat/completions`, { + const res = await fetch(`${url}/chat/completions`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, diff --git a/src/lib/components/chat/ModelSelector.svelte b/src/lib/components/chat/ModelSelector.svelte index 81304cb49..8f4789168 100644 --- a/src/lib/components/chat/ModelSelector.svelte +++ b/src/lib/components/chat/ModelSelector.svelte @@ -25,7 +25,7 @@ $: if (selectedModels.length > 0 && $models.length > 0) { selectedModels = selectedModels.map((model) => - $models.map((m) => m.name).includes(model) ? model : '' + $models.map((m) => m.id).includes(model) ? model : '' ); } @@ -45,7 +45,7 @@ {#if model.name === 'hr'}
{:else} - diff --git a/src/lib/components/chat/SettingsModal.svelte b/src/lib/components/chat/SettingsModal.svelte index 9d631f16d..0e257af90 100644 --- a/src/lib/components/chat/SettingsModal.svelte +++ b/src/lib/components/chat/SettingsModal.svelte @@ -4,6 +4,7 @@ import { getOllamaModels } from '$lib/apis/ollama'; import { getOpenAIModels } from '$lib/apis/openai'; + import { getLiteLLMModels } from '$lib/apis/litellm'; import Modal from '../common/Modal.svelte'; import Account from './Settings/Account.svelte'; @@ -41,7 +42,15 @@ console.log(error); return null; }); + models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); + + const liteLLMModels = await getLiteLLMModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }); + + models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); } return models; diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 614fe9592..11cf38f43 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -5,6 +5,8 @@ export const WEBUI_NAME = 'Open WebUI'; export const WEBUI_BASE_URL = dev ? `http://${location.hostname}:8080` : ``; export const WEBUI_API_BASE_URL = `${WEBUI_BASE_URL}/api/v1`; + +export const LITELLM_API_BASE_URL = `${WEBUI_BASE_URL}/litellm/api`; export const OLLAMA_API_BASE_URL = `${WEBUI_BASE_URL}/ollama/api`; export const OPENAI_API_BASE_URL = `${WEBUI_BASE_URL}/openai/api`; export const AUDIO_API_BASE_URL = `${WEBUI_BASE_URL}/audio/api/v1`; diff --git a/src/routes/(app)/+layout.svelte b/src/routes/(app)/+layout.svelte index af3bb31c4..d34a09b72 100644 --- a/src/routes/(app)/+layout.svelte +++ b/src/routes/(app)/+layout.svelte @@ -12,6 +12,7 @@ import { getPrompts } from '$lib/apis/prompts'; import { getOpenAIModels } from '$lib/apis/openai'; + import { getLiteLLMModels } from '$lib/apis/litellm'; import { user, @@ -59,6 +60,13 @@ models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); + const liteLLMModels = await getLiteLLMModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }); + + models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); + return models; }; diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte index 9783ef5df..0bafd0d3f 100644 --- a/src/routes/(app)/+page.svelte +++ b/src/routes/(app)/+page.svelte @@ -36,6 +36,7 @@ import ModelSelector from '$lib/components/chat/ModelSelector.svelte'; import Navbar from '$lib/components/layout/Navbar.svelte'; import { RAGTemplate } from '$lib/utils/rag'; + import { LITELLM_API_BASE_URL, OPENAI_API_BASE_URL } from '$lib/constants'; let stopResponseFlag = false; let autoScroll = true; @@ -277,9 +278,8 @@ } await Promise.all( - selectedModels.map(async (model) => { - console.log(model); - const modelTag = $models.filter((m) => m.name === model).at(0); + selectedModels.map(async (modelId) => { + const model = $models.filter((m) => m.id === modelId).at(0); // Create response message let responseMessageId = uuidv4(); @@ -289,7 +289,7 @@ childrenIds: [], role: 'assistant', content: '', - model: model, + model: model.id, timestamp: Math.floor(Date.now() / 1000) // Unix epoch }; @@ -305,12 +305,12 @@ ]; } - if (modelTag?.external) { + if (model?.external) { await sendPromptOpenAI(model, prompt, responseMessageId, _chatId); - } else if (modelTag) { + } else if (model) { await sendPromptOllama(model, prompt, responseMessageId, _chatId); } else { - toast.error(`Model ${model} not found`); + toast.error(`Model ${model.id} not found`); } }) ); @@ -319,6 +319,7 @@ }; const sendPromptOllama = async (model, userPrompt, responseMessageId, _chatId) => { + model = model.id; const responseMessage = history.messages[responseMessageId]; // Wait until history/message have been updated @@ -530,54 +531,58 @@ const responseMessage = history.messages[responseMessageId]; scrollToBottom(); - const res = await generateOpenAIChatCompletion(localStorage.token, { - model: model, - stream: true, - messages: [ - $settings.system - ? { - role: 'system', - content: $settings.system - } - : undefined, - ...messages - ] - .filter((message) => message) - .map((message, idx, arr) => ({ - role: message.role, - ...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false + const res = await generateOpenAIChatCompletion( + localStorage.token, + { + model: model.id, + stream: true, + messages: [ + $settings.system ? { - content: [ - { - type: 'text', - text: - arr.length - 1 !== idx - ? message.content - : message?.raContent ?? message.content - }, - ...message.files - .filter((file) => file.type === 'image') - .map((file) => ({ - type: 'image_url', - image_url: { - url: file.url - } - })) - ] + role: 'system', + content: $settings.system } - : { - content: - arr.length - 1 !== idx ? message.content : message?.raContent ?? message.content - }) - })), - seed: $settings?.options?.seed ?? undefined, - stop: $settings?.options?.stop ?? undefined, - temperature: $settings?.options?.temperature ?? undefined, - top_p: $settings?.options?.top_p ?? undefined, - num_ctx: $settings?.options?.num_ctx ?? undefined, - frequency_penalty: $settings?.options?.repeat_penalty ?? undefined, - max_tokens: $settings?.options?.num_predict ?? undefined - }); + : undefined, + ...messages + ] + .filter((message) => message) + .map((message, idx, arr) => ({ + role: message.role, + ...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false + ? { + content: [ + { + type: 'text', + text: + arr.length - 1 !== idx + ? message.content + : message?.raContent ?? message.content + }, + ...message.files + .filter((file) => file.type === 'image') + .map((file) => ({ + type: 'image_url', + image_url: { + url: file.url + } + })) + ] + } + : { + content: + arr.length - 1 !== idx ? message.content : message?.raContent ?? message.content + }) + })), + seed: $settings?.options?.seed ?? undefined, + stop: $settings?.options?.stop ?? undefined, + temperature: $settings?.options?.temperature ?? undefined, + top_p: $settings?.options?.top_p ?? undefined, + num_ctx: $settings?.options?.num_ctx ?? undefined, + frequency_penalty: $settings?.options?.repeat_penalty ?? undefined, + max_tokens: $settings?.options?.num_predict ?? undefined + }, + model.source === 'litellm' ? `${LITELLM_API_BASE_URL}/v1` : `${OPENAI_API_BASE_URL}` + ); if (res && res.ok) { const reader = res.body From af388dfe62ec2102e7c75afcf0878f93bdd2311b Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 23 Feb 2024 22:27:28 -0800 Subject: [PATCH 03/19] fix --- src/routes/(app)/+page.svelte | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte index dc098b880..31312e80c 100644 --- a/src/routes/(app)/+page.svelte +++ b/src/routes/(app)/+page.svelte @@ -539,23 +539,12 @@ stream: true, messages: [ $settings.system - ? { - role: 'system', - content: $settings.system - } - : undefined, - ...messages.filter((message) => !message.deleted) - ] - .filter((message) => message) - .map((message, idx, arr) => ({ - role: message.role, - ...(message.files?.filter((file) => file.type === 'image').length > 0 ?? false ? { role: 'system', content: $settings.system } : undefined, - ...messages + ...messages.filter((message) => !message.deleted) ] .filter((message) => message) .map((message, idx, arr) => ({ From b5bd07a06a7f62e548c5dd18d23ba3c3eb1f19c2 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 23 Feb 2024 22:44:56 -0800 Subject: [PATCH 04/19] feat: secure litellm api --- backend/main.py | 21 +++++++++++++++++++-- backend/utils/utils.py | 11 +++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/main.py b/backend/main.py index b9370a185..94145a975 100644 --- a/backend/main.py +++ b/backend/main.py @@ -4,9 +4,10 @@ import markdown import time -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, Depends from fastapi.staticfiles import StaticFiles from fastapi import HTTPException +from fastapi.responses import JSONResponse from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.middleware.cors import CORSMiddleware from starlette.exceptions import HTTPException as StarletteHTTPException @@ -19,10 +20,11 @@ from apps.openai.main import app as openai_app from apps.audio.main import app as audio_app from apps.images.main import app as images_app from apps.rag.main import app as rag_app - from apps.web.main import app as webui_app + from config import WEBUI_NAME, ENV, VERSION, CHANGELOG, FRONTEND_BUILD_DIR +from utils.utils import get_http_authorization_cred, get_current_user class SPAStaticFiles(StaticFiles): @@ -59,6 +61,21 @@ async def check_url(request: Request, call_next): return response +@litellm_app.middleware("http") +async def auth_middleware(request: Request, call_next): + auth_header = request.headers.get("Authorization", "") + + if ENV != "dev": + try: + user = get_current_user(get_http_authorization_cred(auth_header)) + print(user) + except Exception as e: + return JSONResponse(status_code=400, content={"detail": str(e)}) + + response = await call_next(request) + return response + + app.mount("/api/v1", webui_app) app.mount("/litellm/api", litellm_app) diff --git a/backend/utils/utils.py b/backend/utils/utils.py index c6d018145..7ae7e6944 100644 --- a/backend/utils/utils.py +++ b/backend/utils/utils.py @@ -58,6 +58,17 @@ def extract_token_from_auth_header(auth_header: str): return auth_header[len("Bearer ") :] +def get_http_authorization_cred(auth_header: str): + try: + scheme, credentials = auth_header.split(" ") + return { + "scheme": scheme, + "credentials": credentials, + } + except: + raise ValueError(ERROR_MESSAGES.INVALID_TOKEN) + + def get_current_user( auth_token: HTTPAuthorizationCredentials = Depends(bearer_security), ): From 14dd0d11fb6d670c0e6d3c430541810c484dcdb1 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 00:21:53 -0800 Subject: [PATCH 05/19] feat: litellm yaml --- backend/.gitignore | 2 ++ backend/main.py | 25 +++++++++++++++++++++++-- test.json | 6 ++++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 test.json diff --git a/backend/.gitignore b/backend/.gitignore index 16180123c..1a2bc493f 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -8,4 +8,6 @@ _test Pipfile data/* !data/config.json +!data/litellm/config.yaml + .webui_secret_key \ No newline at end of file diff --git a/backend/main.py b/backend/main.py index 94145a975..a432e9ed6 100644 --- a/backend/main.py +++ b/backend/main.py @@ -2,7 +2,8 @@ from bs4 import BeautifulSoup import json import markdown import time - +import os +import sys from fastapi import FastAPI, Request, Depends from fastapi.staticfiles import StaticFiles @@ -13,6 +14,7 @@ from fastapi.middleware.cors import CORSMiddleware from starlette.exceptions import HTTPException as StarletteHTTPException +from litellm.proxy.proxy_server import ProxyConfig, initialize from litellm.proxy.proxy_server import app as litellm_app from apps.ollama.main import app as ollama_app @@ -38,6 +40,21 @@ class SPAStaticFiles(StaticFiles): raise ex +proxy_config = ProxyConfig() + + +async def config(): + router, model_list, general_settings = await proxy_config.load_config( + router=None, config_file_path="./data/litellm/config.yaml" + ) + + await initialize(config="./data/litellm/config.yaml", telemetry=False) + + +async def startup(): + await config() + + app = FastAPI(docs_url="/docs" if ENV == "dev" else None, redoc_url=None) origins = ["*"] @@ -51,6 +68,11 @@ app.add_middleware( ) +@app.on_event("startup") +async def on_startup(): + await startup() + + @app.middleware("http") async def check_url(request: Request, call_next): start_time = int(time.time()) @@ -79,7 +101,6 @@ async def auth_middleware(request: Request, call_next): app.mount("/api/v1", webui_app) app.mount("/litellm/api", litellm_app) - app.mount("/ollama/api", ollama_app) app.mount("/openai/api", openai_app) diff --git a/test.json b/test.json new file mode 100644 index 000000000..dccb6b269 --- /dev/null +++ b/test.json @@ -0,0 +1,6 @@ +{ + "model_name": "string", + "litellm_params": { + "model": "ollama/mistral" + } +} \ No newline at end of file From 43f9a827d38b039b65163c8517fc59fb9b55e226 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 00:29:03 -0800 Subject: [PATCH 06/19] fix: gitignore --- backend/.gitignore | 7 +++++-- backend/data/litellm/config.yaml | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 backend/data/litellm/config.yaml diff --git a/backend/.gitignore b/backend/.gitignore index 1a2bc493f..ea83b34f4 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -6,8 +6,11 @@ uploads *.db _test Pipfile -data/* -!data/config.json +!/data +/data/* +!/data/litellm +/data/litellm/* !data/litellm/config.yaml +!data/config.json .webui_secret_key \ No newline at end of file diff --git a/backend/data/litellm/config.yaml b/backend/data/litellm/config.yaml new file mode 100644 index 000000000..7d9d2b723 --- /dev/null +++ b/backend/data/litellm/config.yaml @@ -0,0 +1,4 @@ +general_settings: {} +litellm_settings: {} +model_list: [] +router_settings: {} From e8904ac3f7f63bbcdd7facd82f51338b26027fb8 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 01:03:28 -0800 Subject: [PATCH 07/19] feat: remove ollama dependency --- src/routes/(app)/+layout.svelte | 99 +++++++-------------------------- 1 file changed, 20 insertions(+), 79 deletions(-) diff --git a/src/routes/(app)/+layout.svelte b/src/routes/(app)/+layout.svelte index aab34a108..54922fd0f 100644 --- a/src/routes/(app)/+layout.svelte +++ b/src/routes/(app)/+layout.svelte @@ -44,31 +44,28 @@ let showShortcuts = false; const getModels = async () => { - let models = []; - models.push( - ...(await getOllamaModels(localStorage.token).catch((error) => { - toast.error(error); - return []; - })) - ); + let models = await Promise.all([ + await getOllamaModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }), + await getOpenAIModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }), + await getLiteLLMModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }) + ]); - // $settings.OPENAI_API_BASE_URL ?? 'https://api.openai.com/v1', - // $settings.OPENAI_API_KEY - - const openAIModels = await getOpenAIModels(localStorage.token).catch((error) => { - console.log(error); - return null; - }); - - models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); - - const liteLLMModels = await getLiteLLMModels(localStorage.token).catch((error) => { - console.log(error); - return null; - }); - - models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); + models = models + .filter((models) => models) + .reduce((a, e, i, arr) => a.concat(e, ...(i < arr.length - 1 ? [{ name: 'hr' }] : [])), []); + // models.push(...(ollamaModels ? [{ name: 'hr' }, ...ollamaModels] : [])); + // models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); + // models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); return models; }; @@ -125,8 +122,6 @@ await models.set(await getModels()); }); - await setOllamaVersion(); - document.addEventListener('keydown', function (event) { const isCtrlPressed = event.ctrlKey || event.metaKey; // metaKey is for Cmd key on Mac // Check if the Shift key is pressed @@ -258,60 +253,6 @@ - {:else if checkVersion(REQUIRED_OLLAMA_VERSION, ollamaVersion ?? '0')} -
-
-
-
-
- Connection Issue or Update Needed -
- -
- Oops! It seems like your Ollama needs a little attention. We've detected either a connection hiccup or observed that you're using an older - version. Ensure you're on the latest Ollama version - (version - {REQUIRED_OLLAMA_VERSION} or higher) or check your connection. - -
- Trouble accessing Ollama? - - Click here for help. - -
-
- -
- - - -
-
-
-
-
{:else if localDBChats.length > 0}
Date: Sat, 24 Feb 2024 01:19:44 -0800 Subject: [PATCH 08/19] fix: pip dependency --- backend/requirements.txt | 2 +- .../chat/Settings/Connections.svelte | 11 +++--- src/lib/components/chat/SettingsModal.svelte | 36 +++++++++---------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 75d368d8c..7ec45b04a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -16,7 +16,7 @@ aiohttp peewee bcrypt -litellm[proxy] +litellm langchain langchain-community diff --git a/src/lib/components/chat/Settings/Connections.svelte b/src/lib/components/chat/Settings/Connections.svelte index 462d30055..486fd9b78 100644 --- a/src/lib/components/chat/Settings/Connections.svelte +++ b/src/lib/components/chat/Settings/Connections.svelte @@ -3,7 +3,7 @@ import { createEventDispatcher, onMount } from 'svelte'; const dispatch = createEventDispatcher(); - import { getOllamaAPIUrl, updateOllamaAPIUrl } from '$lib/apis/ollama'; + import { getOllamaAPIUrl, getOllamaVersion, updateOllamaAPIUrl } from '$lib/apis/ollama'; import { getOpenAIKey, getOpenAIUrl, updateOpenAIKey, updateOpenAIUrl } from '$lib/apis/openai'; import toast from 'svelte-french-toast'; @@ -24,11 +24,14 @@ const updateOllamaAPIUrlHandler = async () => { API_BASE_URL = await updateOllamaAPIUrl(localStorage.token, API_BASE_URL); - const _models = await getModels('ollama'); - if (_models.length > 0) { + const ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => { + return null; + }); + + if (ollamaVersion) { toast.success('Server connection verified'); - await models.set(_models); + await models.set(await getModels()); } }; diff --git a/src/lib/components/chat/SettingsModal.svelte b/src/lib/components/chat/SettingsModal.svelte index 0e257af90..b112180b3 100644 --- a/src/lib/components/chat/SettingsModal.svelte +++ b/src/lib/components/chat/SettingsModal.svelte @@ -28,31 +28,29 @@ let selectedTab = 'general'; - const getModels = async (type = 'all') => { - const models = []; - models.push( - ...(await getOllamaModels(localStorage.token).catch((error) => { - toast.error(error); - return []; - })) - ); - - if (type === 'all') { - const openAIModels = await getOpenAIModels(localStorage.token).catch((error) => { + const getModels = async () => { + let models = await Promise.all([ + await getOllamaModels(localStorage.token).catch((error) => { console.log(error); return null; - }); - - models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); - - const liteLLMModels = await getLiteLLMModels(localStorage.token).catch((error) => { + }), + await getOpenAIModels(localStorage.token).catch((error) => { console.log(error); return null; - }); + }), + await getLiteLLMModels(localStorage.token).catch((error) => { + console.log(error); + return null; + }) + ]); - models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); - } + models = models + .filter((models) => models) + .reduce((a, e, i, arr) => a.concat(e, ...(i < arr.length - 1 ? [{ name: 'hr' }] : [])), []); + // models.push(...(ollamaModels ? [{ name: 'hr' }, ...ollamaModels] : [])); + // models.push(...(openAIModels ? [{ name: 'hr' }, ...openAIModels] : [])); + // models.push(...(liteLLMModels ? [{ name: 'hr' }, ...liteLLMModels] : [])); return models; }; From a74d47b6fc7a9b7fbd3641c08a516ed7c7d31cf5 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 01:27:44 -0800 Subject: [PATCH 09/19] Update requirements.txt --- backend/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/requirements.txt b/backend/requirements.txt index 7ec45b04a..341369fcb 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -17,6 +17,7 @@ peewee bcrypt litellm +apscheduler langchain langchain-community From fdb6499bfa697c95095b53dba259f4f760875c74 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 01:49:19 -0800 Subject: [PATCH 10/19] fix: error handling --- src/routes/(app)/+page.svelte | 62 ++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte index 31312e80c..79fbf3cdc 100644 --- a/src/routes/(app)/+page.svelte +++ b/src/routes/(app)/+page.svelte @@ -133,6 +133,10 @@ selectedModels = ['']; } + selectedModels = selectedModels.map((modelId) => + $models.map((m) => m.id).includes(modelId) ? modelId : '' + ); + let _settings = JSON.parse(localStorage.getItem('settings') ?? '{}'); settings.set({ ..._settings @@ -151,6 +155,10 @@ const submitPrompt = async (userPrompt, _user = null) => { console.log('submitPrompt', $chatId); + selectedModels = selectedModels.map((modelId) => + $models.map((m) => m.id).includes(modelId) ? modelId : '' + ); + if (selectedModels.includes('')) { toast.error('Model not selected'); } else if (messages.length != 0 && messages.at(-1).done != true) { @@ -282,36 +290,38 @@ selectedModels.map(async (modelId) => { const model = $models.filter((m) => m.id === modelId).at(0); - // Create response message - let responseMessageId = uuidv4(); - let responseMessage = { - parentId: parentId, - id: responseMessageId, - childrenIds: [], - role: 'assistant', - content: '', - model: model.id, - timestamp: Math.floor(Date.now() / 1000) // Unix epoch - }; + if (model) { + // Create response message + let responseMessageId = uuidv4(); + let responseMessage = { + parentId: parentId, + id: responseMessageId, + childrenIds: [], + role: 'assistant', + content: '', + model: model.id, + timestamp: Math.floor(Date.now() / 1000) // Unix epoch + }; - // Add message to history and Set currentId to messageId - history.messages[responseMessageId] = responseMessage; - history.currentId = responseMessageId; + // Add message to history and Set currentId to messageId + history.messages[responseMessageId] = responseMessage; + history.currentId = responseMessageId; - // Append messageId to childrenIds of parent message - if (parentId !== null) { - history.messages[parentId].childrenIds = [ - ...history.messages[parentId].childrenIds, - responseMessageId - ]; - } + // Append messageId to childrenIds of parent message + if (parentId !== null) { + history.messages[parentId].childrenIds = [ + ...history.messages[parentId].childrenIds, + responseMessageId + ]; + } - if (model?.external) { - await sendPromptOpenAI(model, prompt, responseMessageId, _chatId); - } else if (model) { - await sendPromptOllama(model, prompt, responseMessageId, _chatId); + if (model?.external) { + await sendPromptOpenAI(model, prompt, responseMessageId, _chatId); + } else if (model) { + await sendPromptOllama(model, prompt, responseMessageId, _chatId); + } } else { - toast.error(`Model ${model.id} not found`); + toast.error(`Model ${modelId} not found`); } }) ); From 32da2aa511aa895b088e6343e6fb6bd6f2fdd9ce Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 13:11:56 -0800 Subject: [PATCH 11/19] Update config.py --- backend/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/config.py b/backend/config.py index fadae68ce..b16913c12 100644 --- a/backend/config.py +++ b/backend/config.py @@ -83,8 +83,6 @@ for version in soup.find_all("h2"): # Find the next sibling that is a h3 tag (section title) current = version.find_next_sibling() - print(current) - while current and current.name != "h2": if current.name == "h3": section_title = current.get_text().lower() # e.g., "added", "fixed" From 3bcfc7ca76a0b66d5ebb4a5f550ba8c97553a475 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 13:38:06 -0800 Subject: [PATCH 12/19] doc: update --- README.md | 234 ++++-------------------------------------------------- 1 file changed, 16 insertions(+), 218 deletions(-) diff --git a/README.md b/README.md index fdc806b6c..a17dcd458 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,6 @@ User-friendly WebUI for LLMs, supported LLM runners include Ollama and OpenAI-co ![Open WebUI Demo](./demo.gif) -Also check our sibling project, [Open WebUI Community](https://openwebui.com/), where you can discover, download, and explore customized Modelfiles for Ollama! ๐Ÿฆ™๐Ÿ” - ## Features โญ - ๐Ÿ–ฅ๏ธ **Intuitive Interface**: Our chat interface takes inspiration from ChatGPT, ensuring a user-friendly experience. @@ -77,56 +75,17 @@ Also check our sibling project, [Open WebUI Community](https://openwebui.com/), ## ๐Ÿ”— Also Check Out Open WebUI Community! -Don't forget to explore our sibling project, [Open WebUI Community](https://openwebui.com/), where you can discover, download, and explore customized Modelfiles. Open WebUI Community offers a wide range of exciting possibilities for enhancing your chat interactions with Ollama! ๐Ÿš€ +Don't forget to explore our sibling project, [Open WebUI Community](https://openwebui.com/), where you can discover, download, and explore customized Modelfiles. Open WebUI Community offers a wide range of exciting possibilities for enhancing your chat interactions with Open WebUI! ๐Ÿš€ ## How to Install ๐Ÿš€ -๐ŸŒŸ **Important Note on User Roles and Privacy:** +> [!NOTE] +> Please note that for certain Docker environments, additional configurations might be needed. If you encounter any connection issues, our detailed guide on [Open WebUI Documentation](https://docs.openwebui.com/) is ready to assist you. -- **Admin Creation:** The very first account to sign up on Open WebUI will be granted **Administrator privileges**. This account will have comprehensive control over the platform, including user management and system settings. +### Quick Start with Docker ๐Ÿณ -- **User Registrations:** All subsequent users signing up will initially have their accounts set to **Pending** status by default. These accounts will require approval from the Administrator to gain access to the platform functionalities. - -- **Privacy and Data Security:** We prioritize your privacy and data security above all. Please be reassured that all data entered into Open WebUI is stored locally on your device. Our system is designed to be privacy-first, ensuring that no external requests are made, and your data does not leave your local environment. We are committed to maintaining the highest standards of data privacy and security, ensuring that your information remains confidential and under your control. - -### Steps to Install Open WebUI - -#### Before You Begin - -1. **Installing Docker:** - - - **For Windows and Mac Users:** - - - Download Docker Desktop from [Docker's official website](https://www.docker.com/products/docker-desktop). - - Follow the installation instructions provided on the website. After installation, open Docker Desktop to ensure it's running properly. - - - **For Ubuntu and Other Linux Users:** - - Open your terminal. - - Set up your Docker apt repository according to the [Docker documentation](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) - - Update your package index: - ```bash - sudo apt-get update - ``` - - Install Docker using the following command: - ```bash - sudo apt-get install docker-ce docker-ce-cli containerd.io - ``` - - Verify the Docker installation with: - ```bash - sudo docker run hello-world - ``` - This command downloads a test image and runs it in a container, which prints an informational message. - -2. **Ensure You Have the Latest Version of Ollama:** - - - Download the latest version from [https://ollama.com/](https://ollama.com/). - -3. **Verify Ollama Installation:** - - After installing Ollama, check if it's working by visiting [http://127.0.0.1:11434/](http://127.0.0.1:11434/) in your web browser. Remember, the port number might be different for you. - -#### Installing with Docker ๐Ÿณ - -- **Important:** When using Docker to install Open WebUI, make sure to include the `-v open-webui:/app/backend/data` in your Docker command. This step is crucial as it ensures your database is properly mounted and prevents any loss of data. +> [!IMPORTANT] +> When using Docker to install Open WebUI, make sure to include the `-v open-webui:/app/backend/data` in your Docker command. This step is crucial as it ensures your database is properly mounted and prevents any loss of data. - **If Ollama is on your computer**, use this command: @@ -134,16 +93,7 @@ Don't forget to explore our sibling project, [Open WebUI Community](https://open docker run -d -p 3000:8080 --add-host=host.docker.internal:host-gateway -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main ``` -- **To build the container yourself**, follow these steps: - - ```bash - docker build -t open-webui . - docker run -d -p 3000:8080 --add-host=host.docker.internal:host-gateway -v open-webui:/app/backend/data --name open-webui --restart always open-webui - ``` - -- After installation, you can access Open WebUI at [http://localhost:3000](http://localhost:3000). - -#### Using Ollama on a Different Server +- **If Ollama is on a Different Server**, use this command: - To connect to Ollama on another server, change the `OLLAMA_API_BASE_URL` to the server's URL: @@ -151,175 +101,23 @@ Don't forget to explore our sibling project, [Open WebUI Community](https://open docker run -d -p 3000:8080 -e OLLAMA_API_BASE_URL=https://example.com/api -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main ``` - Or for a self-built container: +- After installation, you can access Open WebUI at [http://localhost:3000](http://localhost:3000). Enjoy! ๐Ÿ˜„ - ```bash - docker build -t open-webui . - docker run -d -p 3000:8080 -e OLLAMA_API_BASE_URL=https://example.com/api -v open-webui:/app/backend/data --name open-webui --restart always open-webui - ``` +#### Troubleshooting and Community Support -### Installing Ollama and Open WebUI Together +Encountering connection issues? Our [Open WebUI Documentation](https://docs.openwebui.com/getting-started/troubleshooting/) has got you covered. For further assistance and to join our vibrant community, visit the [Open WebUI Discord](https://discord.gg/5rJgQTnV4s). -#### Using Docker Compose +### Other Installation Methods -- If you don't have Ollama yet, use Docker Compose for easy installation. Run this command: +We offer various installation alternatives, including non-Docker methods, Docker Compose, Kustomize, and Helm. Visit our [Open WebUI Documentation](https://docs.openwebui.com/getting-started/) or join our [Discord community](https://discord.gg/5rJgQTnV4s) for comprehensive guidance. - ```bash - docker compose up -d --build - ``` +### Keeping Your Docker Installation Up-to-Date -- **For GPU Support:** Use an additional Docker Compose file: +Stay current by following our Migration Guide available in our [Open WebUI Documentation](https://docs.openwebui.com/getting-started/). - ```bash - docker compose -f docker-compose.yaml -f docker-compose.gpu.yaml up -d --build - ``` +## What's Next? ๐ŸŒŸ -- **To Expose Ollama API:** Use another Docker Compose file: - - ```bash - docker compose -f docker-compose.yaml -f docker-compose.api.yaml up -d --build - ``` - -#### Using `run-compose.sh` Script (Linux or Docker-Enabled WSL2 on Windows) - -- Give execute permission to the script: - - ```bash - chmod +x run-compose.sh - ``` - -- For CPU-only container: - - ```bash - ./run-compose.sh - ``` - -- For GPU support (read the note about GPU compatibility): - - ```bash - ./run-compose.sh --enable-gpu - ``` - -- To build the latest local version, add `--build`: - - ```bash - ./run-compose.sh --enable-gpu --build - ``` - -### Alternative Installation Methods - -For other ways to install, like using Kustomize or Helm, check out [INSTALLATION.md](/INSTALLATION.md). Join our [Open WebUI Discord community](https://discord.gg/5rJgQTnV4s) for more help and information. - -### Updating your Docker Installation - -In case you want to update your local Docker installation to the latest version, you can do it with [Watchtower](https://containrrr.dev/watchtower/): - -```bash -docker run --rm --volume /var/run/docker.sock:/var/run/docker.sock containrrr/watchtower --run-once open-webui -``` - -In the last part of the command, replace `open-webui` with your container name if it is different. - -### Moving from Ollama WebUI to Open WebUI - -Given recent name changes, the docker image has been renamed. Additional steps are required to update for those people that used Ollama WebUI previously and want to start using the new images. - -#### Updating to Open WebUI without keeping your data - -If you want to update to the new image but don't want to keep any previous data like conversations, prompts, documents, etc. you can perform the following steps: - -```bash -docker rm -f ollama-webui -docker pull ghcr.io/open-webui/open-webui:main -[insert the equivalent command that you used to install with the new Docker image name] -docker volume rm ollama-webui -``` - -For example, for local installation it would be `docker run -d -p 3000:8080 --add-host=host.docker.internal:host-gateway -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main`. For other installation commands, check the relevant parts of this README document. - -#### Migrating your contents from Ollama WebUI to Open WebUI - -If you want to update to the new image migrating all your previous settings like conversations, prompts, documents, etc. you can perform the following steps: - -```bash -docker rm -f ollama-webui -docker pull ghcr.io/open-webui/open-webui:main -# Creates a new volume and uses a temporary container to copy from one volume to another as per https://github.com/moby/moby/issues/31154#issuecomment-360531460 -docker volume create --name open-webui -docker run --rm -v ollama-webui:/from -v open-webui:/to alpine ash -c "cd /from ; cp -av . /to" -[insert the equivalent command that you used to install with the new Docker image name] -``` - -Once you verify that all the data has been migrated you can erase the old volumen using the following command: - -```bash -docker volume rm ollama-webui -``` - -## How to Install Without Docker - -While we strongly recommend using our convenient Docker container installation for optimal support, we understand that some situations may require a non-Docker setup, especially for development purposes. Please note that non-Docker installations are not officially supported, and you might need to troubleshoot on your own. - -### Project Components - -Open WebUI consists of two primary components: the frontend and the backend (which serves as a reverse proxy, handling static frontend files, and additional features). Both need to be running concurrently for the development environment. - -> [!IMPORTANT] -> The backend is required for proper functionality - -### Requirements ๐Ÿ“ฆ - -- ๐Ÿฐ [Bun](https://bun.sh) >= 1.0.21 or ๐Ÿข [Node.js](https://nodejs.org/en) >= 20.10 -- ๐Ÿ [Python](https://python.org) >= 3.11 - -### Build and Install ๐Ÿ› ๏ธ - -Run the following commands to install: - -```sh -git clone https://github.com/open-webui/open-webui.git -cd open-webui/ - -# Copying required .env file -cp -RPp .env.example .env - -# Building Frontend Using Node -npm i -npm run build - -# or Building Frontend Using Bun -# bun install -# bun run build - -# Serving Frontend with the Backend -cd ./backend -pip install -r requirements.txt -U -sh start.sh -``` - -You should have Open WebUI up and running at http://localhost:8080/. Enjoy! ๐Ÿ˜„ - -## Troubleshooting - -See [TROUBLESHOOTING.md](/TROUBLESHOOTING.md) for information on how to troubleshoot and/or join our [Open WebUI Discord community](https://discord.gg/5rJgQTnV4s). - -## What's Next? ๐Ÿš€ - -### Roadmap ๐Ÿ“ - -Here are some exciting tasks on our roadmap: - -- ๐Ÿ”Š **Local Text-to-Speech Integration**: Seamlessly incorporate text-to-speech functionality directly within the platform, allowing for a smoother and more immersive user experience. -- ๐Ÿ›ก๏ธ **Granular Permissions and User Groups**: Empower administrators to finely control access levels and group users according to their roles and responsibilities. This feature ensures robust security measures and streamlined management of user privileges, enhancing overall platform functionality. -- ๐Ÿ”„ **Function Calling**: Empower your interactions by running code directly within the chat. Execute functions and commands effortlessly, enhancing the functionality of your conversations. -- โš™๏ธ **Custom Python Backend Actions**: Empower your Open WebUI by creating or downloading custom Python backend actions. Unleash the full potential of your web interface with tailored actions that suit your specific needs, enhancing functionality and versatility. -- ๐Ÿ”ง **Fine-tune Model (LoRA)**: Fine-tune your model directly from the user interface. This feature allows for precise customization and optimization of the chat experience to better suit your needs and preferences. -- ๐Ÿง  **Long-Term Memory**: Witness the power of persistent memory in our agents. Enjoy conversations that feel continuous as agents remember and reference past interactions, creating a more cohesive and personalized user experience. -- ๐Ÿงช **Research-Centric Features**: Empower researchers in the fields of LLM and HCI with a comprehensive web UI for conducting user studies. Stay tuned for ongoing feature enhancements (e.g., surveys, analytics, and participant tracking) to facilitate their research. -- ๐Ÿ“ˆ **User Study Tools**: Providing specialized tools, like heat maps and behavior tracking modules, to empower researchers in capturing and analyzing user behavior patterns with precision and accuracy. -- ๐Ÿ“š **Enhanced Documentation**: Elevate your setup and customization experience with improved, comprehensive documentation. - -Feel free to contribute and help us make Open WebUI even better! ๐Ÿ™Œ +Discover upcoming features on our roadmap in the [Open WebUI Documentation](https://docs.openwebui.com/roadmap/). ## Supporters โœจ From c8a11dc75f7b0a66702cb7718fb6b1dffbf41f8b Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 13:41:01 -0800 Subject: [PATCH 13/19] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a17dcd458..affa919ff 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Don't forget to explore our sibling project, [Open WebUI Community](https://open - After installation, you can access Open WebUI at [http://localhost:3000](http://localhost:3000). Enjoy! ๐Ÿ˜„ -#### Troubleshooting and Community Support +#### Troubleshooting Encountering connection issues? Our [Open WebUI Documentation](https://docs.openwebui.com/getting-started/troubleshooting/) has got you covered. For further assistance and to join our vibrant community, visit the [Open WebUI Discord](https://discord.gg/5rJgQTnV4s). From 4669099da6f87030f3f584e6d04f9226b088f15d Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 13:48:12 -0800 Subject: [PATCH 14/19] Update README.md --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index affa919ff..134374599 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,17 @@ We offer various installation alternatives, including non-Docker methods, Docker ### Keeping Your Docker Installation Up-to-Date -Stay current by following our Migration Guide available in our [Open WebUI Documentation](https://docs.openwebui.com/getting-started/). +In case you want to update your local Docker installation to the latest version, you can do it with [Watchtower](https://containrrr.dev/watchtower/): + +```bash +docker run --rm --volume /var/run/docker.sock:/var/run/docker.sock containrrr/watchtower --run-once open-webui +``` + +In the last part of the command, replace `open-webui` with your container name if it is different. + +### Moving from Ollama WebUI to Open WebUI + +Check our Migration Guide available in our [Open WebUI Documentation](https://docs.openwebui.com/getting-started/). ## What's Next? ๐ŸŒŸ From e69dfd12e7b96f8bd900886595498c09c264fff7 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 13:52:21 -0800 Subject: [PATCH 15/19] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 134374599..32d7598fc 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ In the last part of the command, replace `open-webui` with your container name i ### Moving from Ollama WebUI to Open WebUI -Check our Migration Guide available in our [Open WebUI Documentation](https://docs.openwebui.com/getting-started/). +Check our Migration Guide available in our [Open WebUI Documentation](https://docs.openwebui.com/migration/). ## What's Next? ๐ŸŒŸ From 42384ea1a90aeb1266c64babeb2fbaabd5526d44 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 13:56:56 -0800 Subject: [PATCH 16/19] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32d7598fc..bed73e09c 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ [![Discord](https://img.shields.io/badge/Discord-Open_WebUI-blue?logo=discord&logoColor=white)](https://discord.gg/5rJgQTnV4s) [![](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/tjbck) -User-friendly WebUI for LLMs, supported LLM runners include Ollama and OpenAI-compatible APIs. +User-friendly WebUI for LLMs, supported LLM runners include Ollama and OpenAI-compatible APIs. For more information, be sure to check out our [Open WebUI Documentation](https://docs.openwebui.com/). ![Open WebUI Demo](./demo.gif) From e15e015db162f7236a7d38bbc7992a8076ac568a Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 15:48:14 -0800 Subject: [PATCH 17/19] fix: images --- backend/apps/images/main.py | 8 ++++---- src/lib/components/chat/Settings/Images.svelte | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/apps/images/main.py b/backend/apps/images/main.py index 39d3f96aa..3ea8b1711 100644 --- a/backend/apps/images/main.py +++ b/backend/apps/images/main.py @@ -49,7 +49,7 @@ async def toggle_enabled(request: Request, user=Depends(get_admin_user)): app.state.ENABLED = not app.state.ENABLED return app.state.ENABLED except Exception as e: - raise HTTPException(status_code=r.status_code, detail=ERROR_MESSAGES.DEFAULT(e)) + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) class UrlUpdateForm(BaseModel): @@ -109,7 +109,7 @@ def get_models(user=Depends(get_current_user)): models = r.json() return models except Exception as e: - raise HTTPException(status_code=r.status_code, detail=ERROR_MESSAGES.DEFAULT(e)) + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) @app.get("/models/default") @@ -120,7 +120,7 @@ async def get_default_model(user=Depends(get_admin_user)): return {"model": options["sd_model_checkpoint"]} except Exception as e: - raise HTTPException(status_code=r.status_code, detail=ERROR_MESSAGES.DEFAULT(e)) + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) class UpdateModelForm(BaseModel): @@ -190,4 +190,4 @@ def generate_image( return r.json() except Exception as e: print(e) - raise HTTPException(status_code=r.status_code, detail=ERROR_MESSAGES.DEFAULT(e)) + raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) diff --git a/src/lib/components/chat/Settings/Images.svelte b/src/lib/components/chat/Settings/Images.svelte index 94cb3213b..cc8274546 100644 --- a/src/lib/components/chat/Settings/Images.svelte +++ b/src/lib/components/chat/Settings/Images.svelte @@ -32,9 +32,11 @@ const getModels = async () => { models = await getDiffusionModels(localStorage.token).catch((error) => { toast.error(error); - return null; + return []; + }); + selectedModel = await getDefaultDiffusionModel(localStorage.token).catch((error) => { + return ''; }); - selectedModel = await getDefaultDiffusionModel(localStorage.token); }; const updateAUTOMATIC1111UrlHandler = async () => { From a453be07e11aada1821f21d8e75b9410992492d8 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 17:22:28 -0800 Subject: [PATCH 18/19] Update CHANGELOG.md Co-Authored-By: Alex Gartner <2747955+gartnera@users.noreply.github.com> --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48a60634c..2a35d7237 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.103] - UNRELEASED + +### Added + +- **Built-in LiteLLM Proxy**: Open WebUI now ships with LiteLLM Proxy. +- **Image Generation Enhancements**: Advanced Settings + Image Preview Feature. + +### Fixed + +- Issue with RAG scan that stops loading documents as soon as it reaches a file with unsupported mime type (or any other exceptions). (#866) + +### Changed + +- Ollama is no longer required to run Open WebUI. +- Our documentation can be found here https://docs.openwebui.com/ + ## [0.1.102] - 2024-02-22 ### Added From ec6f53e29837440da799afe3b51e9d03ef56262a Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 24 Feb 2024 17:56:53 -0800 Subject: [PATCH 19/19] feat: litellm proxy frontend integration --- backend/apps/images/main.py | 2 + package.json | 2 +- src/lib/apis/litellm/index.ts | 108 ++- src/lib/components/chat/Settings/About.svelte | 16 +- .../chat/Settings/Connections.svelte | 164 ++-- .../components/chat/Settings/Models.svelte | 891 ++++++++++++------ 6 files changed, 823 insertions(+), 360 deletions(-) diff --git a/backend/apps/images/main.py b/backend/apps/images/main.py index 3ea8b1711..9322ea7f5 100644 --- a/backend/apps/images/main.py +++ b/backend/apps/images/main.py @@ -109,6 +109,7 @@ def get_models(user=Depends(get_current_user)): models = r.json() return models except Exception as e: + app.state.ENABLED = False raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) @@ -120,6 +121,7 @@ async def get_default_model(user=Depends(get_admin_user)): return {"model": options["sd_model_checkpoint"]} except Exception as e: + app.state.ENABLED = False raise HTTPException(status_code=400, detail=ERROR_MESSAGES.DEFAULT(e)) diff --git a/package.json b/package.json index 30549fdd5..7938558db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-webui", - "version": "0.1.102", + "version": "0.1.103", "private": true, "scripts": { "dev": "vite dev --host", diff --git a/src/lib/apis/litellm/index.ts b/src/lib/apis/litellm/index.ts index 87d2654e5..6466ee35b 100644 --- a/src/lib/apis/litellm/index.ts +++ b/src/lib/apis/litellm/index.ts @@ -17,7 +17,7 @@ export const getLiteLLMModels = async (token: string = '') => { }) .catch((err) => { console.log(err); - error = `OpenAI: ${err?.error?.message ?? 'Network Problem'}`; + error = `LiteLLM: ${err?.error?.message ?? 'Network Problem'}`; return []; }); @@ -40,3 +40,109 @@ export const getLiteLLMModels = async (token: string = '') => { }) : models; }; + +export const getLiteLLMModelInfo = async (token: string = '') => { + let error = null; + + const res = await fetch(`${LITELLM_API_BASE_URL}/model/info`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = `LiteLLM: ${err?.error?.message ?? 'Network Problem'}`; + return []; + }); + + if (error) { + throw error; + } + + const models = Array.isArray(res) ? res : res?.data ?? null; + + return models; +}; + +type AddLiteLLMModelForm = { + name: string; + model: string; + api_base: string; + api_key: string; + rpm: string; +}; + +export const addLiteLLMModel = async (token: string = '', payload: AddLiteLLMModelForm) => { + let error = null; + + const res = await fetch(`${LITELLM_API_BASE_URL}/model/new`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify({ + model_name: payload.name, + litellm_params: { + model: payload.model, + ...(payload.api_base === '' ? {} : { api_base: payload.api_base }), + ...(payload.api_key === '' ? {} : { api_key: payload.api_key }), + ...(isNaN(parseInt(payload.rpm)) ? {} : { rpm: parseInt(payload.rpm) }) + } + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = `LiteLLM: ${err?.error?.message ?? 'Network Problem'}`; + return []; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const deleteLiteLLMModel = async (token: string = '', id: string) => { + let error = null; + + const res = await fetch(`${LITELLM_API_BASE_URL}/model/delete`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...(token && { authorization: `Bearer ${token}` }) + }, + body: JSON.stringify({ + id: id + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + console.log(err); + error = `LiteLLM: ${err?.error?.message ?? 'Network Problem'}`; + return []; + }); + + if (error) { + throw error; + } + + return res; +}; diff --git a/src/lib/components/chat/Settings/About.svelte b/src/lib/components/chat/Settings/About.svelte index 6933ed3b6..60c0d7168 100644 --- a/src/lib/components/chat/Settings/About.svelte +++ b/src/lib/components/chat/Settings/About.svelte @@ -38,16 +38,18 @@
-
+ {#if ollamaVersion} +
-
-
Ollama Version
-
-
- {ollamaVersion ?? 'N/A'} +
+
Ollama Version
+
+
+ {ollamaVersion ?? 'N/A'} +
-
+ {/if}
diff --git a/src/lib/components/chat/Settings/Connections.svelte b/src/lib/components/chat/Settings/Connections.svelte index 486fd9b78..db24c8a03 100644 --- a/src/lib/components/chat/Settings/Connections.svelte +++ b/src/lib/components/chat/Settings/Connections.svelte @@ -15,6 +15,9 @@ let OPENAI_API_KEY = ''; let OPENAI_API_BASE_URL = ''; + let showOpenAI = false; + let showLiteLLM = false; + const updateOpenAIHandler = async () => { OPENAI_API_BASE_URL = await updateOpenAIUrl(localStorage.token, OPENAI_API_BASE_URL); OPENAI_API_KEY = await updateOpenAIKey(localStorage.token, OPENAI_API_KEY); @@ -45,7 +48,7 @@
{ updateOpenAIHandler(); dispatch('save'); @@ -56,81 +59,100 @@ // }); }} > -
-
Ollama API URL
-
-
- +
+
+
+
+
OpenAI API
+ +
+ + {#if showOpenAI} +
+
API Key
+
+
+ +
+
+
+ +
+
API Base URL
+
+
+ +
+
+
+ WebUI will make requests to '{OPENAI_API_BASE_URL}/chat' +
+
+ {/if}
- -
- -
- Trouble accessing Ollama? - - Click here for help. - -
-
- -
- -
-
-
OpenAI API Key
-
-
- -
+ + + +
-
-
-
OpenAI API Base URL
-
-
- -
-
- WebUI will make requests to '{OPENAI_API_BASE_URL}/chat' + Trouble accessing Ollama? + + Click here for help. +
diff --git a/src/lib/components/chat/Settings/Models.svelte b/src/lib/components/chat/Settings/Models.svelte index c7ca79081..455927dfc 100644 --- a/src/lib/components/chat/Settings/Models.svelte +++ b/src/lib/components/chat/Settings/Models.svelte @@ -2,14 +2,33 @@ import queue from 'async/queue'; import toast from 'svelte-french-toast'; - import { createModel, deleteModel, pullModel } from '$lib/apis/ollama'; + import { createModel, deleteModel, getOllamaVersion, pullModel } from '$lib/apis/ollama'; import { WEBUI_API_BASE_URL, WEBUI_BASE_URL } from '$lib/constants'; import { WEBUI_NAME, models, user } from '$lib/stores'; import { splitStream } from '$lib/utils'; + import { onMount } from 'svelte'; + import { addLiteLLMModel, deleteLiteLLMModel, getLiteLLMModelInfo } from '$lib/apis/litellm'; export let getModels: Function; + let showLiteLLM = false; + let showLiteLLMParams = false; + + let liteLLMModelInfo = []; + + let liteLLMModel = ''; + let liteLLMModelName = ''; + let liteLLMAPIBase = ''; + let liteLLMAPIKey = ''; + let liteLLMRPM = ''; + + let deleteLiteLLMModelId = ''; + + $: liteLLMModelName = liteLLMModel; + // Models + let showExperimentalOllama = false; + let ollamaVersion = ''; const MAX_PARALLEL_DOWNLOADS = 3; const modelDownloadQueue = queue( (task: { modelName: string }, cb) => @@ -286,256 +305,184 @@ opts.callback({ success: true, modelName: opts.modelName }); } }; + + const addLiteLLMModelHandler = async () => { + if (!liteLLMModelInfo.find((info) => info.model_name === liteLLMModelName)) { + const res = await addLiteLLMModel(localStorage.token, { + name: liteLLMModelName, + model: liteLLMModel, + api_base: liteLLMAPIBase, + api_key: liteLLMAPIKey, + rpm: liteLLMRPM + }).catch((error) => { + toast.error(error); + return null; + }); + + if (res) { + if (res.message) { + toast.success(res.message); + } + } + } else { + toast.error(`Model ${liteLLMModelName} already exists.`); + } + + liteLLMModelName = ''; + liteLLMModel = ''; + liteLLMAPIBase = ''; + liteLLMAPIKey = ''; + liteLLMRPM = ''; + + liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token); + models.set(await getModels()); + }; + + const deleteLiteLLMModelHandler = async () => { + const res = await deleteLiteLLMModel(localStorage.token, deleteLiteLLMModelId).catch( + (error) => { + toast.error(error); + return null; + } + ); + + if (res) { + if (res.message) { + toast.success(res.message); + } + } + + deleteLiteLLMModelId = ''; + liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token); + models.set(await getModels()); + }; + + onMount(async () => { + ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => false); + liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token); + });
-
-
-
Pull a model from Ollama.com
-
-
- -
- -
- -
- To access the available model names for downloading, click here. -
- - {#if Object.keys(modelDownloadStatus).length > 0} - {#each Object.keys(modelDownloadStatus) as model} -
-
{model}
-
-
- {modelDownloadStatus[model].pullProgress ?? 0}% -
-
- {modelDownloadStatus[model].digest} -
-
-
- {/each} - {/if} -
- -
- -
-
Delete a model
-
-
- -
- -
-
- -
- - { - uploadModelHandler(); - }} - > -
-
- Upload a GGUF model (Experimental) -
- - -
- -
-
- {#if modelUploadMode === 'file'} -
- { - console.log(modelInputFile); - }} - accept=".gguf" - required - hidden - /> - - -
- {:else} -
- -
- {/if} -
- - {#if (modelUploadMode === 'file' && modelInputFile && modelInputFile.length > 0) || (modelUploadMode === 'url' && modelFileUrl !== '')} - +
+ +
+ To access the available model names for downloading, click here. +
+ + {#if Object.keys(modelDownloadStatus).length > 0} + {#each Object.keys(modelDownloadStatus) as model} +
+
{model}
+
+
+ {modelDownloadStatus[model].pullProgress ?? 0}% +
+
+ {modelDownloadStatus[model].digest} +
+
- {:else} + {/each} + {/if} +
+ +
+
Delete a model
+
+
+ +
+ +
+
+ +
+
+
Experimental
+ +
+
+ + {#if showExperimentalOllama} + { + uploadModelHandler(); + }} + > +
+
Upload a GGUF model
+ + +
+ +
+
+ {#if modelUploadMode === 'file'} +
+ { + console.log(modelInputFile); + }} + accept=".gguf" + required + hidden + /> + + +
+ {:else} +
+ +
+ {/if} +
+ + {#if (modelUploadMode === 'file' && modelInputFile && modelInputFile.length > 0) || (modelUploadMode === 'url' && modelFileUrl !== '')} + + {/if} +
+ + {#if (modelUploadMode === 'file' && modelInputFile && modelInputFile.length > 0) || (modelUploadMode === 'url' && modelFileUrl !== '')} +
+
+
Modelfile Content
+