diff --git a/backend/open_webui/apps/webui/main.py b/backend/open_webui/apps/webui/main.py index e93cbfb76..5a0a83961 100644 --- a/backend/open_webui/apps/webui/main.py +++ b/backend/open_webui/apps/webui/main.py @@ -1,6 +1,7 @@ import inspect import json import logging +import time from typing import AsyncGenerator, Generator, Iterator from open_webui.apps.socket.main import get_event_call, get_event_emitter @@ -17,6 +18,7 @@ from open_webui.apps.webui.routers import ( models, knowledge, prompts, + evaluations, tools, users, utils, @@ -32,6 +34,9 @@ from open_webui.config import ( ENABLE_LOGIN_FORM, ENABLE_MESSAGE_RATING, ENABLE_SIGNUP, + ENABLE_EVALUATION_ARENA_MODELS, + EVALUATION_ARENA_MODELS, + DEFAULT_ARENA_MODEL, JWT_EXPIRES_IN, ENABLE_OAUTH_ROLE_MANAGEMENT, OAUTH_ROLES_CLAIM, @@ -94,6 +99,9 @@ app.state.config.BANNERS = WEBUI_BANNERS app.state.config.ENABLE_COMMUNITY_SHARING = ENABLE_COMMUNITY_SHARING app.state.config.ENABLE_MESSAGE_RATING = ENABLE_MESSAGE_RATING +app.state.config.ENABLE_EVALUATION_ARENA_MODELS = ENABLE_EVALUATION_ARENA_MODELS +app.state.config.EVALUATION_ARENA_MODELS = EVALUATION_ARENA_MODELS + app.state.config.OAUTH_USERNAME_CLAIM = OAUTH_USERNAME_CLAIM app.state.config.OAUTH_PICTURE_CLAIM = OAUTH_PICTURE_CLAIM app.state.config.OAUTH_EMAIL_CLAIM = OAUTH_EMAIL_CLAIM @@ -117,20 +125,24 @@ app.add_middleware( app.include_router(configs.router, prefix="/configs", tags=["configs"]) + app.include_router(auths.router, prefix="/auths", tags=["auths"]) app.include_router(users.router, prefix="/users", tags=["users"]) + app.include_router(chats.router, prefix="/chats", tags=["chats"]) -app.include_router(folders.router, prefix="/folders", tags=["folders"]) app.include_router(models.router, prefix="/models", tags=["models"]) app.include_router(knowledge.router, prefix="/knowledge", tags=["knowledge"]) app.include_router(prompts.router, prefix="/prompts", tags=["prompts"]) - -app.include_router(files.router, prefix="/files", tags=["files"]) app.include_router(tools.router, prefix="/tools", tags=["tools"]) app.include_router(functions.router, prefix="/functions", tags=["functions"]) app.include_router(memories.router, prefix="/memories", tags=["memories"]) +app.include_router(evaluations.router, prefix="/evaluations", tags=["evaluations"]) + +app.include_router(folders.router, prefix="/folders", tags=["folders"]) +app.include_router(files.router, prefix="/files", tags=["files"]) + app.include_router(utils.router, prefix="/utils", tags=["utils"]) @@ -145,8 +157,44 @@ async def get_status(): async def get_all_models(): + models = [] pipe_models = await get_pipe_models() - return pipe_models + models = models + pipe_models + + if app.state.config.ENABLE_EVALUATION_ARENA_MODELS: + arena_models = [] + if len(app.state.config.EVALUATION_ARENA_MODELS) > 0: + arena_models = [ + { + "id": model["id"], + "name": model["name"], + "info": { + "meta": model["meta"], + }, + "object": "model", + "created": int(time.time()), + "owned_by": "arena", + "arena": True, + } + for model in app.state.config.EVALUATION_ARENA_MODELS + ] + else: + # Add default arena model + arena_models = [ + { + "id": DEFAULT_ARENA_MODEL["id"], + "name": DEFAULT_ARENA_MODEL["name"], + "info": { + "meta": DEFAULT_ARENA_MODEL["meta"], + }, + "object": "model", + "created": int(time.time()), + "owned_by": "arena", + "arena": True, + } + ] + models = models + arena_models + return models def get_function_module(pipe_id: str): diff --git a/backend/open_webui/apps/webui/routers/evaluations.py b/backend/open_webui/apps/webui/routers/evaluations.py new file mode 100644 index 000000000..b40953e49 --- /dev/null +++ b/backend/open_webui/apps/webui/routers/evaluations.py @@ -0,0 +1,49 @@ +from typing import Optional +from fastapi import APIRouter, Depends, HTTPException, status, Request +from pydantic import BaseModel + + +from open_webui.constants import ERROR_MESSAGES +from open_webui.utils.utils import get_admin_user, get_verified_user + +router = APIRouter() + + +############################ +# GetConfig +############################ + + +@router.get("/config") +async def get_config(request: Request, user=Depends(get_admin_user)): + return { + "ENABLE_EVALUATION_ARENA_MODELS": request.app.state.config.ENABLE_EVALUATION_ARENA_MODELS, + "EVALUATION_ARENA_MODELS": request.app.state.config.EVALUATION_ARENA_MODELS, + } + + +############################ +# UpdateConfig +############################ + + +class UpdateConfigForm(BaseModel): + ENABLE_EVALUATION_ARENA_MODELS: Optional[bool] = None + EVALUATION_ARENA_MODELS: Optional[list[dict]] = None + + +@router.post("/config") +async def update_config( + request: Request, + form_data: UpdateConfigForm, + user=Depends(get_admin_user), +): + config = request.app.state.config + if form_data.ENABLE_EVALUATION_ARENA_MODELS is not None: + config.ENABLE_EVALUATION_ARENA_MODELS = form_data.ENABLE_EVALUATION_ARENA_MODELS + if form_data.EVALUATION_ARENA_MODELS is not None: + config.EVALUATION_ARENA_MODELS = form_data.EVALUATION_ARENA_MODELS + return { + "ENABLE_EVALUATION_ARENA_MODELS": config.ENABLE_EVALUATION_ARENA_MODELS, + "EVALUATION_ARENA_MODELS": config.EVALUATION_ARENA_MODELS, + } diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 779e3d3bc..9d1bd72d8 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -751,6 +751,28 @@ USER_PERMISSIONS = PersistentConfig( }, ) + +ENABLE_EVALUATION_ARENA_MODELS = PersistentConfig( + "ENABLE_EVALUATION_ARENA_MODELS", + "evaluation.arena.enable", + os.environ.get("ENABLE_EVALUATION_ARENA_MODELS", "True").lower() == "true", +) +EVALUATION_ARENA_MODELS = PersistentConfig( + "EVALUATION_ARENA_MODELS", + "evaluation.arena.models", + [], +) + +DEFAULT_ARENA_MODEL = { + "id": "arena-model", + "name": "Arena Model", + "meta": { + "profile_image_url": "/favicon.png", + "description": "Submit your questions to anonymous AI chatbots and vote on the best response.", + "model_ids": None, + }, +} + ENABLE_MODEL_FILTER = PersistentConfig( "ENABLE_MODEL_FILTER", "model_filter.enable", diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index c5c1dd8f0..3f5cab660 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -7,6 +7,7 @@ import os import shutil import sys import time +import random from contextlib import asynccontextmanager from typing import Optional @@ -23,7 +24,7 @@ from fastapi import ( status, ) from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel from sqlalchemy import text @@ -1093,6 +1094,23 @@ async def generate_chat_completions(form_data: dict, user=Depends(get_verified_u ) model = app.state.MODELS[model_id] + + if model["owned_by"] == "arena": + model_ids = model.get("info", {}).get("meta", {}).get("model_ids") + model_id = None + if isinstance(model_ids, list) and model_ids: + model_id = random.choice(model_ids) + else: + model_ids = [ + model["id"] + for model in await get_all_models() + if model.get("owned_by") != "arena" + and not model.get("info", {}).get("meta", {}).get("hidden", False) + ] + model_id = random.choice(model_ids) + + form_data["model"] = model_id + return await generate_chat_completions(form_data, user) if model.get("pipe"): return await generate_function_chat_completion(form_data, user=user) if model["owned_by"] == "ollama": diff --git a/backend/open_webui/utils/payload.py b/backend/open_webui/utils/payload.py index 2789b942f..04e3a98c4 100644 --- a/backend/open_webui/utils/payload.py +++ b/backend/open_webui/utils/payload.py @@ -116,6 +116,9 @@ def convert_messages_openai_to_ollama(messages: list[dict]) -> list[dict]: elif item.get("type") == "image_url": img_url = item.get("image_url", {}).get("url", "") if img_url: + # If the image url starts with data:, it's a base64 image and should be trimmed + if img_url.startswith("data:"): + img_url = img_url.split(",")[-1] images.append(img_url) # Add content text (if any) diff --git a/src/lib/apis/evaluations/index.ts b/src/lib/apis/evaluations/index.ts new file mode 100644 index 000000000..21130bd17 --- /dev/null +++ b/src/lib/apis/evaluations/index.ts @@ -0,0 +1,63 @@ +import { WEBUI_API_BASE_URL } from '$lib/constants'; + +export const getConfig = async (token: string = '') => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/config`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const updateConfig = async (token: string, config: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/config`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + ...config + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; diff --git a/src/lib/components/admin/AddUserModal.svelte b/src/lib/components/admin/AddUserModal.svelte index 8538ba04a..eea77fce5 100644 --- a/src/lib/components/admin/AddUserModal.svelte +++ b/src/lib/components/admin/AddUserModal.svelte @@ -139,7 +139,7 @@ -
+
-
+
-
+
{$i18n.t('Name')}
@@ -198,7 +200,7 @@
-
+
{$i18n.t('Email')}
@@ -209,13 +211,12 @@ type="email" bind:value={_user.email} placeholder={$i18n.t('Enter Your Email')} - autocomplete="off" required />
-
+
{$i18n.t('Password')}
@@ -271,13 +272,13 @@
+ + + +
+
+ +
+ {#if (config?.EVALUATION_ARENA_MODELS ?? []).length > 0} + {#each config.EVALUATION_ARENA_MODELS as model, index} + { + editModelHandler(e.detail, index); + }} + on:delete={(e) => { + deleteModelHandler(index); + }} + /> + {/each} + {:else} +
+ {$i18n.t( + `Using the default arena model with all models. Click the plus button to add custom models.` + )} +
+ {/if} +
+ {/if} +
+ {:else} +
+
+ +
+
+ {/if} +
+ +
+ +
+
diff --git a/src/lib/components/admin/Settings/Evaluations/Model.svelte b/src/lib/components/admin/Settings/Evaluations/Model.svelte new file mode 100644 index 000000000..6c4d417ee --- /dev/null +++ b/src/lib/components/admin/Settings/Evaluations/Model.svelte @@ -0,0 +1,63 @@ + + + { + dispatch('edit', e.detail); + }} + on:delete={async () => { + dispatch('delete'); + }} +/> + +
+
+
+
+ {model.name} + +
+
+
+ {model.name} +
+
+ +
+
+ {model.meta.description} +
+
+
+
+
+ +
+ +
+
+
diff --git a/src/lib/components/admin/Settings/Evaluations/ModelModal.svelte b/src/lib/components/admin/Settings/Evaluations/ModelModal.svelte new file mode 100644 index 000000000..4c2bb9d30 --- /dev/null +++ b/src/lib/components/admin/Settings/Evaluations/ModelModal.svelte @@ -0,0 +1,398 @@ + + + +
+
+
+ {#if edit} + {$i18n.t('Edit Arena Model')} + {:else} + {$i18n.t('Add Arena Model')} + {/if} +
+ +
+ +
+
+
{ + submitHandler(); + }} + > +
+
+ { + const files = e.target.files ?? []; + let reader = new FileReader(); + reader.onload = (event) => { + let originalImageUrl = `${event.target.result}`; + + const img = new Image(); + img.src = originalImageUrl; + + img.onload = function () { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + + // Calculate the aspect ratio of the image + const aspectRatio = img.width / img.height; + + // Calculate the new width and height to fit within 250x250 + let newWidth, newHeight; + if (aspectRatio > 1) { + newWidth = 250 * aspectRatio; + newHeight = 250; + } else { + newWidth = 250; + newHeight = 250 / aspectRatio; + } + + // Set the canvas size + canvas.width = 250; + canvas.height = 250; + + // Calculate the position to center the image + const offsetX = (250 - newWidth) / 2; + const offsetY = (250 - newHeight) / 2; + + // Draw the image on the canvas + ctx.drawImage(img, offsetX, offsetY, newWidth, newHeight); + + // Get the base64 representation of the compressed image + const compressedSrc = canvas.toDataURL('image/jpeg'); + + // Display the compressed image + profileImageUrl = compressedSrc; + + e.target.files = null; + }; + }; + + if ( + files.length > 0 && + ['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes( + files[0]['type'] + ) + ) { + reader.readAsDataURL(files[0]); + } + }} + /> + + +
+
+
+
{$i18n.t('Name')}
+ +
+ +
+
+ +
+
{$i18n.t('ID')}
+ +
+ +
+
+
+ +
+
{$i18n.t('Description')}
+ +
+ +
+
+ +
+ +
+
{$i18n.t('Models')}
+ + {#if modelIds.length > 0} +
+ {#each modelIds as modelId, modelIdx} +
+
+ {$models.find((model) => model.id === modelId)?.name} +
+
+ +
+
+ {/each} +
+ {:else} +
+ {$i18n.t('Leave empty to include all models or select specific models')} +
+ {/if} +
+ +
+ +
+ + +
+ +
+
+
+ +
+ {#if edit} + + {/if} + + +
+
+
+
+
+ diff --git a/src/lib/components/admin/Settings/Interface.svelte b/src/lib/components/admin/Settings/Interface.svelte index a3dbec09d..632d69244 100644 --- a/src/lib/components/admin/Settings/Interface.svelte +++ b/src/lib/components/admin/Settings/Interface.svelte @@ -62,7 +62,7 @@ >
-
+
{$i18n.t('Set Task Model')}
- {#if models[selectedModelIdx]?.info} - {models[selectedModelIdx]?.info?.name} + {#if models[selectedModelIdx]?.name} + {models[selectedModelIdx]?.name} {:else} {$i18n.t('Hello, {{name}}', { name: $user.name })} {/if} diff --git a/src/lib/components/chat/Controls/Controls.svelte b/src/lib/components/chat/Controls/Controls.svelte index 9a90a1495..e2b166fb3 100644 --- a/src/lib/components/chat/Controls/Controls.svelte +++ b/src/lib/components/chat/Controls/Controls.svelte @@ -56,7 +56,7 @@
-
+
{/if} @@ -65,7 +65,7 @@
-
+
@@ -78,7 +78,7 @@
-
+
diff --git a/src/lib/components/chat/Placeholder.svelte b/src/lib/components/chat/Placeholder.svelte index 912d82222..f1d1e0579 100644 --- a/src/lib/components/chat/Placeholder.svelte +++ b/src/lib/components/chat/Placeholder.svelte @@ -134,8 +134,8 @@
- {#if models[selectedModelIdx]?.info} - {models[selectedModelIdx]?.info?.name} + {#if models[selectedModelIdx]?.name} + {models[selectedModelIdx]?.name} {:else} {$i18n.t('Hello, {{name}}', { name: $user.name })} {/if} diff --git a/src/lib/components/chat/Settings/Account.svelte b/src/lib/components/chat/Settings/Account.svelte index a8cd2e53d..e76a54679 100644 --- a/src/lib/components/chat/Settings/Account.svelte +++ b/src/lib/components/chat/Settings/Account.svelte @@ -93,23 +93,23 @@ // Calculate the aspect ratio of the image const aspectRatio = img.width / img.height; - // Calculate the new width and height to fit within 100x100 + // Calculate the new width and height to fit within 250x250 let newWidth, newHeight; if (aspectRatio > 1) { - newWidth = 100 * aspectRatio; - newHeight = 100; + newWidth = 250 * aspectRatio; + newHeight = 250; } else { - newWidth = 100; - newHeight = 100 / aspectRatio; + newWidth = 250; + newHeight = 250 / aspectRatio; } // Set the canvas size - canvas.width = 100; - canvas.height = 100; + canvas.width = 250; + canvas.height = 250; // Calculate the position to center the image - const offsetX = (100 - newWidth) / 2; - const offsetY = (100 - newHeight) / 2; + const offsetX = (250 - newWidth) / 2; + const offsetY = (250 - newHeight) / 2; // Draw the image on the canvas ctx.drawImage(img, offsetX, offsetY, newWidth, newHeight); diff --git a/src/lib/components/common/Switch.svelte b/src/lib/components/common/Switch.svelte index 0f8f45460..829b74629 100644 --- a/src/lib/components/common/Switch.svelte +++ b/src/lib/components/common/Switch.svelte @@ -12,7 +12,7 @@ await tick(); dispatch('change', e); }} - class="flex h-5 min-h-5 w-9 shrink-0 cursor-pointer items-center rounded-full px-[3px] transition {state + class="flex h-5 min-h-5 w-9 shrink-0 cursor-pointer items-center rounded-full px-[3px] mx-[1px] transition {state ? ' bg-emerald-600' : 'bg-gray-200 dark:bg-transparent'} outline outline-1 outline-gray-100 dark:outline-gray-800" > diff --git a/src/lib/components/icons/ChartBar.svelte b/src/lib/components/icons/ChartBar.svelte new file mode 100644 index 000000000..6f2d35821 --- /dev/null +++ b/src/lib/components/icons/ChartBar.svelte @@ -0,0 +1,10 @@ + + + + + diff --git a/src/lib/components/icons/DocumentChartBar.svelte b/src/lib/components/icons/DocumentChartBar.svelte new file mode 100644 index 000000000..bc811bed1 --- /dev/null +++ b/src/lib/components/icons/DocumentChartBar.svelte @@ -0,0 +1,15 @@ + + + + + + diff --git a/src/lib/components/icons/Minus.svelte b/src/lib/components/icons/Minus.svelte new file mode 100644 index 000000000..9b2b7f4ad --- /dev/null +++ b/src/lib/components/icons/Minus.svelte @@ -0,0 +1,15 @@ + + + + + diff --git a/src/lib/components/icons/PencilSolid.svelte b/src/lib/components/icons/PencilSolid.svelte new file mode 100644 index 000000000..f986eeb1c --- /dev/null +++ b/src/lib/components/icons/PencilSolid.svelte @@ -0,0 +1,10 @@ + + + + + diff --git a/src/lib/components/workspace/Models.svelte b/src/lib/components/workspace/Models.svelte index a566ec108..59aa40342 100644 --- a/src/lib/components/workspace/Models.svelte +++ b/src/lib/components/workspace/Models.svelte @@ -243,7 +243,7 @@ onMount(async () => { // Legacy code to sync localModelfiles with models - _models = $models; + _models = $models.filter((m) => m?.owned_by !== 'arena'); localModelfiles = JSON.parse(localStorage.getItem('modelfiles') ?? '[]'); if (localModelfiles) { diff --git a/src/lib/stores/index.ts b/src/lib/stores/index.ts index 785718fd3..501ac9b81 100644 --- a/src/lib/stores/index.ts +++ b/src/lib/stores/index.ts @@ -58,7 +58,7 @@ type BaseModel = { id: string; name: string; info?: ModelConfig; - owned_by: 'ollama' | 'openai'; + owned_by: 'ollama' | 'openai' | 'arena'; }; export interface OpenAIModel extends BaseModel { diff --git a/src/routes/(app)/admin/+layout.svelte b/src/routes/(app)/admin/+layout.svelte index b0bd7dfc1..d7311c917 100644 --- a/src/routes/(app)/admin/+layout.svelte +++ b/src/routes/(app)/admin/+layout.svelte @@ -61,6 +61,15 @@ href="/admin">{$i18n.t('Dashboard')} + {$i18n.t('Evaluations')} + {$i18n.t('Select a base model')} - {#each $models.filter((m) => !m?.preset) as model} + {#each $models.filter((m) => !m?.preset && m?.owned_by !== 'arena') as model} {/each} diff --git a/src/routes/(app)/workspace/models/edit/+page.svelte b/src/routes/(app)/workspace/models/edit/+page.svelte index 3185823e0..424bace80 100644 --- a/src/routes/(app)/workspace/models/edit/+page.svelte +++ b/src/routes/(app)/workspace/models/edit/+page.svelte @@ -139,7 +139,7 @@ const _id = $page.url.searchParams.get('id'); if (_id) { - model = $models.find((m) => m.id === _id); + model = $models.find((m) => m.id === _id && m?.owned_by !== 'arena'); if (model) { id = model.id; name = model.name; @@ -395,7 +395,7 @@ required > - {#each $models.filter((m) => m.id !== model.id && !m?.preset) as model} + {#each $models.filter((m) => m.id !== model.id && !m?.preset && m?.owned_by !== 'arena') as model} {/each}