Merge pull request #6345 from open-webui/leaderboard

feat: leaderboard

backend/open_webui/apps/webui/models/feedbacks.py

@@ -0,0 +1,254 @@
+import logging
+import time
+import uuid
+from typing import Optional
+
+from open_webui.apps.webui.internal.db import Base, get_db
+from open_webui.apps.webui.models.chats import Chats
+
+from open_webui.env import SRC_LOG_LEVELS
+from pydantic import BaseModel, ConfigDict
+from sqlalchemy import BigInteger, Column, Text, JSON, Boolean
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["MODELS"])
+
+
+####################
+# Feedback DB Schema
+####################
+
+
+class Feedback(Base):
+    __tablename__ = "feedback"
+    id = Column(Text, primary_key=True)
+    user_id = Column(Text)
+    version = Column(BigInteger, default=0)
+    type = Column(Text)
+    data = Column(JSON, nullable=True)
+    meta = Column(JSON, nullable=True)
+    snapshot = Column(JSON, nullable=True)
+    created_at = Column(BigInteger)
+    updated_at = Column(BigInteger)
+
+
+class FeedbackModel(BaseModel):
+    id: str
+    user_id: str
+    version: int
+    type: str
+    data: Optional[dict] = None
+    meta: Optional[dict] = None
+    snapshot: Optional[dict] = None
+    created_at: int
+    updated_at: int
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+####################
+# Forms
+####################
+
+
+class FeedbackResponse(BaseModel):
+    id: str
+    user_id: str
+    version: int
+    type: str
+    data: Optional[dict] = None
+    meta: Optional[dict] = None
+    created_at: int
+    updated_at: int
+
+
+class RatingData(BaseModel):
+    rating: Optional[str | int] = None
+    model_id: Optional[str] = None
+    sibling_model_ids: Optional[list[str]] = None
+    reason: Optional[str] = None
+    comment: Optional[str] = None
+    model_config = ConfigDict(extra="allow")
+
+
+class MetaData(BaseModel):
+    arena: Optional[bool] = None
+    chat_id: Optional[str] = None
+    message_id: Optional[str] = None
+    tags: Optional[list[str]] = None
+    model_config = ConfigDict(extra="allow")
+
+
+class SnapshotData(BaseModel):
+    chat: Optional[dict] = None
+    model_config = ConfigDict(extra="allow")
+
+
+class FeedbackForm(BaseModel):
+    type: str
+    data: Optional[RatingData] = None
+    meta: Optional[dict] = None
+    snapshot: Optional[SnapshotData] = None
+    model_config = ConfigDict(extra="allow")
+
+
+class FeedbackTable:
+    def insert_new_feedback(
+        self, user_id: str, form_data: FeedbackForm
+    ) -> Optional[FeedbackModel]:
+        with get_db() as db:
+            id = str(uuid.uuid4())
+            feedback = FeedbackModel(
+                **{
+                    "id": id,
+                    "user_id": user_id,
+                    "version": 0,
+                    **form_data.model_dump(),
+                    "created_at": int(time.time()),
+                    "updated_at": int(time.time()),
+                }
+            )
+            try:
+                result = Feedback(**feedback.model_dump())
+                db.add(result)
+                db.commit()
+                db.refresh(result)
+                if result:
+                    return FeedbackModel.model_validate(result)
+                else:
+                    return None
+            except Exception as e:
+                print(e)
+                return None
+
+    def get_feedback_by_id(self, id: str) -> Optional[FeedbackModel]:
+        try:
+            with get_db() as db:
+                feedback = db.query(Feedback).filter_by(id=id).first()
+                if not feedback:
+                    return None
+                return FeedbackModel.model_validate(feedback)
+        except Exception:
+            return None
+
+    def get_feedback_by_id_and_user_id(
+        self, id: str, user_id: str
+    ) -> Optional[FeedbackModel]:
+        try:
+            with get_db() as db:
+                feedback = db.query(Feedback).filter_by(id=id, user_id=user_id).first()
+                if not feedback:
+                    return None
+                return FeedbackModel.model_validate(feedback)
+        except Exception:
+            return None
+
+    def get_all_feedbacks(self) -> list[FeedbackModel]:
+        with get_db() as db:
+            return [
+                FeedbackModel.model_validate(feedback)
+                for feedback in db.query(Feedback)
+                .order_by(Feedback.updated_at.desc())
+                .all()
+            ]
+
+    def get_feedbacks_by_type(self, type: str) -> list[FeedbackModel]:
+        with get_db() as db:
+            return [
+                FeedbackModel.model_validate(feedback)
+                for feedback in db.query(Feedback)
+                .filter_by(type=type)
+                .order_by(Feedback.updated_at.desc())
+                .all()
+            ]
+
+    def get_feedbacks_by_user_id(self, user_id: str) -> list[FeedbackModel]:
+        with get_db() as db:
+            return [
+                FeedbackModel.model_validate(feedback)
+                for feedback in db.query(Feedback)
+                .filter_by(user_id=user_id)
+                .order_by(Feedback.updated_at.desc())
+                .all()
+            ]
+
+    def update_feedback_by_id(
+        self, id: str, form_data: FeedbackForm
+    ) -> Optional[FeedbackModel]:
+        with get_db() as db:
+            feedback = db.query(Feedback).filter_by(id=id).first()
+            if not feedback:
+                return None
+
+            if form_data.data:
+                feedback.data = form_data.data.model_dump()
+            if form_data.meta:
+                feedback.meta = form_data.meta
+            if form_data.snapshot:
+                feedback.snapshot = form_data.snapshot.model_dump()
+
+            feedback.updated_at = int(time.time())
+
+            db.commit()
+            return FeedbackModel.model_validate(feedback)
+
+    def update_feedback_by_id_and_user_id(
+        self, id: str, user_id: str, form_data: FeedbackForm
+    ) -> Optional[FeedbackModel]:
+        with get_db() as db:
+            feedback = db.query(Feedback).filter_by(id=id, user_id=user_id).first()
+            if not feedback:
+                return None
+
+            if form_data.data:
+                feedback.data = form_data.data.model_dump()
+            if form_data.meta:
+                feedback.meta = form_data.meta
+            if form_data.snapshot:
+                feedback.snapshot = form_data.snapshot.model_dump()
+
+            feedback.updated_at = int(time.time())
+
+            db.commit()
+            return FeedbackModel.model_validate(feedback)
+
+    def delete_feedback_by_id(self, id: str) -> bool:
+        with get_db() as db:
+            feedback = db.query(Feedback).filter_by(id=id).first()
+            if not feedback:
+                return False
+            db.delete(feedback)
+            db.commit()
+            return True
+
+    def delete_feedback_by_id_and_user_id(self, id: str, user_id: str) -> bool:
+        with get_db() as db:
+            feedback = db.query(Feedback).filter_by(id=id, user_id=user_id).first()
+            if not feedback:
+                return False
+            db.delete(feedback)
+            db.commit()
+            return True
+
+    def delete_feedbacks_by_user_id(self, user_id: str) -> bool:
+        with get_db() as db:
+            feedbacks = db.query(Feedback).filter_by(user_id=user_id).all()
+            if not feedbacks:
+                return False
+            for feedback in feedbacks:
+                db.delete(feedback)
+            db.commit()
+            return True
+
+    def delete_all_feedbacks(self) -> bool:
+        with get_db() as db:
+            feedbacks = db.query(Feedback).all()
+            if not feedbacks:
+                return False
+            for feedback in feedbacks:
+                db.delete(feedback)
+            db.commit()
+            return True
+
+
+Feedbacks = FeedbackTable()

backend/open_webui/apps/webui/routers/evaluations.py

@@ -2,6 +2,12 @@ from typing import Optional
 from fastapi import APIRouter, Depends, HTTPException, status, Request
 from pydantic import BaseModel
 
+from open_webui.apps.webui.models.users import Users, UserModel
+from open_webui.apps.webui.models.feedbacks import (
+    FeedbackModel,
+    FeedbackForm,
+    Feedbacks,
+)
 
 from open_webui.constants import ERROR_MESSAGES
 from open_webui.utils.utils import get_admin_user, get_verified_user
@@ -47,3 +53,95 @@ async def update_config(
         "ENABLE_EVALUATION_ARENA_MODELS": config.ENABLE_EVALUATION_ARENA_MODELS,
         "EVALUATION_ARENA_MODELS": config.EVALUATION_ARENA_MODELS,
     }
+
+
+@router.get("/feedbacks", response_model=list[FeedbackModel])
+async def get_feedbacks(user=Depends(get_verified_user)):
+    feedbacks = Feedbacks.get_feedbacks_by_user_id(user.id)
+    return feedbacks
+
+
+@router.delete("/feedbacks", response_model=bool)
+async def delete_feedbacks(user=Depends(get_verified_user)):
+    success = Feedbacks.delete_feedbacks_by_user_id(user.id)
+    return success
+
+
+class FeedbackUserModel(FeedbackModel):
+    user: Optional[UserModel] = None
+
+
+@router.get("/feedbacks/all", response_model=list[FeedbackUserModel])
+async def get_all_feedbacks(user=Depends(get_admin_user)):
+    feedbacks = Feedbacks.get_all_feedbacks()
+    return [
+        FeedbackUserModel(
+            **feedback.model_dump(), user=Users.get_user_by_id(feedback.user_id)
+        )
+        for feedback in feedbacks
+    ]
+
+
+@router.delete("/feedbacks/all")
+async def delete_all_feedbacks(user=Depends(get_admin_user)):
+    success = Feedbacks.delete_all_feedbacks()
+    return success
+
+
+@router.post("/feedback", response_model=FeedbackModel)
+async def create_feedback(
+    request: Request,
+    form_data: FeedbackForm,
+    user=Depends(get_verified_user),
+):
+    feedback = Feedbacks.insert_new_feedback(user_id=user.id, form_data=form_data)
+    if not feedback:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.DEFAULT(),
+        )
+
+    return feedback
+
+
+@router.get("/feedback/{id}", response_model=FeedbackModel)
+async def get_feedback_by_id(id: str, user=Depends(get_verified_user)):
+    feedback = Feedbacks.get_feedback_by_id_and_user_id(id=id, user_id=user.id)
+
+    if not feedback:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    return feedback
+
+
+@router.post("/feedback/{id}", response_model=FeedbackModel)
+async def update_feedback_by_id(
+    id: str, form_data: FeedbackForm, user=Depends(get_verified_user)
+):
+    feedback = Feedbacks.update_feedback_by_id_and_user_id(
+        id=id, user_id=user.id, form_data=form_data
+    )
+
+    if not feedback:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    return feedback
+
+
+@router.delete("/feedback/{id}")
+async def delete_feedback_by_id(id: str, user=Depends(get_verified_user)):
+    if user.role == "admin":
+        success = Feedbacks.delete_feedback_by_id(id=id)
+    else:
+        success = Feedbacks.delete_feedback_by_id_and_user_id(id=id, user_id=user.id)
+
+    if not success:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND
+        )
+
+    return success

backend/open_webui/migrations/versions/af906e964978_add_feedback_table.py

@@ -0,0 +1,51 @@
+"""Add feedback table
+
+Revision ID: af906e964978
+Revises: c29facfe716b
+Create Date: 2024-10-20 17:02:35.241684
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# Revision identifiers, used by Alembic.
+revision = "af906e964978"
+down_revision = "c29facfe716b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### Create feedback table ###
+    op.create_table(
+        "feedback",
+        sa.Column(
+            "id", sa.Text(), primary_key=True
+        ),  # Unique identifier for each feedback (TEXT type)
+        sa.Column(
+            "user_id", sa.Text(), nullable=True
+        ),  # ID of the user providing the feedback (TEXT type)
+        sa.Column(
+            "version", sa.BigInteger(), default=0
+        ),  # Version of feedback (BIGINT type)
+        sa.Column("type", sa.Text(), nullable=True),  # Type of feedback (TEXT type)
+        sa.Column("data", sa.JSON(), nullable=True),  # Feedback data (JSON type)
+        sa.Column(
+            "meta", sa.JSON(), nullable=True
+        ),  # Metadata for feedback (JSON type)
+        sa.Column(
+            "snapshot", sa.JSON(), nullable=True
+        ),  # snapshot data for feedback (JSON type)
+        sa.Column(
+            "created_at", sa.BigInteger(), nullable=False
+        ),  # Feedback creation timestamp (BIGINT representing epoch)
+        sa.Column(
+            "updated_at", sa.BigInteger(), nullable=False
+        ),  # Feedback update timestamp (BIGINT representing epoch)
+    )
+
+
+def downgrade():
+    # ### Drop feedback table ###
+    op.drop_table("feedback")

src/lib/apis/evaluations/index.ts

@@ -61,3 +61,155 @@ export const updateConfig = async (token: string, config: object) => {
 
 	return res;
 };
+
+export const getAllFeedbacks = async (token: string = '') => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedbacks/all`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const createNewFeedback = async (token: string, feedback: object) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedback`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			...feedback
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const getFeedbackById = async (token: string, feedbackId: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedback/${feedbackId}`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const updateFeedbackById = async (token: string, feedbackId: string, feedback: object) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedback/${feedbackId}`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			...feedback
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const deleteFeedbackById = async (token: string, feedbackId: string) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedback/${feedbackId}`, {
+		method: 'DELETE',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};

src/lib/components/admin/Evaluations.svelte

@@ -1,27 +1,339 @@
 <script lang="ts">
 	import { onMount, getContext } from 'svelte';
+
+	import dayjs from 'dayjs';
+	import relativeTime from 'dayjs/plugin/relativeTime';
+	dayjs.extend(relativeTime);
+
+	import { models } from '$lib/stores';
+	import { getAllFeedbacks } from '$lib/apis/evaluations';
+
+	import FeedbackMenu from './Evaluations/FeedbackMenu.svelte';
+	import EllipsisHorizontal from '../icons/EllipsisHorizontal.svelte';
 	import Tooltip from '../common/Tooltip.svelte';
-	import Plus from '../icons/Plus.svelte';
+	import Badge from '../common/Badge.svelte';
-	import Collapsible from '../common/Collapsible.svelte';
+
-	import Switch from '../common/Switch.svelte';
-	import ChevronUp from '../icons/ChevronUp.svelte';
-	import ChevronDown from '../icons/ChevronDown.svelte';
 	const i18n = getContext('i18n');
 
+	let rankedModels = [];
+	let feedbacks = [];
+
+	type Feedback = {
+		model_id: string;
+		sibling_model_ids?: string[];
+		rating: number;
+	};
+
+	type ModelStats = {
+		rating: number;
+		won: number;
+		draw: number;
+		lost: number;
+	};
+
+	function calculateModelStats(feedbacks: Feedback[]): Map<string, ModelStats> {
+		const stats = new Map<string, ModelStats>();
+		const K = 32;
+
+		function getOrDefaultStats(modelId: string): ModelStats {
+			return stats.get(modelId) || { rating: 1000, won: 0, draw: 0, lost: 0 };
+		}
+
+		function updateStats(modelId: string, ratingChange: number, outcome: number) {
+			const currentStats = getOrDefaultStats(modelId);
+			currentStats.rating += ratingChange;
+			if (outcome === 1) currentStats.won++;
+			else if (outcome === 0.5) currentStats.draw++;
+			else if (outcome === 0) currentStats.lost++;
+			stats.set(modelId, currentStats);
+		}
+
+		function calculateEloChange(ratingA: number, ratingB: number, outcome: number): number {
+			const expectedScore = 1 / (1 + Math.pow(10, (ratingB - ratingA) / 400));
+			return K * (outcome - expectedScore);
+		}
+
+		feedbacks.forEach((feedback) => {
+			const modelA = feedback.data.model_id;
+			const statsA = getOrDefaultStats(modelA);
+			let outcome: number;
+
+			switch (feedback.data.rating.toString()) {
+				case '1':
+					outcome = 1;
+					break;
+				case '0':
+					outcome = 0.5;
+					break;
+				case '-1':
+					outcome = 0;
+					break;
+				default:
+					return; // Skip invalid ratings
+			}
+
+			const opponents = feedback.data.sibling_model_ids || [];
+			opponents.forEach((modelB) => {
+				const statsB = getOrDefaultStats(modelB);
+				const changeA = calculateEloChange(statsA.rating, statsB.rating, outcome);
+				const changeB = calculateEloChange(statsB.rating, statsA.rating, 1 - outcome);
+
+				updateStats(modelA, changeA, outcome);
+				updateStats(modelB, changeB, 1 - outcome);
+			});
+		});
+
+		return stats;
+	}
+
 	let loaded = false;
-	let evaluationEnabled = true;
+	onMount(async () => {
+		feedbacks = await getAllFeedbacks(localStorage.token);
+		const modelStats = calculateModelStats(feedbacks);
 
-	let showModels = false;
+		rankedModels = $models
+			.filter((m) => m?.owned_by !== 'arena' && (m?.info?.meta?.hidden ?? false) !== true)
+			.map((model) => {
+				const stats = modelStats.get(model.name);
+				return {
+					...model,
+					rating: stats ? Math.round(stats.rating) : '-',
+					stats: {
+						count: stats ? stats.won + stats.draw + stats.lost : 0,
+						won: stats ? stats.won.toString() : '-',
+						lost: stats ? stats.lost.toString() : '-'
+					}
+				};
+			})
+			.sort((a, b) => {
+				// Handle sorting by rating ('-' goes to the end)
+				if (a.rating === '-' && b.rating !== '-') return 1;
+				if (b.rating === '-' && a.rating !== '-') return -1;
+
+				// If both have ratings (non '-'), sort by rating numerically (descending)
+				if (a.rating !== '-' && b.rating !== '-') return b.rating - a.rating;
+
+				// If both ratings are '-', sort alphabetically (by 'name')
+				return a.name.localeCompare(b.name);
+			});
 
-	onMount(() => {
 		loaded = true;
 	});
 </script>
 
 {#if loaded}
-	<div class="my-0.5 gap-1 flex flex-col md:flex-row justify-between">
+	<div class="mt-0.5 mb-2 gap-1 flex flex-col md:flex-row justify-between">
 		<div class="flex md:self-center text-lg font-medium px-0.5">
 			{$i18n.t('Leaderboard')}
+
+			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-50 dark:bg-gray-850" />
+
+			<span class="text-lg font-medium text-gray-500 dark:text-gray-300">{rankedModels.length}</span
+			>
 		</div>
 	</div>
+
+	<div
+		class="scrollbar-hidden relative whitespace-nowrap overflow-x-auto max-w-full rounded pt-0.5"
+	>
+		{#if (rankedModels ?? []).length === 0}
+			<div class="text-center text-xs text-gray-500 dark:text-gray-400 py-1">
+				{$i18n.t('No models found')}
+			</div>
+		{:else}
+			<table
+				class="w-full text-sm text-left text-gray-500 dark:text-gray-400 table-auto max-w-full rounded"
+			>
+				<thead
+					class="text-xs text-gray-700 uppercase bg-gray-50 dark:bg-gray-850 dark:text-gray-400 -translate-y-0.5"
+				>
+					<tr class="">
+						<th scope="col" class="px-3 py-1.5 cursor-pointer select-none w-3">
+							{$i18n.t('RK')}
+						</th>
+						<th scope="col" class="px-3 py-1.5 cursor-pointer select-none">
+							{$i18n.t('Model')}
+						</th>
+						<th scope="col" class="px-3 py-1.5 text-right cursor-pointer select-none w-fit">
+							{$i18n.t('Rating')}
+						</th>
+						<th scope="col" class="px-3 py-1.5 text-right cursor-pointer select-none w-5">
+							{$i18n.t('Won')}
+						</th>
+						<th scope="col" class="px-3 py-1.5 text-right cursor-pointer select-none w-5">
+							{$i18n.t('Lost')}
+						</th>
+					</tr>
+				</thead>
+				<tbody class="">
+					{#each rankedModels as model, modelIdx (model.id)}
+						<tr class="bg-white dark:bg-gray-900 dark:border-gray-850 text-xs group">
+							<td class="px-3 py-1.5 text-left font-medium text-gray-900 dark:text-white w-fit">
+								<div class=" line-clamp-1">
+									{model?.rating !== '-' ? modelIdx + 1 : '-'}
+								</div>
+							</td>
+							<td class="px-3 py-1.5 flex flex-col justify-center">
+								<div class="flex items-center gap-2">
+									<div class="flex-shrink-0">
+										<img
+											src={model?.info?.meta?.profile_image_url ?? '/favicon.png'}
+											alt={model.name}
+											class="size-5 rounded-full object-cover shrink-0"
+										/>
+									</div>
+
+									<div class="font-medium text-gray-800 dark:text-gray-200 pr-4">
+										{model.name}
+									</div>
+								</div>
+							</td>
+							<td class="px-3 py-1.5 text-right font-medium text-gray-900 dark:text-white w-max">
+								{model.rating}
+							</td>
+
+							<td class=" px-3 py-1.5 text-right font-semibold text-green-500">
+								<div class=" w-10">
+									{#if model.stats.won === '-'}
+										-
+									{:else}
+										<span class="hidden group-hover:inline"
+											>{((model.stats.won / model.stats.count) * 100).toFixed(1)}%</span
+										>
+										<span class=" group-hover:hidden">{model.stats.won}</span>
+									{/if}
+								</div>
+							</td>
+
+							<td class="px-3 py-1.5 text-right font-semibold text-red-500">
+								<div class=" w-10">
+									{#if model.stats.lost === '-'}
+										-
+									{:else}
+										<span class="hidden group-hover:inline"
+											>{((model.stats.lost / model.stats.count) * 100).toFixed(1)}%</span
+										>
+										<span class=" group-hover:hidden">{model.stats.lost}</span>
+									{/if}
+								</div>
+							</td>
+						</tr>
+					{/each}
+				</tbody>
+			</table>
+		{/if}
+	</div>
+
+	<div class="pb-4"></div>
+
+	<div class="mt-0.5 mb-2 gap-1 flex flex-col md:flex-row justify-between">
+		<div class="flex md:self-center text-lg font-medium px-0.5">
+			{$i18n.t('Feedback History')}
+		</div>
+	</div>
+
+	<div
+		class="scrollbar-hidden relative whitespace-nowrap overflow-x-auto max-w-full rounded pt-0.5"
+	>
+		{#if (feedbacks ?? []).length === 0}
+			<div class="text-center text-xs text-gray-500 dark:text-gray-400 py-1">
+				{$i18n.t('No feedbacks found')}
+			</div>
+		{:else}
+			<table
+				class="w-full text-sm text-left text-gray-500 dark:text-gray-400 table-auto max-w-full rounded"
+			>
+				<thead
+					class="text-xs text-gray-700 uppercase bg-gray-50 dark:bg-gray-850 dark:text-gray-400 -translate-y-0.5"
+				>
+					<tr class="">
+						<th scope="col" class="px-3 text-right cursor-pointer select-none w-0">
+							{$i18n.t('User')}
+						</th>
+
+						<th scope="col" class="px-3 pr-1.5 cursor-pointer select-none">
+							{$i18n.t('Models')}
+						</th>
+
+						<th scope="col" class="px-3 py-1.5 text-right cursor-pointer select-none w-fit">
+							{$i18n.t('Result')}
+						</th>
+
+						<th scope="col" class="px-3 py-1.5 text-right cursor-pointer select-none w-0">
+							{$i18n.t('Updated At')}
+						</th>
+
+						<th scope="col" class="px-3 py-1.5 text-right cursor-pointer select-none w-0"> </th>
+					</tr>
+				</thead>
+				<tbody class="">
+					{#each feedbacks as feedback (feedback.id)}
+						<tr class="bg-white dark:bg-gray-900 dark:border-gray-850 text-xs">
+							<td class=" py-0.5 text-right font-semibold">
+								<div class="flex justify-center">
+									<Tooltip content={feedback?.user?.name}>
+										<div class="flex-shrink-0">
+											<img
+												src={feedback?.user?.profile_image_url ?? '/user.png'}
+												alt={feedback?.user?.name}
+												class="size-5 rounded-full object-cover shrink-0"
+											/>
+										</div>
+									</Tooltip>
+								</div>
+							</td>
+
+							<td class=" py-1 pl-3 flex flex-col">
+								<div class="flex flex-col items-start gap-0.5 h-full">
+									<div class="flex flex-col h-full">
+										{#if feedback.data?.sibling_model_ids}
+											<div class="font-semibold text-gray-600 dark:text-gray-400 flex-1">
+												{feedback.data?.model_id}
+											</div>
+											<div class=" text-[0.65rem] text-gray-600 dark:text-gray-400 line-clamp-1">
+												{feedback.data.sibling_model_ids.join(', ')}
+											</div>
+										{:else}
+											<div
+												class=" text-sm font-medium text-gray-600 dark:text-gray-400 flex-1 py-1.5"
+											>
+												{feedback.data?.model_id}
+											</div>
+										{/if}
+									</div>
+								</div>
+							</td>
+							<td class="px-3 py-1 text-right font-medium text-gray-900 dark:text-white w-max">
+								<div class=" flex justify-end">
+									{#if feedback.data.rating.toString() === '1'}
+										<Badge type="info" content={$i18n.t('Won')} />
+									{:else if feedback.data.rating.toString() === '0'}
+										<Badge type="muted" content={$i18n.t('Draw')} />
+									{:else if feedback.data.rating.toString() === '-1'}
+										<Badge type="error" content={$i18n.t('Lost')} />
+									{/if}
+								</div>
+							</td>
+
+							<td class=" px-3 py-1 text-right font-medium">
+								{dayjs(feedback.updated_at * 1000).fromNow()}
+							</td>
+
+							<td class=" px-3 py-1 text-right font-semibold">
+								<FeedbackMenu>
+									<button
+										class="self-center w-fit text-sm p-1.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-xl"
+									>
+										<EllipsisHorizontal />
+									</button>
+								</FeedbackMenu>
+							</td>
+						</tr>
+					{/each}
+				</tbody>
+			</table>
+		{/if}
+	</div>
+
+	<div class="pb-8"></div>
 {/if}

src/lib/components/admin/Evaluations/FeedbackMenu.svelte

@@ -0,0 +1,46 @@
+<script lang="ts">
+	import { DropdownMenu } from 'bits-ui';
+	import { flyAndScale } from '$lib/utils/transitions';
+	import { getContext, createEventDispatcher } from 'svelte';
+
+	import fileSaver from 'file-saver';
+	const { saveAs } = fileSaver;
+
+	const dispatch = createEventDispatcher();
+	const i18n = getContext('i18n');
+
+	import Dropdown from '$lib/components/common/Dropdown.svelte';
+	import GarbageBin from '$lib/components/icons/GarbageBin.svelte';
+	import Pencil from '$lib/components/icons/Pencil.svelte';
+	import Tooltip from '$lib/components/common/Tooltip.svelte';
+	import Download from '$lib/components/icons/Download.svelte';
+
+	let show = false;
+</script>
+
+<Dropdown bind:show on:change={(e) => {}}>
+	<Tooltip content={$i18n.t('More')}>
+		<slot />
+	</Tooltip>
+
+	<div slot="content">
+		<DropdownMenu.Content
+			class="w-full max-w-[150px] rounded-xl px-1 py-1.5 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
+			sideOffset={-2}
+			side="bottom"
+			align="start"
+			transition={flyAndScale}
+		>
+			<DropdownMenu.Item
+				class="flex  gap-2  items-center px-3 py-1.5 text-sm  cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
+				on:click={() => {
+					dispatch('delete');
+					show = false;
+				}}
+			>
+				<GarbageBin strokeWidth="2" />
+				<div class="flex items-center">{$i18n.t('Delete')}</div>
+			</DropdownMenu.Item>
+		</DropdownMenu.Content>
+	</div>
+</Dropdown>

src/lib/components/chat/Messages/Message.svelte

@@ -66,6 +66,7 @@
 			/>
 		{:else if (history.messages[history.messages[messageId].parentId]?.models?.length ?? 1) === 1}
 			<ResponseMessage
+				{chatId}
 				{history}
 				{messageId}
 				isLastMessage={messageId === history.currentId}

src/lib/components/chat/Messages/MultiResponseMessages.svelte

@@ -200,6 +200,7 @@
 						{#key history.currentId}
 							{#if message}
 								<ResponseMessage
+									{chatId}
 									{history}
 									messageId={_messageId}
 									isLastMessage={true}

src/lib/components/chat/Messages/RateComment.svelte

@@ -140,7 +140,7 @@
 	</div>
 
 	<div class="mt-2 gap-1.5 flex justify-end">
-		{#if $config?.features.enable_community_sharing}
+		{#if $config?.features.enable_community_sharing && selectedModel}
 			<button
 				class=" self-center px-3.5 py-2 rounded-xl text-sm font-medium bg-gray-50 hover:bg-gray-100 text-gray-800 dark:bg-gray-850 dark:hover:bg-gray-800 dark:text-white transition"
 				type="button"

src/lib/components/chat/Messages/ResponseMessage.svelte

@@ -40,6 +40,8 @@
 	import type { Writable } from 'svelte/store';
 	import type { i18n as i18nType } from 'i18next';
 	import ContentRenderer from './ContentRenderer.svelte';
+	import { createNewFeedback, getFeedbackById, updateFeedbackById } from '$lib/apis/evaluations';
+	import { getChatById } from '$lib/apis/chats';
 
 	interface MessageType {
 		id: string;
@@ -92,6 +94,7 @@
 		annotation?: { type: string; rating: number };
 	}
 
+	export let chatId = '';
 	export let history;
 	export let messageId;
 
@@ -330,6 +333,80 @@
 		generatingImage = false;
 	};
 
+	const feedbackHandler = async (
+		rating: number | null = null,
+		annotation: object | null = null
+	) => {
+		console.log('Feedback', rating, annotation);
+
+		const updatedMessage = {
+			...message,
+			annotation: {
+				...(message?.annotation ?? {}),
+				...(rating !== null ? { rating: rating } : {}),
+				...(annotation ? annotation : {})
+			}
+		};
+
+		const chat = await getChatById(localStorage.token, chatId).catch((error) => {
+			toast.error(error);
+		});
+		if (!chat) {
+			return;
+		}
+
+		let feedbackItem = {
+			type: 'rating',
+			data: {
+				...(updatedMessage?.annotation ? updatedMessage.annotation : {}),
+				model_id: message?.selectedModelId ?? message.model,
+				...(history.messages[message.parentId].childrenIds.length > 1
+					? {
+							sibling_model_ids: history.messages[message.parentId].childrenIds
+								.filter((id) => id !== message.id)
+								.map((id) => history.messages[id]?.selectedModelId ?? history.messages[id].model)
+						}
+					: {})
+			},
+			meta: {
+				arena: message ? message.arena : false,
+				message_id: message.id,
+				chat_id: chatId
+			},
+			snapshot: {
+				chat: chat
+			}
+		};
+
+		let feedback = null;
+		if (message?.feedbackId) {
+			feedback = await updateFeedbackById(
+				localStorage.token,
+				message.feedbackId,
+				feedbackItem
+			).catch((error) => {
+				toast.error(error);
+			});
+		} else {
+			feedback = await createNewFeedback(localStorage.token, feedbackItem).catch((error) => {
+				toast.error(error);
+			});
+
+			if (feedback) {
+				updatedMessage.feedbackId = feedback.id;
+			}
+		}
+
+		console.log(updatedMessage);
+		dispatch('save', updatedMessage);
+
+		await tick();
+
+		if (!annotation) {
+			showRateComment = true;
+		}
+	};
+
 	$: if (!edit) {
 		(async () => {
 			await tick();
@@ -880,12 +957,13 @@
 											<button
 												class="{isLastMessage
 													? 'visible'
-													: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg {(message
+													: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg {(
-													?.annotation?.rating ?? null) === 1
+													message?.annotation?.rating ?? ''
+												).toString() === '1'
 													? 'bg-gray-100 dark:bg-gray-800'
 													: ''} dark:hover:text-white hover:text-black transition"
 												on:click={async () => {
-													await rateMessage(message.id, 1);
+													await feedbackHandler(1);
 
 													(model?.actions ?? [])
 														.filter((action) => action?.__webui__ ?? false)
@@ -901,7 +979,6 @@
 															});
 														});
 
-													showRateComment = true;
 													window.setTimeout(() => {
 														document
 															.getElementById(`message-feedback-${message.id}`)
@@ -930,12 +1007,13 @@
 											<button
 												class="{isLastMessage
 													? 'visible'
-													: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg {(message
+													: 'invisible group-hover:visible'} p-1.5 hover:bg-black/5 dark:hover:bg-white/5 rounded-lg {(
-													?.annotation?.rating ?? null) === -1
+													message?.annotation?.rating ?? ''
+												).toString() === '-1'
 													? 'bg-gray-100 dark:bg-gray-800'
 													: ''} dark:hover:text-white hover:text-black transition"
 												on:click={async () => {
-													await rateMessage(message.id, -1);
+													await feedbackHandler(-1);
 
 													(model?.actions ?? [])
 														.filter((action) => action?.__webui__ ?? false)
@@ -951,7 +1029,6 @@
 															});
 														});
 
-													showRateComment = true;
 													window.setTimeout(() => {
 														document
 															.getElementById(`message-feedback-${message.id}`)
@@ -1103,15 +1180,12 @@
 						<RateComment
 							bind:message
 							bind:show={showRateComment}
-							on:save={(e) => {
+							on:save={async (e) => {
-								dispatch('save', {
+								await feedbackHandler(null, {
-									...message,
-									annotation: {
-										...message.annotation,
 									comment: e.detail.comment,
 									reason: e.detail.reason
-									}
 								});
+
 								(model?.actions ?? [])
 									.filter((action) => action?.__webui__ ?? false)
 									.forEach((action) => {

src/lib/components/workspace/Functions.svelte

@@ -228,7 +228,7 @@
 	<div class="flex justify-between items-center">
 		<div class="flex md:self-center text-base font-medium px-0.5">
 			{$i18n.t('Functions')}
-			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-200 dark:bg-gray-700" />
+			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-50 dark:bg-gray-850" />
 			<span class="text-base font-medium text-gray-500 dark:text-gray-300"
 				>{filteredItems.length}</span
 			>

src/lib/components/workspace/Knowledge.svelte

@@ -122,7 +122,7 @@
 	<div class="flex justify-between items-center">
 		<div class="flex md:self-center text-base font-medium px-0.5">
 			{$i18n.t('Knowledge')}
-			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-200 dark:bg-gray-700" />
+			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-50 dark:bg-gray-850" />
 			<span class="text-base font-medium text-gray-500 dark:text-gray-300"
 				>{filteredItems.length}</span
 			>

src/lib/components/workspace/Models.svelte

@@ -348,7 +348,7 @@
 	<div class="flex justify-between items-center">
 		<div class="flex md:self-center text-base font-medium px-0.5">
 			{$i18n.t('Models')}
-			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-200 dark:bg-gray-850" />
+			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-50 dark:bg-gray-850" />
 			<span class="text-base font-medium text-gray-500 dark:text-gray-300"
 				>{filteredModels.length}</span
 			>

src/lib/components/workspace/Prompts.svelte

@@ -125,7 +125,7 @@
 	<div class="flex justify-between items-center">
 		<div class="flex md:self-center text-base font-medium px-0.5">
 			{$i18n.t('Prompts')}
-			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-200 dark:bg-gray-700" />
+			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-50 dark:bg-gray-850" />
 			<span class="text-base font-medium text-gray-500 dark:text-gray-300"
 				>{filteredItems.length}</span
 			>

src/lib/components/workspace/Tools.svelte

@@ -200,7 +200,7 @@
 	<div class="flex justify-between items-center">
 		<div class="flex md:self-center text-base font-medium px-0.5">
 			{$i18n.t('Tools')}
-			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-200 dark:bg-gray-700" />
+			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-50 dark:bg-gray-850" />
 			<span class="text-base font-medium text-gray-500 dark:text-gray-300"
 				>{filteredItems.length}</span
 			>

src/routes/(app)/admin/+page.svelte

@@ -21,6 +21,9 @@
 	import ConfirmDialog from '$lib/components/common/ConfirmDialog.svelte';
 	import Badge from '$lib/components/common/Badge.svelte';
 	import Plus from '$lib/components/icons/Plus.svelte';
+	import ChevronUp from '$lib/components/icons/ChevronUp.svelte';
+	import ChevronDown from '$lib/components/icons/ChevronDown.svelte';
+	import About from '$lib/components/chat/Settings/About.svelte';
 
 	const i18n = getContext('i18n');
 
@@ -138,10 +141,11 @@
 <UserChatsModal bind:show={showUserChatsModal} user={selectedUser} />
 
 {#if loaded}
-	<div class="mt-0.5 mb-3 gap-1 flex flex-col md:flex-row justify-between">
+	<div class="mt-0.5 mb-2 gap-1 flex flex-col md:flex-row justify-between">
 		<div class="flex md:self-center text-lg font-medium px-0.5">
 			{$i18n.t('Users')}
-			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-200 dark:bg-gray-700" />
+			<div class="flex self-center w-[1px] h-6 mx-2.5 bg-gray-50 dark:bg-gray-850" />
+
 			<span class="text-lg font-medium text-gray-500 dark:text-gray-300">{users.length}</span>
 		</div>
 
@@ -200,36 +204,69 @@
 						class="px-3 py-1.5 cursor-pointer select-none"
 						on:click={() => setSortKey('role')}
 					>
+						<div class="flex gap-1.5 items-center">
 							{$i18n.t('Role')}
+
 							{#if sortKey === 'role'}
-							<span class="font-normal ml-1">{sortOrder === 'asc' ? '▲' : '▼'}</span>
+								<span class="font-normal"
+									>{#if sortOrder === 'asc'}
+										<ChevronUp className="size-2" />
 									{:else}
-							<span class="invisible">▲</span>
+										<ChevronDown className="size-2" />
 									{/if}
+								</span>
+							{:else}
+								<span class="invisible">
+									<ChevronUp className="size-2" />
+								</span>
+							{/if}
+						</div>
 					</th>
 					<th
 						scope="col"
 						class="px-3 py-1.5 cursor-pointer select-none"
 						on:click={() => setSortKey('name')}
 					>
+						<div class="flex gap-1.5 items-center">
 							{$i18n.t('Name')}
+
 							{#if sortKey === 'name'}
-							<span class="font-normal ml-1">{sortOrder === 'asc' ? '▲' : '▼'}</span>
+								<span class="font-normal"
+									>{#if sortOrder === 'asc'}
+										<ChevronUp className="size-2" />
 									{:else}
-							<span class="invisible">▲</span>
+										<ChevronDown className="size-2" />
 									{/if}
+								</span>
+							{:else}
+								<span class="invisible">
+									<ChevronUp className="size-2" />
+								</span>
+							{/if}
+						</div>
 					</th>
 					<th
 						scope="col"
 						class="px-3 py-1.5 cursor-pointer select-none"
 						on:click={() => setSortKey('email')}
 					>
+						<div class="flex gap-1.5 items-center">
 							{$i18n.t('Email')}
+
 							{#if sortKey === 'email'}
-							<span class="font-normal ml-1">{sortOrder === 'asc' ? '▲' : '▼'}</span>
+								<span class="font-normal"
+									>{#if sortOrder === 'asc'}
+										<ChevronUp className="size-2" />
 									{:else}
-							<span class="invisible">▲</span>
+										<ChevronDown className="size-2" />
 									{/if}
+								</span>
+							{:else}
+								<span class="invisible">
+									<ChevronUp className="size-2" />
+								</span>
+							{/if}
+						</div>
 					</th>
 
 					<th
@@ -237,24 +274,45 @@
 						class="px-3 py-1.5 cursor-pointer select-none"
 						on:click={() => setSortKey('last_active_at')}
 					>
+						<div class="flex gap-1.5 items-center">
 							{$i18n.t('Last Active')}
+
 							{#if sortKey === 'last_active_at'}
-							<span class="font-normal ml-1">{sortOrder === 'asc' ? '▲' : '▼'}</span>
+								<span class="font-normal"
+									>{#if sortOrder === 'asc'}
+										<ChevronUp className="size-2" />
 									{:else}
-							<span class="invisible">▲</span>
+										<ChevronDown className="size-2" />
 									{/if}
+								</span>
+							{:else}
+								<span class="invisible">
+									<ChevronUp className="size-2" />
+								</span>
+							{/if}
+						</div>
 					</th>
 					<th
 						scope="col"
 						class="px-3 py-1.5 cursor-pointer select-none"
 						on:click={() => setSortKey('created_at')}
 					>
+						<div class="flex gap-1.5 items-center">
 							{$i18n.t('Created at')}
 							{#if sortKey === 'created_at'}
-							<span class="font-normal ml-1">{sortOrder === 'asc' ? '▲' : '▼'}</span>
+								<span class="font-normal"
+									>{#if sortOrder === 'asc'}
+										<ChevronUp className="size-2" />
 									{:else}
-							<span class="invisible">▲</span>
+										<ChevronDown className="size-2" />
 									{/if}
+								</span>
+							{:else}
+								<span class="invisible">
+									<ChevronUp className="size-2" />
+								</span>
+							{/if}
+						</div>
 					</th>
 
 					<th
@@ -262,12 +320,23 @@
 						class="px-3 py-1.5 cursor-pointer select-none"
 						on:click={() => setSortKey('oauth_sub')}
 					>
+						<div class="flex gap-1.5 items-center">
 							{$i18n.t('OAuth ID')}
+
 							{#if sortKey === 'oauth_sub'}
-							<span class="font-normal ml-1">{sortOrder === 'asc' ? '▲' : '▼'}</span>
+								<span class="font-normal"
+									>{#if sortOrder === 'asc'}
+										<ChevronUp className="size-2" />
 									{:else}
-							<span class="invisible">▲</span>
+										<ChevronDown className="size-2" />
 									{/if}
+								</span>
+							{:else}
+								<span class="invisible">
+									<ChevronUp className="size-2" />
+								</span>
+							{/if}
+						</div>
 					</th>
 
 					<th scope="col" class="px-3 py-2 text-right" />