diff --git a/backend/open_webui/apps/webui/models/feedbacks.py b/backend/open_webui/apps/webui/models/feedbacks.py new file mode 100644 index 000000000..72b5fd79b --- /dev/null +++ b/backend/open_webui/apps/webui/models/feedbacks.py @@ -0,0 +1,254 @@ +import logging +import time +import uuid +from typing import Optional + +from open_webui.apps.webui.internal.db import Base, get_db +from open_webui.apps.webui.models.chats import Chats + +from open_webui.env import SRC_LOG_LEVELS +from pydantic import BaseModel, ConfigDict +from sqlalchemy import BigInteger, Column, Text, JSON, Boolean + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["MODELS"]) + + +#################### +# Feedback DB Schema +#################### + + +class Feedback(Base): + __tablename__ = "feedback" + id = Column(Text, primary_key=True) + user_id = Column(Text) + version = Column(BigInteger, default=0) + type = Column(Text) + data = Column(JSON, nullable=True) + meta = Column(JSON, nullable=True) + snapshot = Column(JSON, nullable=True) + created_at = Column(BigInteger) + updated_at = Column(BigInteger) + + +class FeedbackModel(BaseModel): + id: str + user_id: str + version: int + type: str + data: Optional[dict] = None + meta: Optional[dict] = None + snapshot: Optional[dict] = None + created_at: int + updated_at: int + + model_config = ConfigDict(from_attributes=True) + + +#################### +# Forms +#################### + + +class FeedbackResponse(BaseModel): + id: str + user_id: str + version: int + type: str + data: Optional[dict] = None + meta: Optional[dict] = None + created_at: int + updated_at: int + + +class RatingData(BaseModel): + rating: Optional[str | int] = None + model_id: Optional[str] = None + sibling_model_ids: Optional[list[str]] = None + reason: Optional[str] = None + comment: Optional[str] = None + model_config = ConfigDict(extra="allow") + + +class MetaData(BaseModel): + arena: Optional[bool] = None + chat_id: Optional[str] = None + message_id: Optional[str] = None + tags: Optional[list[str]] = None + model_config = ConfigDict(extra="allow") + + +class SnapshotData(BaseModel): + chat: Optional[dict] = None + model_config = ConfigDict(extra="allow") + + +class FeedbackForm(BaseModel): + type: str + data: Optional[RatingData] = None + meta: Optional[dict] = None + snapshot: Optional[SnapshotData] = None + model_config = ConfigDict(extra="allow") + + +class FeedbackTable: + def insert_new_feedback( + self, user_id: str, form_data: FeedbackForm + ) -> Optional[FeedbackModel]: + with get_db() as db: + id = str(uuid.uuid4()) + feedback = FeedbackModel( + **{ + "id": id, + "user_id": user_id, + "version": 0, + **form_data.model_dump(), + "created_at": int(time.time()), + "updated_at": int(time.time()), + } + ) + try: + result = Feedback(**feedback.model_dump()) + db.add(result) + db.commit() + db.refresh(result) + if result: + return FeedbackModel.model_validate(result) + else: + return None + except Exception as e: + print(e) + return None + + def get_feedback_by_id(self, id: str) -> Optional[FeedbackModel]: + try: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id).first() + if not feedback: + return None + return FeedbackModel.model_validate(feedback) + except Exception: + return None + + def get_feedback_by_id_and_user_id( + self, id: str, user_id: str + ) -> Optional[FeedbackModel]: + try: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id, user_id=user_id).first() + if not feedback: + return None + return FeedbackModel.model_validate(feedback) + except Exception: + return None + + def get_all_feedbacks(self) -> list[FeedbackModel]: + with get_db() as db: + return [ + FeedbackModel.model_validate(feedback) + for feedback in db.query(Feedback) + .order_by(Feedback.updated_at.desc()) + .all() + ] + + def get_feedbacks_by_type(self, type: str) -> list[FeedbackModel]: + with get_db() as db: + return [ + FeedbackModel.model_validate(feedback) + for feedback in db.query(Feedback) + .filter_by(type=type) + .order_by(Feedback.updated_at.desc()) + .all() + ] + + def get_feedbacks_by_user_id(self, user_id: str) -> list[FeedbackModel]: + with get_db() as db: + return [ + FeedbackModel.model_validate(feedback) + for feedback in db.query(Feedback) + .filter_by(user_id=user_id) + .order_by(Feedback.updated_at.desc()) + .all() + ] + + def update_feedback_by_id( + self, id: str, form_data: FeedbackForm + ) -> Optional[FeedbackModel]: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id).first() + if not feedback: + return None + + if form_data.data: + feedback.data = form_data.data.model_dump() + if form_data.meta: + feedback.meta = form_data.meta + if form_data.snapshot: + feedback.snapshot = form_data.snapshot.model_dump() + + feedback.updated_at = int(time.time()) + + db.commit() + return FeedbackModel.model_validate(feedback) + + def update_feedback_by_id_and_user_id( + self, id: str, user_id: str, form_data: FeedbackForm + ) -> Optional[FeedbackModel]: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id, user_id=user_id).first() + if not feedback: + return None + + if form_data.data: + feedback.data = form_data.data.model_dump() + if form_data.meta: + feedback.meta = form_data.meta + if form_data.snapshot: + feedback.snapshot = form_data.snapshot.model_dump() + + feedback.updated_at = int(time.time()) + + db.commit() + return FeedbackModel.model_validate(feedback) + + def delete_feedback_by_id(self, id: str) -> bool: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id).first() + if not feedback: + return False + db.delete(feedback) + db.commit() + return True + + def delete_feedback_by_id_and_user_id(self, id: str, user_id: str) -> bool: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id, user_id=user_id).first() + if not feedback: + return False + db.delete(feedback) + db.commit() + return True + + def delete_feedbacks_by_user_id(self, user_id: str) -> bool: + with get_db() as db: + feedbacks = db.query(Feedback).filter_by(user_id=user_id).all() + if not feedbacks: + return False + for feedback in feedbacks: + db.delete(feedback) + db.commit() + return True + + def delete_all_feedbacks(self) -> bool: + with get_db() as db: + feedbacks = db.query(Feedback).all() + if not feedbacks: + return False + for feedback in feedbacks: + db.delete(feedback) + db.commit() + return True + + +Feedbacks = FeedbackTable() diff --git a/backend/open_webui/apps/webui/routers/evaluations.py b/backend/open_webui/apps/webui/routers/evaluations.py index b40953e49..9a6845efa 100644 --- a/backend/open_webui/apps/webui/routers/evaluations.py +++ b/backend/open_webui/apps/webui/routers/evaluations.py @@ -2,6 +2,12 @@ from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status, Request from pydantic import BaseModel +from open_webui.apps.webui.models.users import Users, UserModel +from open_webui.apps.webui.models.feedbacks import ( + FeedbackModel, + FeedbackForm, + Feedbacks, +) from open_webui.constants import ERROR_MESSAGES from open_webui.utils.utils import get_admin_user, get_verified_user @@ -47,3 +53,95 @@ async def update_config( "ENABLE_EVALUATION_ARENA_MODELS": config.ENABLE_EVALUATION_ARENA_MODELS, "EVALUATION_ARENA_MODELS": config.EVALUATION_ARENA_MODELS, } + + +@router.get("/feedbacks", response_model=list[FeedbackModel]) +async def get_feedbacks(user=Depends(get_verified_user)): + feedbacks = Feedbacks.get_feedbacks_by_user_id(user.id) + return feedbacks + + +@router.delete("/feedbacks", response_model=bool) +async def delete_feedbacks(user=Depends(get_verified_user)): + success = Feedbacks.delete_feedbacks_by_user_id(user.id) + return success + + +class FeedbackUserModel(FeedbackModel): + user: Optional[UserModel] = None + + +@router.get("/feedbacks/all", response_model=list[FeedbackUserModel]) +async def get_all_feedbacks(user=Depends(get_admin_user)): + feedbacks = Feedbacks.get_all_feedbacks() + return [ + FeedbackUserModel( + **feedback.model_dump(), user=Users.get_user_by_id(feedback.user_id) + ) + for feedback in feedbacks + ] + + +@router.delete("/feedbacks/all") +async def delete_all_feedbacks(user=Depends(get_admin_user)): + success = Feedbacks.delete_all_feedbacks() + return success + + +@router.post("/feedback", response_model=FeedbackModel) +async def create_feedback( + request: Request, + form_data: FeedbackForm, + user=Depends(get_verified_user), +): + feedback = Feedbacks.insert_new_feedback(user_id=user.id, form_data=form_data) + if not feedback: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.DEFAULT(), + ) + + return feedback + + +@router.get("/feedback/{id}", response_model=FeedbackModel) +async def get_feedback_by_id(id: str, user=Depends(get_verified_user)): + feedback = Feedbacks.get_feedback_by_id_and_user_id(id=id, user_id=user.id) + + if not feedback: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND + ) + + return feedback + + +@router.post("/feedback/{id}", response_model=FeedbackModel) +async def update_feedback_by_id( + id: str, form_data: FeedbackForm, user=Depends(get_verified_user) +): + feedback = Feedbacks.update_feedback_by_id_and_user_id( + id=id, user_id=user.id, form_data=form_data + ) + + if not feedback: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND + ) + + return feedback + + +@router.delete("/feedback/{id}") +async def delete_feedback_by_id(id: str, user=Depends(get_verified_user)): + if user.role == "admin": + success = Feedbacks.delete_feedback_by_id(id=id) + else: + success = Feedbacks.delete_feedback_by_id_and_user_id(id=id, user_id=user.id) + + if not success: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND + ) + + return success diff --git a/backend/open_webui/migrations/versions/af906e964978_add_feedback_table.py b/backend/open_webui/migrations/versions/af906e964978_add_feedback_table.py new file mode 100644 index 000000000..9116aa388 --- /dev/null +++ b/backend/open_webui/migrations/versions/af906e964978_add_feedback_table.py @@ -0,0 +1,51 @@ +"""Add feedback table + +Revision ID: af906e964978 +Revises: c29facfe716b +Create Date: 2024-10-20 17:02:35.241684 + +""" + +from alembic import op +import sqlalchemy as sa + +# Revision identifiers, used by Alembic. +revision = "af906e964978" +down_revision = "c29facfe716b" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### Create feedback table ### + op.create_table( + "feedback", + sa.Column( + "id", sa.Text(), primary_key=True + ), # Unique identifier for each feedback (TEXT type) + sa.Column( + "user_id", sa.Text(), nullable=True + ), # ID of the user providing the feedback (TEXT type) + sa.Column( + "version", sa.BigInteger(), default=0 + ), # Version of feedback (BIGINT type) + sa.Column("type", sa.Text(), nullable=True), # Type of feedback (TEXT type) + sa.Column("data", sa.JSON(), nullable=True), # Feedback data (JSON type) + sa.Column( + "meta", sa.JSON(), nullable=True + ), # Metadata for feedback (JSON type) + sa.Column( + "snapshot", sa.JSON(), nullable=True + ), # snapshot data for feedback (JSON type) + sa.Column( + "created_at", sa.BigInteger(), nullable=False + ), # Feedback creation timestamp (BIGINT representing epoch) + sa.Column( + "updated_at", sa.BigInteger(), nullable=False + ), # Feedback update timestamp (BIGINT representing epoch) + ) + + +def downgrade(): + # ### Drop feedback table ### + op.drop_table("feedback") diff --git a/src/lib/apis/evaluations/index.ts b/src/lib/apis/evaluations/index.ts index 21130bd17..854b3abb8 100644 --- a/src/lib/apis/evaluations/index.ts +++ b/src/lib/apis/evaluations/index.ts @@ -61,3 +61,155 @@ export const updateConfig = async (token: string, config: object) => { return res; }; + +export const getAllFeedbacks = async (token: string = '') => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedbacks/all`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const createNewFeedback = async (token: string, feedback: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedback`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + ...feedback + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const getFeedbackById = async (token: string, feedbackId: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedback/${feedbackId}`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .then((json) => { + return json; + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const updateFeedbackById = async (token: string, feedbackId: string, feedback: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedback/${feedbackId}`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ + ...feedback + }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const deleteFeedbackById = async (token: string, feedbackId: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/evaluations/feedback/${feedbackId}`, { + method: 'DELETE', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.log(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; diff --git a/src/lib/components/admin/Evaluations.svelte b/src/lib/components/admin/Evaluations.svelte index c764a958d..1d7019d32 100644 --- a/src/lib/components/admin/Evaluations.svelte +++ b/src/lib/components/admin/Evaluations.svelte @@ -1,27 +1,339 @@ {#if loaded} -
+
{$i18n.t('Leaderboard')} + +
+ + {rankedModels.length}
+ +
+ {#if (rankedModels ?? []).length === 0} +
+ {$i18n.t('No models found')} +
+ {:else} + + + + + + + + + + + + {#each rankedModels as model, modelIdx (model.id)} + + + + + + + + + + {/each} + +
+ {$i18n.t('RK')} + + {$i18n.t('Model')} + + {$i18n.t('Rating')} + + {$i18n.t('Won')} + + {$i18n.t('Lost')} +
+
+ {model?.rating !== '-' ? modelIdx + 1 : '-'} +
+
+
+
+ {model.name} +
+ +
+ {model.name} +
+
+
+ {model.rating} + +
+ {#if model.stats.won === '-'} + - + {:else} + + {model.stats.won} + {/if} +
+
+
+ {#if model.stats.lost === '-'} + - + {:else} + + {model.stats.lost} + {/if} +
+
+ {/if} +
+ +
+ +
+
+ {$i18n.t('Feedback History')} +
+
+ +
+ {#if (feedbacks ?? []).length === 0} +
+ {$i18n.t('No feedbacks found')} +
+ {:else} + + + + + + + + + + + + + + + + {#each feedbacks as feedback (feedback.id)} + + + + + + + + + + + {/each} + +
+ {$i18n.t('User')} + + {$i18n.t('Models')} + + {$i18n.t('Result')} + + {$i18n.t('Updated At')} +
+
+ +
+ {feedback?.user?.name} +
+
+
+
+
+
+ {#if feedback.data?.sibling_model_ids} +
+ {feedback.data?.model_id} +
+
+ {feedback.data.sibling_model_ids.join(', ')} +
+ {:else} +
+ {feedback.data?.model_id} +
+ {/if} +
+
+
+
+ {#if feedback.data.rating.toString() === '1'} + + {:else if feedback.data.rating.toString() === '0'} + + {:else if feedback.data.rating.toString() === '-1'} + + {/if} +
+
+ {dayjs(feedback.updated_at * 1000).fromNow()} + + + + +
+ {/if} +
+ +
{/if} diff --git a/src/lib/components/admin/Evaluations/FeedbackMenu.svelte b/src/lib/components/admin/Evaluations/FeedbackMenu.svelte new file mode 100644 index 000000000..83defd804 --- /dev/null +++ b/src/lib/components/admin/Evaluations/FeedbackMenu.svelte @@ -0,0 +1,46 @@ + + + {}}> + + + + +
+ + { + dispatch('delete'); + show = false; + }} + > + +
{$i18n.t('Delete')}
+
+
+
+
diff --git a/src/lib/components/chat/Messages/Message.svelte b/src/lib/components/chat/Messages/Message.svelte index 742658390..283af001a 100644 --- a/src/lib/components/chat/Messages/Message.svelte +++ b/src/lib/components/chat/Messages/Message.svelte @@ -66,6 +66,7 @@ /> {:else if (history.messages[history.messages[messageId].parentId]?.models?.length ?? 1) === 1}
- {#if $config?.features.enable_community_sharing} + {#if $config?.features.enable_community_sharing && selectedModel}