From bc95e62600862b67e17d5b002becb99fa39bdbed Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 22 Oct 2024 20:14:10 -0700 Subject: [PATCH] feat: leaderboard --- .../open_webui/apps/webui/models/feedbacks.py | 158 ++++++++++++++++++ .../af906e964978_add_feedback_table.py | 45 +++++ src/lib/components/admin/Evaluations.svelte | 119 +++++++++++-- .../chat/Messages/RateComment.svelte | 2 +- src/lib/components/workspace/Functions.svelte | 2 +- src/lib/components/workspace/Knowledge.svelte | 2 +- src/lib/components/workspace/Models.svelte | 2 +- src/lib/components/workspace/Prompts.svelte | 2 +- src/lib/components/workspace/Tools.svelte | 2 +- src/routes/(app)/admin/+page.svelte | 3 +- 10 files changed, 320 insertions(+), 17 deletions(-) create mode 100644 backend/open_webui/apps/webui/models/feedbacks.py create mode 100644 backend/open_webui/migrations/versions/af906e964978_add_feedback_table.py diff --git a/backend/open_webui/apps/webui/models/feedbacks.py b/backend/open_webui/apps/webui/models/feedbacks.py new file mode 100644 index 000000000..76410f417 --- /dev/null +++ b/backend/open_webui/apps/webui/models/feedbacks.py @@ -0,0 +1,158 @@ +import logging +import time +import uuid +from typing import Optional + +from open_webui.apps.webui.internal.db import Base, get_db +from open_webui.apps.webui.models.chats import Chats + +from open_webui.env import SRC_LOG_LEVELS +from pydantic import BaseModel, ConfigDict +from sqlalchemy import BigInteger, Column, Text, JSON, Boolean + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["MODELS"]) + + +#################### +# Feedback DB Schema +#################### + + +class Feedback(Base): + __tablename__ = "feedback" + id = Column(Text, primary_key=True) + user_id = Column(Text) + type = Column(Text) + data = Column(JSON, nullable=True) + meta = Column(JSON, nullable=True) + created_at = Column(BigInteger) + updated_at = Column(BigInteger) + + +class FeedbackModel(BaseModel): + id: str + user_id: str + type: str + data: Optional[dict] = None + meta: Optional[dict] = None + created_at: int + updated_at: int + + model_config = ConfigDict(from_attributes=True) + + +#################### +# Forms +#################### + + +class RatingData(BaseModel): + rating: str + comment: str + model_config = ConfigDict(extra="allow") + + +class VoteData(BaseModel): + rating: str + model_id: str + model_ids: list[str] + model_config = ConfigDict(extra="allow") + + +class MetaData(BaseModel): + chat: Optional[dict] = None + message_id: Optional[str] = None + tags: Optional[list[str]] = None + model_config = ConfigDict(extra="allow") + + +class FeedbackForm(BaseModel): + type: str + data: Optional[RatingData | VoteData] = None + meta: Optional[dict] = None + model_config = ConfigDict(extra="allow") + + +class FeedbackTable: + def insert_new_feedback( + self, user_id: str, form_data: FeedbackForm + ) -> Optional[FeedbackModel]: + with get_db() as db: + id = str(uuid.uuid4()) + feedback = FeedbackModel( + **{ + "id": id, + "user_id": user_id, + "type": form_data.type, + "data": form_data.data, + "meta": form_data.meta, + "created_at": int(time.time()), + } + ) + try: + result = Feedback(**feedback.model_dump()) + db.add(result) + db.commit() + db.refresh(result) + if result: + return FeedbackModel.model_validate(result) + else: + return None + except Exception as e: + print(e) + return None + + def get_feedback_by_id(self, id: str) -> Optional[FeedbackModel]: + try: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id).first() + if not feedback: + return None + return FeedbackModel.model_validate(feedback) + except Exception: + return None + + def get_feedbacks_by_type(self, type: str) -> list[FeedbackModel]: + with get_db() as db: + return [ + FeedbackModel.model_validate(feedback) + for feedback in db.query(Feedback).filter_by(type=type).all() + ] + + def get_feedbacks_by_user_id(self, user_id: str) -> list[FeedbackModel]: + with get_db() as db: + return [ + FeedbackModel.model_validate(feedback) + for feedback in db.query(Feedback).filter_by(user_id=user_id).all() + ] + + def update_feedback_by_id( + self, id: str, form_data: FeedbackForm + ) -> Optional[FeedbackModel]: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id).first() + if not feedback: + return None + + if form_data.data: + feedback.data = form_data.data + if form_data.meta: + feedback.meta = form_data.meta + + feedback.updated_at = int(time.time()) + + db.commit() + return FeedbackModel.model_validate(feedback) + + def delete_feedback_by_id(self, id: str) -> bool: + with get_db() as db: + feedback = db.query(Feedback).filter_by(id=id).first() + if not feedback: + return False + db.delete(feedback) + db.commit() + return True + + +Feedbacks = FeedbackTable() diff --git a/backend/open_webui/migrations/versions/af906e964978_add_feedback_table.py b/backend/open_webui/migrations/versions/af906e964978_add_feedback_table.py new file mode 100644 index 000000000..8119c9396 --- /dev/null +++ b/backend/open_webui/migrations/versions/af906e964978_add_feedback_table.py @@ -0,0 +1,45 @@ +"""Add feedback table + +Revision ID: af906e964978 +Revises: c29facfe716b +Create Date: 2024-10-20 17:02:35.241684 + +""" + +from alembic import op +import sqlalchemy as sa + +# Revision identifiers, used by Alembic. +revision = "af906e964978" +down_revision = "c29facfe716b" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### Create feedback table ### + op.create_table( + "feedback", + sa.Column( + "id", sa.Text(), primary_key=True + ), # Unique identifier for each feedback (TEXT type) + sa.Column( + "user_id", sa.Text(), nullable=True + ), # ID of the user providing the feedback (TEXT type) + sa.Column("type", sa.Text(), nullable=True), # Type of feedback (TEXT type) + sa.Column("data", sa.JSON(), nullable=True), # Feedback data (JSON type) + sa.Column( + "meta", sa.JSON(), nullable=True + ), # Metadata for feedback (JSON type) + sa.Column( + "created_at", sa.BigInteger(), nullable=False + ), # Feedback creation timestamp (BIGINT representing epoch) + sa.Column( + "updated_at", sa.BigInteger(), nullable=False + ), # Feedback update timestamp (BIGINT representing epoch) + ) + + +def downgrade(): + # ### Drop feedback table ### + op.drop_table("feedback") diff --git a/src/lib/components/admin/Evaluations.svelte b/src/lib/components/admin/Evaluations.svelte index c764a958d..7fe3c5756 100644 --- a/src/lib/components/admin/Evaluations.svelte +++ b/src/lib/components/admin/Evaluations.svelte @@ -1,27 +1,126 @@ {#if loaded} -
+
{$i18n.t('Leaderboard')} + +
+ + {rankedModels.length}
+ +
+ + + + + + + + + + + + + {#each rankedModels as model (model.id)} + + + + + + + + + + + {/each} + +
+ {$i18n.t('Model')} + + {$i18n.t('Rating')} + + {$i18n.t('Won')} + + {$i18n.t('Draw')} + + {$i18n.t('Lost')} +
+
+
+ {model.name} +
+ +
+ {model.name} +
+
+ 		+ {model.rating} + {model.stats.won} + {model.stats.draw} + + {model.stats.lost} +
+
+ +
+ +
+
+ {$i18n.t('Rating History')} +
+
+ +
{/if} diff --git a/src/lib/components/chat/Messages/RateComment.svelte b/src/lib/components/chat/Messages/RateComment.svelte index 7c6955ed9..6c95c4271 100644 --- a/src/lib/components/chat/Messages/RateComment.svelte +++ b/src/lib/components/chat/Messages/RateComment.svelte @@ -140,7 +140,7 @@
- {#if $config?.features.enable_community_sharing} + {#if $config?.features.enable_community_sharing && selectedModel}