Merge remote-tracking branch 'upstream/dev' into feat/oauth

backend/apps/webui/internal/migrations/012_add_tools.py

@@ -0,0 +1,61 @@
+"""Peewee migrations -- 009_add_models.py.
+
+Some examples (model - class or model name)::
+
+    > Model = migrator.orm['table_name']            # Return model in current state by name
+    > Model = migrator.ModelClass                   # Return model in current state by name
+
+    > migrator.sql(sql)                             # Run custom SQL
+    > migrator.run(func, *args, **kwargs)           # Run python function with the given args
+    > migrator.create_model(Model)                  # Create a model (could be used as decorator)
+    > migrator.remove_model(model, cascade=True)    # Remove a model
+    > migrator.add_fields(model, **fields)          # Add fields to a model
+    > migrator.change_fields(model, **fields)       # Change fields
+    > migrator.remove_fields(model, *field_names, cascade=True)
+    > migrator.rename_field(model, old_field_name, new_field_name)
+    > migrator.rename_table(model, new_table_name)
+    > migrator.add_index(model, *col_names, unique=False)
+    > migrator.add_not_null(model, *field_names)
+    > migrator.add_default(model, field_name, default)
+    > migrator.add_constraint(model, name, sql)
+    > migrator.drop_index(model, *col_names)
+    > migrator.drop_not_null(model, *field_names)
+    > migrator.drop_constraints(model, *constraints)
+
+"""
+
+from contextlib import suppress
+
+import peewee as pw
+from peewee_migrate import Migrator
+
+
+with suppress(ImportError):
+    import playhouse.postgres_ext as pw_pext
+
+
+def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your migrations here."""
+
+    @migrator.create_model
+    class Tool(pw.Model):
+        id = pw.TextField(unique=True)
+        user_id = pw.TextField()
+
+        name = pw.TextField()
+        content = pw.TextField()
+        specs = pw.TextField()
+
+        meta = pw.TextField()
+
+        created_at = pw.BigIntegerField(null=False)
+        updated_at = pw.BigIntegerField(null=False)
+
+        class Meta:
+            table_name = "tool"
+
+
+def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
+    """Write your rollback migrations here."""
+
+    migrator.remove_model("tool")

backend/apps/webui/internal/migrations/013_add_user_oauth_sub.py

@@ -1,4 +1,4 @@
-"""Peewee migrations -- 011_add_user_oauth_sub.py.
+"""Peewee migrations -- 013_add_user_oauth_sub.py.
 
 Some examples (model - class or model name)::
 

backend/apps/webui/main.py

@@ -8,6 +8,7 @@ from apps.webui.routers import (
     users,
     chats,
     documents,
+    tools,
     models,
     prompts,
     configs,
@@ -27,9 +28,9 @@ from config import (
     WEBHOOK_URL,
     WEBUI_AUTH_TRUSTED_EMAIL_HEADER,
     JWT_EXPIRES_IN,
-    AppConfig,
-    ENABLE_COMMUNITY_SHARING,
     WEBUI_BANNERS,
+    ENABLE_COMMUNITY_SHARING,
+    AppConfig,
 )
 
 app = FastAPI()
@@ -40,6 +41,7 @@ app.state.config = AppConfig()
 
 app.state.config.ENABLE_SIGNUP = ENABLE_SIGNUP
 app.state.config.JWT_EXPIRES_IN = JWT_EXPIRES_IN
+app.state.AUTH_TRUSTED_EMAIL_HEADER = WEBUI_AUTH_TRUSTED_EMAIL_HEADER
 
 
 app.state.config.SHOW_ADMIN_DETAILS = SHOW_ADMIN_DETAILS
@@ -56,7 +58,7 @@ app.state.config.BANNERS = WEBUI_BANNERS
 app.state.config.ENABLE_COMMUNITY_SHARING = ENABLE_COMMUNITY_SHARING
 
 app.state.MODELS = {}
-app.state.AUTH_TRUSTED_EMAIL_HEADER = WEBUI_AUTH_TRUSTED_EMAIL_HEADER
+app.state.TOOLS = {}
 
 
 app.add_middleware(
@@ -72,6 +74,7 @@ app.include_router(users.router, prefix="/users", tags=["users"])
 app.include_router(chats.router, prefix="/chats", tags=["chats"])
 
 app.include_router(documents.router, prefix="/documents", tags=["documents"])
+app.include_router(tools.router, prefix="/tools", tags=["tools"])
 app.include_router(models.router, prefix="/models", tags=["models"])
 app.include_router(prompts.router, prefix="/prompts", tags=["prompts"])
 app.include_router(memories.router, prefix="/memories", tags=["memories"])

backend/apps/webui/models/memories.py

@@ -65,6 +65,20 @@ class MemoriesTable:
         else:
             return None
 
+    def update_memory_by_id(
+        self,
+        id: str,
+        content: str,
+    ) -> Optional[MemoryModel]:
+        try:
+            memory = Memory.get(Memory.id == id)
+            memory.content = content
+            memory.updated_at = int(time.time())
+            memory.save()
+            return MemoryModel(**model_to_dict(memory))
+        except:
+            return None
+
     def get_memories(self) -> List[MemoryModel]:
         try:
             memories = Memory.select()

backend/apps/webui/models/tools.py

@@ -0,0 +1,132 @@
+from pydantic import BaseModel
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+from typing import List, Union, Optional
+import time
+import logging
+from apps.webui.internal.db import DB, JSONField
+
+import json
+
+from config import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["MODELS"])
+
+####################
+# Tools DB Schema
+####################
+
+
+class Tool(Model):
+    id = CharField(unique=True)
+    user_id = CharField()
+    name = TextField()
+    content = TextField()
+    specs = JSONField()
+    meta = JSONField()
+    updated_at = BigIntegerField()
+    created_at = BigIntegerField()
+
+    class Meta:
+        database = DB
+
+
+class ToolMeta(BaseModel):
+    description: Optional[str] = None
+
+
+class ToolModel(BaseModel):
+    id: str
+    user_id: str
+    name: str
+    content: str
+    specs: List[dict]
+    meta: ToolMeta
+    updated_at: int  # timestamp in epoch
+    created_at: int  # timestamp in epoch
+
+
+####################
+# Forms
+####################
+
+
+class ToolResponse(BaseModel):
+    id: str
+    user_id: str
+    name: str
+    meta: ToolMeta
+    updated_at: int  # timestamp in epoch
+    created_at: int  # timestamp in epoch
+
+
+class ToolForm(BaseModel):
+    id: str
+    name: str
+    content: str
+    meta: ToolMeta
+
+
+class ToolsTable:
+    def __init__(self, db):
+        self.db = db
+        self.db.create_tables([Tool])
+
+    def insert_new_tool(
+        self, user_id: str, form_data: ToolForm, specs: List[dict]
+    ) -> Optional[ToolModel]:
+        tool = ToolModel(
+            **{
+                **form_data.model_dump(),
+                "specs": specs,
+                "user_id": user_id,
+                "updated_at": int(time.time()),
+                "created_at": int(time.time()),
+            }
+        )
+
+        try:
+            result = Tool.create(**tool.model_dump())
+            if result:
+                return tool
+            else:
+                return None
+        except Exception as e:
+            print(f"Error creating tool: {e}")
+            return None
+
+    def get_tool_by_id(self, id: str) -> Optional[ToolModel]:
+        try:
+            tool = Tool.get(Tool.id == id)
+            return ToolModel(**model_to_dict(tool))
+        except:
+            return None
+
+    def get_tools(self) -> List[ToolModel]:
+        return [ToolModel(**model_to_dict(tool)) for tool in Tool.select()]
+
+    def update_tool_by_id(self, id: str, updated: dict) -> Optional[ToolModel]:
+        try:
+            query = Tool.update(
+                **updated,
+                updated_at=int(time.time()),
+            ).where(Tool.id == id)
+            query.execute()
+
+            tool = Tool.get(Tool.id == id)
+            return ToolModel(**model_to_dict(tool))
+        except:
+            return None
+
+    def delete_tool_by_id(self, id: str) -> bool:
+        try:
+            query = Tool.delete().where((Tool.id == id))
+            query.execute()  # Remove the rows, return number of rows removed.
+
+            return True
+        except:
+            return False
+
+
+Tools = ToolsTable(DB)

backend/apps/webui/routers/chats.py

@@ -161,7 +161,7 @@ async def get_archived_session_user_chat_list(
 ############################
 
 
-@router.post("/archive/all", response_model=List[ChatTitleIdResponse])
+@router.post("/archive/all", response_model=bool)
 async def archive_all_chats(user=Depends(get_current_user)):
     return Chats.archive_all_chats_by_user_id(user.id)
 

backend/apps/webui/routers/documents.py

@@ -73,7 +73,7 @@ async def create_new_doc(form_data: DocumentForm, user=Depends(get_admin_user)):
 ############################
 
 
-@router.get("/name/{name}", response_model=Optional[DocumentResponse])
+@router.get("/doc", response_model=Optional[DocumentResponse])
 async def get_doc_by_name(name: str, user=Depends(get_current_user)):
     doc = Documents.get_doc_by_name(name)
 
@@ -105,7 +105,7 @@ class TagDocumentForm(BaseModel):
     tags: List[dict]
 
 
-@router.post("/name/{name}/tags", response_model=Optional[DocumentResponse])
+@router.post("/doc/tags", response_model=Optional[DocumentResponse])
 async def tag_doc_by_name(form_data: TagDocumentForm, user=Depends(get_current_user)):
     doc = Documents.update_doc_content_by_name(form_data.name, {"tags": form_data.tags})
 
@@ -128,7 +128,7 @@ async def tag_doc_by_name(form_data: TagDocumentForm, user=Depends(get_current_u
 ############################
 
 
-@router.post("/name/{name}/update", response_model=Optional[DocumentResponse])
+@router.post("/doc/update", response_model=Optional[DocumentResponse])
 async def update_doc_by_name(
     name: str, form_data: DocumentUpdateForm, user=Depends(get_admin_user)
 ):
@@ -152,7 +152,7 @@ async def update_doc_by_name(
 ############################
 
 
-@router.delete("/name/{name}/delete", response_model=bool)
+@router.delete("/doc/delete", response_model=bool)
 async def delete_doc_by_name(name: str, user=Depends(get_admin_user)):
     result = Documents.delete_doc_by_name(name)
     return result

backend/apps/webui/routers/memories.py

@@ -44,6 +44,10 @@ class AddMemoryForm(BaseModel):
     content: str
 
 
+class MemoryUpdateModel(BaseModel):
+    content: Optional[str] = None
+
+
 @router.post("/add", response_model=Optional[MemoryModel])
 async def add_memory(
     request: Request, form_data: AddMemoryForm, user=Depends(get_verified_user)
@@ -62,6 +66,34 @@ async def add_memory(
     return memory
 
 
+@router.post("/{memory_id}/update", response_model=Optional[MemoryModel])
+async def update_memory_by_id(
+    memory_id: str,
+    request: Request,
+    form_data: MemoryUpdateModel,
+    user=Depends(get_verified_user),
+):
+    memory = Memories.update_memory_by_id(memory_id, form_data.content)
+    if memory is None:
+        raise HTTPException(status_code=404, detail="Memory not found")
+
+    if form_data.content is not None:
+        memory_embedding = request.app.state.EMBEDDING_FUNCTION(form_data.content)
+        collection = CHROMA_CLIENT.get_or_create_collection(
+            name=f"user-memory-{user.id}"
+        )
+        collection.upsert(
+            documents=[form_data.content],
+            ids=[memory.id],
+            embeddings=[memory_embedding],
+            metadatas=[
+                {"created_at": memory.created_at, "updated_at": memory.updated_at}
+            ],
+        )
+
+    return memory
+
+
 ############################
 # QueryMemory
 ############################

backend/apps/webui/routers/tools.py

@@ -0,0 +1,183 @@
+from fastapi import Depends, FastAPI, HTTPException, status, Request
+from datetime import datetime, timedelta
+from typing import List, Union, Optional
+
+from fastapi import APIRouter
+from pydantic import BaseModel
+import json
+
+from apps.webui.models.tools import Tools, ToolForm, ToolModel, ToolResponse
+from apps.webui.utils import load_toolkit_module_by_id
+
+from utils.utils import get_current_user, get_admin_user
+from utils.tools import get_tools_specs
+from constants import ERROR_MESSAGES
+
+from importlib import util
+import os
+
+from config import DATA_DIR
+
+
+TOOLS_DIR = f"{DATA_DIR}/tools"
+os.makedirs(TOOLS_DIR, exist_ok=True)
+
+
+router = APIRouter()
+
+############################
+# GetToolkits
+############################
+
+
+@router.get("/", response_model=List[ToolResponse])
+async def get_toolkits(user=Depends(get_current_user)):
+    toolkits = [toolkit for toolkit in Tools.get_tools()]
+    return toolkits
+
+
+############################
+# ExportToolKits
+############################
+
+
+@router.get("/export", response_model=List[ToolModel])
+async def get_toolkits(user=Depends(get_admin_user)):
+    toolkits = [toolkit for toolkit in Tools.get_tools()]
+    return toolkits
+
+
+############################
+# CreateNewToolKit
+############################
+
+
+@router.post("/create", response_model=Optional[ToolResponse])
+async def create_new_toolkit(
+    request: Request, form_data: ToolForm, user=Depends(get_admin_user)
+):
+    if not form_data.id.isidentifier():
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Only alphanumeric characters and underscores are allowed in the id",
+        )
+
+    form_data.id = form_data.id.lower()
+
+    toolkit = Tools.get_tool_by_id(form_data.id)
+    if toolkit == None:
+        toolkit_path = os.path.join(TOOLS_DIR, f"{form_data.id}.py")
+        try:
+            with open(toolkit_path, "w") as tool_file:
+                tool_file.write(form_data.content)
+
+            toolkit_module = load_toolkit_module_by_id(form_data.id)
+
+            TOOLS = request.app.state.TOOLS
+            TOOLS[form_data.id] = toolkit_module
+
+            specs = get_tools_specs(TOOLS[form_data.id])
+            toolkit = Tools.insert_new_tool(user.id, form_data, specs)
+
+            if toolkit:
+                return toolkit
+            else:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=ERROR_MESSAGES.DEFAULT("Error creating toolkit"),
+                )
+        except Exception as e:
+            print(e)
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT(e),
+            )
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.ID_TAKEN,
+        )
+
+
+############################
+# GetToolkitById
+############################
+
+
+@router.get("/id/{id}", response_model=Optional[ToolModel])
+async def get_toolkit_by_id(id: str, user=Depends(get_admin_user)):
+    toolkit = Tools.get_tool_by_id(id)
+
+    if toolkit:
+        return toolkit
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=ERROR_MESSAGES.NOT_FOUND,
+        )
+
+
+############################
+# UpdateToolkitById
+############################
+
+
+@router.post("/id/{id}/update", response_model=Optional[ToolModel])
+async def update_toolkit_by_id(
+    request: Request, id: str, form_data: ToolForm, user=Depends(get_admin_user)
+):
+    toolkit_path = os.path.join(TOOLS_DIR, f"{id}.py")
+
+    try:
+        with open(toolkit_path, "w") as tool_file:
+            tool_file.write(form_data.content)
+
+        toolkit_module = load_toolkit_module_by_id(id)
+
+        TOOLS = request.app.state.TOOLS
+        TOOLS[id] = toolkit_module
+
+        specs = get_tools_specs(TOOLS[id])
+
+        updated = {
+            **form_data.model_dump(exclude={"id"}),
+            "specs": specs,
+        }
+
+        print(updated)
+        toolkit = Tools.update_tool_by_id(id, updated)
+
+        if toolkit:
+            return toolkit
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.DEFAULT("Error updating toolkit"),
+            )
+
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=ERROR_MESSAGES.DEFAULT(e),
+        )
+
+
+############################
+# DeleteToolkitById
+############################
+
+
+@router.delete("/id/{id}/delete", response_model=bool)
+async def delete_toolkit_by_id(request: Request, id: str, user=Depends(get_admin_user)):
+    result = Tools.delete_tool_by_id(id)
+
+    if result:
+        TOOLS = request.app.state.TOOLS
+        if id in TOOLS:
+            del TOOLS[id]
+
+        # delete the toolkit file
+        toolkit_path = os.path.join(TOOLS_DIR, f"{id}.py")
+        os.remove(toolkit_path)
+
+    return result

backend/apps/webui/routers/utils.py

@@ -7,6 +7,8 @@ from pydantic import BaseModel
 
 from fpdf import FPDF
 import markdown
+import black
+
 
 from apps.webui.internal.db import DB
 from utils.utils import get_admin_user
@@ -26,6 +28,21 @@ async def get_gravatar(
     return get_gravatar_url(email)
 
 
+class CodeFormatRequest(BaseModel):
+    code: str
+
+
+@router.post("/code/format")
+async def format_code(request: CodeFormatRequest):
+    try:
+        formatted_code = black.format_str(request.code, mode=black.Mode())
+        return {"code": formatted_code}
+    except black.NothingChanged:
+        return {"code": request.code}
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
 class MarkdownForm(BaseModel):
     md: str
 

backend/apps/webui/utils.py

@@ -0,0 +1,23 @@
+from importlib import util
+import os
+
+from config import TOOLS_DIR
+
+
+def load_toolkit_module_by_id(toolkit_id):
+    toolkit_path = os.path.join(TOOLS_DIR, f"{toolkit_id}.py")
+    spec = util.spec_from_file_location(toolkit_id, toolkit_path)
+    module = util.module_from_spec(spec)
+
+    try:
+        spec.loader.exec_module(module)
+        print(f"Loaded module: {module.__name__}")
+        if hasattr(module, "Tools"):
+            return module.Tools()
+        else:
+            raise Exception("No Tools class found")
+    except Exception as e:
+        print(f"Error loading module: {toolkit_id}")
+        # Move the file to the error folder
+        os.rename(toolkit_path, f"{toolkit_path}.error")
+        raise e