diff --git a/backend/open_webui/tools/__init__.py b/backend/open_webui/tools/__init__.py new file mode 100644 index 000000000..112324b56 --- /dev/null +++ b/backend/open_webui/tools/__init__.py @@ -0,0 +1,6 @@ +""" +Open WebUI Tools Package. + +This package contains built-in tools that are automatically available +when native function calling is enabled. +""" diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py new file mode 100644 index 000000000..922f9a10e --- /dev/null +++ b/backend/open_webui/tools/builtin.py @@ -0,0 +1,241 @@ +""" +Built-in tools for Open WebUI. + +These tools are automatically available when native function calling is enabled. +""" + +import json +import logging +import time +from typing import Optional + +from fastapi import Request + +from open_webui.models.users import UserModel +from open_webui.routers.retrieval import search_web +from open_webui.retrieval.utils import get_content_from_url +from open_webui.routers.images import image_generations, image_edits, CreateImageForm, EditImageForm +from open_webui.routers.memories import query_memory, add_memory, QueryMemoryForm, AddMemoryForm + +log = logging.getLogger(__name__) + + +async def web_search( + query: str, + count: int = 5, + __request__: Request = None, + __user__: dict = None, +) -> str: + """ + Search the web for information on a given topic. + + :param query: The search query to look up + :param count: Number of results to return (default: 5) + :return: JSON with search results containing title, link, and snippet for each result + """ + if __request__ is None: + return json.dumps({"error": "Request context not available"}) + + try: + engine = __request__.app.state.config.WEB_SEARCH_ENGINE + user = UserModel(**__user__) if __user__ else None + + results = search_web(__request__, engine, query, user) + + # Limit results + results = results[:count] if results else [] + + return json.dumps( + [{"title": r.title, "link": r.link, "snippet": r.snippet} for r in results], + ensure_ascii=False, + ) + except Exception as e: + log.exception(f"web_search error: {e}") + return json.dumps({"error": str(e)}) + + +async def fetch_url( + url: str, + __request__: Request = None, + __user__: dict = None, +) -> str: + """ + Fetch and extract the main text content from a web page URL. + + :param url: The URL to fetch content from + :return: The extracted text content from the page + """ + if __request__ is None: + return json.dumps({"error": "Request context not available"}) + + try: + content, _ = get_content_from_url(__request__, url) + + # Truncate if too long (avoid overwhelming context) + max_length = 50000 + if len(content) > max_length: + content = content[:max_length] + "\n\n[Content truncated...]" + + return content + except Exception as e: + log.exception(f"fetch_url error: {e}") + return json.dumps({"error": str(e)}) + + +async def generate_image( + prompt: str, + __request__: Request = None, + __user__: dict = None, + __event_emitter__: callable = None, +) -> str: + """ + Generate an image based on a text prompt. + + :param prompt: A detailed description of the image to generate + :return: Confirmation that the image was generated, or an error message + """ + if __request__ is None: + return json.dumps({"error": "Request context not available"}) + + try: + user = UserModel(**__user__) if __user__ else None + + images = await image_generations( + request=__request__, + form_data=CreateImageForm(prompt=prompt), + user=user, + ) + + # Emit the images to the UI if event emitter is available + if __event_emitter__ and images: + await __event_emitter__( + { + "type": "files", + "data": { + "files": [ + {"type": "image", "url": img["url"]} + for img in images + ] + }, + } + ) + + return json.dumps({"status": "success", "images": images}, ensure_ascii=False) + except Exception as e: + log.exception(f"generate_image error: {e}") + return json.dumps({"error": str(e)}) + + +async def edit_image( + prompt: str, + image_url: str, + __request__: Request = None, + __user__: dict = None, + __event_emitter__: callable = None, +) -> str: + """ + Edit an existing image based on a text prompt. + + :param prompt: A description of the changes to make to the image + :param image_url: The URL of the image to edit + :return: Confirmation that the image was edited, or an error message + """ + if __request__ is None: + return json.dumps({"error": "Request context not available"}) + + try: + user = UserModel(**__user__) if __user__ else None + + images = await image_edits( + request=__request__, + form_data=EditImageForm(prompt=prompt, image=image_url), + user=user, + ) + + # Emit the images to the UI if event emitter is available + if __event_emitter__ and images: + await __event_emitter__( + { + "type": "files", + "data": { + "files": [ + {"type": "image", "url": img["url"]} + for img in images + ] + }, + } + ) + + return json.dumps({"status": "success", "images": images}, ensure_ascii=False) + except Exception as e: + log.exception(f"edit_image error: {e}") + return json.dumps({"error": str(e)}) + + +async def memory_query( + query: str, + __request__: Request = None, + __user__: dict = None, +) -> str: + """ + Search the user's stored memories for relevant information. + + :param query: The search query to find relevant memories + :return: JSON with matching memories and their dates + """ + if __request__ is None: + return json.dumps({"error": "Request context not available"}) + + try: + user = UserModel(**__user__) if __user__ else None + + results = await query_memory( + __request__, + QueryMemoryForm(content=query, k=5), + user, + ) + + if results and hasattr(results, "documents") and results.documents: + memories = [] + for doc_idx, doc in enumerate(results.documents[0]): + created_at = "Unknown" + if results.metadatas and results.metadatas[0][doc_idx].get("created_at"): + created_at = time.strftime( + "%Y-%m-%d", time.localtime(results.metadatas[0][doc_idx]["created_at"]) + ) + memories.append({"date": created_at, "content": doc}) + return json.dumps(memories, ensure_ascii=False) + else: + return json.dumps([]) + except Exception as e: + log.exception(f"memory_query error: {e}") + return json.dumps({"error": str(e)}) + + +async def memory_add( + content: str, + __request__: Request = None, + __user__: dict = None, +) -> str: + """ + Store a new memory for the user. + + :param content: The memory content to store + :return: Confirmation that the memory was stored + """ + if __request__ is None: + return json.dumps({"error": "Request context not available"}) + + try: + user = UserModel(**__user__) if __user__ else None + + memory = await add_memory( + __request__, + AddMemoryForm(content=content), + user, + ) + + return json.dumps({"status": "success", "id": memory.id}, ensure_ascii=False) + except Exception as e: + log.exception(f"memory_add error: {e}") + return json.dumps({"error": str(e)}) diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index 510529ad3..15414aa00 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -44,6 +44,7 @@ from open_webui.routers.retrieval import ( process_web_search, SearchForm, ) +from open_webui.utils.tools import get_builtin_tools from open_webui.routers.images import ( image_generations, CreateImageForm, @@ -1305,7 +1306,8 @@ async def process_chat_payload(request, form_data, user, metadata, model): except Exception as e: raise Exception(f"{e}") - features = form_data.pop("features", None) + features = form_data.pop("features", None) or {} + extra_params["__features__"] = features if features: if "voice" in features and features["voice"]: if request.app.state.config.VOICE_MODE_PROMPT_TEMPLATE != None: @@ -1320,19 +1322,25 @@ async def process_chat_payload(request, form_data, user, metadata, model): ) if "memory" in features and features["memory"]: - form_data = await chat_memory_handler( - request, form_data, extra_params, user - ) + # Skip forced memory injection when native FC is enabled - model can use memory tools + if metadata.get("params", {}).get("function_calling") != "native": + form_data = await chat_memory_handler( + request, form_data, extra_params, user + ) if "web_search" in features and features["web_search"]: - form_data = await chat_web_search_handler( - request, form_data, extra_params, user - ) + # Skip forced RAG web search when native FC is enabled - model can use web_search tool + if metadata.get("params", {}).get("function_calling") != "native": + form_data = await chat_web_search_handler( + request, form_data, extra_params, user + ) if "image_generation" in features and features["image_generation"]: - form_data = await chat_image_generation_handler( - request, form_data, extra_params, user - ) + # Skip forced image generation when native FC is enabled - model can use generate_image tool + if metadata.get("params", {}).get("function_calling") != "native": + form_data = await chat_image_generation_handler( + request, form_data, extra_params, user + ) if "code_interpreter" in features and features["code_interpreter"]: form_data["messages"] = add_or_update_user_message( @@ -1543,6 +1551,20 @@ async def process_chat_payload(request, form_data, user, metadata, model): if mcp_clients: metadata["mcp_clients"] = mcp_clients + # Always inject builtin tools for native function calling based on enabled features + if metadata.get("params", {}).get("function_calling") == "native": + builtin_tools = get_builtin_tools( + request, + { + **extra_params, + "__event_emitter__": event_emitter, + }, + features, + ) + for name, tool_dict in builtin_tools.items(): + if name not in tools_dict: + tools_dict[name] = tool_dict + if tools_dict: if metadata.get("params", {}).get("function_calling") == "native": # If the function calling is native, then call the tools function calling handler diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index f517e8999..4379d9f4d 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -43,6 +43,10 @@ from open_webui.env import ( AIOHTTP_CLIENT_TIMEOUT_TOOL_SERVER_DATA, AIOHTTP_CLIENT_SESSION_TOOL_SERVER_SSL, ) +from open_webui.tools.builtin import ( + web_search, fetch_url, generate_image, edit_image, + memory_query, memory_add +) import copy @@ -320,6 +324,56 @@ async def get_tools( return tools_dict +def get_builtin_tools(request: Request, extra_params: dict, features: dict = None) -> dict[str, dict]: + """ + Get built-in tools for native function calling. + Only returns tools when BOTH the global config is enabled AND the feature is enabled for this chat. + """ + tools_dict = {} + builtin_functions = [] + features = features or {} + + # Add web search tools if enabled globally AND for this chat + if (getattr(request.app.state.config, "ENABLE_WEB_SEARCH", False) + and features.get("web_search")): + builtin_functions.extend([web_search, fetch_url]) + + # Add image generation/edit tools if enabled globally AND for this chat + if (getattr(request.app.state.config, "ENABLE_IMAGE_GENERATION", False) + and features.get("image_generation")): + builtin_functions.append(generate_image) + if (getattr(request.app.state.config, "ENABLE_IMAGE_EDIT", False) + and features.get("image_generation")): + builtin_functions.append(edit_image) + + # Add memory tools if enabled for this chat + if features.get("memory"): + builtin_functions.extend([memory_query, memory_add]) + + for func in builtin_functions: + callable = get_async_tool_function_and_apply_extra_params( + func, + { + "__request__": request, + "__user__": extra_params.get("__user__", {}), + "__event_emitter__": extra_params.get("__event_emitter__"), + }, + ) + + # Generate spec from function + pydantic_model = convert_function_to_pydantic_model(func) + spec = convert_pydantic_model_to_openai_function_spec(pydantic_model) + + tools_dict[func.__name__] = { + "tool_id": f"builtin:{func.__name__}", + "callable": callable, + "spec": spec, + "type": "builtin", + } + + return tools_dict + + def parse_description(docstring: str | None) -> str: """ Parse a function's docstring to extract the description.