From ffd7d74f779f4a0ab85c8a0655ffb7f264d81cdc Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 20 Sep 2024 23:43:22 +0200 Subject: [PATCH 01/20] enh: websocket redis support --- backend/open_webui/apps/socket/main.py | 29 +++++++++++++++++++------- backend/open_webui/env.py | 4 ++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/backend/open_webui/apps/socket/main.py b/backend/open_webui/apps/socket/main.py index e41ef8412..8355cbcd8 100644 --- a/backend/open_webui/apps/socket/main.py +++ b/backend/open_webui/apps/socket/main.py @@ -2,16 +2,29 @@ import asyncio import socketio from open_webui.apps.webui.models.users import Users -from open_webui.env import ENABLE_WEBSOCKET_SUPPORT +from open_webui.env import ( + ENABLE_WEBSOCKET_SUPPORT, + WEBSOCKET_MANAGER, + WEBSOCKET_REDIS_URL, +) from open_webui.utils.utils import decode_token -sio = socketio.AsyncServer( - cors_allowed_origins=[], - async_mode="asgi", - transports=(["polling", "websocket"] if ENABLE_WEBSOCKET_SUPPORT else ["polling"]), - allow_upgrades=ENABLE_WEBSOCKET_SUPPORT, - always_connect=True, -) + +if WEBSOCKET_MANAGER == "redis": + mgr = socketio.AsyncRedisManager(WEBSOCKET_REDIS_URL) + sio = socketio.AsyncServer(client_manager=mgr) +else: + sio = socketio.AsyncServer( + cors_allowed_origins=[], + async_mode="asgi", + transports=( + ["polling", "websocket"] if ENABLE_WEBSOCKET_SUPPORT else ["polling"] + ), + allow_upgrades=ENABLE_WEBSOCKET_SUPPORT, + always_connect=True, + ) + + app = socketio.ASGIApp(sio, socketio_path="/ws/socket.io") # Dictionary to maintain the user pool diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 89422e57b..504eeea54 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -302,3 +302,7 @@ if WEBUI_AUTH and WEBUI_SECRET_KEY == "": ENABLE_WEBSOCKET_SUPPORT = ( os.environ.get("ENABLE_WEBSOCKET_SUPPORT", "True").lower() == "true" ) + +WEBSOCKET_MANAGER = os.environ.get("WEBSOCKET_MANAGER", "") + +WEBSOCKET_REDIS_URL = os.environ.get("WEBSOCKET_REDIS_URL", "redis://localhost:6379/0") From 585b9eb84a783c1135f0e739b0434ab8a1fbaf55 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 20 Sep 2024 23:48:03 +0200 Subject: [PATCH 02/20] refac --- backend/open_webui/main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 319d95165..78bb587dd 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -1620,8 +1620,8 @@ async def generate_moa_response(form_data: dict, user=Depends(get_verified_user) # Check if the user has a custom task model # If the user has a custom task model, use that model - model_id = get_task_model_id(model_id) - print(model_id) + task_model_id = get_task_model_id(model_id) + print(task_model_id) template = """You have been provided with a set of responses from various models to the latest user query: "{{prompt}}" @@ -1635,14 +1635,18 @@ Responses from models: {{responses}}""" form_data["responses"], ) + + payload = { - "model": model_id, + "model": task_model_id, "messages": [{"role": "user", "content": content}], "stream": form_data.get("stream", False), "chat_id": form_data.get("chat_id", None), "metadata": {"task": str(TASKS.MOA_RESPONSE_GENERATION)}, } + + log.debug(payload) try: From 41926172d3858cbef5115de014333f21304caa06 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 21 Sep 2024 00:30:13 +0200 Subject: [PATCH 03/20] fix/refac: use ollama /api/chat endpoint for tasks --- backend/open_webui/main.py | 72 ++++++++++++++++++++-------- backend/open_webui/utils/payload.py | 46 ++++++++++++++++++ backend/open_webui/utils/response.py | 14 ++++++ 3 files changed, 111 insertions(+), 21 deletions(-) create mode 100644 backend/open_webui/utils/response.py diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 78bb587dd..55f9acfdf 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -19,7 +19,9 @@ from open_webui.apps.audio.main import app as audio_app from open_webui.apps.images.main import app as images_app from open_webui.apps.ollama.main import app as ollama_app from open_webui.apps.ollama.main import ( - generate_openai_chat_completion as generate_ollama_chat_completion, + GenerateChatCompletionForm, + generate_chat_completion as generate_ollama_chat_completion, + generate_openai_chat_completion as generate_ollama_openai_chat_completion, ) from open_webui.apps.ollama.main import get_all_models as get_ollama_models from open_webui.apps.openai.main import app as openai_app @@ -135,6 +137,9 @@ from open_webui.utils.utils import ( ) from open_webui.utils.webhook import post_webhook +from open_webui.utils.payload import convert_payload_openai_to_ollama +from open_webui.utils.response import convert_response_ollama_to_openai + if SAFE_MODE: print("SAFE MODE ENABLED") Functions.deactivate_all_functions() @@ -1048,7 +1053,7 @@ async def generate_chat_completions(form_data: dict, user=Depends(get_verified_u if model.get("pipe"): return await generate_function_chat_completion(form_data, user=user) if model["owned_by"] == "ollama": - return await generate_ollama_chat_completion(form_data, user=user) + return await generate_ollama_openai_chat_completion(form_data, user=user) else: return await generate_openai_chat_completion(form_data, user=user) @@ -1399,9 +1404,10 @@ async def generate_title(form_data: dict, user=Depends(get_verified_user)): # Check if the user has a custom task model # If the user has a custom task model, use that model task_model_id = get_task_model_id(model_id) - print(task_model_id) + model = app.state.MODELS[task_model_id] + if app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE != "": template = app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE else: @@ -1440,9 +1446,9 @@ Prompt: {{prompt:middletruncate:8000}}""" "chat_id": form_data.get("chat_id", None), "metadata": {"task": str(TASKS.TITLE_GENERATION)}, } - log.debug(payload) + # Handle pipeline filters try: payload = filter_pipeline(payload, user) except Exception as e: @@ -1456,11 +1462,17 @@ Prompt: {{prompt:middletruncate:8000}}""" status_code=status.HTTP_400_BAD_REQUEST, content={"detail": str(e)}, ) - if "chat_id" in payload: del payload["chat_id"] - return await generate_chat_completions(form_data=payload, user=user) + # Check if task model is ollama model + if model["owned_by"] == "ollama": + payload = convert_payload_openai_to_ollama(payload) + form_data = GenerateChatCompletionForm(**payload) + response = await generate_ollama_chat_completion(form_data=form_data, user=user) + return convert_response_ollama_to_openai(response) + else: + return await generate_chat_completions(form_data=payload, user=user) @app.post("/api/task/query/completions") @@ -1484,6 +1496,8 @@ async def generate_search_query(form_data: dict, user=Depends(get_verified_user) task_model_id = get_task_model_id(model_id) print(task_model_id) + model = app.state.MODELS[task_model_id] + if app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE != "": template = app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE else: @@ -1516,9 +1530,9 @@ Search Query:""" ), "metadata": {"task": str(TASKS.QUERY_GENERATION)}, } + log.debug(payload) - print(payload) - + # Handle pipeline filters try: payload = filter_pipeline(payload, user) except Exception as e: @@ -1532,11 +1546,17 @@ Search Query:""" status_code=status.HTTP_400_BAD_REQUEST, content={"detail": str(e)}, ) - if "chat_id" in payload: del payload["chat_id"] - return await generate_chat_completions(form_data=payload, user=user) + # Check if task model is ollama model + if model["owned_by"] == "ollama": + payload = convert_payload_openai_to_ollama(payload) + form_data = GenerateChatCompletionForm(**payload) + response = await generate_ollama_chat_completion(form_data=form_data, user=user) + return convert_response_ollama_to_openai(response) + else: + return await generate_chat_completions(form_data=payload, user=user) @app.post("/api/task/emoji/completions") @@ -1555,12 +1575,13 @@ async def generate_emoji(form_data: dict, user=Depends(get_verified_user)): task_model_id = get_task_model_id(model_id) print(task_model_id) + model = app.state.MODELS[task_model_id] + template = ''' Your task is to reflect the speaker's likely facial expression through a fitting emoji. Interpret emotions from the message and reflect their facial expression using fitting, diverse emojis (e.g., 😊, 😢, 😡, 😱). Message: """{{prompt}}""" ''' - content = title_generation_template( template, form_data["prompt"], @@ -1584,9 +1605,9 @@ Message: """{{prompt}}""" "chat_id": form_data.get("chat_id", None), "metadata": {"task": str(TASKS.EMOJI_GENERATION)}, } - log.debug(payload) + # Handle pipeline filters try: payload = filter_pipeline(payload, user) except Exception as e: @@ -1600,11 +1621,17 @@ Message: """{{prompt}}""" status_code=status.HTTP_400_BAD_REQUEST, content={"detail": str(e)}, ) - if "chat_id" in payload: del payload["chat_id"] - return await generate_chat_completions(form_data=payload, user=user) + # Check if task model is ollama model + if model["owned_by"] == "ollama": + payload = convert_payload_openai_to_ollama(payload) + form_data = GenerateChatCompletionForm(**payload) + response = await generate_ollama_chat_completion(form_data=form_data, user=user) + return convert_response_ollama_to_openai(response) + else: + return await generate_chat_completions(form_data=payload, user=user) @app.post("/api/task/moa/completions") @@ -1623,6 +1650,8 @@ async def generate_moa_response(form_data: dict, user=Depends(get_verified_user) task_model_id = get_task_model_id(model_id) print(task_model_id) + model = app.state.MODELS[task_model_id] + template = """You have been provided with a set of responses from various models to the latest user query: "{{prompt}}" Your task is to synthesize these responses into a single, high-quality response. It is crucial to critically evaluate the information provided in these responses, recognizing that some of it may be biased or incorrect. Your response should not simply replicate the given answers but should offer a refined, accurate, and comprehensive reply to the instruction. Ensure your response is well-structured, coherent, and adheres to the highest standards of accuracy and reliability. @@ -1635,8 +1664,6 @@ Responses from models: {{responses}}""" form_data["responses"], ) - - payload = { "model": task_model_id, "messages": [{"role": "user", "content": content}], @@ -1644,9 +1671,6 @@ Responses from models: {{responses}}""" "chat_id": form_data.get("chat_id", None), "metadata": {"task": str(TASKS.MOA_RESPONSE_GENERATION)}, } - - - log.debug(payload) try: @@ -1662,11 +1686,17 @@ Responses from models: {{responses}}""" status_code=status.HTTP_400_BAD_REQUEST, content={"detail": str(e)}, ) - if "chat_id" in payload: del payload["chat_id"] - return await generate_chat_completions(form_data=payload, user=user) + # Check if task model is ollama model + if model["owned_by"] == "ollama": + payload = convert_payload_openai_to_ollama(payload) + form_data = GenerateChatCompletionForm(**payload) + response = await generate_ollama_chat_completion(form_data=form_data, user=user) + return convert_response_ollama_to_openai(response) + else: + return await generate_chat_completions(form_data=payload, user=user) ################################## diff --git a/backend/open_webui/utils/payload.py b/backend/open_webui/utils/payload.py index b2654cd25..72aec6a6c 100644 --- a/backend/open_webui/utils/payload.py +++ b/backend/open_webui/utils/payload.py @@ -86,3 +86,49 @@ def apply_model_params_to_body_ollama(params: dict, form_data: dict) -> dict: form_data[value] = param return form_data + + +def convert_payload_openai_to_ollama(openai_payload: dict) -> dict: + """ + Converts a payload formatted for OpenAI's API to be compatible with Ollama's API endpoint for chat completions. + + Args: + openai_payload (dict): The payload originally designed for OpenAI API usage. + + Returns: + dict: A modified payload compatible with the Ollama API. + """ + ollama_payload = {} + + # Mapping basic model and message details + ollama_payload["model"] = openai_payload.get("model") + ollama_payload["messages"] = openai_payload.get("messages") + ollama_payload["stream"] = openai_payload.get("stream", False) + + # If there are advanced parameters in the payload, format them in Ollama's options field + ollama_options = {} + + # Handle parameters which map directly + for param in ["temperature", "top_p", "seed"]: + if param in openai_payload: + ollama_options[param] = openai_payload[param] + + # Mapping OpenAI's `max_tokens` -> Ollama's `num_predict` + if "max_completion_tokens" in openai_payload: + ollama_options["num_predict"] = openai_payload["max_completion_tokens"] + elif "max_tokens" in openai_payload: + ollama_options["num_predict"] = openai_payload["max_tokens"] + + # Handle frequency / presence_penalty, which needs renaming and checking + if "frequency_penalty" in openai_payload: + ollama_options["repeat_penalty"] = openai_payload["frequency_penalty"] + + if "presence_penalty" in openai_payload and "penalty" not in ollama_options: + # We are assuming presence penalty uses a similar concept in Ollama, which needs custom handling if exists. + ollama_options["new_topic_penalty"] = openai_payload["presence_penalty"] + + # Add options to payload if any have been set + if ollama_options: + ollama_payload["options"] = ollama_options + + return ollama_payload diff --git a/backend/open_webui/utils/response.py b/backend/open_webui/utils/response.py new file mode 100644 index 000000000..22275488f --- /dev/null +++ b/backend/open_webui/utils/response.py @@ -0,0 +1,14 @@ +from open_webui.utils.task import prompt_template +from open_webui.utils.misc import ( + openai_chat_completion_message_template, +) + +from typing import Callable, Optional + + +def convert_response_ollama_to_openai(ollama_response: dict) -> dict: + model = ollama_response.get("model", "ollama") + message_content = ollama_response.get("message", {}).get("content", "") + + response = openai_chat_completion_message_template(model, message_content) + return response From 3a0a1aca1184f5c8c7a7cf3a7217a7980d80d4f6 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 21 Sep 2024 01:07:57 +0200 Subject: [PATCH 04/20] refac: task ollama stream support --- backend/open_webui/main.py | 41 ++++++++++++++++++++++++---- backend/open_webui/utils/misc.py | 16 ++++++++--- backend/open_webui/utils/response.py | 24 ++++++++++++++-- 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 55f9acfdf..a6141484f 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -138,7 +138,10 @@ from open_webui.utils.utils import ( from open_webui.utils.webhook import post_webhook from open_webui.utils.payload import convert_payload_openai_to_ollama -from open_webui.utils.response import convert_response_ollama_to_openai +from open_webui.utils.response import ( + convert_response_ollama_to_openai, + convert_streaming_response_ollama_to_openai, +) if SAFE_MODE: print("SAFE MODE ENABLED") @@ -1470,7 +1473,14 @@ Prompt: {{prompt:middletruncate:8000}}""" payload = convert_payload_openai_to_ollama(payload) form_data = GenerateChatCompletionForm(**payload) response = await generate_ollama_chat_completion(form_data=form_data, user=user) - return convert_response_ollama_to_openai(response) + if form_data.stream: + response.headers["content-type"] = "text/event-stream" + return StreamingResponse( + convert_streaming_response_ollama_to_openai(response), + headers=dict(response.headers), + ) + else: + return convert_response_ollama_to_openai(response) else: return await generate_chat_completions(form_data=payload, user=user) @@ -1554,7 +1564,14 @@ Search Query:""" payload = convert_payload_openai_to_ollama(payload) form_data = GenerateChatCompletionForm(**payload) response = await generate_ollama_chat_completion(form_data=form_data, user=user) - return convert_response_ollama_to_openai(response) + if form_data.stream: + response.headers["content-type"] = "text/event-stream" + return StreamingResponse( + convert_streaming_response_ollama_to_openai(response), + headers=dict(response.headers), + ) + else: + return convert_response_ollama_to_openai(response) else: return await generate_chat_completions(form_data=payload, user=user) @@ -1629,7 +1646,14 @@ Message: """{{prompt}}""" payload = convert_payload_openai_to_ollama(payload) form_data = GenerateChatCompletionForm(**payload) response = await generate_ollama_chat_completion(form_data=form_data, user=user) - return convert_response_ollama_to_openai(response) + if form_data.stream: + response.headers["content-type"] = "text/event-stream" + return StreamingResponse( + convert_streaming_response_ollama_to_openai(response), + headers=dict(response.headers), + ) + else: + return convert_response_ollama_to_openai(response) else: return await generate_chat_completions(form_data=payload, user=user) @@ -1694,7 +1718,14 @@ Responses from models: {{responses}}""" payload = convert_payload_openai_to_ollama(payload) form_data = GenerateChatCompletionForm(**payload) response = await generate_ollama_chat_completion(form_data=form_data, user=user) - return convert_response_ollama_to_openai(response) + if form_data.stream: + response.headers["content-type"] = "text/event-stream" + return StreamingResponse( + convert_streaming_response_ollama_to_openai(response), + headers=dict(response.headers), + ) + else: + return convert_response_ollama_to_openai(response) else: return await generate_chat_completions(form_data=payload, user=user) diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index d1b340044..bdce74b05 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -105,17 +105,25 @@ def openai_chat_message_template(model: str): } -def openai_chat_chunk_message_template(model: str, message: str) -> dict: +def openai_chat_chunk_message_template( + model: str, message: Optional[str] = None +) -> dict: template = openai_chat_message_template(model) template["object"] = "chat.completion.chunk" - template["choices"][0]["delta"] = {"content": message} + if message: + template["choices"][0]["delta"] = {"content": message} + else: + template["choices"][0]["finish_reason"] = "stop" return template -def openai_chat_completion_message_template(model: str, message: str) -> dict: +def openai_chat_completion_message_template( + model: str, message: Optional[str] = None +) -> dict: template = openai_chat_message_template(model) template["object"] = "chat.completion" - template["choices"][0]["message"] = {"content": message, "role": "assistant"} + if message: + template["choices"][0]["message"] = {"content": message, "role": "assistant"} template["choices"][0]["finish_reason"] = "stop" return template diff --git a/backend/open_webui/utils/response.py b/backend/open_webui/utils/response.py index 22275488f..3debe63af 100644 --- a/backend/open_webui/utils/response.py +++ b/backend/open_webui/utils/response.py @@ -1,10 +1,9 @@ -from open_webui.utils.task import prompt_template +import json from open_webui.utils.misc import ( + openai_chat_chunk_message_template, openai_chat_completion_message_template, ) -from typing import Callable, Optional - def convert_response_ollama_to_openai(ollama_response: dict) -> dict: model = ollama_response.get("model", "ollama") @@ -12,3 +11,22 @@ def convert_response_ollama_to_openai(ollama_response: dict) -> dict: response = openai_chat_completion_message_template(model, message_content) return response + + +async def convert_streaming_response_ollama_to_openai(ollama_streaming_response): + async for data in ollama_streaming_response.body_iterator: + data = json.loads(data) + + model = data.get("model", "ollama") + message_content = data.get("message", {}).get("content", "") + done = data.get("done", False) + + data = openai_chat_chunk_message_template( + model, message_content if not done else None + ) + + line = f"data: {json.dumps(data)}\n\n" + if done: + line += "data: [DONE]\n\n" + + yield line From 95985e7bbb3d1aa2342d0b476fe254d43d18cc3e Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 21 Sep 2024 02:03:58 +0200 Subject: [PATCH 05/20] refac --- backend/open_webui/main.py | 77 ++++++++------------------------------ 1 file changed, 16 insertions(+), 61 deletions(-) diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index a6141484f..64b7f2153 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -1056,7 +1056,18 @@ async def generate_chat_completions(form_data: dict, user=Depends(get_verified_u if model.get("pipe"): return await generate_function_chat_completion(form_data, user=user) if model["owned_by"] == "ollama": - return await generate_ollama_openai_chat_completion(form_data, user=user) + # Using /ollama/api/chat endpoint + form_data = convert_payload_openai_to_ollama(form_data) + form_data = GenerateChatCompletionForm(**form_data) + response = await generate_ollama_chat_completion(form_data=form_data, user=user) + if form_data.stream: + response.headers["content-type"] = "text/event-stream" + return StreamingResponse( + convert_streaming_response_ollama_to_openai(response), + headers=dict(response.headers), + ) + else: + return convert_response_ollama_to_openai(response) else: return await generate_openai_chat_completion(form_data, user=user) @@ -1468,21 +1479,7 @@ Prompt: {{prompt:middletruncate:8000}}""" if "chat_id" in payload: del payload["chat_id"] - # Check if task model is ollama model - if model["owned_by"] == "ollama": - payload = convert_payload_openai_to_ollama(payload) - form_data = GenerateChatCompletionForm(**payload) - response = await generate_ollama_chat_completion(form_data=form_data, user=user) - if form_data.stream: - response.headers["content-type"] = "text/event-stream" - return StreamingResponse( - convert_streaming_response_ollama_to_openai(response), - headers=dict(response.headers), - ) - else: - return convert_response_ollama_to_openai(response) - else: - return await generate_chat_completions(form_data=payload, user=user) + return await generate_chat_completions(form_data=payload, user=user) @app.post("/api/task/query/completions") @@ -1559,21 +1556,7 @@ Search Query:""" if "chat_id" in payload: del payload["chat_id"] - # Check if task model is ollama model - if model["owned_by"] == "ollama": - payload = convert_payload_openai_to_ollama(payload) - form_data = GenerateChatCompletionForm(**payload) - response = await generate_ollama_chat_completion(form_data=form_data, user=user) - if form_data.stream: - response.headers["content-type"] = "text/event-stream" - return StreamingResponse( - convert_streaming_response_ollama_to_openai(response), - headers=dict(response.headers), - ) - else: - return convert_response_ollama_to_openai(response) - else: - return await generate_chat_completions(form_data=payload, user=user) + return await generate_chat_completions(form_data=payload, user=user) @app.post("/api/task/emoji/completions") @@ -1641,21 +1624,7 @@ Message: """{{prompt}}""" if "chat_id" in payload: del payload["chat_id"] - # Check if task model is ollama model - if model["owned_by"] == "ollama": - payload = convert_payload_openai_to_ollama(payload) - form_data = GenerateChatCompletionForm(**payload) - response = await generate_ollama_chat_completion(form_data=form_data, user=user) - if form_data.stream: - response.headers["content-type"] = "text/event-stream" - return StreamingResponse( - convert_streaming_response_ollama_to_openai(response), - headers=dict(response.headers), - ) - else: - return convert_response_ollama_to_openai(response) - else: - return await generate_chat_completions(form_data=payload, user=user) + return await generate_chat_completions(form_data=payload, user=user) @app.post("/api/task/moa/completions") @@ -1713,21 +1682,7 @@ Responses from models: {{responses}}""" if "chat_id" in payload: del payload["chat_id"] - # Check if task model is ollama model - if model["owned_by"] == "ollama": - payload = convert_payload_openai_to_ollama(payload) - form_data = GenerateChatCompletionForm(**payload) - response = await generate_ollama_chat_completion(form_data=form_data, user=user) - if form_data.stream: - response.headers["content-type"] = "text/event-stream" - return StreamingResponse( - convert_streaming_response_ollama_to_openai(response), - headers=dict(response.headers), - ) - else: - return convert_response_ollama_to_openai(response) - else: - return await generate_chat_completions(form_data=payload, user=user) + return await generate_chat_completions(form_data=payload, user=user) ################################## From 657d443a3edcc9c3a126d9174bedbbbb496f2eff Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 21 Sep 2024 02:24:30 +0200 Subject: [PATCH 06/20] fix --- backend/open_webui/apps/socket/main.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/open_webui/apps/socket/main.py b/backend/open_webui/apps/socket/main.py index 8355cbcd8..c353c8e6f 100644 --- a/backend/open_webui/apps/socket/main.py +++ b/backend/open_webui/apps/socket/main.py @@ -12,7 +12,16 @@ from open_webui.utils.utils import decode_token if WEBSOCKET_MANAGER == "redis": mgr = socketio.AsyncRedisManager(WEBSOCKET_REDIS_URL) - sio = socketio.AsyncServer(client_manager=mgr) + sio = socketio.AsyncServer( + cors_allowed_origins=[], + async_mode="asgi", + transports=( + ["polling", "websocket"] if ENABLE_WEBSOCKET_SUPPORT else ["polling"] + ), + allow_upgrades=ENABLE_WEBSOCKET_SUPPORT, + always_connect=True, + client_manager=mgr, + ) else: sio = socketio.AsyncServer( cors_allowed_origins=[], From e1ea0c23ebb4a5d35b9406d21bc4fe747c344705 Mon Sep 17 00:00:00 2001 From: kivvi Date: Sat, 21 Sep 2024 09:06:28 +0800 Subject: [PATCH 07/20] Fixed the issue of being unable to generate titles --- backend/open_webui/apps/openai/main.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/open_webui/apps/openai/main.py b/backend/open_webui/apps/openai/main.py index 9a27c46a3..87bcc042c 100644 --- a/backend/open_webui/apps/openai/main.py +++ b/backend/open_webui/apps/openai/main.py @@ -405,14 +405,19 @@ async def generate_chat_completion( "role": user.role, } + url = app.state.config.OPENAI_API_BASE_URLS[idx] + key = app.state.config.OPENAI_API_KEYS[idx] + + # Change max_completion_tokens to max_tokens (Backward compatible) + if "api.openai.com" not in url and not payload["model"].lower().startswith("o1-"): + if "max_completion_tokens" in payload: + payload["max_tokens"] = payload.pop("max_completion_tokens") + # Convert the modified body back to JSON payload = json.dumps(payload) log.debug(payload) - url = app.state.config.OPENAI_API_BASE_URLS[idx] - key = app.state.config.OPENAI_API_KEYS[idx] - headers = {} headers["Authorization"] = f"Bearer {key}" headers["Content-Type"] = "application/json" From 692f04d45737e1d927cf30913caff8945ccff4ca Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 21 Sep 2024 03:33:06 +0200 Subject: [PATCH 08/20] enh: width adjustable chat controls --- package-lock.json | 12 ++ package.json | 1 + src/lib/components/chat/Chat.svelte | 212 ++++++++++---------- src/lib/components/chat/ChatControls.svelte | 76 ++++--- src/lib/components/layout/Navbar.svelte | 13 +- 5 files changed, 182 insertions(+), 132 deletions(-) diff --git a/package-lock.json b/package-lock.json index a0048a211..84924afea 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "katex": "^0.16.9", "marked": "^9.1.0", "mermaid": "^10.9.1", + "paneforge": "^0.0.6", "pyodide": "^0.26.1", "socket.io-client": "^4.2.0", "sortablejs": "^1.15.2", @@ -6986,6 +6987,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/paneforge": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/paneforge/-/paneforge-0.0.6.tgz", + "integrity": "sha512-jYeN/wdREihja5c6nK3S5jritDQ+EbCqC5NrDo97qCZzZ9GkmEcN5C0ZCjF4nmhBwkDKr6tLIgz4QUKWxLXjAw==", + "dependencies": { + "nanoid": "^5.0.4" + }, + "peerDependencies": { + "svelte": "^4.0.0 || ^5.0.0-next.1" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/package.json b/package.json index 371507789..25c2289a2 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "katex": "^0.16.9", "marked": "^9.1.0", "mermaid": "^10.9.1", + "paneforge": "^0.0.6", "pyodide": "^0.26.1", "socket.io-client": "^4.2.0", "sortablejs": "^1.15.2", diff --git a/src/lib/components/chat/Chat.svelte b/src/lib/components/chat/Chat.svelte index e0ec62b52..de1eb0131 100644 --- a/src/lib/components/chat/Chat.svelte +++ b/src/lib/components/chat/Chat.svelte @@ -2,6 +2,7 @@ import { v4 as uuidv4 } from 'uuid'; import { toast } from 'svelte-sonner'; import mermaid from 'mermaid'; + import { PaneGroup, Pane, PaneResizer } from 'paneforge'; import { getContext, onDestroy, onMount, tick } from 'svelte'; import { goto } from '$app/navigation'; @@ -26,7 +27,8 @@ showControls, showCallOverlay, currentChatPage, - temporaryChatEnabled + temporaryChatEnabled, + mobile } from '$lib/stores'; import { convertMessagesToHistory, @@ -64,12 +66,14 @@ import Navbar from '$lib/components/layout/Navbar.svelte'; import ChatControls from './ChatControls.svelte'; import EventConfirmDialog from '../common/ConfirmDialog.svelte'; + import EllipsisVertical from '../icons/EllipsisVertical.svelte'; const i18n: Writable = getContext('i18n'); export let chatIdProp = ''; let loaded = false; const eventTarget = new EventTarget(); + let controlPane; let stopResponseFlag = false; let autoScroll = true; @@ -1760,117 +1764,117 @@ bind:selectedModels bind:showModelSelector shareEnabled={messages.length > 0} + {controlPane} {chat} {initNewChat} /> - {#if $banners.length > 0 && messages.length === 0 && !$chatId && selectedModels.length <= 1} -
-
- {#each $banners.filter( (b) => (b.dismissible ? !JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]').includes(b.id) : true) ) as banner} - { - const bannerId = e.detail; + + + {#if $banners.length > 0 && messages.length === 0 && !$chatId && selectedModels.length <= 1} +
+
+ {#each $banners.filter( (b) => (b.dismissible ? !JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]').includes(b.id) : true) ) as banner} + { + const bannerId = e.detail; - localStorage.setItem( - 'dismissedBannerIds', - JSON.stringify( - [ - bannerId, - ...JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]') - ].filter((id) => $banners.find((b) => b.id === id)) - ) - ); + localStorage.setItem( + 'dismissedBannerIds', + JSON.stringify( + [ + bannerId, + ...JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]') + ].filter((id) => $banners.find((b) => b.id === id)) + ) + ); + }} + /> + {/each} +
+
+ {/if} + +
+ + +
+ { + const model = $models.find((m) => m.id === e); + if (model?.info?.meta?.toolIds ?? false) { + return [...new Set([...a, ...model.info.meta.toolIds])]; + } + return a; + }, [])} + transparentBackground={$settings?.backgroundImageUrl ?? false} + {selectedModels} + {messages} + {submitPrompt} + {stopResponse} + on:call={() => { + showControls.set(true); }} /> - {/each} +
-
- {/if} + -
- - -
- { - const model = $models.find((m) => m.id === e); - if (model?.info?.meta?.toolIds ?? false) { - return [...new Set([...a, ...model.info.meta.toolIds])]; - } - return a; - }, [])} - transparentBackground={$settings?.backgroundImageUrl ?? false} - {selectedModels} - {messages} - {submitPrompt} - {stopResponse} - on:call={() => { - showControls.set(true); - }} - /> -
-
+ { + const model = $models.find((m) => m.id === e); + if (model) { + return [...a, model]; + } + return a; + }, [])} + bind:history + bind:chatFiles + bind:params + bind:files + bind:pane={controlPane} + {submitPrompt} + {stopResponse} + {showMessage} + modelId={selectedModelIds?.at(0) ?? null} + chatId={$chatId} + {eventTarget} + /> +
{/if} - - { - const model = $models.find((m) => m.id === e); - if (model) { - return [...a, model]; - } - return a; - }, [])} - bind:history - bind:chatFiles - bind:params - bind:files - {submitPrompt} - {stopResponse} - {showMessage} - modelId={selectedModelIds?.at(0) ?? null} - chatId={$chatId} - {eventTarget} -/> diff --git a/src/lib/components/chat/ChatControls.svelte b/src/lib/components/chat/ChatControls.svelte index b9268cebc..5c5962d7f 100644 --- a/src/lib/components/chat/ChatControls.svelte +++ b/src/lib/components/chat/ChatControls.svelte @@ -10,6 +10,9 @@ import CallOverlay from './MessageInput/CallOverlay.svelte'; import Drawer from '../common/Drawer.svelte'; import Overview from './Overview.svelte'; + import { Pane, PaneResizer } from 'paneforge'; + import EllipsisVertical from '../icons/EllipsisVertical.svelte'; + import { get } from 'svelte/store'; export let history; export let models = []; @@ -25,7 +28,9 @@ export let files; export let modelId; + export let pane; let largeScreen = false; + onMount(() => { // listen to resize 1024px const mediaQuery = window.matchMedia('(min-width: 1024px)'); @@ -58,33 +63,33 @@ {#if !largeScreen} - {#if $showCallOverlay} -
-
- { - showControls.set(false); - }} - /> -
-
- {:else if $showControls} + {#if $showControls} { showControls.set(false); }} > -
- {#if $showOverview} +
+ {#if $showCallOverlay} +
+ { + showControls.set(false); + }} + /> +
+ {:else if $showOverview} { @@ -107,11 +112,30 @@
{/if} - {:else if $showControls} -
-
+ {:else} + + +
+ +
+ + { + if (size === 0) { + showControls.set(false); + } else { + if (!$showControls) { + showControls.set(true); + } + localStorage.setItem('chat-controls-size', size); + } + }} + > +
@@ -149,6 +173,6 @@ {/if}
-
+ {/if} diff --git a/src/lib/components/layout/Navbar.svelte b/src/lib/components/layout/Navbar.svelte index e5ec5eb34..fec277a0c 100644 --- a/src/lib/components/layout/Navbar.svelte +++ b/src/lib/components/layout/Navbar.svelte @@ -29,6 +29,7 @@ export let initNewChat: Function; export let title: string = $WEBUI_NAME; export let shareEnabled: boolean = false; + export let controlPane; export let chat; export let selectedModels; @@ -109,8 +110,16 @@
diff --git a/src/lib/components/chat/ChatControls.svelte b/src/lib/components/chat/ChatControls.svelte index 925f19aa4..12a9905b7 100644 --- a/src/lib/components/chat/ChatControls.svelte +++ b/src/lib/components/chat/ChatControls.svelte @@ -140,17 +140,19 @@ : 'px-5 py-4 bg-white dark:shadow-lg dark:bg-gray-850 border border-gray-50 dark:border-gray-800'} rounded-lg z-50 pointer-events-auto overflow-y-auto scrollbar-hidden" > {#if $showCallOverlay} - { - showControls.set(false); - }} - /> +
+ { + showControls.set(false); + }} + /> +
{:else if $showOverview} { await showControls.set(!$showControls); - - if (controlPane) { - if ($showControls) { - controlPane.resize( - parseInt(localStorage.getItem('chat-controls-size') || '35') - ); - } else { - controlPane.resize(0); - } - } }} aria-label="Controls" > From ee6b1376c3248016a3156fb1221f4de228a46b49 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 21 Sep 2024 03:53:53 +0200 Subject: [PATCH 12/20] fix: rag duplicate collection issue --- backend/open_webui/apps/rag/main.py | 52 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/backend/open_webui/apps/rag/main.py b/backend/open_webui/apps/rag/main.py index 981a6fe5b..74855b336 100644 --- a/backend/open_webui/apps/rag/main.py +++ b/backend/open_webui/apps/rag/main.py @@ -1099,35 +1099,35 @@ def store_docs_in_vector_db( log.info(f"deleting existing collection {collection_name}") VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name) - embedding_function = get_embedding_function( - app.state.config.RAG_EMBEDDING_ENGINE, - app.state.config.RAG_EMBEDDING_MODEL, - app.state.sentence_transformer_ef, - app.state.config.OPENAI_API_KEY, - app.state.config.OPENAI_API_BASE_URL, - app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, - ) - - VECTOR_DB_CLIENT.insert( - collection_name=collection_name, - items=[ - { - "id": str(uuid.uuid4()), - "text": text, - "vector": embedding_function(text.replace("\n", " ")), - "metadata": metadatas[idx], - } - for idx, text in enumerate(texts) - ], - ) - - return True - except Exception as e: - if e.__class__.__name__ == "UniqueConstraintError": + if VECTOR_DB_CLIENT.has_collection(collection_name=collection_name): + log.info(f"collection {collection_name} already exists") return True + else: + embedding_function = get_embedding_function( + app.state.config.RAG_EMBEDDING_ENGINE, + app.state.config.RAG_EMBEDDING_MODEL, + app.state.sentence_transformer_ef, + app.state.config.OPENAI_API_KEY, + app.state.config.OPENAI_API_BASE_URL, + app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, + ) + VECTOR_DB_CLIENT.insert( + collection_name=collection_name, + items=[ + { + "id": str(uuid.uuid4()), + "text": text, + "vector": embedding_function(text.replace("\n", " ")), + "metadata": metadatas[idx], + } + for idx, text in enumerate(texts) + ], + ) + + return True + except Exception as e: log.exception(e) - return False From 70d16c3904a282de91c69ecd753757373e4ada8c Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 21 Sep 2024 03:55:17 +0200 Subject: [PATCH 13/20] refac: styling --- src/lib/components/chat/Messages/Citations.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/chat/Messages/Citations.svelte b/src/lib/components/chat/Messages/Citations.svelte index 1f7c69425..2c23e87a4 100644 --- a/src/lib/components/chat/Messages/Citations.svelte +++ b/src/lib/components/chat/Messages/Citations.svelte @@ -48,7 +48,7 @@ {#each _citations as citation, idx}