diff --git a/CHANGELOG.md b/CHANGELOG.md index 46c1c644e..0008d21ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.23] - 2024-09-21 + +### Added + +- **🚀 WebSocket Redis Support**: Enhanced load balancing capabilities for multiple instance setups, promoting better performance and reliability in WebUI. +- **🔧 Adjustable Chat Controls**: Introduced width-adjustable chat controls, enabling a personalized and more comfortable user interface. +- **🌎 i18n Updates**: Improved and updated the Chinese translations. + +### Fixed + +- **🌐 Task Model Unloading Issue**: Modified task handling to use the Ollama /api/chat endpoint instead of OpenAI compatible endpoint, ensuring models stay loaded and ready with custom parameters, thus minimizing delays in task execution. +- **📝 Title Generation Fix for OpenAI Compatible APIs**: Resolved an issue preventing the generation of titles, enhancing consistency and reliability when using multiple API providers. +- **🗃️ RAG Duplicate Collection Issue**: Fixed a bug causing repeated processing of the same uploaded file. Now utilizes indexed files to prevent unnecessary duplications, optimizing resource usage. +- **🖼️ Image Generation Enhancement**: Refactored OpenAI image generation endpoint to be asynchronous, preventing the WebUI from becoming unresponsive during processing, thus enhancing user experience. +- **🔓 Downgrade Authlib**: Reverted Authlib to version 1.3.1 to address and resolve issues concerning OAuth functionality. + +### Changed + +- **🔍 Improved Message Interaction**: Enhanced the message node interface to allow for easier focus redirection with a simple click, streamlining user interaction. +- **✨ Styling Refactor**: Updated WebUI styling for a cleaner, more modern look, enhancing user experience across the platform. + ## [0.3.22] - 2024-09-19 ### Added diff --git a/backend/open_webui/apps/openai/main.py b/backend/open_webui/apps/openai/main.py index 9a27c46a3..87bcc042c 100644 --- a/backend/open_webui/apps/openai/main.py +++ b/backend/open_webui/apps/openai/main.py @@ -405,14 +405,19 @@ async def generate_chat_completion( "role": user.role, } + url = app.state.config.OPENAI_API_BASE_URLS[idx] + key = app.state.config.OPENAI_API_KEYS[idx] + + # Change max_completion_tokens to max_tokens (Backward compatible) + if "api.openai.com" not in url and not payload["model"].lower().startswith("o1-"): + if "max_completion_tokens" in payload: + payload["max_tokens"] = payload.pop("max_completion_tokens") + # Convert the modified body back to JSON payload = json.dumps(payload) log.debug(payload) - url = app.state.config.OPENAI_API_BASE_URLS[idx] - key = app.state.config.OPENAI_API_KEYS[idx] - headers = {} headers["Authorization"] = f"Bearer {key}" headers["Content-Type"] = "application/json" diff --git a/backend/open_webui/apps/rag/main.py b/backend/open_webui/apps/rag/main.py index 981a6fe5b..74855b336 100644 --- a/backend/open_webui/apps/rag/main.py +++ b/backend/open_webui/apps/rag/main.py @@ -1099,35 +1099,35 @@ def store_docs_in_vector_db( log.info(f"deleting existing collection {collection_name}") VECTOR_DB_CLIENT.delete_collection(collection_name=collection_name) - embedding_function = get_embedding_function( - app.state.config.RAG_EMBEDDING_ENGINE, - app.state.config.RAG_EMBEDDING_MODEL, - app.state.sentence_transformer_ef, - app.state.config.OPENAI_API_KEY, - app.state.config.OPENAI_API_BASE_URL, - app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, - ) - - VECTOR_DB_CLIENT.insert( - collection_name=collection_name, - items=[ - { - "id": str(uuid.uuid4()), - "text": text, - "vector": embedding_function(text.replace("\n", " ")), - "metadata": metadatas[idx], - } - for idx, text in enumerate(texts) - ], - ) - - return True - except Exception as e: - if e.__class__.__name__ == "UniqueConstraintError": + if VECTOR_DB_CLIENT.has_collection(collection_name=collection_name): + log.info(f"collection {collection_name} already exists") return True + else: + embedding_function = get_embedding_function( + app.state.config.RAG_EMBEDDING_ENGINE, + app.state.config.RAG_EMBEDDING_MODEL, + app.state.sentence_transformer_ef, + app.state.config.OPENAI_API_KEY, + app.state.config.OPENAI_API_BASE_URL, + app.state.config.RAG_EMBEDDING_OPENAI_BATCH_SIZE, + ) + VECTOR_DB_CLIENT.insert( + collection_name=collection_name, + items=[ + { + "id": str(uuid.uuid4()), + "text": text, + "vector": embedding_function(text.replace("\n", " ")), + "metadata": metadatas[idx], + } + for idx, text in enumerate(texts) + ], + ) + + return True + except Exception as e: log.exception(e) - return False diff --git a/backend/open_webui/apps/socket/main.py b/backend/open_webui/apps/socket/main.py index e41ef8412..c353c8e6f 100644 --- a/backend/open_webui/apps/socket/main.py +++ b/backend/open_webui/apps/socket/main.py @@ -2,16 +2,38 @@ import asyncio import socketio from open_webui.apps.webui.models.users import Users -from open_webui.env import ENABLE_WEBSOCKET_SUPPORT +from open_webui.env import ( + ENABLE_WEBSOCKET_SUPPORT, + WEBSOCKET_MANAGER, + WEBSOCKET_REDIS_URL, +) from open_webui.utils.utils import decode_token -sio = socketio.AsyncServer( - cors_allowed_origins=[], - async_mode="asgi", - transports=(["polling", "websocket"] if ENABLE_WEBSOCKET_SUPPORT else ["polling"]), - allow_upgrades=ENABLE_WEBSOCKET_SUPPORT, - always_connect=True, -) + +if WEBSOCKET_MANAGER == "redis": + mgr = socketio.AsyncRedisManager(WEBSOCKET_REDIS_URL) + sio = socketio.AsyncServer( + cors_allowed_origins=[], + async_mode="asgi", + transports=( + ["polling", "websocket"] if ENABLE_WEBSOCKET_SUPPORT else ["polling"] + ), + allow_upgrades=ENABLE_WEBSOCKET_SUPPORT, + always_connect=True, + client_manager=mgr, + ) +else: + sio = socketio.AsyncServer( + cors_allowed_origins=[], + async_mode="asgi", + transports=( + ["polling", "websocket"] if ENABLE_WEBSOCKET_SUPPORT else ["polling"] + ), + allow_upgrades=ENABLE_WEBSOCKET_SUPPORT, + always_connect=True, + ) + + app = socketio.ASGIApp(sio, socketio_path="/ws/socket.io") # Dictionary to maintain the user pool diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index 89422e57b..504eeea54 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -302,3 +302,7 @@ if WEBUI_AUTH and WEBUI_SECRET_KEY == "": ENABLE_WEBSOCKET_SUPPORT = ( os.environ.get("ENABLE_WEBSOCKET_SUPPORT", "True").lower() == "true" ) + +WEBSOCKET_MANAGER = os.environ.get("WEBSOCKET_MANAGER", "") + +WEBSOCKET_REDIS_URL = os.environ.get("WEBSOCKET_REDIS_URL", "redis://localhost:6379/0") diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 319d95165..64b7f2153 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -19,7 +19,9 @@ from open_webui.apps.audio.main import app as audio_app from open_webui.apps.images.main import app as images_app from open_webui.apps.ollama.main import app as ollama_app from open_webui.apps.ollama.main import ( - generate_openai_chat_completion as generate_ollama_chat_completion, + GenerateChatCompletionForm, + generate_chat_completion as generate_ollama_chat_completion, + generate_openai_chat_completion as generate_ollama_openai_chat_completion, ) from open_webui.apps.ollama.main import get_all_models as get_ollama_models from open_webui.apps.openai.main import app as openai_app @@ -135,6 +137,12 @@ from open_webui.utils.utils import ( ) from open_webui.utils.webhook import post_webhook +from open_webui.utils.payload import convert_payload_openai_to_ollama +from open_webui.utils.response import ( + convert_response_ollama_to_openai, + convert_streaming_response_ollama_to_openai, +) + if SAFE_MODE: print("SAFE MODE ENABLED") Functions.deactivate_all_functions() @@ -1048,7 +1056,18 @@ async def generate_chat_completions(form_data: dict, user=Depends(get_verified_u if model.get("pipe"): return await generate_function_chat_completion(form_data, user=user) if model["owned_by"] == "ollama": - return await generate_ollama_chat_completion(form_data, user=user) + # Using /ollama/api/chat endpoint + form_data = convert_payload_openai_to_ollama(form_data) + form_data = GenerateChatCompletionForm(**form_data) + response = await generate_ollama_chat_completion(form_data=form_data, user=user) + if form_data.stream: + response.headers["content-type"] = "text/event-stream" + return StreamingResponse( + convert_streaming_response_ollama_to_openai(response), + headers=dict(response.headers), + ) + else: + return convert_response_ollama_to_openai(response) else: return await generate_openai_chat_completion(form_data, user=user) @@ -1399,9 +1418,10 @@ async def generate_title(form_data: dict, user=Depends(get_verified_user)): # Check if the user has a custom task model # If the user has a custom task model, use that model task_model_id = get_task_model_id(model_id) - print(task_model_id) + model = app.state.MODELS[task_model_id] + if app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE != "": template = app.state.config.TITLE_GENERATION_PROMPT_TEMPLATE else: @@ -1440,9 +1460,9 @@ Prompt: {{prompt:middletruncate:8000}}""" "chat_id": form_data.get("chat_id", None), "metadata": {"task": str(TASKS.TITLE_GENERATION)}, } - log.debug(payload) + # Handle pipeline filters try: payload = filter_pipeline(payload, user) except Exception as e: @@ -1456,7 +1476,6 @@ Prompt: {{prompt:middletruncate:8000}}""" status_code=status.HTTP_400_BAD_REQUEST, content={"detail": str(e)}, ) - if "chat_id" in payload: del payload["chat_id"] @@ -1484,6 +1503,8 @@ async def generate_search_query(form_data: dict, user=Depends(get_verified_user) task_model_id = get_task_model_id(model_id) print(task_model_id) + model = app.state.MODELS[task_model_id] + if app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE != "": template = app.state.config.SEARCH_QUERY_GENERATION_PROMPT_TEMPLATE else: @@ -1516,9 +1537,9 @@ Search Query:""" ), "metadata": {"task": str(TASKS.QUERY_GENERATION)}, } + log.debug(payload) - print(payload) - + # Handle pipeline filters try: payload = filter_pipeline(payload, user) except Exception as e: @@ -1532,7 +1553,6 @@ Search Query:""" status_code=status.HTTP_400_BAD_REQUEST, content={"detail": str(e)}, ) - if "chat_id" in payload: del payload["chat_id"] @@ -1555,12 +1575,13 @@ async def generate_emoji(form_data: dict, user=Depends(get_verified_user)): task_model_id = get_task_model_id(model_id) print(task_model_id) + model = app.state.MODELS[task_model_id] + template = ''' Your task is to reflect the speaker's likely facial expression through a fitting emoji. Interpret emotions from the message and reflect their facial expression using fitting, diverse emojis (e.g., 😊, 😢, 😡, 😱). Message: """{{prompt}}""" ''' - content = title_generation_template( template, form_data["prompt"], @@ -1584,9 +1605,9 @@ Message: """{{prompt}}""" "chat_id": form_data.get("chat_id", None), "metadata": {"task": str(TASKS.EMOJI_GENERATION)}, } - log.debug(payload) + # Handle pipeline filters try: payload = filter_pipeline(payload, user) except Exception as e: @@ -1600,7 +1621,6 @@ Message: """{{prompt}}""" status_code=status.HTTP_400_BAD_REQUEST, content={"detail": str(e)}, ) - if "chat_id" in payload: del payload["chat_id"] @@ -1620,8 +1640,10 @@ async def generate_moa_response(form_data: dict, user=Depends(get_verified_user) # Check if the user has a custom task model # If the user has a custom task model, use that model - model_id = get_task_model_id(model_id) - print(model_id) + task_model_id = get_task_model_id(model_id) + print(task_model_id) + + model = app.state.MODELS[task_model_id] template = """You have been provided with a set of responses from various models to the latest user query: "{{prompt}}" @@ -1636,13 +1658,12 @@ Responses from models: {{responses}}""" ) payload = { - "model": model_id, + "model": task_model_id, "messages": [{"role": "user", "content": content}], "stream": form_data.get("stream", False), "chat_id": form_data.get("chat_id", None), "metadata": {"task": str(TASKS.MOA_RESPONSE_GENERATION)}, } - log.debug(payload) try: @@ -1658,7 +1679,6 @@ Responses from models: {{responses}}""" status_code=status.HTTP_400_BAD_REQUEST, content={"detail": str(e)}, ) - if "chat_id" in payload: del payload["chat_id"] diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index d1b340044..bdce74b05 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -105,17 +105,25 @@ def openai_chat_message_template(model: str): } -def openai_chat_chunk_message_template(model: str, message: str) -> dict: +def openai_chat_chunk_message_template( + model: str, message: Optional[str] = None +) -> dict: template = openai_chat_message_template(model) template["object"] = "chat.completion.chunk" - template["choices"][0]["delta"] = {"content": message} + if message: + template["choices"][0]["delta"] = {"content": message} + else: + template["choices"][0]["finish_reason"] = "stop" return template -def openai_chat_completion_message_template(model: str, message: str) -> dict: +def openai_chat_completion_message_template( + model: str, message: Optional[str] = None +) -> dict: template = openai_chat_message_template(model) template["object"] = "chat.completion" - template["choices"][0]["message"] = {"content": message, "role": "assistant"} + if message: + template["choices"][0]["message"] = {"content": message, "role": "assistant"} template["choices"][0]["finish_reason"] = "stop" return template diff --git a/backend/open_webui/utils/payload.py b/backend/open_webui/utils/payload.py index b2654cd25..72aec6a6c 100644 --- a/backend/open_webui/utils/payload.py +++ b/backend/open_webui/utils/payload.py @@ -86,3 +86,49 @@ def apply_model_params_to_body_ollama(params: dict, form_data: dict) -> dict: form_data[value] = param return form_data + + +def convert_payload_openai_to_ollama(openai_payload: dict) -> dict: + """ + Converts a payload formatted for OpenAI's API to be compatible with Ollama's API endpoint for chat completions. + + Args: + openai_payload (dict): The payload originally designed for OpenAI API usage. + + Returns: + dict: A modified payload compatible with the Ollama API. + """ + ollama_payload = {} + + # Mapping basic model and message details + ollama_payload["model"] = openai_payload.get("model") + ollama_payload["messages"] = openai_payload.get("messages") + ollama_payload["stream"] = openai_payload.get("stream", False) + + # If there are advanced parameters in the payload, format them in Ollama's options field + ollama_options = {} + + # Handle parameters which map directly + for param in ["temperature", "top_p", "seed"]: + if param in openai_payload: + ollama_options[param] = openai_payload[param] + + # Mapping OpenAI's `max_tokens` -> Ollama's `num_predict` + if "max_completion_tokens" in openai_payload: + ollama_options["num_predict"] = openai_payload["max_completion_tokens"] + elif "max_tokens" in openai_payload: + ollama_options["num_predict"] = openai_payload["max_tokens"] + + # Handle frequency / presence_penalty, which needs renaming and checking + if "frequency_penalty" in openai_payload: + ollama_options["repeat_penalty"] = openai_payload["frequency_penalty"] + + if "presence_penalty" in openai_payload and "penalty" not in ollama_options: + # We are assuming presence penalty uses a similar concept in Ollama, which needs custom handling if exists. + ollama_options["new_topic_penalty"] = openai_payload["presence_penalty"] + + # Add options to payload if any have been set + if ollama_options: + ollama_payload["options"] = ollama_options + + return ollama_payload diff --git a/backend/open_webui/utils/response.py b/backend/open_webui/utils/response.py new file mode 100644 index 000000000..3debe63af --- /dev/null +++ b/backend/open_webui/utils/response.py @@ -0,0 +1,32 @@ +import json +from open_webui.utils.misc import ( + openai_chat_chunk_message_template, + openai_chat_completion_message_template, +) + + +def convert_response_ollama_to_openai(ollama_response: dict) -> dict: + model = ollama_response.get("model", "ollama") + message_content = ollama_response.get("message", {}).get("content", "") + + response = openai_chat_completion_message_template(model, message_content) + return response + + +async def convert_streaming_response_ollama_to_openai(ollama_streaming_response): + async for data in ollama_streaming_response.body_iterator: + data = json.loads(data) + + model = data.get("model", "ollama") + message_content = data.get("message", {}).get("content", "") + done = data.get("done", False) + + data = openai_chat_chunk_message_template( + model, message_content if not done else None + ) + + line = f"data: {json.dumps(data)}\n\n" + if done: + line += "data: [DONE]\n\n" + + yield line diff --git a/package-lock.json b/package-lock.json index a0048a211..754c89c95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "open-webui", - "version": "0.3.22", + "version": "0.3.23", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "open-webui", - "version": "0.3.22", + "version": "0.3.23", "dependencies": { "@codemirror/lang-javascript": "^6.2.2", "@codemirror/lang-python": "^6.1.6", @@ -32,6 +32,7 @@ "katex": "^0.16.9", "marked": "^9.1.0", "mermaid": "^10.9.1", + "paneforge": "^0.0.6", "pyodide": "^0.26.1", "socket.io-client": "^4.2.0", "sortablejs": "^1.15.2", @@ -6986,6 +6987,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/paneforge": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/paneforge/-/paneforge-0.0.6.tgz", + "integrity": "sha512-jYeN/wdREihja5c6nK3S5jritDQ+EbCqC5NrDo97qCZzZ9GkmEcN5C0ZCjF4nmhBwkDKr6tLIgz4QUKWxLXjAw==", + "dependencies": { + "nanoid": "^5.0.4" + }, + "peerDependencies": { + "svelte": "^4.0.0 || ^5.0.0-next.1" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/package.json b/package.json index 371507789..8b314654e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-webui", - "version": "0.3.22", + "version": "0.3.23", "private": true, "scripts": { "dev": "npm run pyodide:fetch && vite dev --host", @@ -72,6 +72,7 @@ "katex": "^0.16.9", "marked": "^9.1.0", "mermaid": "^10.9.1", + "paneforge": "^0.0.6", "pyodide": "^0.26.1", "socket.io-client": "^4.2.0", "sortablejs": "^1.15.2", diff --git a/src/lib/components/chat/Chat.svelte b/src/lib/components/chat/Chat.svelte index e0ec62b52..48acf8d3f 100644 --- a/src/lib/components/chat/Chat.svelte +++ b/src/lib/components/chat/Chat.svelte @@ -2,6 +2,7 @@ import { v4 as uuidv4 } from 'uuid'; import { toast } from 'svelte-sonner'; import mermaid from 'mermaid'; + import { PaneGroup, Pane, PaneResizer } from 'paneforge'; import { getContext, onDestroy, onMount, tick } from 'svelte'; import { goto } from '$app/navigation'; @@ -26,7 +27,9 @@ showControls, showCallOverlay, currentChatPage, - temporaryChatEnabled + temporaryChatEnabled, + mobile, + showOverview } from '$lib/stores'; import { convertMessagesToHistory, @@ -64,12 +67,14 @@ import Navbar from '$lib/components/layout/Navbar.svelte'; import ChatControls from './ChatControls.svelte'; import EventConfirmDialog from '../common/ConfirmDialog.svelte'; + import EllipsisVertical from '../icons/EllipsisVertical.svelte'; const i18n: Writable = getContext('i18n'); export let chatIdProp = ''; let loaded = false; const eventTarget = new EventTarget(); + let controlPane; let stopResponseFlag = false; let autoScroll = true; @@ -279,6 +284,29 @@ await goto('/'); } } + + showControls.subscribe(async (value) => { + if (controlPane && !$mobile) { + try { + if (value) { + controlPane.resize( + parseInt(localStorage.getItem('chat-controls-size') || '35') + ? parseInt(localStorage.getItem('chat-controls-size') || '35') + : 35 + ); + } else { + controlPane.resize(0); + } + } catch (e) { + // ignore + } + } + + if (!value) { + showCallOverlay.set(false); + showOverview.set(false); + } + }); }); onDestroy(() => { @@ -1764,113 +1792,112 @@ {initNewChat} /> - {#if $banners.length > 0 && messages.length === 0 && !$chatId && selectedModels.length <= 1} -
-
- {#each $banners.filter( (b) => (b.dismissible ? !JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]').includes(b.id) : true) ) as banner} - { - const bannerId = e.detail; + + + {#if $banners.length > 0 && messages.length === 0 && !$chatId && selectedModels.length <= 1} +
+
+ {#each $banners.filter( (b) => (b.dismissible ? !JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]').includes(b.id) : true) ) as banner} + { + const bannerId = e.detail; - localStorage.setItem( - 'dismissedBannerIds', - JSON.stringify( - [ - bannerId, - ...JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]') - ].filter((id) => $banners.find((b) => b.id === id)) - ) - ); + localStorage.setItem( + 'dismissedBannerIds', + JSON.stringify( + [ + bannerId, + ...JSON.parse(localStorage.getItem('dismissedBannerIds') ?? '[]') + ].filter((id) => $banners.find((b) => b.id === id)) + ) + ); + }} + /> + {/each} +
+
+ {/if} + +
+ + +
+ { + const model = $models.find((m) => m.id === e); + if (model?.info?.meta?.toolIds ?? false) { + return [...new Set([...a, ...model.info.meta.toolIds])]; + } + return a; + }, [])} + transparentBackground={$settings?.backgroundImageUrl ?? false} + {selectedModels} + {messages} + {submitPrompt} + {stopResponse} + on:call={async () => { + await showControls.set(true); }} /> - {/each} +
-
- {/if} + -
- - -
- { - const model = $models.find((m) => m.id === e); - if (model?.info?.meta?.toolIds ?? false) { - return [...new Set([...a, ...model.info.meta.toolIds])]; - } - return a; - }, [])} - transparentBackground={$settings?.backgroundImageUrl ?? false} - {selectedModels} - {messages} - {submitPrompt} - {stopResponse} - on:call={() => { - showControls.set(true); - }} - /> -
-
+ { + const model = $models.find((m) => m.id === e); + if (model) { + return [...a, model]; + } + return a; + }, [])} + bind:history + bind:chatFiles + bind:params + bind:files + bind:pane={controlPane} + {submitPrompt} + {stopResponse} + {showMessage} + modelId={selectedModelIds?.at(0) ?? null} + chatId={$chatId} + {eventTarget} + /> +
{/if} - - { - const model = $models.find((m) => m.id === e); - if (model) { - return [...a, model]; - } - return a; - }, [])} - bind:history - bind:chatFiles - bind:params - bind:files - {submitPrompt} - {stopResponse} - {showMessage} - modelId={selectedModelIds?.at(0) ?? null} - chatId={$chatId} - {eventTarget} -/> diff --git a/src/lib/components/chat/ChatControls.svelte b/src/lib/components/chat/ChatControls.svelte index b9268cebc..aae0bc573 100644 --- a/src/lib/components/chat/ChatControls.svelte +++ b/src/lib/components/chat/ChatControls.svelte @@ -10,6 +10,9 @@ import CallOverlay from './MessageInput/CallOverlay.svelte'; import Drawer from '../common/Drawer.svelte'; import Overview from './Overview.svelte'; + import { Pane, PaneResizer } from 'paneforge'; + import EllipsisVertical from '../icons/EllipsisVertical.svelte'; + import { get } from 'svelte/store'; export let history; export let models = []; @@ -25,7 +28,9 @@ export let files; export let modelId; + export let pane; let largeScreen = false; + onMount(() => { // listen to resize 1024px const mediaQuery = window.matchMedia('(min-width: 1024px)'); @@ -35,6 +40,7 @@ largeScreen = true; } else { largeScreen = false; + pane = null; } }; @@ -58,33 +64,33 @@ {#if !largeScreen} - {#if $showCallOverlay} -
-
- { - showControls.set(false); - }} - /> -
-
- {:else if $showControls} + {#if $showControls} { showControls.set(false); }} > -
- {#if $showOverview} +
+ {#if $showCallOverlay} +
+ { + showControls.set(false); + }} + /> +
+ {:else if $showOverview} { @@ -107,48 +113,75 @@
{/if} - {:else if $showControls} -
-
-
- {#if $showCallOverlay} - { - showControls.set(false); - }} - /> - {:else if $showOverview} - { - showMessage(e.detail.node.data.message); - }} - on:close={() => { - showControls.set(false); - }} - /> - {:else} - { - showControls.set(false); - }} - {models} - bind:chatFiles - bind:params - /> - {/if} -
+ {:else} + + +
+
-
+ + { + if (size === 0) { + showControls.set(false); + } else { + if (!$showControls) { + showControls.set(true); + } + localStorage.setItem('chat-controls-size', size); + } + }} + > + {#if $showControls} +
+
+ {#if $showCallOverlay} +
+ { + showControls.set(false); + }} + /> +
+ {:else if $showOverview} + { + showMessage(e.detail.node.data.message); + }} + on:close={() => { + showControls.set(false); + }} + /> + {:else} + { + showControls.set(false); + }} + {models} + bind:chatFiles + bind:params + /> + {/if} +
+
+ {/if} + {/if} diff --git a/src/lib/components/chat/MessageInput/CallOverlay.svelte b/src/lib/components/chat/MessageInput/CallOverlay.svelte index 18427434a..3c6ea7b9a 100644 --- a/src/lib/components/chat/MessageInput/CallOverlay.svelte +++ b/src/lib/components/chat/MessageInput/CallOverlay.svelte @@ -220,7 +220,9 @@ }; const startRecording = async () => { - audioStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (!audioStream) { + audioStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } mediaRecorder = new MediaRecorder(audioStream); mediaRecorder.onstart = () => { @@ -236,7 +238,7 @@ }; mediaRecorder.onstop = (e) => { - console.log('Recording stopped', e); + console.log('Recording stopped', audioStream, e); stopRecordingCallback(); }; @@ -244,10 +246,11 @@ }; const stopAudioStream = async () => { - if (audioStream) { - const tracks = audioStream.getTracks(); - tracks.forEach((track) => track.stop()); - } + if (!audioStream) return; + + audioStream.getAudioTracks().forEach(function (track) { + track.stop(); + }); audioStream = null; }; @@ -525,6 +528,60 @@ console.log(`Audio monitoring and playing stopped for message ID ${id}`); }; + const chatStartHandler = async (e) => { + const { id } = e.detail; + + chatStreaming = true; + + if (currentMessageId !== id) { + console.log(`Received chat start event for message ID ${id}`); + + currentMessageId = id; + if (audioAbortController) { + audioAbortController.abort(); + } + audioAbortController = new AbortController(); + + assistantSpeaking = true; + // Start monitoring and playing audio for the message ID + monitorAndPlayAudio(id, audioAbortController.signal); + } + }; + + const chatEventHandler = async (e) => { + const { id, content } = e.detail; + // "id" here is message id + // if "id" is not the same as "currentMessageId" then do not process + // "content" here is a sentence from the assistant, + // there will be many sentences for the same "id" + + if (currentMessageId === id) { + console.log(`Received chat event for message ID ${id}: ${content}`); + + try { + if (messages[id] === undefined) { + messages[id] = [content]; + } else { + messages[id].push(content); + } + + console.log(content); + + fetchAudio(content); + } catch (error) { + console.error('Failed to fetch or play audio:', error); + } + } + }; + + const chatFinishHandler = async (e) => { + const { id, content } = e.detail; + // "content" here is the entire message from the assistant + finishedMessages[id] = true; + + chatStreaming = false; + }; + onMount(async () => { const setWakeLock = async () => { try { @@ -558,65 +615,15 @@ startRecording(); - const chatStartHandler = async (e) => { - const { id } = e.detail; - - chatStreaming = true; - - if (currentMessageId !== id) { - console.log(`Received chat start event for message ID ${id}`); - - currentMessageId = id; - if (audioAbortController) { - audioAbortController.abort(); - } - audioAbortController = new AbortController(); - - assistantSpeaking = true; - // Start monitoring and playing audio for the message ID - monitorAndPlayAudio(id, audioAbortController.signal); - } - }; - - const chatEventHandler = async (e) => { - const { id, content } = e.detail; - // "id" here is message id - // if "id" is not the same as "currentMessageId" then do not process - // "content" here is a sentence from the assistant, - // there will be many sentences for the same "id" - - if (currentMessageId === id) { - console.log(`Received chat event for message ID ${id}: ${content}`); - - try { - if (messages[id] === undefined) { - messages[id] = [content]; - } else { - messages[id].push(content); - } - - console.log(content); - - fetchAudio(content); - } catch (error) { - console.error('Failed to fetch or play audio:', error); - } - } - }; - - const chatFinishHandler = async (e) => { - const { id, content } = e.detail; - // "content" here is the entire message from the assistant - finishedMessages[id] = true; - - chatStreaming = false; - }; - eventTarget.addEventListener('chat:start', chatStartHandler); eventTarget.addEventListener('chat', chatEventHandler); eventTarget.addEventListener('chat:finish', chatFinishHandler); return async () => { + await stopAllAudio(); + + stopAudioStream(); + eventTarget.removeEventListener('chat:start', chatStartHandler); eventTarget.removeEventListener('chat', chatEventHandler); eventTarget.removeEventListener('chat:finish', chatFinishHandler); @@ -633,6 +640,17 @@ onDestroy(async () => { await stopAllAudio(); + stopAudioStream(); + + eventTarget.removeEventListener('chat:start', chatStartHandler); + eventTarget.removeEventListener('chat', chatEventHandler); + eventTarget.removeEventListener('chat:finish', chatFinishHandler); + + audioAbortController.abort(); + await tick(); + + await stopAllAudio(); + await stopRecordingCallback(false); await stopCamera(); }); @@ -924,6 +942,10 @@ on:click={async () => { await stopAudioStream(); await stopVideoStream(); + + console.log(audioStream); + console.log(cameraStream); + showCallOverlay.set(false); dispatch('close'); }} diff --git a/src/lib/components/chat/MessageInput/VoiceRecording.svelte b/src/lib/components/chat/MessageInput/VoiceRecording.svelte index b2e859649..f98fc2cb1 100644 --- a/src/lib/components/chat/MessageInput/VoiceRecording.svelte +++ b/src/lib/components/chat/MessageInput/VoiceRecording.svelte @@ -44,6 +44,7 @@ return `${minutes}:${formattedSeconds}`; }; + let stream; let speechRecognition; let mediaRecorder; @@ -159,7 +160,7 @@ const startRecording = async () => { startDurationCounter(); - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); mediaRecorder = new MediaRecorder(stream); mediaRecorder.onstart = () => { console.log('Recording started'); @@ -251,6 +252,13 @@ } stopDurationCounter(); audioChunks = []; + + if (stream) { + const tracks = stream.getTracks(); + tracks.forEach((track) => track.stop()); + } + + stream = null; }; const confirmRecording = async () => { diff --git a/src/lib/components/chat/Messages/Citations.svelte b/src/lib/components/chat/Messages/Citations.svelte index 1f7c69425..2c23e87a4 100644 --- a/src/lib/components/chat/Messages/Citations.svelte +++ b/src/lib/components/chat/Messages/Citations.svelte @@ -48,7 +48,7 @@ {#each _citations as citation, idx}