enh: image tool response

backend/open_webui/utils/middleware.py

@@ -1201,13 +1201,15 @@ async def process_chat_response(
                                 )
 
                                 tool_result = None
+                                tool_result_files = None
                                 for result in results:
                                     if tool_call_id == result.get("tool_call_id", ""):
                                         tool_result = result.get("content", None)
+                                        tool_result_files = result.get("files", None)
                                         break
 
                                 if tool_result:
-                                    tool_calls_display_content = f'{tool_calls_display_content}\n<details type="tool_calls" done="true" id="{tool_call_id}" name="{tool_name}" arguments="{html.escape(json.dumps(tool_arguments))}" result="{html.escape(json.dumps(tool_result))}">\n<summary>Tool Executed</summary>\n</details>'
+                                    tool_calls_display_content = f'{tool_calls_display_content}\n<details type="tool_calls" done="true" id="{tool_call_id}" name="{tool_name}" arguments="{html.escape(json.dumps(tool_arguments))}" result="{html.escape(json.dumps(tool_result))}" files="{html.escape(json.dumps(tool_result_files)) if tool_result_files else ""}">\n<summary>Tool Executed</summary>\n</details>\n'
                                 else:
                                     tool_calls_display_content = f'{tool_calls_display_content}\n<details type="tool_calls" done="false" id="{tool_call_id}" name="{tool_name}" arguments="{html.escape(json.dumps(tool_arguments))}">\n<summary>Executing...</summary>\n</details>'
 
@@ -1901,6 +1903,13 @@ async def process_chat_response(
                             except Exception as e:
                                 tool_result = str(e)
 
+                        tool_result_files = []
+                        if isinstance(tool_result, list):
+                            for item in tool_result:
+                                if item.startswith("data:"):
+                                    tool_result_files.append(item)
+                                    tool_result.remove(item)
+
                         if isinstance(tool_result, dict) or isinstance(
                             tool_result, list
                         ):
@@ -1910,6 +1919,11 @@ async def process_chat_response(
                             {
                                 "tool_call_id": tool_call_id,
                                 "content": tool_result,
+                                **(
+                                    {"files": tool_result_files}
+                                    if tool_result_files
+                                    else {}
+                                ),
                             }
                         )
 

src/lib/components/chat/Chat.svelte

@@ -48,7 +48,8 @@
 		splitStream,
 		sleep,
 		removeDetails,
-		getPromptVariables
+		getPromptVariables,
+		processDetails
 	} from '$lib/utils';
 
 	import { generateChatCompletion } from '$lib/apis/ollama';
@@ -1514,7 +1515,7 @@
 				: undefined,
 			...createMessagesList(_history, responseMessageId).map((message) => ({
 				...message,
-				content: removeDetails(message.content, ['reasoning', 'code_interpreter'])
+				content: processDetails(message.content)
 			}))
 		].filter((message) => message);
 

src/lib/components/common/Collapsible.svelte

@@ -36,6 +36,7 @@
 	import Spinner from './Spinner.svelte';
 	import CodeBlock from '../chat/Messages/CodeBlock.svelte';
 	import Markdown from '../chat/Messages/Markdown.svelte';
+	import Image from './Image.svelte';
 
 	export let open = false;
 
@@ -53,9 +54,17 @@
 	export let disabled = false;
 	export let hide = false;
 
-	function formatJSONString(obj) {
+	function parseJSONString(str) {
 		try {
-			const parsed = JSON.parse(JSON.parse(obj));
+			return parseJSONString(JSON.parse(str));
+		} catch (e) {
+			return str;
+		}
+	}
+
+	function formatJSONString(str) {
+		try {
+			const parsed = parseJSONString(str);
 			// If parsed is an object/array, then it's valid JSON
 			if (typeof parsed === 'object') {
 				return JSON.stringify(parsed, null, 2);
@@ -65,7 +74,7 @@
 			}
 		} catch (e) {
 			// Not valid JSON, return as-is
-			return obj;
+			return str;
 		}
 	}
 </script>
@@ -120,14 +129,14 @@
 						{#if attributes?.done === 'true'}
 							<Markdown
 								id={`tool-calls-${attributes?.id}`}
-								content={$i18n.t('View Result from `{{NAME}}`', {
+								content={$i18n.t('View Result from **{{NAME}}**', {
 									NAME: attributes.name
 								})}
 							/>
 						{:else}
 							<Markdown
-								id={`tool-calls-${attributes?.id}`}
+								id={`tool-calls-${attributes?.id}-executing`}
-								content={$i18n.t('Executing `{{NAME}}`...', {
+								content={$i18n.t('Executing **{{NAME}}**...', {
 									NAME: attributes.name
 								})}
 							/>
@@ -194,6 +203,7 @@
 				{#if attributes?.type === 'tool_calls'}
 					{@const args = decode(attributes?.arguments)}
 					{@const result = decode(attributes?.result ?? '')}
+					{@const files = parseJSONString(decode(attributes?.files ?? ''))}
 
 					{#if attributes?.done === 'true'}
 						<Markdown
@@ -203,6 +213,14 @@
 > ${formatJSONString(result)}
 > \`\`\``}
 						/>
+
+						{#if typeof files === 'object'}
+							{#each files ?? [] as file, idx}
+								{#if file.startsWith('data:image/')}
+									<Image id={`tool-calls-${attributes?.id}-result-${idx}`} src={file} alt="Image" />
+								{/if}
+							{/each}
+						{/if}
 					{:else}
 						<Markdown
 							id={`tool-calls-${attributes?.id}-result`}

src/lib/utils/index.ts

@@ -683,6 +683,31 @@ export const removeDetails = (content, types) => {
 	return content;
 };
 
+export const processDetails = (content) => {
+	content = removeDetails(content, ['reasoning', 'code_interpreter']);
+
+	// This regex matches <details> tags with type="tool_calls" and captures their attributes to convert them to <tool_calls> tags
+	const detailsRegex = /<details\s+type="tool_calls"([^>]*)>([\s\S]*?)<\/details>/gis;
+	const matches = content.match(detailsRegex);
+	if (matches) {
+		for (const match of matches) {
+			const attributesRegex = /(\w+)="([^"]*)"/g;
+			const attributes = {};
+			let attributeMatch;
+			while ((attributeMatch = attributesRegex.exec(match)) !== null) {
+				attributes[attributeMatch[1]] = attributeMatch[2];
+			}
+
+			content = content.replace(
+				match,
+				`<tool_calls name="${attributes.name}" result="${attributes.result}"/>`
+			);
+		}
+	}
+
+	return content;
+};
+
 // This regular expression matches code blocks marked by triple backticks
 const codeBlockRegex = /```[\s\S]*?```/g;