feat: chat_file table

backend/open_webui/utils/files.py

@@ -22,11 +22,43 @@ import base64
 import io
 import re
 
+import requests
 
 BASE64_IMAGE_URL_PREFIX = re.compile(r"data:image/\w+;base64,", re.IGNORECASE)
 MARKDOWN_IMAGE_URL_PATTERN = re.compile(r"!\[(.*?)\]\((.+?)\)", re.IGNORECASE)
 
 
+def get_image_base64_from_url(url: str) -> Optional[str]:
+    try:
+        if url.startswith("http"):
+            # Download the image from the URL
+            response = requests.get(url)
+            response.raise_for_status()
+            image_data = response.content
+            encoded_string = base64.b64encode(image_data).decode("utf-8")
+            content_type = response.headers.get("Content-Type", "image/png")
+            return f"data:{content_type};base64,{encoded_string}"
+        else:
+            file = Files.get_file_by_id(url)
+
+            if not file:
+                return None
+
+            file_path = Storage.get_file(file.path)
+            file_path = Path(file_path)
+
+            if file_path.is_file():
+                with open(file_path, "rb") as image_file:
+                    encoded_string = base64.b64encode(image_file.read()).decode("utf-8")
+                    content_type, _ = mimetypes.guess_type(file_path.name)
+                    return f"data:{content_type};base64,{encoded_string}"
+            else:
+                return None
+
+    except Exception as e:
+        return None
+
+
 def get_image_url_from_base64(request, base64_image_string, metadata, user):
     if BASE64_IMAGE_URL_PREFIX.match(base64_image_string):
         image_url = ""

backend/open_webui/utils/middleware.py

@@ -60,6 +60,7 @@ from open_webui.utils.webhook import post_webhook
 from open_webui.utils.files import (
     convert_markdown_base64_images,
     get_file_url_from_base64,
+    get_image_base64_from_url,
     get_image_url_from_base64,
 )
 
@@ -1108,6 +1109,45 @@ def apply_params_to_form_data(form_data, model):
     return form_data
 
 
+async def convert_url_images_to_base64(form_data):
+    messages = form_data.get("messages", [])
+
+    for message in messages:
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+
+        new_content = []
+
+        for item in content:
+            if not isinstance(item, dict) or item.get("type") != "image_url":
+                new_content.append(item)
+                continue
+
+            image_url = item.get("image_url", {}).get("url", "")
+            if image_url.startswith("data:image/"):
+                new_content.append(item)
+                continue
+
+            try:
+                base64_data = await asyncio.to_thread(
+                    get_image_base64_from_url, image_url
+                )
+                new_content.append(
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": base64_data},
+                    }
+                )
+            except Exception as e:
+                log.debug(f"Error converting image URL to base64: {e}")
+                new_content.append(item)
+
+        message["content"] = new_content
+
+    return form_data
+
+
 async def process_chat_payload(request, form_data, user, metadata, model):
     # Pipeline Inlet -> Filter Inlet -> Chat Memory -> Chat Web Search -> Chat Image Generation
     # -> Chat Code Interpreter (Form Data Update) -> (Default) Chat Tools Function Calling
@@ -1125,6 +1165,8 @@ async def process_chat_payload(request, form_data, user, metadata, model):
         except:
             pass
 
+    form_data = await convert_url_images_to_base64(form_data)
+
     event_emitter = get_event_emitter(metadata)
     event_caller = get_event_call(metadata)