OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-06-23 02:16:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix: Added support for multiple pages in external document loader, added filename in api request header

Browse Source
This commit is contained in:
sree 2025-06-15 19:59:05 +05:30
parent 53060a44ae
commit 62bfe73964
1 changed files with 35 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
backend/open_webui/retrieval/loaders/external_document.py
View File

@ -1,5 +1,5 @@
import requests import requests
import logging import logging, os
from typing import Iterator, List, Union from typing import Iterator, List, Union
from langchain_core.document_loaders import BaseLoader from langchain_core.document_loaders import BaseLoader
@ -25,7 +25,7 @@ class ExternalDocumentLoader(BaseLoader):
self.file_path = file_path self.file_path = file_path
self.mime_type = mime_type self.mime_type = mime_type
def load(self) -> list[Document]: def load(self) -> List[Document]:
with open(self.file_path, "rb") as f: with open(self.file_path, "rb") as f:
data = f.read() data = f.read()
@ -36,23 +36,44 @@ class ExternalDocumentLoader(BaseLoader):
if self.api_key is not None: if self.api_key is not None:
headers["Authorization"] = f"Bearer {self.api_key}" headers["Authorization"] = f"Bearer {self.api_key}"
try:
headers["X-Filename"] = os.path.basename(self.file_path)
except:
pass
url = self.url url = self.url
if url.endswith("/"): if url.endswith("/"):
url = url[:-1] url = url[:-1]
r = requests.put(f"{url}/process", data=data, headers=headers) try:
response = requests.put(f"{url}/process", data=data, headers=headers)
except Exception as e:
log.error(f"Error connecting to endpoint: {e}")
raise Exception(f"Error connecting to endpoint: {e}")
if r.ok: if response.ok:
res = r.json()
if res: response_data = response.json()
if response_data:
if isinstance(response_data, dict):
return [ return [
Document( Document(
page_content=res.get("page_content"), page_content=response_data.get("page_content"),
metadata=res.get("metadata"), metadata=response_data.get("metadata"),
) )
] ]
elif isinstance(response_data, list):
documents = []
for document in response_data:
documents.append(Document(
page_content=document.get("page_content"),
metadata=document.get("metadata"),
))
return documents
else:
raise Exception("Error loading document: Unable to parse content")
else: else:
raise Exception("Error loading document: No content returned") raise Exception("Error loading document: No content returned")
else: else:
raise Exception(f"Error loading document: {r.status_code} {r.text}") raise Exception(f"Error loading document: {response.status_code} {response.text}")