OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-06-23 02:16:52 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #14992 from sreesdas/dev

Browse Source 
Fix: Added support for multiple pages in external document loader
This commit is contained in:
Tim Jaeryang Baek 2025-06-16 11:01:33 +04:00 committed by GitHub
commit c5b48ec551
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 35 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
backend/open_webui/retrieval/loaders/external_document.py
View File

@ -1,5 +1,5 @@
import requests
import logging
import logging, os
from typing import Iterator, List, Union
from langchain_core.document_loaders import BaseLoader
@ -25,7 +25,7 @@ class ExternalDocumentLoader(BaseLoader):
self.file_path = file_path
self.mime_type = mime_type
def load(self) -> list[Document]:
def load(self) -> List[Document]:
with open(self.file_path, "rb") as f:
data = f.read()
@ -36,23 +36,44 @@ class ExternalDocumentLoader(BaseLoader):
if self.api_key is not None:
headers["Authorization"] = f"Bearer {self.api_key}"
try:
headers["X-Filename"] = os.path.basename(self.file_path)
except:
pass
url = self.url
if url.endswith("/"):
url = url[:-1]
r = requests.put(f"{url}/process", data=data, headers=headers)
try:
response = requests.put(f"{url}/process", data=data, headers=headers)
except Exception as e:
log.error(f"Error connecting to endpoint: {e}")
raise Exception(f"Error connecting to endpoint: {e}")
if r.ok:
res = r.json()
if res:
return [
Document(
page_content=res.get("page_content"),
metadata=res.get("metadata"),
)
]
if response.ok:
response_data = response.json()
if response_data:
if isinstance(response_data, dict):
return [
Document(
page_content=response_data.get("page_content"),
metadata=response_data.get("metadata"),
)
]
elif isinstance(response_data, list):
documents = []
for document in response_data:
documents.append(Document(
page_content=document.get("page_content"),
metadata=document.get("metadata"),
))
return documents
else:
raise Exception("Error loading document: Unable to parse content")
else:
raise Exception("Error loading document: No content returned")
else:
raise Exception(f"Error loading document: {r.status_code} {r.text}")
raise Exception(f"Error loading document: {response.status_code} {response.text}")