OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-05-12 09:31:34 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

Fix process/files/batch

Browse Source
This commit is contained in:
Gabriel Ecegi 2024-12-14 10:45:27 +01:00
parent f2e2b59c18
commit 440894f8d3
2 changed files with 18 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
backend/open_webui/apps/retrieval/main.py
View File

@ -1062,30 +1062,24 @@ def process_files_batch(
Process a batch of files and save them to the vector database. Process a batch of files and save them to the vector database.
""" """
results: List[BatchProcessFilesResult] = [] results: List[BatchProcessFilesResult] = []
errors: List[BatchProcessFilesResult] = [] errors: List[BatchProcessFilesResult] = []
collection_name = form_data.collection_name collection_name = form_data.collection_name
# Prepare all documents first # Prepare all documents first
all_docs: List[Document] = [] all_docs: List[Document] = []
for file_request in form_data.files: for file in form_data.files:
try: try:
file = Files.get_file_by_id(file_request.file_id) text_content = file.data.get("content", "")
if not file:
log.error(f"process_files_batch: File {file_request.file_id} not found")
raise ValueError(f"File {file_request.file_id} not found")
text_content = file_request.content
docs: List[Document] = [ docs: List[Document] = [
Document( Document(
page_content=text_content.replace("<br/>", "\n"), page_content=text_content.replace("<br/>", "\n"),
metadata={ metadata={
**file.meta, **file.meta,
"name": file_request.filename, "name": file.filename,
"created_by": file.user_id, "created_by": file.user_id,
"file_id": file.id, "file_id": file.id,
"source": file_request.filename, "source": file.filename,
}, },
) )
] ]
@ -1101,9 +1095,9 @@ def process_files_batch(
)) ))
except Exception as e: except Exception as e:
log.error(f"process_files_batch: Error processing file {file_request.file_id}: {str(e)}") log.error(f"process_files_batch: Error processing file {file.id}: {str(e)}")
errors.append(BatchProcessFilesResult( errors.append(BatchProcessFilesResult(
file_id=file_request.file_id, file_id=file.id,
status="failed", status="failed",
error=str(e) error=str(e)
)) ))
@ -1139,7 +1133,6 @@ def process_files_batch(
errors=errors errors=errors
) )
class ProcessTextForm(BaseModel): class ProcessTextForm(BaseModel):
name: str name: str
content: str content: str

15
backend/open_webui/apps/webui/routers/knowledge.py
View File

@ -548,10 +548,17 @@ def add_files_to_knowledge_batch(
files.append(file) files.append(file)
# Process files # Process files
result = process_files_batch(BatchProcessFilesForm( try:
files=files, result = process_files_batch(BatchProcessFilesForm(
collection_name=id files=files,
)) collection_name=id
))
except Exception as e:
log.error(f"add_files_to_knowledge_batch: Exception occurred: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=str(e)
)
# Add successful files to knowledge base # Add successful files to knowledge base
data = knowledge.data or {} data = knowledge.data or {}