From 0577dc299066374934d37311d7a6a719d8db6821 Mon Sep 17 00:00:00 2001 From: Felipe Hernandez Date: Fri, 18 Apr 2025 01:40:30 +0200 Subject: [PATCH 1/3] feat: Lazy load file content on click --- backend/open_webui/routers/knowledge.py | 6 ++--- .../workspace/Knowledge/KnowledgeBase.svelte | 24 +++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 15547afa7..408d1c7ed 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -9,7 +9,7 @@ from open_webui.models.knowledge import ( KnowledgeResponse, KnowledgeUserResponse, ) -from open_webui.models.files import Files, FileModel +from open_webui.models.files import Files, FileModel, FileMetadataResponse from open_webui.retrieval.vector.connector import VECTOR_DB_CLIENT from open_webui.routers.retrieval import ( process_file, @@ -235,7 +235,7 @@ async def reindex_knowledge_files(request: Request, user=Depends(get_verified_us class KnowledgeFilesResponse(KnowledgeResponse): - files: list[FileModel] + files: list[FileMetadataResponse] @router.get("/{id}", response_model=Optional[KnowledgeFilesResponse]) @@ -251,7 +251,7 @@ async def get_knowledge_by_id(id: str, user=Depends(get_verified_user)): ): file_ids = knowledge.data.get("file_ids", []) if knowledge.data else [] - files = Files.get_files_by_ids(file_ids) + files = Files.get_file_metadatas_by_ids(file_ids) return KnowledgeFilesResponse( **knowledge.model_dump(), diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index dc0e354ec..420673500 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -11,7 +11,7 @@ import { page } from '$app/stores'; import { mobile, showSidebar, knowledge as _knowledge, config, user } from '$lib/stores'; - import { updateFileDataContentById, uploadFile, deleteFileById } from '$lib/apis/files'; + import { updateFileDataContentById, uploadFile, deleteFileById, getFileById } from '$lib/apis/files'; import { addFileToKnowledgeById, getKnowledgeById, @@ -84,12 +84,12 @@ let selectedFile = null; let selectedFileId = null; + let selectedFileContent = ''; $: if (selectedFileId) { const file = (knowledge?.files ?? []).find((file) => file.id === selectedFileId); if (file) { - file.data = file.data ?? { content: '' }; - selectedFile = file; + handleFileClick(file); } else { selectedFile = null; } @@ -450,6 +450,20 @@ } }; + const handleFileClick = async (file) => { + try { + selectedFile = file; + const response = await getFileById(localStorage.token, file.id); + if (response) { + selectedFileContent = response.data.content; + } else { + toast.error($i18n.t('No content found in file.')); + } + } catch (e) { + toast.error($i18n.t('Failed to load file content.')); + } + }; + const onDragOver = (e) => { e.preventDefault(); @@ -728,7 +742,7 @@ {#key selectedFile.id} @@ -786,7 +800,7 @@ {#key selectedFile.id} From 88f97372d9cbbe8315bd19002eb742c4368c6cfc Mon Sep 17 00:00:00 2001 From: Felipe Hernandez Date: Fri, 18 Apr 2025 02:01:47 +0200 Subject: [PATCH 2/3] fix: Show content on upload file --- backend/open_webui/routers/knowledge.py | 10 +++++----- .../workspace/Knowledge/KnowledgeBase.svelte | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 408d1c7ed..926b35b17 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -379,7 +379,7 @@ def add_file_to_knowledge_by_id( knowledge = Knowledges.update_knowledge_data_by_id(id=id, data=data) if knowledge: - files = Files.get_files_by_ids(file_ids) + files = Files.get_file_metadatas_by_ids(file_ids) return KnowledgeFilesResponse( **knowledge.model_dump(), @@ -456,7 +456,7 @@ def update_file_from_knowledge_by_id( data = knowledge.data or {} file_ids = data.get("file_ids", []) - files = Files.get_files_by_ids(file_ids) + files = Files.get_file_metadatas_by_ids(file_ids) return KnowledgeFilesResponse( **knowledge.model_dump(), @@ -538,7 +538,7 @@ def remove_file_from_knowledge_by_id( knowledge = Knowledges.update_knowledge_data_by_id(id=id, data=data) if knowledge: - files = Files.get_files_by_ids(file_ids) + files = Files.get_file_metadatas_by_ids(file_ids) return KnowledgeFilesResponse( **knowledge.model_dump(), @@ -734,7 +734,7 @@ def add_files_to_knowledge_batch( error_details = [f"{err.file_id}: {err.error}" for err in result.errors] return KnowledgeFilesResponse( **knowledge.model_dump(), - files=Files.get_files_by_ids(existing_file_ids), + files=Files.get_file_metadatas_by_ids(existing_file_ids), warnings={ "message": "Some files failed to process", "errors": error_details, @@ -742,5 +742,5 @@ def add_files_to_knowledge_batch( ) return KnowledgeFilesResponse( - **knowledge.model_dump(), files=Files.get_files_by_ids(existing_file_ids) + **knowledge.model_dump(), files=Files.get_file_metadatas_by_ids(existing_file_ids) ) diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index 420673500..46348b2bc 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -394,7 +394,7 @@ const updateFileContentHandler = async () => { const fileId = selectedFile.id; - const content = selectedFile.data.content; + const content = selectedFileContent const res = updateFileDataContentById(localStorage.token, fileId, content).catch((e) => { toast.error(`${e}`); From 14fa7d16f9a8623e41c150e1611fb7a0b5538e9b Mon Sep 17 00:00:00 2001 From: Felipe Hernandez Date: Fri, 18 Apr 2025 11:38:15 +0200 Subject: [PATCH 3/3] feat: Cache file content --- .../workspace/Knowledge/KnowledgeBase.svelte | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index 46348b2bc..a38dfe0d7 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -86,10 +86,13 @@ let selectedFileId = null; let selectedFileContent = ''; + // Add cache object + let fileContentCache = new Map(); + $: if (selectedFileId) { const file = (knowledge?.files ?? []).find((file) => file.id === selectedFileId); if (file) { - handleFileClick(file); + fileSelectHandler(file); } else { selectedFile = null; } @@ -394,7 +397,10 @@ const updateFileContentHandler = async () => { const fileId = selectedFile.id; - const content = selectedFileContent + const content = selectedFileContent; + + // Clear the cache for this file since we're updating it + fileContentCache.delete(fileId); const res = updateFileDataContentById(localStorage.token, fileId, content).catch((e) => { toast.error(`${e}`); @@ -450,12 +456,21 @@ } }; - const handleFileClick = async (file) => { + const fileSelectHandler = async (file) => { try { selectedFile = file; + + // Check cache first + if (fileContentCache.has(file.id)) { + selectedFileContent = fileContentCache.get(file.id); + return; + } + const response = await getFileById(localStorage.token, file.id); if (response) { selectedFileContent = response.data.content; + // Cache the content + fileContentCache.set(file.id, response.data.content); } else { toast.error($i18n.t('No content found in file.')); }