From 79c005a041b1ec2d39d1fb536666c2530858d31c Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Fri, 4 Oct 2024 18:22:55 -0700 Subject: [PATCH] refac: deprecate docs_dir --- .../open_webui/apps/webui/routers/files.py | 4 +- .../apps/webui/routers/knowledge.py | 4 + .../admin/Settings/Documents.svelte | 62 ------------- src/lib/components/icons/FolderOpen.svelte | 19 ++++ .../workspace/Knowledge/Collection.svelte | 88 +++++++++++++++++-- .../Collection/AddContentMenu.svelte | 15 +++- 6 files changed, 122 insertions(+), 70 deletions(-) create mode 100644 src/lib/components/icons/FolderOpen.svelte diff --git a/backend/open_webui/apps/webui/routers/files.py b/backend/open_webui/apps/webui/routers/files.py index 70d58cc40..a4062d142 100644 --- a/backend/open_webui/apps/webui/routers/files.py +++ b/backend/open_webui/apps/webui/routers/files.py @@ -92,6 +92,7 @@ def upload_file(file: UploadFile = File(...), user=Depends(get_verified_user)): @router.post("/upload/dir") def upload_dir(user=Depends(get_admin_user)): + file_ids = [] for path in Path(DOCS_DIR).rglob("./**/*"): if path.is_file() and not path.name.startswith("."): try: @@ -126,13 +127,14 @@ def upload_dir(user=Depends(get_admin_user)): try: process_file(ProcessFileForm(file_id=id)) log.debug(f"File processed: {path}, {file.id}") + file_ids.append(file.id) except Exception as e: log.exception(e) log.error(f"Error processing file: {file.id}") except Exception as e: log.exception(e) pass - return True + return file_ids ############################ diff --git a/backend/open_webui/apps/webui/routers/knowledge.py b/backend/open_webui/apps/webui/routers/knowledge.py index 203ec95c7..4a73f1528 100644 --- a/backend/open_webui/apps/webui/routers/knowledge.py +++ b/backend/open_webui/apps/webui/routers/knowledge.py @@ -160,6 +160,10 @@ def add_file_to_knowledge_by_id( process_file(ProcessFileForm(file_id=form_data.file_id, collection_name=id)) except Exception as e: log.debug(e) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e), + ) if knowledge: data = knowledge.data or {} diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index 5c44938f7..d6f7dc987 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -62,16 +62,6 @@ hybrid: false }; - const scanHandler = async () => { - scanDirLoading = true; - const res = await uploadDir(localStorage.token); - scanDirLoading = false; - - if (res) { - toast.success($i18n.t('Scan complete!')); - } - }; - const embeddingModelUpdateHandler = async () => { if (embeddingEngine === '' && embeddingModel.split('/').length - 1 > 1) { toast.error( @@ -284,58 +274,6 @@
{$i18n.t('General Settings')}
-
-
- {$i18n.t('Scan for documents from {{path}}', { path: 'DOCS_DIR (/data/docs)' })} -
- - -
-
{$i18n.t('Embedding Model Engine')}
diff --git a/src/lib/components/icons/FolderOpen.svelte b/src/lib/components/icons/FolderOpen.svelte new file mode 100644 index 000000000..f6b3c64b3 --- /dev/null +++ b/src/lib/components/icons/FolderOpen.svelte @@ -0,0 +1,19 @@ + + + + + diff --git a/src/lib/components/workspace/Knowledge/Collection.svelte b/src/lib/components/workspace/Knowledge/Collection.svelte index 178ce0219..275cc5283 100644 --- a/src/lib/components/workspace/Knowledge/Collection.svelte +++ b/src/lib/components/workspace/Knowledge/Collection.svelte @@ -125,6 +125,81 @@ } }; + const uploadDirectoryHandler = async () => { + try { + // Get directory handle through picker + const dirHandle = await window.showDirectoryPicker(); + + let totalFiles = 0; + let uploadedFiles = 0; + + // Function to update the UI with the progress + const updateProgress = () => { + const percentage = (uploadedFiles / totalFiles) * 100; + toast.info(`Upload Progress: ${uploadedFiles}/${totalFiles} (${percentage.toFixed(2)}%)`); + }; + + // Recursive function to count all files excluding hidden ones + async function countFiles(dirHandle) { + for await (const entry of dirHandle.values()) { + if (entry.name.startsWith('.')) continue; // Skip hidden files and directories + + if (entry.kind === 'file') { + totalFiles++; + } else if (entry.kind === 'directory') { + await countFiles(entry); + } + } + } + + // Recursive function to process directories excluding hidden files + async function processDirectory(dirHandle, path = '') { + for await (const entry of dirHandle.values()) { + if (entry.name.startsWith('.')) continue; // Skip hidden files and directories + + const entryPath = path ? `${path}/${entry.name}` : entry.name; + + if (entry.kind === 'file') { + // Get file from handle + const file = await entry.getFile(); + // Create a new file with the path information + const fileWithPath = new File([file], entryPath, { type: file.type }); + + await uploadFileHandler(fileWithPath); + uploadedFiles++; + updateProgress(); + } else if (entry.kind === 'directory') { + // Recursively process subdirectories + await processDirectory(entry, entryPath); + } + } + } + + // First count all files excluding hidden ones + await countFiles(dirHandle); + updateProgress(); + + // Start processing from root directory + if (totalFiles > 0) { + await processDirectory(dirHandle); + } else { + console.log('No files to upload.'); + } + } catch (error) { + if (error.name === 'AbortError') { + toast.info('Directory selection was cancelled'); + } else { + toast.error('Error accessing directory'); + console.error('Directory access error:', error); + } + } + }; + + // Helper function to maintain file paths within zip + const getRelativePath = (fullPath, basePath) => { + return fullPath.substring(basePath.length + 1); + }; + const addFileHandler = async (fileId) => { const updatedKnowledge = await addFileToKnowledgeById(localStorage.token, id, fileId).catch( (e) => { @@ -417,11 +492,14 @@
{ - document.getElementById('files-input').click(); - }} - on:text={() => { - showAddTextContentModal = true; + on:upload={(e) => { + if (e.detail.type === 'directory') { + uploadDirectoryHandler(); + } else if (e.detail.type === 'text') { + showAddTextContentModal = true; + } else { + document.getElementById('files-input').click(); + } }} />
diff --git a/src/lib/components/workspace/Knowledge/Collection/AddContentMenu.svelte b/src/lib/components/workspace/Knowledge/Collection/AddContentMenu.svelte index cc1ca54d6..245793979 100644 --- a/src/lib/components/workspace/Knowledge/Collection/AddContentMenu.svelte +++ b/src/lib/components/workspace/Knowledge/Collection/AddContentMenu.svelte @@ -16,6 +16,7 @@ import ArrowUpCircle from '$lib/components/icons/ArrowUpCircle.svelte'; import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte'; import BarsArrowUp from '$lib/components/icons/BarsArrowUp.svelte'; + import FolderOpen from '$lib/components/icons/FolderOpen.svelte'; const i18n = getContext('i18n'); @@ -65,7 +66,7 @@ { - dispatch('files'); + dispatch('upload', { type: 'files' }); }} > @@ -75,7 +76,17 @@ { - dispatch('text'); + dispatch('upload', { type: 'directory' }); + }} + > + +
{$i18n.t('Upload directory')}
+ + + { + dispatch('upload', { type: 'text' }); }} >