UniqAI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-04-19 13:57:13 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

refac: deprecate docs_dir

Browse Source
This commit is contained in:
Timothy J. Baek 2024-10-04 18:22:55 -07:00
parent a6c797d4c2
commit 79c005a041
6 changed files with 122 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/open_webui/apps/webui/routers/files.py
View File

@ -92,6 +92,7 @@ def upload_file(file: UploadFile = File(...), user=Depends(get_verified_user)):
@router.post("/upload/dir")
def upload_dir(user=Depends(get_admin_user)):
file_ids = []
for path in Path(DOCS_DIR).rglob("./**/*"):
if path.is_file() and not path.name.startswith("."):
try:
@ -126,13 +127,14 @@ def upload_dir(user=Depends(get_admin_user)):
try:
process_file(ProcessFileForm(file_id=id))
log.debug(f"File processed: {path}, {file.id}")
file_ids.append(file.id)
except Exception as e:
log.exception(e)
log.error(f"Error processing file: {file.id}")
except Exception as e:
log.exception(e)
pass
return True
return file_ids
############################

4
backend/open_webui/apps/webui/routers/knowledge.py
View File

@ -160,6 +160,10 @@ def add_file_to_knowledge_by_id(
process_file(ProcessFileForm(file_id=form_data.file_id, collection_name=id))
except Exception as e:
log.debug(e)
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=str(e),
)
if knowledge:
data = knowledge.data or {}

62
src/lib/components/admin/Settings/Documents.svelte
View File

@ -62,16 +62,6 @@
hybrid: false
};
const scanHandler = async () => {
scanDirLoading = true;
const res = await uploadDir(localStorage.token);
scanDirLoading = false;
if (res) {
toast.success($i18n.t('Scan complete!'));
}
};
const embeddingModelUpdateHandler = async () => {
if (embeddingEngine === '' && embeddingModel.split('/').length - 1 > 1) {
toast.error(
@ -284,58 +274,6 @@
<div class="flex flex-col gap-0.5">
<div class=" mb-0.5 text-sm font-medium">{$i18n.t('General Settings')}</div>
<div class=" flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Scan for documents from {{path}}', { path: 'DOCS_DIR (/data/docs)' })}
</div>
<button
class=" self-center text-xs p-1 px-3 bg-gray-50 dark:bg-gray-800 dark:hover:bg-gray-700 rounded-lg flex flex-row space-x-1 items-center {scanDirLoading
? ' cursor-not-allowed'
: ''}"
on:click={() => {
scanHandler();
console.log('check');
}}
type="button"
disabled={scanDirLoading}
>
<div class="self-center font-medium">{$i18n.t('Scan')}</div>
{#if scanDirLoading}
<div class="ml-3 self-center">
<svg
class=" w-3 h-3"
viewBox="0 0 24 24"
fill="currentColor"
xmlns="http://www.w3.org/2000/svg"
>
<style>
.spinner_ajPY {
transform-origin: center;
animation: spinner_AtaB 0.75s infinite linear;
}
@keyframes spinner_AtaB {
100% {
transform: rotate(360deg);
}
}
</style>
<path
d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,19a8,8,0,1,1,8-8A8,8,0,0,1,12,20Z"
opacity=".25"
/>
<path
d="M10.14,1.16a11,11,0,0,0-9,8.92A1.59,1.59,0,0,0,2.46,12,1.52,1.52,0,0,0,4.11,10.7a8,8,0,0,1,6.66-6.61A1.42,1.42,0,0,0,12,2.69h0A1.57,1.57,0,0,0,10.14,1.16Z"
class="spinner_ajPY"
/>
</svg>
</div>
{/if}
</button>
</div>
<div class=" flex w-full justify-between">
<div class=" self-center text-xs font-medium">{$i18n.t('Embedding Model Engine')}</div>
<div class="flex items-center relative">

19
src/lib/components/icons/FolderOpen.svelte Normal file
View File

@ -0,0 +1,19 @@
<script lang="ts">
export let className = 'size-4';
export let strokeWidth = '1.5';
</script>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
stroke-width={strokeWidth}
stroke="currentColor"
class={className}
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M3.75 9.776c.112-.017.227-.026.344-.026h15.812c.117 0 .232.009.344.026m-16.5 0a2.25 2.25 0 0 0-1.883 2.542l.857 6a2.25 2.25 0 0 0 2.227 1.932H19.05a2.25 2.25 0 0 0 2.227-1.932l.857-6a2.25 2.25 0 0 0-1.883-2.542m-16.5 0V6A2.25 2.25 0 0 1 6 3.75h3.879a1.5 1.5 0 0 1 1.06.44l2.122 2.12a1.5 1.5 0 0 0 1.06.44H18A2.25 2.25 0 0 1 20.25 9v.776"
/>
</svg>

86
src/lib/components/workspace/Knowledge/Collection.svelte
View File

@ -125,6 +125,81 @@
}
};
const uploadDirectoryHandler = async () => {
try {
// Get directory handle through picker
const dirHandle = await window.showDirectoryPicker();
let totalFiles = 0;
let uploadedFiles = 0;
// Function to update the UI with the progress
const updateProgress = () => {
const percentage = (uploadedFiles / totalFiles) * 100;
toast.info(`Upload Progress: ${uploadedFiles}/${totalFiles} (${percentage.toFixed(2)}%)`);
};
// Recursive function to count all files excluding hidden ones
async function countFiles(dirHandle) {
for await (const entry of dirHandle.values()) {
if (entry.name.startsWith('.')) continue; // Skip hidden files and directories
if (entry.kind === 'file') {
totalFiles++;
} else if (entry.kind === 'directory') {
await countFiles(entry);
}
}
}
// Recursive function to process directories excluding hidden files
async function processDirectory(dirHandle, path = '') {
for await (const entry of dirHandle.values()) {
if (entry.name.startsWith('.')) continue; // Skip hidden files and directories
const entryPath = path ? `${path}/${entry.name}` : entry.name;
if (entry.kind === 'file') {
// Get file from handle
const file = await entry.getFile();
// Create a new file with the path information
const fileWithPath = new File([file], entryPath, { type: file.type });
await uploadFileHandler(fileWithPath);
uploadedFiles++;
updateProgress();
} else if (entry.kind === 'directory') {
// Recursively process subdirectories
await processDirectory(entry, entryPath);
}
}
}
// First count all files excluding hidden ones
await countFiles(dirHandle);
updateProgress();
// Start processing from root directory
if (totalFiles > 0) {
await processDirectory(dirHandle);
} else {
console.log('No files to upload.');
}
} catch (error) {
if (error.name === 'AbortError') {
toast.info('Directory selection was cancelled');
} else {
toast.error('Error accessing directory');
console.error('Directory access error:', error);
}
}
};
// Helper function to maintain file paths within zip
const getRelativePath = (fullPath, basePath) => {
return fullPath.substring(basePath.length + 1);
};
const addFileHandler = async (fileId) => {
const updatedKnowledge = await addFileToKnowledgeById(localStorage.token, id, fileId).catch(
(e) => {
@ -417,11 +492,14 @@
<div>
<AddContentMenu
on:files={() => {
document.getElementById('files-input').click();
}}
on:text={() => {
on:upload={(e) => {
if (e.detail.type === 'directory') {
uploadDirectoryHandler();
} else if (e.detail.type === 'text') {
showAddTextContentModal = true;
} else {
document.getElementById('files-input').click();
}
}}
/>
</div>

15
src/lib/components/workspace/Knowledge/Collection/AddContentMenu.svelte
View File

@ -16,6 +16,7 @@
import ArrowUpCircle from '$lib/components/icons/ArrowUpCircle.svelte';
import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte';
import BarsArrowUp from '$lib/components/icons/BarsArrowUp.svelte';
import FolderOpen from '$lib/components/icons/FolderOpen.svelte';
const i18n = getContext('i18n');
@ -65,7 +66,7 @@
<DropdownMenu.Item
class="flex gap-2 items-center px-3 py-2 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
on:click={() => {
dispatch('files');
dispatch('upload', { type: 'files' });
}}
>
<ArrowUpCircle strokeWidth="2" />
@ -75,7 +76,17 @@
<DropdownMenu.Item
class="flex gap-2 items-center px-3 py-2 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
on:click={() => {
dispatch('text');
dispatch('upload', { type: 'directory' });
}}
>
<FolderOpen strokeWidth="2" />
<div class="flex items-center">{$i18n.t('Upload directory')}</div>
</DropdownMenu.Item>
<DropdownMenu.Item
class="flex gap-2 items-center px-3 py-2 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-md"
on:click={() => {
dispatch('upload', { type: 'text' });
}}
>
<BarsArrowUp strokeWidth="2" />