diff --git a/backend/apps/rag/main.py b/backend/apps/rag/main.py index 78e43d9e7..15eff3d84 100644 --- a/backend/apps/rag/main.py +++ b/backend/apps/rag/main.py @@ -145,8 +145,8 @@ app.state.config = AppConfig() app.state.config.TOP_K = RAG_TOP_K app.state.config.RELEVANCE_THRESHOLD = RAG_RELEVANCE_THRESHOLD -app.state.config.MAX_FILE_SIZE = RAG_FILE_MAX_SIZE -app.state.config.MAX_FILE_COUNT = RAG_FILE_MAX_COUNT +app.state.config.FILE_MAX_SIZE = RAG_FILE_MAX_SIZE +app.state.config.FILE_MAX_COUNT = RAG_FILE_MAX_COUNT app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = ( @@ -397,6 +397,10 @@ async def get_rag_config(user=Depends(get_admin_user)): return { "status": True, "pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES, + "file": { + "max_size": app.state.config.FILE_MAX_SIZE, + "max_count": app.state.config.FILE_MAX_COUNT, + }, "content_extraction": { "engine": app.state.config.CONTENT_EXTRACTION_ENGINE, "tika_server_url": app.state.config.TIKA_SERVER_URL, @@ -522,6 +526,10 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_ return { "status": True, + "file": { + "max_size": app.state.config.FILE_MAX_SIZE, + "max_count": app.state.config.FILE_MAX_COUNT, + }, "pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES, "content_extraction": { "engine": app.state.config.CONTENT_EXTRACTION_ENGINE, @@ -571,8 +579,6 @@ async def get_query_settings(user=Depends(get_admin_user)): "template": app.state.config.RAG_TEMPLATE, "k": app.state.config.TOP_K, "r": app.state.config.RELEVANCE_THRESHOLD, - "max_file_size": app.state.config.MAX_FILE_SIZE, - "max_file_count": app.state.config.MAX_FILE_COUNT, "hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH, } @@ -580,16 +586,16 @@ async def get_query_settings(user=Depends(get_admin_user)): @app.get("/file/limit/settings") async def get_query_settings(user=Depends(get_verified_user)): return { - "max_file_size": app.state.config.MAX_FILE_SIZE, - "max_file_count": app.state.config.MAX_FILE_COUNT, + "FILE_MAX_SIZE": app.state.config.FILE_MAX_SIZE, + "FILE_MAX_COUNT": app.state.config.FILE_MAX_COUNT, } class QuerySettingsForm(BaseModel): k: Optional[int] = None r: Optional[float] = None - max_file_size: Optional[int] = None - max_file_count: Optional[int] = None + FILE_MAX_SIZE: Optional[int] = None + FILE_MAX_COUNT: Optional[int] = None template: Optional[str] = None hybrid: Optional[bool] = None @@ -606,11 +612,11 @@ async def update_query_settings( app.state.config.ENABLE_RAG_HYBRID_SEARCH = ( form_data.hybrid if form_data.hybrid else False ) - app.state.config.MAX_FILE_SIZE = ( - form_data.max_file_size if form_data.max_file_size else 10 + app.state.config.FILE_MAX_SIZE = ( + form_data.FILE_MAX_SIZE if form_data.FILE_MAX_SIZE else 10 ) - app.state.config.MAX_FILE_COUNT = ( - form_data.max_file_count if form_data.max_file_count else 5 + app.state.config.FILE_MAX_COUNT = ( + form_data.FILE_MAX_COUNT if form_data.FILE_MAX_COUNT else 5 ) return { @@ -618,8 +624,8 @@ async def update_query_settings( "template": app.state.config.RAG_TEMPLATE, "k": app.state.config.TOP_K, "r": app.state.config.RELEVANCE_THRESHOLD, - "max_file_size": app.state.config.MAX_FILE_SIZE, - "max_file_count": app.state.config.MAX_FILE_COUNT, + "FILE_MAX_SIZE": app.state.config.FILE_MAX_SIZE, + "FILE_MAX_COUNT": app.state.config.FILE_MAX_COUNT, "hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH, } diff --git a/src/lib/apis/rag/index.ts b/src/lib/apis/rag/index.ts index 153d9ab99..5c0a47b35 100644 --- a/src/lib/apis/rag/index.ts +++ b/src/lib/apis/rag/index.ts @@ -134,38 +134,9 @@ export const getQuerySettings = async (token: string) => { return res; }; -export const getFileLimitSettings = async (token: string) => { - let error = null; - - const res = await fetch(`${RAG_API_BASE_URL}/file/limit/settings`, { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${token}` - } - }) - .then(async (res) => { - if (!res.ok) throw await res.json(); - return res.json(); - }) - .catch((err) => { - console.log(err); - error = err.detail; - return null; - }); - - if (error) { - throw error; - } - - return res; -}; - type QuerySettings = { k: number | null; r: number | null; - max_file_size: number | null; - max_file_count: number | null; template: string | null; }; diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index 57ddffcb1..e8c7f1f7f 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -37,6 +37,9 @@ let embeddingModel = ''; let rerankingModel = ''; + let fileMaxSize = null; + let fileMaxCount = null; + let contentExtractionEngine = 'default'; let tikaServerUrl = ''; let showTikaServerUrl = false; @@ -53,8 +56,6 @@ template: '', r: 0.0, k: 4, - max_file_size: 10, - max_file_count: 5, hybrid: false }; @@ -220,7 +221,6 @@ await setRerankingConfig(); querySettings = await getQuerySettings(localStorage.token); - const res = await getRAGConfig(localStorage.token); if (res) { @@ -232,6 +232,9 @@ contentExtractionEngine = res.content_extraction.engine; tikaServerUrl = res.content_extraction.tika_server_url; showTikaServerUrl = contentExtractionEngine === 'tika'; + + fileMaxSize = res.file.file_max_size; + fileMaxCount = res.file.file_max_count; } }); @@ -388,41 +391,6 @@ {/if} -
-
-
- {$i18n.t('Max File Count')} -
-
- -
-
- -
-
- {$i18n.t('Max File Size(MB)')} -
- -
- -
-
-
-
{$i18n.t('Hybrid Search')}
@@ -647,6 +615,48 @@
{/if} + +
+ +
+
{$i18n.t('Files')}
+ +
+
+
+ {$i18n.t('Max Upload Count')} +
+
+ +
+
+ +
+
+ {$i18n.t('Max Upload Size')} +
+ +
+ +
+
+
+
+
diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index 4d47a9cf5..53b7dd044 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -17,7 +17,7 @@ import { blobToFile, findWordIndices } from '$lib/utils'; import { transcribeAudio } from '$lib/apis/audio'; - import { getFileLimitSettings, processDocToVectorDB } from '$lib/apis/rag'; + import { processDocToVectorDB } from '$lib/apis/rag'; import { uploadFile } from '$lib/apis/files'; import { @@ -56,7 +56,6 @@ let commandsElement; let inputFiles; - let fileLimitSettings; let dragged = false; let user = null; @@ -173,7 +172,7 @@ }; const processFileCountLimit = async (querySettings, inputFiles) => { - const maxFiles = querySettings.max_file_count; + const maxFiles = querySettings.FILE_MAX_COUNT; const currentFilesCount = files.length; const inputFilesCount = inputFiles.length; const totalFilesCount = currentFilesCount + inputFilesCount; @@ -190,8 +189,8 @@ return [true, inputFiles]; }; - const processFileSizeLimit = async (fileLimitSettings, file) => { - if (file.size <= fileLimitSettings.max_file_size * 1024 * 1024) { + const inputFilesHandler = async (inputFiles) => { + inputFiles.forEach((file) => { if (['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])) { if (visionCapableModels.length === 0) { toast.error($i18n.t('Selected model(s) do not support image inputs')); @@ -211,25 +210,10 @@ } else { uploadFileHandler(file); } - } else { - toast.error( - $i18n.t('File size exceeds the limit of {{size}}MB', { - size: fileLimitSettings.max_file_size - }) - ); - } + }); }; onMount(() => { - const initFileLimitSettings = async () => { - try { - fileLimitSettings = await getFileLimitSettings(localStorage.token); - } catch (error) { - console.error('Error fetching query settings:', error); - } - }; - initFileLimitSettings(); - window.setTimeout(() => chatTextAreaElement?.focus(), 0); const dropZone = document.querySelector('body'); @@ -256,22 +240,10 @@ if (e.dataTransfer?.files) { const inputFiles = Array.from(e.dataTransfer?.files); - + console.log(file, file.name.split('.').at(-1)); if (inputFiles && inputFiles.length > 0) { console.log(inputFiles); - const [canProcess, filesToProcess] = await processFileCountLimit( - fileLimitSettings, - inputFiles - ); - if (!canProcess) { - dragged = false; - return; - } - console.log(filesToProcess); - filesToProcess.forEach((file) => { - console.log(file, file.name.split('.').at(-1)); - processFileSizeLimit(fileLimitSettings, file); - }); + inputFilesHandler(inputFiles); } else { toast.error($i18n.t(`File not found.`)); } @@ -392,21 +364,7 @@ multiple on:change={async () => { if (inputFiles && inputFiles.length > 0) { - const _inputFiles = Array.from(inputFiles); - console.log(_inputFiles); - const [canProcess, filesToProcess] = await processFileCountLimit( - fileLimitSettings, - _inputFiles - ); - if (!canProcess) { - filesInputElement.value = ''; - return; - } - console.log(filesToProcess); - filesToProcess.forEach((file) => { - console.log(file, file.name.split('.').at(-1)); - processFileSizeLimit(fileLimitSettings, file); - }); + inputFilesHandler(inputFiles); } else { toast.error($i18n.t(`File not found.`)); } @@ -715,17 +673,7 @@ .map((item) => item.getAsFile()) .filter((file) => file); - const [canProcess, filesToProcess] = await processFileCountLimit( - fileLimitSettings, - inputFiles - ); - if (!canProcess) { - return; - } - filesToProcess.forEach((file) => { - console.log(file, file.name.split('.').at(-1)); - processFileSizeLimit(fileLimitSettings, file); - }); + inputFilesHandler(inputFiles); } else { toast.error($i18n.t(`File not found.`)); } diff --git a/src/lib/components/workspace/Documents.svelte b/src/lib/components/workspace/Documents.svelte index f738967a0..be3be082c 100644 --- a/src/lib/components/workspace/Documents.svelte +++ b/src/lib/components/workspace/Documents.svelte @@ -8,7 +8,7 @@ import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents'; import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants'; - import { getFileLimitSettings, processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag'; + import { processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag'; import { blobToFile, transformFileName } from '$lib/utils'; import Checkbox from '$lib/components/common/Checkbox.svelte'; @@ -24,7 +24,7 @@ let importFiles = ''; let inputFiles = ''; - let fileLimitSettings; + let query = ''; let documentsImportInputElement: HTMLInputElement; let tags = []; @@ -99,17 +99,7 @@ } }; - const initFileLimitSettings = async () => { - try { - fileLimitSettings = await getFileLimitSettings(localStorage.token); - } catch (error) { - console.error('Error fetching query settings:', error); - } - }; - onMount(() => { - initFileLimitSettings(); - documents.subscribe((docs) => { tags = docs.reduce((a, e, i, arr) => { return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])]; @@ -147,24 +137,16 @@ if (inputFiles && inputFiles.length > 0) { for (const file of inputFiles) { console.log(file, file.name.split('.').at(-1)); - if (file.size <= fileLimitSettings.max_file_size * 1024 * 1024) { - if ( - SUPPORTED_FILE_TYPE.includes(file['type']) || - SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1)) - ) { - uploadDoc(file); - } else { - toast.error( - `Unknown File Type '${file['type']}', but accepting and treating as plain text` - ); - uploadDoc(file); - } + if ( + SUPPORTED_FILE_TYPE.includes(file['type']) || + SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1)) + ) { + uploadDoc(file); } else { toast.error( - $i18n.t('File size exceeds the limit of {{size}}MB', { - size: fileLimitSettings.max_file_size - }) + `Unknown File Type '${file['type']}', but accepting and treating as plain text` ); + uploadDoc(file); } } } else {