OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-06-23 02:16:52 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

refac: rm sub standard code

Browse Source
This commit is contained in:
Timothy J. Baek 2024-08-27 15:51:40 +02:00
parent 600409682e
commit 09cba5b87a
5 changed files with 86 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
backend/apps/rag/main.py
View File

@ -145,8 +145,8 @@ app.state.config = AppConfig()
app.state.config.TOP_K = RAG_TOP_K app.state.config.TOP_K = RAG_TOP_K
app.state.config.RELEVANCE_THRESHOLD = RAG_RELEVANCE_THRESHOLD app.state.config.RELEVANCE_THRESHOLD = RAG_RELEVANCE_THRESHOLD
app.state.config.MAX_FILE_SIZE = RAG_FILE_MAX_SIZE app.state.config.FILE_MAX_SIZE = RAG_FILE_MAX_SIZE
app.state.config.MAX_FILE_COUNT = RAG_FILE_MAX_COUNT app.state.config.FILE_MAX_COUNT = RAG_FILE_MAX_COUNT
app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH
app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = ( app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
@ -397,6 +397,10 @@ async def get_rag_config(user=Depends(get_admin_user)):
return { return {
"status": True, "status": True,
"pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES, "pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES,
"file": {
"max_size": app.state.config.FILE_MAX_SIZE,
"max_count": app.state.config.FILE_MAX_COUNT,
},
"content_extraction": { "content_extraction": {
"engine": app.state.config.CONTENT_EXTRACTION_ENGINE, "engine": app.state.config.CONTENT_EXTRACTION_ENGINE,
"tika_server_url": app.state.config.TIKA_SERVER_URL, "tika_server_url": app.state.config.TIKA_SERVER_URL,
@ -522,6 +526,10 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_
return { return {
"status": True, "status": True,
"file": {
"max_size": app.state.config.FILE_MAX_SIZE,
"max_count": app.state.config.FILE_MAX_COUNT,
},
"pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES, "pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES,
"content_extraction": { "content_extraction": {
"engine": app.state.config.CONTENT_EXTRACTION_ENGINE, "engine": app.state.config.CONTENT_EXTRACTION_ENGINE,
@ -571,8 +579,6 @@ async def get_query_settings(user=Depends(get_admin_user)):
"template": app.state.config.RAG_TEMPLATE, "template": app.state.config.RAG_TEMPLATE,
"k": app.state.config.TOP_K, "k": app.state.config.TOP_K,
"r": app.state.config.RELEVANCE_THRESHOLD, "r": app.state.config.RELEVANCE_THRESHOLD,
"max_file_size": app.state.config.MAX_FILE_SIZE,
"max_file_count": app.state.config.MAX_FILE_COUNT,
"hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH, "hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH,
} }
@ -580,16 +586,16 @@ async def get_query_settings(user=Depends(get_admin_user)):
@app.get("/file/limit/settings") @app.get("/file/limit/settings")
async def get_query_settings(user=Depends(get_verified_user)): async def get_query_settings(user=Depends(get_verified_user)):
return { return {
"max_file_size": app.state.config.MAX_FILE_SIZE, "FILE_MAX_SIZE": app.state.config.FILE_MAX_SIZE,
"max_file_count": app.state.config.MAX_FILE_COUNT, "FILE_MAX_COUNT": app.state.config.FILE_MAX_COUNT,
} }
class QuerySettingsForm(BaseModel): class QuerySettingsForm(BaseModel):
k: Optional[int] = None k: Optional[int] = None
r: Optional[float] = None r: Optional[float] = None
max_file_size: Optional[int] = None FILE_MAX_SIZE: Optional[int] = None
max_file_count: Optional[int] = None FILE_MAX_COUNT: Optional[int] = None
template: Optional[str] = None template: Optional[str] = None
hybrid: Optional[bool] = None hybrid: Optional[bool] = None
@ -606,11 +612,11 @@ async def update_query_settings(
app.state.config.ENABLE_RAG_HYBRID_SEARCH = ( app.state.config.ENABLE_RAG_HYBRID_SEARCH = (
form_data.hybrid if form_data.hybrid else False form_data.hybrid if form_data.hybrid else False
) )
app.state.config.MAX_FILE_SIZE = ( app.state.config.FILE_MAX_SIZE = (
form_data.max_file_size if form_data.max_file_size else 10 form_data.FILE_MAX_SIZE if form_data.FILE_MAX_SIZE else 10
) )
app.state.config.MAX_FILE_COUNT = ( app.state.config.FILE_MAX_COUNT = (
form_data.max_file_count if form_data.max_file_count else 5 form_data.FILE_MAX_COUNT if form_data.FILE_MAX_COUNT else 5
) )
return { return {
@ -618,8 +624,8 @@ async def update_query_settings(
"template": app.state.config.RAG_TEMPLATE, "template": app.state.config.RAG_TEMPLATE,
"k": app.state.config.TOP_K, "k": app.state.config.TOP_K,
"r": app.state.config.RELEVANCE_THRESHOLD, "r": app.state.config.RELEVANCE_THRESHOLD,
"max_file_size": app.state.config.MAX_FILE_SIZE, "FILE_MAX_SIZE": app.state.config.FILE_MAX_SIZE,
"max_file_count": app.state.config.MAX_FILE_COUNT, "FILE_MAX_COUNT": app.state.config.FILE_MAX_COUNT,
"hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH, "hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH,
} }

29
src/lib/apis/rag/index.ts
View File

@ -134,38 +134,9 @@ export const getQuerySettings = async (token: string) => {
return res; return res;
}; };
export const getFileLimitSettings = async (token: string) => {
let error = null;
const res = await fetch(`${RAG_API_BASE_URL}/file/limit/settings`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
console.log(err);
error = err.detail;
return null;
});
if (error) {
throw error;
}
return res;
};
type QuerySettings = { type QuerySettings = {
k: number | null; k: number | null;
r: number | null; r: number | null;
max_file_size: number | null;
max_file_count: number | null;
template: string | null; template: string | null;
}; };

86
src/lib/components/admin/Settings/Documents.svelte
View File

@ -37,6 +37,9 @@
let embeddingModel = ''; let embeddingModel = '';
let rerankingModel = ''; let rerankingModel = '';
let fileMaxSize = null;
let fileMaxCount = null;
let contentExtractionEngine = 'default'; let contentExtractionEngine = 'default';
let tikaServerUrl = ''; let tikaServerUrl = '';
let showTikaServerUrl = false; let showTikaServerUrl = false;
@ -53,8 +56,6 @@
template: '', template: '',
r: 0.0, r: 0.0,
k: 4, k: 4,
max_file_size: 10,
max_file_count: 5,
hybrid: false hybrid: false
}; };
@ -220,7 +221,6 @@
await setRerankingConfig(); await setRerankingConfig();
querySettings = await getQuerySettings(localStorage.token); querySettings = await getQuerySettings(localStorage.token);
const res = await getRAGConfig(localStorage.token); const res = await getRAGConfig(localStorage.token);
if (res) { if (res) {
@ -232,6 +232,9 @@
contentExtractionEngine = res.content_extraction.engine; contentExtractionEngine = res.content_extraction.engine;
tikaServerUrl = res.content_extraction.tika_server_url; tikaServerUrl = res.content_extraction.tika_server_url;
showTikaServerUrl = contentExtractionEngine === 'tika'; showTikaServerUrl = contentExtractionEngine === 'tika';
fileMaxSize = res.file.file_max_size;
fileMaxCount = res.file.file_max_count;
} }
}); });
</script> </script>
@ -388,41 +391,6 @@
</div> </div>
{/if} {/if}
<div class=" my-2 flex gap-1.5">
<div class=" w-full justify-between">
<div class="self-center text-xs font-medium min-w-fit mb-1">
{$i18n.t('Max File Count')}
</div>
<div class="self-center">
<input
class=" w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
type="number"
placeholder={$i18n.t('Enter Max File Count')}
bind:value={querySettings.max_file_count}
autocomplete="off"
min="0"
/>
</div>
</div>
<div class="w-full">
<div class=" self-center text-xs font-medium min-w-fit mb-1">
{$i18n.t('Max File Size(MB)')}
</div>
<div class="self-center">
<input
class="w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
type="number"
placeholder={$i18n.t('Enter Max File Size(MB)')}
bind:value={querySettings.max_file_size}
autocomplete="off"
min="0"
/>
</div>
</div>
</div>
<div class=" flex w-full justify-between"> <div class=" flex w-full justify-between">
<div class=" self-center text-xs font-medium">{$i18n.t('Hybrid Search')}</div> <div class=" self-center text-xs font-medium">{$i18n.t('Hybrid Search')}</div>
@ -647,6 +615,48 @@
</div> </div>
{/if} {/if}
</div> </div>
<hr class=" dark:border-gray-850" />
<div class="">
<div class="text-sm font-medium">{$i18n.t('Files')}</div>
<div class=" my-2 flex gap-1.5">
<div class=" w-full justify-between">
<div class="self-center text-xs font-medium min-w-fit mb-1">
{$i18n.t('Max Upload Count')}
</div>
<div class="self-center">
<input
class=" w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
type="number"
placeholder={$i18n.t('Enter File Count')}
bind:value={fileMaxCount}
autocomplete="off"
min="0"
/>
</div>
</div>
<div class="w-full">
<div class=" self-center text-xs font-medium min-w-fit mb-1">
{$i18n.t('Max Upload Size')}
</div>
<div class="self-center">
<input
class="w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
type="number"
placeholder={$i18n.t('Enter File Size (MB)')}
bind:value={fileMaxSize}
autocomplete="off"
min="0"
/>
</div>
</div>
</div>
</div>
<hr class=" dark:border-gray-850" /> <hr class=" dark:border-gray-850" />
<div class=" "> <div class=" ">

70
src/lib/components/chat/MessageInput.svelte
View File

@ -17,7 +17,7 @@
import { blobToFile, findWordIndices } from '$lib/utils'; import { blobToFile, findWordIndices } from '$lib/utils';
import { transcribeAudio } from '$lib/apis/audio'; import { transcribeAudio } from '$lib/apis/audio';
import { getFileLimitSettings, processDocToVectorDB } from '$lib/apis/rag'; import { processDocToVectorDB } from '$lib/apis/rag';
import { uploadFile } from '$lib/apis/files'; import { uploadFile } from '$lib/apis/files';
import { import {
@ -56,7 +56,6 @@
let commandsElement; let commandsElement;
let inputFiles; let inputFiles;
let fileLimitSettings;
let dragged = false; let dragged = false;
let user = null; let user = null;
@ -173,7 +172,7 @@
}; };
const processFileCountLimit = async (querySettings, inputFiles) => { const processFileCountLimit = async (querySettings, inputFiles) => {
const maxFiles = querySettings.max_file_count; const maxFiles = querySettings.FILE_MAX_COUNT;
const currentFilesCount = files.length; const currentFilesCount = files.length;
const inputFilesCount = inputFiles.length; const inputFilesCount = inputFiles.length;
const totalFilesCount = currentFilesCount + inputFilesCount; const totalFilesCount = currentFilesCount + inputFilesCount;
@ -190,8 +189,8 @@
return [true, inputFiles]; return [true, inputFiles];
}; };
const processFileSizeLimit = async (fileLimitSettings, file) => { const inputFilesHandler = async (inputFiles) => {
if (file.size <= fileLimitSettings.max_file_size * 1024 * 1024) { inputFiles.forEach((file) => {
if (['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])) { if (['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])) {
if (visionCapableModels.length === 0) { if (visionCapableModels.length === 0) {
toast.error($i18n.t('Selected model(s) do not support image inputs')); toast.error($i18n.t('Selected model(s) do not support image inputs'));
@ -211,25 +210,10 @@
} else { } else {
uploadFileHandler(file); uploadFileHandler(file);
} }
} else { });
toast.error(
$i18n.t('File size exceeds the limit of {{size}}MB', {
size: fileLimitSettings.max_file_size
})
);
}
}; };
onMount(() => { onMount(() => {
const initFileLimitSettings = async () => {
try {
fileLimitSettings = await getFileLimitSettings(localStorage.token);
} catch (error) {
console.error('Error fetching query settings:', error);
}
};
initFileLimitSettings();
window.setTimeout(() => chatTextAreaElement?.focus(), 0); window.setTimeout(() => chatTextAreaElement?.focus(), 0);
const dropZone = document.querySelector('body'); const dropZone = document.querySelector('body');
@ -256,22 +240,10 @@
if (e.dataTransfer?.files) { if (e.dataTransfer?.files) {
const inputFiles = Array.from(e.dataTransfer?.files); const inputFiles = Array.from(e.dataTransfer?.files);
console.log(file, file.name.split('.').at(-1));
if (inputFiles && inputFiles.length > 0) { if (inputFiles && inputFiles.length > 0) {
console.log(inputFiles); console.log(inputFiles);
const [canProcess, filesToProcess] = await processFileCountLimit( inputFilesHandler(inputFiles);
fileLimitSettings,
inputFiles
);
if (!canProcess) {
dragged = false;
return;
}
console.log(filesToProcess);
filesToProcess.forEach((file) => {
console.log(file, file.name.split('.').at(-1));
processFileSizeLimit(fileLimitSettings, file);
});
} else { } else {
toast.error($i18n.t(`File not found.`)); toast.error($i18n.t(`File not found.`));
} }
@ -392,21 +364,7 @@
multiple multiple
on:change={async () => { on:change={async () => {
if (inputFiles && inputFiles.length > 0) { if (inputFiles && inputFiles.length > 0) {
const _inputFiles = Array.from(inputFiles); inputFilesHandler(inputFiles);
console.log(_inputFiles);
const [canProcess, filesToProcess] = await processFileCountLimit(
fileLimitSettings,
_inputFiles
);
if (!canProcess) {
filesInputElement.value = '';
return;
}
console.log(filesToProcess);
filesToProcess.forEach((file) => {
console.log(file, file.name.split('.').at(-1));
processFileSizeLimit(fileLimitSettings, file);
});
} else { } else {
toast.error($i18n.t(`File not found.`)); toast.error($i18n.t(`File not found.`));
} }
@ -715,17 +673,7 @@
.map((item) => item.getAsFile()) .map((item) => item.getAsFile())
.filter((file) => file); .filter((file) => file);
const [canProcess, filesToProcess] = await processFileCountLimit( inputFilesHandler(inputFiles);
fileLimitSettings,
inputFiles
);
if (!canProcess) {
return;
}
filesToProcess.forEach((file) => {
console.log(file, file.name.split('.').at(-1));
processFileSizeLimit(fileLimitSettings, file);
});
} else { } else {
toast.error($i18n.t(`File not found.`)); toast.error($i18n.t(`File not found.`));
} }

36
src/lib/components/workspace/Documents.svelte
View File

@ -8,7 +8,7 @@
import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents'; import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents';
import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants'; import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants';
import { getFileLimitSettings, processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag'; import { processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag';
import { blobToFile, transformFileName } from '$lib/utils'; import { blobToFile, transformFileName } from '$lib/utils';
import Checkbox from '$lib/components/common/Checkbox.svelte'; import Checkbox from '$lib/components/common/Checkbox.svelte';
@ -24,7 +24,7 @@
let importFiles = ''; let importFiles = '';
let inputFiles = ''; let inputFiles = '';
let fileLimitSettings;
let query = ''; let query = '';
let documentsImportInputElement: HTMLInputElement; let documentsImportInputElement: HTMLInputElement;
let tags = []; let tags = [];
@ -99,17 +99,7 @@
} }
}; };
const initFileLimitSettings = async () => {
try {
fileLimitSettings = await getFileLimitSettings(localStorage.token);
} catch (error) {
console.error('Error fetching query settings:', error);
}
};
onMount(() => { onMount(() => {
initFileLimitSettings();
documents.subscribe((docs) => { documents.subscribe((docs) => {
tags = docs.reduce((a, e, i, arr) => { tags = docs.reduce((a, e, i, arr) => {
return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])]; return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])];
@ -147,24 +137,16 @@
if (inputFiles && inputFiles.length > 0) { if (inputFiles && inputFiles.length > 0) {
for (const file of inputFiles) { for (const file of inputFiles) {
console.log(file, file.name.split('.').at(-1)); console.log(file, file.name.split('.').at(-1));
if (file.size <= fileLimitSettings.max_file_size * 1024 * 1024) { if (
if ( SUPPORTED_FILE_TYPE.includes(file['type']) ||
SUPPORTED_FILE_TYPE.includes(file['type']) || SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1)) ) {
) { uploadDoc(file);
uploadDoc(file);
} else {
toast.error(
`Unknown File Type '${file['type']}', but accepting and treating as plain text`
);
uploadDoc(file);
}
} else { } else {
toast.error( toast.error(
$i18n.t('File size exceeds the limit of {{size}}MB', { `Unknown File Type '${file['type']}', but accepting and treating as plain text`
size: fileLimitSettings.max_file_size
})
); );
uploadDoc(file);
} }
} }
} else { } else {