mirror of
https://github.com/open-webui/open-webui
synced 2025-03-03 02:41:56 +00:00
refac: rm sub standard code
This commit is contained in:
Timothy J. Baek
parent
600409682e
commit
09cba5b87a
@ -145,8 +145,8 @@ app.state.config = AppConfig()
|
||||
|
||||
app.state.config.TOP_K = RAG_TOP_K
|
||||
app.state.config.RELEVANCE_THRESHOLD = RAG_RELEVANCE_THRESHOLD
|
||||
app.state.config.MAX_FILE_SIZE = RAG_FILE_MAX_SIZE
|
||||
app.state.config.MAX_FILE_COUNT = RAG_FILE_MAX_COUNT
|
||||
app.state.config.FILE_MAX_SIZE = RAG_FILE_MAX_SIZE
|
||||
app.state.config.FILE_MAX_COUNT = RAG_FILE_MAX_COUNT
|
||||
|
||||
app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH
|
||||
app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
|
||||
@ -397,6 +397,10 @@ async def get_rag_config(user=Depends(get_admin_user)):
|
||||
return {
|
||||
"status": True,
|
||||
"pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES,
|
||||
"file": {
|
||||
"max_size": app.state.config.FILE_MAX_SIZE,
|
||||
"max_count": app.state.config.FILE_MAX_COUNT,
|
||||
},
|
||||
"content_extraction": {
|
||||
"engine": app.state.config.CONTENT_EXTRACTION_ENGINE,
|
||||
"tika_server_url": app.state.config.TIKA_SERVER_URL,
|
||||
@ -522,6 +526,10 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_
|
||||
|
||||
return {
|
||||
"status": True,
|
||||
"file": {
|
||||
"max_size": app.state.config.FILE_MAX_SIZE,
|
||||
"max_count": app.state.config.FILE_MAX_COUNT,
|
||||
},
|
||||
"pdf_extract_images": app.state.config.PDF_EXTRACT_IMAGES,
|
||||
"content_extraction": {
|
||||
"engine": app.state.config.CONTENT_EXTRACTION_ENGINE,
|
||||
@ -571,8 +579,6 @@ async def get_query_settings(user=Depends(get_admin_user)):
|
||||
"template": app.state.config.RAG_TEMPLATE,
|
||||
"k": app.state.config.TOP_K,
|
||||
"r": app.state.config.RELEVANCE_THRESHOLD,
|
||||
"max_file_size": app.state.config.MAX_FILE_SIZE,
|
||||
"max_file_count": app.state.config.MAX_FILE_COUNT,
|
||||
"hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH,
|
||||
}
|
||||
|
||||
@ -580,16 +586,16 @@ async def get_query_settings(user=Depends(get_admin_user)):
|
||||
@app.get("/file/limit/settings")
|
||||
async def get_query_settings(user=Depends(get_verified_user)):
|
||||
return {
|
||||
"max_file_size": app.state.config.MAX_FILE_SIZE,
|
||||
"max_file_count": app.state.config.MAX_FILE_COUNT,
|
||||
"FILE_MAX_SIZE": app.state.config.FILE_MAX_SIZE,
|
||||
"FILE_MAX_COUNT": app.state.config.FILE_MAX_COUNT,
|
||||
}
|
||||
|
||||
|
||||
class QuerySettingsForm(BaseModel):
|
||||
k: Optional[int] = None
|
||||
r: Optional[float] = None
|
||||
max_file_size: Optional[int] = None
|
||||
max_file_count: Optional[int] = None
|
||||
FILE_MAX_SIZE: Optional[int] = None
|
||||
FILE_MAX_COUNT: Optional[int] = None
|
||||
template: Optional[str] = None
|
||||
hybrid: Optional[bool] = None
|
||||
|
||||
@ -606,11 +612,11 @@ async def update_query_settings(
|
||||
app.state.config.ENABLE_RAG_HYBRID_SEARCH = (
|
||||
form_data.hybrid if form_data.hybrid else False
|
||||
)
|
||||
app.state.config.MAX_FILE_SIZE = (
|
||||
form_data.max_file_size if form_data.max_file_size else 10
|
||||
app.state.config.FILE_MAX_SIZE = (
|
||||
form_data.FILE_MAX_SIZE if form_data.FILE_MAX_SIZE else 10
|
||||
)
|
||||
app.state.config.MAX_FILE_COUNT = (
|
||||
form_data.max_file_count if form_data.max_file_count else 5
|
||||
app.state.config.FILE_MAX_COUNT = (
|
||||
form_data.FILE_MAX_COUNT if form_data.FILE_MAX_COUNT else 5
|
||||
)
|
||||
|
||||
return {
|
||||
@ -618,8 +624,8 @@ async def update_query_settings(
|
||||
"template": app.state.config.RAG_TEMPLATE,
|
||||
"k": app.state.config.TOP_K,
|
||||
"r": app.state.config.RELEVANCE_THRESHOLD,
|
||||
"max_file_size": app.state.config.MAX_FILE_SIZE,
|
||||
"max_file_count": app.state.config.MAX_FILE_COUNT,
|
||||
"FILE_MAX_SIZE": app.state.config.FILE_MAX_SIZE,
|
||||
"FILE_MAX_COUNT": app.state.config.FILE_MAX_COUNT,
|
||||
"hybrid": app.state.config.ENABLE_RAG_HYBRID_SEARCH,
|
||||
}
|
||||
|
||||
|
||||
@ -134,38 +134,9 @@ export const getQuerySettings = async (token: string) => {
|
||||
return res;
|
||||
};
|
||||
|
||||
export const getFileLimitSettings = async (token: string) => {
|
||||
let error = null;
|
||||
|
||||
const res = await fetch(`${RAG_API_BASE_URL}/file/limit/settings`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${token}`
|
||||
}
|
||||
})
|
||||
.then(async (res) => {
|
||||
if (!res.ok) throw await res.json();
|
||||
return res.json();
|
||||
})
|
||||
.catch((err) => {
|
||||
console.log(err);
|
||||
error = err.detail;
|
||||
return null;
|
||||
});
|
||||
|
||||
if (error) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
return res;
|
||||
};
|
||||
|
||||
type QuerySettings = {
|
||||
k: number | null;
|
||||
r: number | null;
|
||||
max_file_size: number | null;
|
||||
max_file_count: number | null;
|
||||
template: string | null;
|
||||
};
|
||||
|
||||
|
||||
@ -37,6 +37,9 @@
|
||||
let embeddingModel = '';
|
||||
let rerankingModel = '';
|
||||
|
||||
let fileMaxSize = null;
|
||||
let fileMaxCount = null;
|
||||
|
||||
let contentExtractionEngine = 'default';
|
||||
let tikaServerUrl = '';
|
||||
let showTikaServerUrl = false;
|
||||
@ -53,8 +56,6 @@
|
||||
template: '',
|
||||
r: 0.0,
|
||||
k: 4,
|
||||
max_file_size: 10,
|
||||
max_file_count: 5,
|
||||
hybrid: false
|
||||
};
|
||||
|
||||
@ -220,7 +221,6 @@
|
||||
await setRerankingConfig();
|
||||
|
||||
querySettings = await getQuerySettings(localStorage.token);
|
||||
|
||||
const res = await getRAGConfig(localStorage.token);
|
||||
|
||||
if (res) {
|
||||
@ -232,6 +232,9 @@
|
||||
contentExtractionEngine = res.content_extraction.engine;
|
||||
tikaServerUrl = res.content_extraction.tika_server_url;
|
||||
showTikaServerUrl = contentExtractionEngine === 'tika';
|
||||
|
||||
fileMaxSize = res.file.file_max_size;
|
||||
fileMaxCount = res.file.file_max_count;
|
||||
}
|
||||
});
|
||||
</script>
|
||||
@ -388,41 +391,6 @@
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class=" my-2 flex gap-1.5">
|
||||
<div class=" w-full justify-between">
|
||||
<div class="self-center text-xs font-medium min-w-fit mb-1">
|
||||
{$i18n.t('Max File Count')}
|
||||
</div>
|
||||
<div class="self-center">
|
||||
<input
|
||||
class=" w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
|
||||
type="number"
|
||||
placeholder={$i18n.t('Enter Max File Count')}
|
||||
bind:value={querySettings.max_file_count}
|
||||
autocomplete="off"
|
||||
min="0"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="w-full">
|
||||
<div class=" self-center text-xs font-medium min-w-fit mb-1">
|
||||
{$i18n.t('Max File Size(MB)')}
|
||||
</div>
|
||||
|
||||
<div class="self-center">
|
||||
<input
|
||||
class="w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
|
||||
type="number"
|
||||
placeholder={$i18n.t('Enter Max File Size(MB)')}
|
||||
bind:value={querySettings.max_file_size}
|
||||
autocomplete="off"
|
||||
min="0"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class=" flex w-full justify-between">
|
||||
<div class=" self-center text-xs font-medium">{$i18n.t('Hybrid Search')}</div>
|
||||
|
||||
@ -647,6 +615,48 @@
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<hr class=" dark:border-gray-850" />
|
||||
|
||||
<div class="">
|
||||
<div class="text-sm font-medium">{$i18n.t('Files')}</div>
|
||||
|
||||
<div class=" my-2 flex gap-1.5">
|
||||
<div class=" w-full justify-between">
|
||||
<div class="self-center text-xs font-medium min-w-fit mb-1">
|
||||
{$i18n.t('Max Upload Count')}
|
||||
</div>
|
||||
<div class="self-center">
|
||||
<input
|
||||
class=" w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
|
||||
type="number"
|
||||
placeholder={$i18n.t('Enter File Count')}
|
||||
bind:value={fileMaxCount}
|
||||
autocomplete="off"
|
||||
min="0"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="w-full">
|
||||
<div class=" self-center text-xs font-medium min-w-fit mb-1">
|
||||
{$i18n.t('Max Upload Size')}
|
||||
</div>
|
||||
|
||||
<div class="self-center">
|
||||
<input
|
||||
class="w-full rounded-lg py-1.5 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-none"
|
||||
type="number"
|
||||
placeholder={$i18n.t('Enter File Size (MB)')}
|
||||
bind:value={fileMaxSize}
|
||||
autocomplete="off"
|
||||
min="0"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr class=" dark:border-gray-850" />
|
||||
|
||||
<div class=" ">
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
import { blobToFile, findWordIndices } from '$lib/utils';
|
||||
import { transcribeAudio } from '$lib/apis/audio';
|
||||
|
||||
import { getFileLimitSettings, processDocToVectorDB } from '$lib/apis/rag';
|
||||
import { processDocToVectorDB } from '$lib/apis/rag';
|
||||
|
||||
import { uploadFile } from '$lib/apis/files';
|
||||
import {
|
||||
@ -56,7 +56,6 @@
|
||||
let commandsElement;
|
||||
|
||||
let inputFiles;
|
||||
let fileLimitSettings;
|
||||
let dragged = false;
|
||||
|
||||
let user = null;
|
||||
@ -173,7 +172,7 @@
|
||||
};
|
||||
|
||||
const processFileCountLimit = async (querySettings, inputFiles) => {
|
||||
const maxFiles = querySettings.max_file_count;
|
||||
const maxFiles = querySettings.FILE_MAX_COUNT;
|
||||
const currentFilesCount = files.length;
|
||||
const inputFilesCount = inputFiles.length;
|
||||
const totalFilesCount = currentFilesCount + inputFilesCount;
|
||||
@ -190,8 +189,8 @@
|
||||
return [true, inputFiles];
|
||||
};
|
||||
|
||||
const processFileSizeLimit = async (fileLimitSettings, file) => {
|
||||
if (file.size <= fileLimitSettings.max_file_size * 1024 * 1024) {
|
||||
const inputFilesHandler = async (inputFiles) => {
|
||||
inputFiles.forEach((file) => {
|
||||
if (['image/gif', 'image/webp', 'image/jpeg', 'image/png'].includes(file['type'])) {
|
||||
if (visionCapableModels.length === 0) {
|
||||
toast.error($i18n.t('Selected model(s) do not support image inputs'));
|
||||
@ -211,25 +210,10 @@
|
||||
} else {
|
||||
uploadFileHandler(file);
|
||||
}
|
||||
} else {
|
||||
toast.error(
|
||||
$i18n.t('File size exceeds the limit of {{size}}MB', {
|
||||
size: fileLimitSettings.max_file_size
|
||||
})
|
||||
);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
onMount(() => {
|
||||
const initFileLimitSettings = async () => {
|
||||
try {
|
||||
fileLimitSettings = await getFileLimitSettings(localStorage.token);
|
||||
} catch (error) {
|
||||
console.error('Error fetching query settings:', error);
|
||||
}
|
||||
};
|
||||
initFileLimitSettings();
|
||||
|
||||
window.setTimeout(() => chatTextAreaElement?.focus(), 0);
|
||||
|
||||
const dropZone = document.querySelector('body');
|
||||
@ -256,22 +240,10 @@
|
||||
|
||||
if (e.dataTransfer?.files) {
|
||||
const inputFiles = Array.from(e.dataTransfer?.files);
|
||||
|
||||
console.log(file, file.name.split('.').at(-1));
|
||||
if (inputFiles && inputFiles.length > 0) {
|
||||
console.log(inputFiles);
|
||||
const [canProcess, filesToProcess] = await processFileCountLimit(
|
||||
fileLimitSettings,
|
||||
inputFiles
|
||||
);
|
||||
if (!canProcess) {
|
||||
dragged = false;
|
||||
return;
|
||||
}
|
||||
console.log(filesToProcess);
|
||||
filesToProcess.forEach((file) => {
|
||||
console.log(file, file.name.split('.').at(-1));
|
||||
processFileSizeLimit(fileLimitSettings, file);
|
||||
});
|
||||
inputFilesHandler(inputFiles);
|
||||
} else {
|
||||
toast.error($i18n.t(`File not found.`));
|
||||
}
|
||||
@ -392,21 +364,7 @@
|
||||
multiple
|
||||
on:change={async () => {
|
||||
if (inputFiles && inputFiles.length > 0) {
|
||||
const _inputFiles = Array.from(inputFiles);
|
||||
console.log(_inputFiles);
|
||||
const [canProcess, filesToProcess] = await processFileCountLimit(
|
||||
fileLimitSettings,
|
||||
_inputFiles
|
||||
);
|
||||
if (!canProcess) {
|
||||
filesInputElement.value = '';
|
||||
return;
|
||||
}
|
||||
console.log(filesToProcess);
|
||||
filesToProcess.forEach((file) => {
|
||||
console.log(file, file.name.split('.').at(-1));
|
||||
processFileSizeLimit(fileLimitSettings, file);
|
||||
});
|
||||
inputFilesHandler(inputFiles);
|
||||
} else {
|
||||
toast.error($i18n.t(`File not found.`));
|
||||
}
|
||||
@ -715,17 +673,7 @@
|
||||
.map((item) => item.getAsFile())
|
||||
.filter((file) => file);
|
||||
|
||||
const [canProcess, filesToProcess] = await processFileCountLimit(
|
||||
fileLimitSettings,
|
||||
inputFiles
|
||||
);
|
||||
if (!canProcess) {
|
||||
return;
|
||||
}
|
||||
filesToProcess.forEach((file) => {
|
||||
console.log(file, file.name.split('.').at(-1));
|
||||
processFileSizeLimit(fileLimitSettings, file);
|
||||
});
|
||||
inputFilesHandler(inputFiles);
|
||||
} else {
|
||||
toast.error($i18n.t(`File not found.`));
|
||||
}
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
import { createNewDoc, deleteDocByName, getDocs } from '$lib/apis/documents';
|
||||
|
||||
import { SUPPORTED_FILE_TYPE, SUPPORTED_FILE_EXTENSIONS } from '$lib/constants';
|
||||
import { getFileLimitSettings, processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag';
|
||||
import { processDocToVectorDB, uploadDocToVectorDB } from '$lib/apis/rag';
|
||||
import { blobToFile, transformFileName } from '$lib/utils';
|
||||
|
||||
import Checkbox from '$lib/components/common/Checkbox.svelte';
|
||||
@ -24,7 +24,7 @@
|
||||
let importFiles = '';
|
||||
|
||||
let inputFiles = '';
|
||||
let fileLimitSettings;
|
||||
|
||||
let query = '';
|
||||
let documentsImportInputElement: HTMLInputElement;
|
||||
let tags = [];
|
||||
@ -99,17 +99,7 @@
|
||||
}
|
||||
};
|
||||
|
||||
const initFileLimitSettings = async () => {
|
||||
try {
|
||||
fileLimitSettings = await getFileLimitSettings(localStorage.token);
|
||||
} catch (error) {
|
||||
console.error('Error fetching query settings:', error);
|
||||
}
|
||||
};
|
||||
|
||||
onMount(() => {
|
||||
initFileLimitSettings();
|
||||
|
||||
documents.subscribe((docs) => {
|
||||
tags = docs.reduce((a, e, i, arr) => {
|
||||
return [...new Set([...a, ...(e?.content?.tags ?? []).map((tag) => tag.name)])];
|
||||
@ -147,24 +137,16 @@
|
||||
if (inputFiles && inputFiles.length > 0) {
|
||||
for (const file of inputFiles) {
|
||||
console.log(file, file.name.split('.').at(-1));
|
||||
if (file.size <= fileLimitSettings.max_file_size * 1024 * 1024) {
|
||||
if (
|
||||
SUPPORTED_FILE_TYPE.includes(file['type']) ||
|
||||
SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
|
||||
) {
|
||||
uploadDoc(file);
|
||||
} else {
|
||||
toast.error(
|
||||
`Unknown File Type '${file['type']}', but accepting and treating as plain text`
|
||||
);
|
||||
uploadDoc(file);
|
||||
}
|
||||
if (
|
||||
SUPPORTED_FILE_TYPE.includes(file['type']) ||
|
||||
SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1))
|
||||
) {
|
||||
uploadDoc(file);
|
||||
} else {
|
||||
toast.error(
|
||||
$i18n.t('File size exceeds the limit of {{size}}MB', {
|
||||
size: fileLimitSettings.max_file_size
|
||||
})
|
||||
`Unknown File Type '${file['type']}', but accepting and treating as plain text`
|
||||
);
|
||||
uploadDoc(file);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user