mirror of
https://github.com/open-webui/open-webui
synced 2025-04-02 12:09:06 +00:00
feat: web_loader_ssl_verification setting
This commit is contained in:
parent
07b39d1cb9
commit
cecb87b8c2
@ -313,6 +313,7 @@ async def get_rag_config(user=Depends(get_admin_user)):
|
||||
"chunk_size": app.state.CHUNK_SIZE,
|
||||
"chunk_overlap": app.state.CHUNK_OVERLAP,
|
||||
},
|
||||
"web_loader_ssl_verification": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
|
||||
}
|
||||
|
||||
|
||||
@ -322,15 +323,34 @@ class ChunkParamUpdateForm(BaseModel):
|
||||
|
||||
|
||||
class ConfigUpdateForm(BaseModel):
|
||||
pdf_extract_images: bool
|
||||
chunk: ChunkParamUpdateForm
|
||||
pdf_extract_images: Optional[bool] = None
|
||||
chunk: Optional[ChunkParamUpdateForm] = None
|
||||
web_loader_ssl_verification: Optional[bool] = None
|
||||
|
||||
|
||||
@app.post("/config/update")
|
||||
async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_user)):
|
||||
app.state.PDF_EXTRACT_IMAGES = form_data.pdf_extract_images
|
||||
app.state.CHUNK_SIZE = form_data.chunk.chunk_size
|
||||
app.state.CHUNK_OVERLAP = form_data.chunk.chunk_overlap
|
||||
app.state.PDF_EXTRACT_IMAGES = (
|
||||
form_data.pdf_extract_images
|
||||
if form_data.pdf_extract_images != None
|
||||
else app.state.PDF_EXTRACT_IMAGES
|
||||
)
|
||||
|
||||
app.state.CHUNK_SIZE = (
|
||||
form_data.chunk.chunk_size if form_data.chunk != None else app.state.CHUNK_SIZE
|
||||
)
|
||||
|
||||
app.state.CHUNK_OVERLAP = (
|
||||
form_data.chunk.chunk_overlap
|
||||
if form_data.chunk != None
|
||||
else app.state.CHUNK_OVERLAP
|
||||
)
|
||||
|
||||
app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
|
||||
form_data.web_loader_ssl_verification
|
||||
if form_data.web_loader_ssl_verification != None
|
||||
else app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
|
||||
)
|
||||
|
||||
return {
|
||||
"status": True,
|
||||
@ -339,6 +359,7 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_
|
||||
"chunk_size": app.state.CHUNK_SIZE,
|
||||
"chunk_overlap": app.state.CHUNK_OVERLAP,
|
||||
},
|
||||
"web_loader_ssl_verification": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
|
||||
}
|
||||
|
||||
|
||||
@ -490,10 +511,9 @@ def store_youtube_video(form_data: UrlForm, user=Depends(get_current_user)):
|
||||
def store_web(form_data: UrlForm, user=Depends(get_current_user)):
|
||||
# "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm"
|
||||
try:
|
||||
loader = get_web_loader(form_data.url)
|
||||
loader.requests_kwargs = {
|
||||
"verify": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
|
||||
}
|
||||
loader = get_web_loader(
|
||||
form_data.url, verify_ssl=app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
|
||||
)
|
||||
data = loader.load()
|
||||
|
||||
collection_name = form_data.collection_name
|
||||
@ -514,7 +534,7 @@ def store_web(form_data: UrlForm, user=Depends(get_current_user)):
|
||||
)
|
||||
|
||||
|
||||
def get_web_loader(url: str):
|
||||
def get_web_loader(url: str, verify_ssl: bool = True):
|
||||
# Check if the URL is valid
|
||||
if isinstance(validators.url(url), validators.ValidationError):
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
@ -531,7 +551,7 @@ def get_web_loader(url: str):
|
||||
for ip in ipv6_addresses:
|
||||
if validators.ipv6(ip, private=True):
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
return WebBaseLoader(url)
|
||||
return WebBaseLoader(url, verify_ssl=verify_ssl)
|
||||
|
||||
|
||||
def resolve_hostname(hostname):
|
||||
|
@ -456,7 +456,7 @@ ENABLE_RAG_HYBRID_SEARCH = (
|
||||
|
||||
|
||||
ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
|
||||
os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "").lower() == "true"
|
||||
os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "True").lower() == "true"
|
||||
)
|
||||
|
||||
RAG_EMBEDDING_ENGINE = os.environ.get("RAG_EMBEDDING_ENGINE", "")
|
||||
|
@ -33,8 +33,9 @@ type ChunkConfigForm = {
|
||||
};
|
||||
|
||||
type RAGConfigForm = {
|
||||
pdf_extract_images: boolean;
|
||||
chunk: ChunkConfigForm;
|
||||
pdf_extract_images?: boolean;
|
||||
chunk?: ChunkConfigForm;
|
||||
web_loader_ssl_verification?: boolean;
|
||||
};
|
||||
|
||||
export const updateRAGConfig = async (token: string, payload: RAGConfigForm) => {
|
||||
|
74
src/lib/components/documents/Settings/WebParams.svelte
Normal file
74
src/lib/components/documents/Settings/WebParams.svelte
Normal file
@ -0,0 +1,74 @@
|
||||
<script lang="ts">
|
||||
import { getRAGConfig, updateRAGConfig } from '$lib/apis/rag';
|
||||
|
||||
import { documents, models } from '$lib/stores';
|
||||
import { onMount, getContext } from 'svelte';
|
||||
import { toast } from 'svelte-sonner';
|
||||
|
||||
const i18n = getContext('i18n');
|
||||
|
||||
export let saveHandler: Function;
|
||||
|
||||
let webLoaderSSLVerification = true;
|
||||
|
||||
const submitHandler = async () => {
|
||||
const res = await updateRAGConfig(localStorage.token, {
|
||||
web_loader_ssl_verification: webLoaderSSLVerification
|
||||
});
|
||||
};
|
||||
|
||||
onMount(async () => {
|
||||
const res = await getRAGConfig(localStorage.token);
|
||||
|
||||
if (res) {
|
||||
webLoaderSSLVerification = res.web_loader_ssl_verification;
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
<form
|
||||
class="flex flex-col h-full justify-between space-y-3 text-sm"
|
||||
on:submit|preventDefault={() => {
|
||||
submitHandler();
|
||||
saveHandler();
|
||||
}}
|
||||
>
|
||||
<div class=" space-y-3 pr-1.5 overflow-y-scroll h-full max-h-[22rem]">
|
||||
<div>
|
||||
<div class=" mb-1 text-sm font-medium">
|
||||
{$i18n.t('Retrieval Augmented Generation Settings')}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<div class=" py-0.5 flex w-full justify-between">
|
||||
<div class=" self-center text-xs font-medium">
|
||||
{$i18n.t('Bypass SSL verification for Websites')}
|
||||
</div>
|
||||
|
||||
<button
|
||||
class="p-1 px-3 text-xs flex rounded transition"
|
||||
on:click={() => {
|
||||
webLoaderSSLVerification = !webLoaderSSLVerification;
|
||||
submitHandler();
|
||||
}}
|
||||
type="button"
|
||||
>
|
||||
{#if webLoaderSSLVerification === true}
|
||||
<span class="ml-2 self-center">{$i18n.t('On')}</span>
|
||||
{:else}
|
||||
<span class="ml-2 self-center">{$i18n.t('Off')}</span>
|
||||
{/if}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex justify-end pt-3 text-sm font-medium">
|
||||
<button
|
||||
class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
|
||||
type="submit"
|
||||
>
|
||||
{$i18n.t('Save')}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
@ -4,6 +4,8 @@
|
||||
import General from './Settings/General.svelte';
|
||||
import ChunkParams from './Settings/ChunkParams.svelte';
|
||||
import QueryParams from './Settings/QueryParams.svelte';
|
||||
import WebParams from './Settings/WebParams.svelte';
|
||||
import { toast } from 'svelte-sonner';
|
||||
|
||||
const i18n = getContext('i18n');
|
||||
|
||||
@ -123,14 +125,56 @@
|
||||
</div>
|
||||
<div class=" self-center">{$i18n.t('Query Params')}</div>
|
||||
</button>
|
||||
|
||||
<button
|
||||
class="px-2.5 py-2.5 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
|
||||
'web'
|
||||
? 'bg-gray-200 dark:bg-gray-700'
|
||||
: ' hover:bg-gray-300 dark:hover:bg-gray-800'}"
|
||||
on:click={() => {
|
||||
selectedTab = 'web';
|
||||
}}
|
||||
>
|
||||
<div class=" self-center mr-2">
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
class="w-4 h-4"
|
||||
>
|
||||
<path
|
||||
d="M21.721 12.752a9.711 9.711 0 0 0-.945-5.003 12.754 12.754 0 0 1-4.339 2.708 18.991 18.991 0 0 1-.214 4.772 17.165 17.165 0 0 0 5.498-2.477ZM14.634 15.55a17.324 17.324 0 0 0 .332-4.647c-.952.227-1.945.347-2.966.347-1.021 0-2.014-.12-2.966-.347a17.515 17.515 0 0 0 .332 4.647 17.385 17.385 0 0 0 5.268 0ZM9.772 17.119a18.963 18.963 0 0 0 4.456 0A17.182 17.182 0 0 1 12 21.724a17.18 17.18 0 0 1-2.228-4.605ZM7.777 15.23a18.87 18.87 0 0 1-.214-4.774 12.753 12.753 0 0 1-4.34-2.708 9.711 9.711 0 0 0-.944 5.004 17.165 17.165 0 0 0 5.498 2.477ZM21.356 14.752a9.765 9.765 0 0 1-7.478 6.817 18.64 18.64 0 0 0 1.988-4.718 18.627 18.627 0 0 0 5.49-2.098ZM2.644 14.752c1.682.971 3.53 1.688 5.49 2.099a18.64 18.64 0 0 0 1.988 4.718 9.765 9.765 0 0 1-7.478-6.816ZM13.878 2.43a9.755 9.755 0 0 1 6.116 3.986 11.267 11.267 0 0 1-3.746 2.504 18.63 18.63 0 0 0-2.37-6.49ZM12 2.276a17.152 17.152 0 0 1 2.805 7.121c-.897.23-1.837.353-2.805.353-.968 0-1.908-.122-2.805-.353A17.151 17.151 0 0 1 12 2.276ZM10.122 2.43a18.629 18.629 0 0 0-2.37 6.49 11.266 11.266 0 0 1-3.746-2.504 9.754 9.754 0 0 1 6.116-3.985Z"
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
<div class=" self-center">{$i18n.t('Web Params')}</div>
|
||||
</button>
|
||||
</div>
|
||||
<div class="flex-1 md:min-h-[380px]">
|
||||
{#if selectedTab === 'general'}
|
||||
<General saveHandler={() => {}} />
|
||||
<General
|
||||
saveHandler={() => {
|
||||
toast.success($i18n.t('Settings saved successfully!'));
|
||||
}}
|
||||
/>
|
||||
{:else if selectedTab === 'chunk'}
|
||||
<ChunkParams saveHandler={() => {}} />
|
||||
<ChunkParams
|
||||
saveHandler={() => {
|
||||
toast.success($i18n.t('Settings saved successfully!'));
|
||||
}}
|
||||
/>
|
||||
{:else if selectedTab === 'query'}
|
||||
<QueryParams saveHandler={() => {}} />
|
||||
<QueryParams
|
||||
saveHandler={() => {
|
||||
toast.success($i18n.t('Settings saved successfully!'));
|
||||
}}
|
||||
/>
|
||||
{:else if selectedTab === 'web'}
|
||||
<WebParams
|
||||
saveHandler={() => {
|
||||
toast.success($i18n.t('Settings saved successfully!'));
|
||||
}}
|
||||
/>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
Loading…
Reference in New Issue
Block a user