UniqAI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-04-27 17:51:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: web_loader_ssl_verification setting

Browse Source
This commit is contained in:
Timothy J. Baek 2024-05-06 14:50:55 -07:00
parent 07b39d1cb9
commit cecb87b8c2
6 changed files with 156 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
backend/apps/rag/main.py
View File

@ -313,6 +313,7 @@ async def get_rag_config(user=Depends(get_admin_user)):
"chunk_size": app.state.CHUNK_SIZE,
"chunk_overlap": app.state.CHUNK_OVERLAP,
},
"web_loader_ssl_verification": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
}
@ -322,15 +323,34 @@ class ChunkParamUpdateForm(BaseModel):
class ConfigUpdateForm(BaseModel):
pdf_extract_images: bool
chunk: ChunkParamUpdateForm
pdf_extract_images: Optional[bool] = None
chunk: Optional[ChunkParamUpdateForm] = None
web_loader_ssl_verification: Optional[bool] = None
@app.post("/config/update")
async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_user)):
app.state.PDF_EXTRACT_IMAGES = form_data.pdf_extract_images
app.state.CHUNK_SIZE = form_data.chunk.chunk_size
app.state.CHUNK_OVERLAP = form_data.chunk.chunk_overlap
app.state.PDF_EXTRACT_IMAGES = (
form_data.pdf_extract_images
if form_data.pdf_extract_images != None
else app.state.PDF_EXTRACT_IMAGES
)
app.state.CHUNK_SIZE = (
form_data.chunk.chunk_size if form_data.chunk != None else app.state.CHUNK_SIZE
)
app.state.CHUNK_OVERLAP = (
form_data.chunk.chunk_overlap
if form_data.chunk != None
else app.state.CHUNK_OVERLAP
)
app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
form_data.web_loader_ssl_verification
if form_data.web_loader_ssl_verification != None
else app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
)
return {
"status": True,
@ -339,6 +359,7 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_
"chunk_size": app.state.CHUNK_SIZE,
"chunk_overlap": app.state.CHUNK_OVERLAP,
},
"web_loader_ssl_verification": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
}
@ -490,10 +511,9 @@ def store_youtube_video(form_data: UrlForm, user=Depends(get_current_user)):
def store_web(form_data: UrlForm, user=Depends(get_current_user)):
# "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm"
try:
loader = get_web_loader(form_data.url)
loader.requests_kwargs = {
"verify": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
}
loader = get_web_loader(
form_data.url, verify_ssl=app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
)
data = loader.load()
collection_name = form_data.collection_name
@ -514,7 +534,7 @@ def store_web(form_data: UrlForm, user=Depends(get_current_user)):
)
def get_web_loader(url: str):
def get_web_loader(url: str, verify_ssl: bool = True):
# Check if the URL is valid
if isinstance(validators.url(url), validators.ValidationError):
raise ValueError(ERROR_MESSAGES.INVALID_URL)
@ -531,7 +551,7 @@ def get_web_loader(url: str):
for ip in ipv6_addresses:
if validators.ipv6(ip, private=True):
raise ValueError(ERROR_MESSAGES.INVALID_URL)
return WebBaseLoader(url)
return WebBaseLoader(url, verify_ssl=verify_ssl)
def resolve_hostname(hostname):

2
backend/config.py
View File

@ -456,7 +456,7 @@ ENABLE_RAG_HYBRID_SEARCH = (
ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "").lower() == "true"
os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "True").lower() == "true"
)
RAG_EMBEDDING_ENGINE = os.environ.get("RAG_EMBEDDING_ENGINE", "")

5
src/lib/apis/rag/index.ts
View File

@ -33,8 +33,9 @@ type ChunkConfigForm = {
};
type RAGConfigForm = {
pdf_extract_images: boolean;
chunk: ChunkConfigForm;
pdf_extract_images?: boolean;
chunk?: ChunkConfigForm;
web_loader_ssl_verification?: boolean;
};
export const updateRAGConfig = async (token: string, payload: RAGConfigForm) => {

0
src/lib/components/documents/Settings/WebLoader.svelte
View File

74
src/lib/components/documents/Settings/WebParams.svelte Normal file
View File

@ -0,0 +1,74 @@
<script lang="ts">
import { getRAGConfig, updateRAGConfig } from '$lib/apis/rag';
import { documents, models } from '$lib/stores';
import { onMount, getContext } from 'svelte';
import { toast } from 'svelte-sonner';
const i18n = getContext('i18n');
export let saveHandler: Function;
let webLoaderSSLVerification = true;
const submitHandler = async () => {
const res = await updateRAGConfig(localStorage.token, {
web_loader_ssl_verification: webLoaderSSLVerification
});
};
onMount(async () => {
const res = await getRAGConfig(localStorage.token);
if (res) {
webLoaderSSLVerification = res.web_loader_ssl_verification;
}
});
</script>
<form
class="flex flex-col h-full justify-between space-y-3 text-sm"
on:submit|preventDefault={() => {
submitHandler();
saveHandler();
}}
>
<div class=" space-y-3 pr-1.5 overflow-y-scroll h-full max-h-[22rem]">
<div>
<div class=" mb-1 text-sm font-medium">
{$i18n.t('Retrieval Augmented Generation Settings')}
</div>
<div>
<div class=" py-0.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Bypass SSL verification for Websites')}
</div>
<button
class="p-1 px-3 text-xs flex rounded transition"
on:click={() => {
webLoaderSSLVerification = !webLoaderSSLVerification;
submitHandler();
}}
type="button"
>
{#if webLoaderSSLVerification === true}
<span class="ml-2 self-center">{$i18n.t('On')}</span>
{:else}
<span class="ml-2 self-center">{$i18n.t('Off')}</span>
{/if}
</button>
</div>
</div>
</div>
</div>
<div class="flex justify-end pt-3 text-sm font-medium">
<button
class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
type="submit"
>
{$i18n.t('Save')}
</button>
</div>
</form>

50
src/lib/components/documents/SettingsModal.svelte
View File

@ -4,6 +4,8 @@
import General from './Settings/General.svelte';
import ChunkParams from './Settings/ChunkParams.svelte';
import QueryParams from './Settings/QueryParams.svelte';
import WebParams from './Settings/WebParams.svelte';
import { toast } from 'svelte-sonner';
const i18n = getContext('i18n');
@ -123,14 +125,56 @@
</div>
<div class=" self-center">{$i18n.t('Query Params')}</div>
</button>
<button
class="px-2.5 py-2.5 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
'web'
? 'bg-gray-200 dark:bg-gray-700'
: ' hover:bg-gray-300 dark:hover:bg-gray-800'}"
on:click={() => {
selectedTab = 'web';
}}
>
<div class=" self-center mr-2">
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="currentColor"
class="w-4 h-4"
>
<path
d="M21.721 12.752a9.711 9.711 0 0 0-.945-5.003 12.754 12.754 0 0 1-4.339 2.708 18.991 18.991 0 0 1-.214 4.772 17.165 17.165 0 0 0 5.498-2.477ZM14.634 15.55a17.324 17.324 0 0 0 .332-4.647c-.952.227-1.945.347-2.966.347-1.021 0-2.014-.12-2.966-.347a17.515 17.515 0 0 0 .332 4.647 17.385 17.385 0 0 0 5.268 0ZM9.772 17.119a18.963 18.963 0 0 0 4.456 0A17.182 17.182 0 0 1 12 21.724a17.18 17.18 0 0 1-2.228-4.605ZM7.777 15.23a18.87 18.87 0 0 1-.214-4.774 12.753 12.753 0 0 1-4.34-2.708 9.711 9.711 0 0 0-.944 5.004 17.165 17.165 0 0 0 5.498 2.477ZM21.356 14.752a9.765 9.765 0 0 1-7.478 6.817 18.64 18.64 0 0 0 1.988-4.718 18.627 18.627 0 0 0 5.49-2.098ZM2.644 14.752c1.682.971 3.53 1.688 5.49 2.099a18.64 18.64 0 0 0 1.988 4.718 9.765 9.765 0 0 1-7.478-6.816ZM13.878 2.43a9.755 9.755 0 0 1 6.116 3.986 11.267 11.267 0 0 1-3.746 2.504 18.63 18.63 0 0 0-2.37-6.49ZM12 2.276a17.152 17.152 0 0 1 2.805 7.121c-.897.23-1.837.353-2.805.353-.968 0-1.908-.122-2.805-.353A17.151 17.151 0 0 1 12 2.276ZM10.122 2.43a18.629 18.629 0 0 0-2.37 6.49 11.266 11.266 0 0 1-3.746-2.504 9.754 9.754 0 0 1 6.116-3.985Z"
/>
</svg>
</div>
<div class=" self-center">{$i18n.t('Web Params')}</div>
</button>
</div>
<div class="flex-1 md:min-h-[380px]">
{#if selectedTab === 'general'}
<General saveHandler={() => {}} />
<General
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{:else if selectedTab === 'chunk'}
<ChunkParams saveHandler={() => {}} />
<ChunkParams
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{:else if selectedTab === 'query'}
<QueryParams saveHandler={() => {}} />
<QueryParams
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{:else if selectedTab === 'web'}
<WebParams
saveHandler={() => {
toast.success($i18n.t('Settings saved successfully!'));
}}
/>
{/if}
</div>
</div>