feat: web_loader_ssl_verification setting

backend/apps/rag/main.py

@@ -313,6 +313,7 @@ async def get_rag_config(user=Depends(get_admin_user)):
             "chunk_size": app.state.CHUNK_SIZE,
             "chunk_overlap": app.state.CHUNK_OVERLAP,
         },
+        "web_loader_ssl_verification": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
     }
 
 
@@ -322,15 +323,34 @@ class ChunkParamUpdateForm(BaseModel):
 
 
 class ConfigUpdateForm(BaseModel):
-    pdf_extract_images: bool
+    pdf_extract_images: Optional[bool] = None
-    chunk: ChunkParamUpdateForm
+    chunk: Optional[ChunkParamUpdateForm] = None
+    web_loader_ssl_verification: Optional[bool] = None
 
 
 @app.post("/config/update")
 async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_user)):
-    app.state.PDF_EXTRACT_IMAGES = form_data.pdf_extract_images
+    app.state.PDF_EXTRACT_IMAGES = (
-    app.state.CHUNK_SIZE = form_data.chunk.chunk_size
+        form_data.pdf_extract_images
-    app.state.CHUNK_OVERLAP = form_data.chunk.chunk_overlap
+        if form_data.pdf_extract_images != None
+        else app.state.PDF_EXTRACT_IMAGES
+    )
+
+    app.state.CHUNK_SIZE = (
+        form_data.chunk.chunk_size if form_data.chunk != None else app.state.CHUNK_SIZE
+    )
+
+    app.state.CHUNK_OVERLAP = (
+        form_data.chunk.chunk_overlap
+        if form_data.chunk != None
+        else app.state.CHUNK_OVERLAP
+    )
+
+    app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
+        form_data.web_loader_ssl_verification
+        if form_data.web_loader_ssl_verification != None
+        else app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
+    )
 
     return {
         "status": True,
@@ -339,6 +359,7 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_
             "chunk_size": app.state.CHUNK_SIZE,
             "chunk_overlap": app.state.CHUNK_OVERLAP,
         },
+        "web_loader_ssl_verification": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
     }
 
 
@@ -490,10 +511,9 @@ def store_youtube_video(form_data: UrlForm, user=Depends(get_current_user)):
 def store_web(form_data: UrlForm, user=Depends(get_current_user)):
     # "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm"
     try:
-        loader = get_web_loader(form_data.url)
+        loader = get_web_loader(
-        loader.requests_kwargs = {
+            form_data.url, verify_ssl=app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
-            "verify": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
+        )
-        }
         data = loader.load()
 
         collection_name = form_data.collection_name
@@ -514,7 +534,7 @@ def store_web(form_data: UrlForm, user=Depends(get_current_user)):
         )
 
 
-def get_web_loader(url: str):
+def get_web_loader(url: str, verify_ssl: bool = True):
     # Check if the URL is valid
     if isinstance(validators.url(url), validators.ValidationError):
         raise ValueError(ERROR_MESSAGES.INVALID_URL)
@@ -531,7 +551,7 @@ def get_web_loader(url: str):
         for ip in ipv6_addresses:
             if validators.ipv6(ip, private=True):
                 raise ValueError(ERROR_MESSAGES.INVALID_URL)
-    return WebBaseLoader(url)
+    return WebBaseLoader(url, verify_ssl=verify_ssl)
 
 
 def resolve_hostname(hostname):

backend/config.py

@@ -456,7 +456,7 @@ ENABLE_RAG_HYBRID_SEARCH = (
 
 
 ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = (
-    os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "").lower() == "true"
+    os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "True").lower() == "true"
 )
 
 RAG_EMBEDDING_ENGINE = os.environ.get("RAG_EMBEDDING_ENGINE", "")

src/lib/apis/rag/index.ts

@@ -33,8 +33,9 @@ type ChunkConfigForm = {
 };
 
 type RAGConfigForm = {
-	pdf_extract_images: boolean;
+	pdf_extract_images?: boolean;
-	chunk: ChunkConfigForm;
+	chunk?: ChunkConfigForm;
+	web_loader_ssl_verification?: boolean;
 };
 
 export const updateRAGConfig = async (token: string, payload: RAGConfigForm) => {

src/lib/components/documents/Settings/WebParams.svelte

@@ -0,0 +1,74 @@
+<script lang="ts">
+	import { getRAGConfig, updateRAGConfig } from '$lib/apis/rag';
+
+	import { documents, models } from '$lib/stores';
+	import { onMount, getContext } from 'svelte';
+	import { toast } from 'svelte-sonner';
+
+	const i18n = getContext('i18n');
+
+	export let saveHandler: Function;
+
+	let webLoaderSSLVerification = true;
+
+	const submitHandler = async () => {
+		const res = await updateRAGConfig(localStorage.token, {
+			web_loader_ssl_verification: webLoaderSSLVerification
+		});
+	};
+
+	onMount(async () => {
+		const res = await getRAGConfig(localStorage.token);
+
+		if (res) {
+			webLoaderSSLVerification = res.web_loader_ssl_verification;
+		}
+	});
+</script>
+
+<form
+	class="flex flex-col h-full justify-between space-y-3 text-sm"
+	on:submit|preventDefault={() => {
+		submitHandler();
+		saveHandler();
+	}}
+>
+	<div class=" space-y-3 pr-1.5 overflow-y-scroll h-full max-h-[22rem]">
+		<div>
+			<div class=" mb-1 text-sm font-medium">
+				{$i18n.t('Retrieval Augmented Generation Settings')}
+			</div>
+
+			<div>
+				<div class=" py-0.5 flex w-full justify-between">
+					<div class=" self-center text-xs font-medium">
+						{$i18n.t('Bypass SSL verification for Websites')}
+					</div>
+
+					<button
+						class="p-1 px-3 text-xs flex rounded transition"
+						on:click={() => {
+							webLoaderSSLVerification = !webLoaderSSLVerification;
+							submitHandler();
+						}}
+						type="button"
+					>
+						{#if webLoaderSSLVerification === true}
+							<span class="ml-2 self-center">{$i18n.t('On')}</span>
+						{:else}
+							<span class="ml-2 self-center">{$i18n.t('Off')}</span>
+						{/if}
+					</button>
+				</div>
+			</div>
+		</div>
+	</div>
+	<div class="flex justify-end pt-3 text-sm font-medium">
+		<button
+			class=" px-4 py-2 bg-emerald-700 hover:bg-emerald-800 text-gray-100 transition rounded-lg"
+			type="submit"
+		>
+			{$i18n.t('Save')}
+		</button>
+	</div>
+</form>

src/lib/components/documents/SettingsModal.svelte

@@ -4,6 +4,8 @@
 	import General from './Settings/General.svelte';
 	import ChunkParams from './Settings/ChunkParams.svelte';
 	import QueryParams from './Settings/QueryParams.svelte';
+	import WebParams from './Settings/WebParams.svelte';
+	import { toast } from 'svelte-sonner';
 
 	const i18n = getContext('i18n');
 
@@ -123,14 +125,56 @@
 					</div>
 					<div class=" self-center">{$i18n.t('Query Params')}</div>
 				</button>
+
+				<button
+					class="px-2.5 py-2.5 min-w-fit rounded-lg flex-1 md:flex-none flex text-right transition {selectedTab ===
+					'web'
+						? 'bg-gray-200 dark:bg-gray-700'
+						: ' hover:bg-gray-300 dark:hover:bg-gray-800'}"
+					on:click={() => {
+						selectedTab = 'web';
+					}}
+				>
+					<div class=" self-center mr-2">
+						<svg
+							xmlns="http://www.w3.org/2000/svg"
+							viewBox="0 0 24 24"
+							fill="currentColor"
+							class="w-4 h-4"
+						>
+							<path
+								d="M21.721 12.752a9.711 9.711 0 0 0-.945-5.003 12.754 12.754 0 0 1-4.339 2.708 18.991 18.991 0 0 1-.214 4.772 17.165 17.165 0 0 0 5.498-2.477ZM14.634 15.55a17.324 17.324 0 0 0 .332-4.647c-.952.227-1.945.347-2.966.347-1.021 0-2.014-.12-2.966-.347a17.515 17.515 0 0 0 .332 4.647 17.385 17.385 0 0 0 5.268 0ZM9.772 17.119a18.963 18.963 0 0 0 4.456 0A17.182 17.182 0 0 1 12 21.724a17.18 17.18 0 0 1-2.228-4.605ZM7.777 15.23a18.87 18.87 0 0 1-.214-4.774 12.753 12.753 0 0 1-4.34-2.708 9.711 9.711 0 0 0-.944 5.004 17.165 17.165 0 0 0 5.498 2.477ZM21.356 14.752a9.765 9.765 0 0 1-7.478 6.817 18.64 18.64 0 0 0 1.988-4.718 18.627 18.627 0 0 0 5.49-2.098ZM2.644 14.752c1.682.971 3.53 1.688 5.49 2.099a18.64 18.64 0 0 0 1.988 4.718 9.765 9.765 0 0 1-7.478-6.816ZM13.878 2.43a9.755 9.755 0 0 1 6.116 3.986 11.267 11.267 0 0 1-3.746 2.504 18.63 18.63 0 0 0-2.37-6.49ZM12 2.276a17.152 17.152 0 0 1 2.805 7.121c-.897.23-1.837.353-2.805.353-.968 0-1.908-.122-2.805-.353A17.151 17.151 0 0 1 12 2.276ZM10.122 2.43a18.629 18.629 0 0 0-2.37 6.49 11.266 11.266 0 0 1-3.746-2.504 9.754 9.754 0 0 1 6.116-3.985Z"
+							/>
+						</svg>
+					</div>
+					<div class=" self-center">{$i18n.t('Web Params')}</div>
+				</button>
 			</div>
 			<div class="flex-1 md:min-h-[380px]">
 				{#if selectedTab === 'general'}
-					<General saveHandler={() => {}} />
+					<General
+						saveHandler={() => {
+							toast.success($i18n.t('Settings saved successfully!'));
+						}}
+					/>
 				{:else if selectedTab === 'chunk'}
-					<ChunkParams saveHandler={() => {}} />
+					<ChunkParams
+						saveHandler={() => {
+							toast.success($i18n.t('Settings saved successfully!'));
+						}}
+					/>
 				{:else if selectedTab === 'query'}
-					<QueryParams saveHandler={() => {}} />
+					<QueryParams
+						saveHandler={() => {
+							toast.success($i18n.t('Settings saved successfully!'));
+						}}
+					/>
+				{:else if selectedTab === 'web'}
+					<WebParams
+						saveHandler={() => {
+							toast.success($i18n.t('Settings saved successfully!'));
+						}}
+					/>
 				{/if}
 			</div>
 		</div>