refac: web/rag config

2025-06-26 18:26:48 +00:00 · 2025-04-12 16:33:36 -07:00
parent c3497da5dd
commit 48a23ce3fe
11 changed files with 1367 additions and 1530 deletions
--- a/src/lib/apis/retrieval/index.ts
+++ b/src/lib/apis/retrieval/index.ts
@@ -50,9 +50,9 @@ type YoutubeConfigForm = {
 };

 type RAGConfigForm = {
-	pdf_extract_images?: boolean;
-	enable_google_drive_integration?: boolean;
-	enable_onedrive_integration?: boolean;
+	PDF_EXTRACT_IMAGES?: boolean;
+	ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
+	ENABLE_ONEDRIVE_INTEGRATION?: boolean;
 	chunk?: ChunkConfigForm;
 	content_extraction?: ContentExtractConfigForm;
 	web_loader_ssl_verification?: boolean;
@@ -89,33 +89,6 @@ export const updateRAGConfig = async (token: string, payload: RAGConfigForm) =>
 	return res;
 };

-export const getRAGTemplate = async (token: string) => {
-	let error = null;
-
-	const res = await fetch(`${RETRIEVAL_API_BASE_URL}/template`, {
-		method: 'GET',
-		headers: {
-			'Content-Type': 'application/json',
-			Authorization: `Bearer ${token}`
-		}
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			console.log(err);
-			error = err.detail;
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res?.template ?? '';
-};
-
 export const getQuerySettings = async (token: string) => {
 	let error = null;

--- a/src/lib/components/admin/Settings/Documents.svelte
+++ b/src/lib/components/admin/Settings/Documents.svelte
--- a/src/lib/components/admin/Settings/WebSearch.svelte
+++ b/src/lib/components/admin/Settings/WebSearch.svelte
@@ -12,12 +12,6 @@

 	export let saveHandler: Function;

-	let webConfig = null;
-
-	let bypass_ssl_verification = null;
-	let tavily_api_key = null;
-	let youtube_language = null;
-
 	let webSearchEngines = [
 		'searxng',
 		'google_pse',
@@ -38,33 +32,25 @@
 		'perplexity',
 		'sougou'
 	];
-	let webLoaderEngines = ['safe_web', 'playwright', 'firecrawl', 'tavily'];
+	let webLoaderEngines = ['playwright', 'firecrawl', 'tavily'];
+
+	let webConfig = null;

 	const submitHandler = async () => {
 		// Convert domain filter string to array before sending
-		if (webConfig.search.domain_filter_list) {
-			webConfig.search.domain_filter_list = webConfig.search.domain_filter_list
-				.split(',')
+		if (webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST) {
+			webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST = webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST.split(',')
 				.map((domain) => domain.trim())
 				.filter((domain) => domain.length > 0);
 		} else {
-			webConfig.search.domain_filter_list = [];
+			webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST = [];
 		}

-		// Set the enable_ssl_verification flag based on the switch state
-		webConfig.loader.enable_ssl_verification = !bypass_ssl_verification;
-
-		// Set shared tavily_api_key
-		webConfig.search.tavily_api_key = tavily_api_key;
-		webConfig.loader.tavily_api_key = tavily_api_key;
-		webConfig.loader.youtube.language = youtube_language.split(',').map((lang) => lang.trim());
-
 		const res = await updateRAGConfig(localStorage.token, {
 			web: webConfig
 		});

-		webConfig.search.domain_filter_list = webConfig.search.domain_filter_list.join(', ');
-		youtube_language = webConfig.loader.youtube.language.join(', ');
+		webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST = webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST.join(', ');
 	};

 	onMount(async () => {
@@ -72,13 +58,14 @@

 		if (res) {
 			webConfig = res.web;
+
 			// Convert array back to comma-separated string for display
-			if (webConfig?.search?.domain_filter_list) {
-				webConfig.search.domain_filter_list = webConfig.search.domain_filter_list.join(', ');
+			if (webConfig?.WEB_SEARCH_DOMAIN_FILTER_LIST) {
+				webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST =
+					webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST.join(', ');
 			}
-			bypass_ssl_verification = !webConfig.loader.enable_ssl_verification;
-			tavily_api_key = webConfig.search.tavily_api_key || webConfig.loader.tavily_api_key;
-			youtube_language = webConfig.loader.youtube.language.join(', ');
+
+			webConfig.YOUTUBE_LOADER_LANGUAGE = webConfig.YOUTUBE_LOADER_LANGUAGE.join(',');
 		}
 	});
 </script>
@@ -100,10 +87,10 @@

 					<div class="  mb-2.5 flex w-full justify-between">
 						<div class=" self-center text-xs font-medium">
-							{$i18n.t('Enable Web Search')}
+							{$i18n.t('Web Search')}
 						</div>
 						<div class="flex items-center relative">
-							<Switch bind:state={webConfig.ENABLE_RAG_WEB_SEARCH} />
+							<Switch bind:state={webConfig.ENABLE_WEB_SEARCH} />
 						</div>
 					</div>

@@ -114,7 +101,7 @@
 						<div class="flex items-center relative">
 							<select
 								class="dark:bg-gray-900 w-fit pr-8 rounded-sm px-2 p-1 text-xs bg-transparent outline-hidden text-right"
-								bind:value={webConfig.search.engine}
+								bind:value={webConfig.WEB_SEARCH_ENGINE}
 								placeholder={$i18n.t('Select a engine')}
 								required
 							>
@@ -126,8 +113,8 @@
 						</div>
 					</div>

-					{#if webConfig.search.engine !== ''}
-						{#if webConfig.search.engine === 'searxng'}
+					{#if webConfig.WEB_SEARCH_ENGINE !== ''}
+						{#if webConfig.WEB_SEARCH_ENGINE === 'searxng'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -140,14 +127,14 @@
 												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
 												type="text"
 												placeholder={$i18n.t('Enter Searxng Query URL')}
-												bind:value={webConfig.search.searxng_query_url}
+												bind:value={webConfig.SEARXNG_QUERY_URL}
 												autocomplete="off"
 											/>
 										</div>
 									</div>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'google_pse'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'google_pse'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -156,7 +143,7 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Google PSE API Key')}
-										bind:value={webConfig.search.google_pse_api_key}
+										bind:value={webConfig.GOOGLE_PSE_API_KEY}
 									/>
 								</div>
 								<div class="mt-1.5">
@@ -170,14 +157,14 @@
 												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
 												type="text"
 												placeholder={$i18n.t('Enter Google PSE Engine Id')}
-												bind:value={webConfig.search.google_pse_engine_id}
+												bind:value={webConfig.GOOGLE_PSE_ENGINE_ID}
 												autocomplete="off"
 											/>
 										</div>
 									</div>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'brave'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'brave'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -186,11 +173,11 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Brave Search API Key')}
-										bind:value={webConfig.search.brave_search_api_key}
+										bind:value={webConfig.BRAVE_SEARCH_API_KEY}
 									/>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'kagi'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'kagi'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -199,11 +186,12 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Kagi Search API Key')}
-										bind:value={webConfig.search.kagi_search_api_key}
+										bind:value={webConfig.KAGI_SEARCH_API_KEY}
 									/>
 								</div>
+								.
 							</div>
-						{:else if webConfig.search.engine === 'mojeek'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'mojeek'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -212,11 +200,11 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Mojeek Search API Key')}
-										bind:value={webConfig.search.mojeek_search_api_key}
+										bind:value={webConfig.MOJEEK_SEARCH_API_KEY}
 									/>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'bocha'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'bocha'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -225,11 +213,11 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Bocha Search API Key')}
-										bind:value={webConfig.search.bocha_search_api_key}
+										bind:value={webConfig.BOCHA_SEARCH_API_KEY}
 									/>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'serpstack'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'serpstack'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -238,11 +226,11 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Serpstack API Key')}
-										bind:value={webConfig.search.serpstack_api_key}
+										bind:value={webConfig.SERPSTACK_API_KEY}
 									/>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'serper'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'serper'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -251,11 +239,11 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Serper API Key')}
-										bind:value={webConfig.search.serper_api_key}
+										bind:value={webConfig.SERPER_API_KEY}
 									/>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'serply'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'serply'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -264,11 +252,24 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Serply API Key')}
-										bind:value={webConfig.search.serply_api_key}
+										bind:value={webConfig.SERPLY_API_KEY}
 									/>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'searchapi'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'tavily'}
+							<div class="mb-2.5 flex w-full flex-col">
+								<div>
+									<div class=" self-center text-xs font-medium mb-1">
+										{$i18n.t('Tavily API Key')}
+									</div>
+
+									<SensitiveInput
+										placeholder={$i18n.t('Enter Tavily API Key')}
+										bind:value={webConfig.TAVILY_API_KEY}
+									/>
+								</div>
+							</div>
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'searchapi'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -277,7 +278,7 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter SearchApi API Key')}
-										bind:value={webConfig.search.searchapi_api_key}
+										bind:value={webConfig.SEARCHAPI_API_KEY}
 									/>
 								</div>
 								<div class="mt-1.5">
@@ -291,14 +292,14 @@
 												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
 												type="text"
 												placeholder={$i18n.t('Enter SearchApi Engine')}
-												bind:value={webConfig.search.searchapi_engine}
+												bind:value={webConfig.SEARCHAPI_ENGINE}
 												autocomplete="off"
 											/>
 										</div>
 									</div>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'serpapi'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'serpapi'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -307,7 +308,7 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter SerpApi API Key')}
-										bind:value={webConfig.search.serpapi_api_key}
+										bind:value={webConfig.SERPAPI_API_KEY}
 									/>
 								</div>
 								<div class="mt-1.5">
@@ -321,27 +322,14 @@
 												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
 												type="text"
 												placeholder={$i18n.t('Enter SerpApi Engine')}
-												bind:value={webConfig.search.serpapi_engine}
+												bind:value={webConfig.SERPAPI_ENGINE}
 												autocomplete="off"
 											/>
 										</div>
 									</div>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'tavily'}
-							<div class="mb-2.5 flex w-full flex-col">
-								<div>
-									<div class=" self-center text-xs font-medium mb-1">
-										{$i18n.t('Tavily API Key')}
-									</div>
-
-									<SensitiveInput
-										placeholder={$i18n.t('Enter Tavily API Key')}
-										bind:value={tavily_api_key}
-									/>
-								</div>
-							</div>
-						{:else if webConfig.search.engine === 'jina'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'jina'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -350,35 +338,11 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Jina API Key')}
-										bind:value={webConfig.search.jina_api_key}
+										bind:value={webConfig.JINA_API_KEY}
 									/>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'exa'}
-							<div class="mb-2.5 flex w-full flex-col">
-								<div>
-									<div class=" self-center text-xs font-medium mb-1">
-										{$i18n.t('Exa API Key')}
-									</div>
-
-									<SensitiveInput
-										placeholder={$i18n.t('Enter Exa API Key')}
-										bind:value={webConfig.search.exa_api_key}
-									/>
-								</div>
-							</div>
-						{:else if webConfig.search.engine === 'perplexity'}
-							<div>
-								<div class=" self-center text-xs font-medium mb-1">
-									{$i18n.t('Perplexity API Key')}
-								</div>
-
-								<SensitiveInput
-									placeholder={$i18n.t('Enter Perplexity API Key')}
-									bind:value={webConfig.search.perplexity_api_key}
-								/>
-							</div>
-						{:else if webConfig.search.engine === 'bing'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'bing'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -391,7 +355,7 @@
 												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
 												type="text"
 												placeholder={$i18n.t('Enter Bing Search V7 Endpoint')}
-												bind:value={webConfig.search.bing_search_v7_endpoint}
+												bind:value={webConfig.BING_SEARCH_V7_ENDPOINT}
 												autocomplete="off"
 											/>
 										</div>
@@ -405,11 +369,35 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Bing Search V7 Subscription Key')}
-										bind:value={webConfig.search.bing_search_v7_subscription_key}
+										bind:value={webConfig.BING_SEARCH_V7_SUBSCRIPTION_KEY}
 									/>
 								</div>
 							</div>
-						{:else if webConfig.search.engine === 'sougou'}
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'exa'}
+							<div class="mb-2.5 flex w-full flex-col">
+								<div>
+									<div class=" self-center text-xs font-medium mb-1">
+										{$i18n.t('Exa API Key')}
+									</div>
+
+									<SensitiveInput
+										placeholder={$i18n.t('Enter Exa API Key')}
+										bind:value={webConfig.EXA_API_KEY}
+									/>
+								</div>
+							</div>
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'perplexity'}
+							<div>
+								<div class=" self-center text-xs font-medium mb-1">
+									{$i18n.t('Perplexity API Key')}
+								</div>
+
+								<SensitiveInput
+									placeholder={$i18n.t('Enter Perplexity API Key')}
+									bind:value={webConfig.PERPLEXITY_API_KEY}
+								/>
+							</div>
+						{:else if webConfig.WEB_SEARCH_ENGINE === 'sougou'}
 							<div class="mb-2.5 flex w-full flex-col">
 								<div>
 									<div class=" self-center text-xs font-medium mb-1">
@@ -418,7 +406,7 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Sougou Search API sID')}
-										bind:value={webConfig.search.sougou_api_sid}
+										bind:value={webConfig.SOUGOU_API_SID}
 									/>
 								</div>
 							</div>
@@ -430,57 +418,99 @@

 									<SensitiveInput
 										placeholder={$i18n.t('Enter Sougou Search API SK')}
-										bind:value={webConfig.search.sougou_api_sk}
+										bind:value={webConfig.SOUGOU_API_SK}
 									/>
 								</div>
 							</div>
 						{/if}
 					{/if}

-					<div class="mb-2.5 flex w-full flex-col">
-						<div class="flex gap-2">
-							<div class="w-full">
-								<div class=" self-center text-xs font-medium mb-1">
-									{$i18n.t('Search Result Count')}
+					{#if webConfig.ENABLE_WEB_SEARCH}
+						<div class="mb-2.5 flex w-full flex-col">
+							<div class="flex gap-2">
+								<div class="w-full">
+									<div class=" self-center text-xs font-medium mb-1">
+										{$i18n.t('Search Result Count')}
+									</div>
+
+									<input
+										class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
+										placeholder={$i18n.t('Search Result Count')}
+										bind:value={webConfig.WEB_SEARCH_RESULT_COUNT}
+										required
+									/>
 								</div>

-								<input
-									class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
-									placeholder={$i18n.t('Search Result Count')}
-									bind:value={webConfig.search.result_count}
-									required
-								/>
-							</div>
+								<div class="w-full">
+									<div class=" self-center text-xs font-medium mb-1">
+										{$i18n.t('Concurrent Requests')}
+									</div>

-							<div class="w-full">
-								<div class=" self-center text-xs font-medium mb-1">
-									{$i18n.t('Concurrent Requests')}
+									<input
+										class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
+										placeholder={$i18n.t('Concurrent Requests')}
+										bind:value={webConfig.WEB_SEARCH_CONCURRENT_REQUESTS}
+										required
+									/>
 								</div>
-
-								<input
-									class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
-									placeholder={$i18n.t('Concurrent Requests')}
-									bind:value={webConfig.search.concurrent_requests}
-									required
-								/>
 							</div>
 						</div>
+
+						<div class="mb-2.5 flex w-full flex-col">
+							<div class="  text-xs font-medium mb-1">
+								{$i18n.t('Domain Filter List')}
+							</div>
+
+							<input
+								class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
+								placeholder={$i18n.t(
+									'Enter domains separated by commas (e.g., example.com,site.org)'
+								)}
+								bind:value={webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST}
+							/>
+						</div>
+					{/if}
+
+					<div class="  mb-2.5 flex w-full justify-between">
+						<div class=" self-center text-xs font-medium">
+							<Tooltip content={$i18n.t('Full Context Mode')} placement="top-start">
+								{$i18n.t('Bypass Embedding and Retrieval')}
+							</Tooltip>
+						</div>
+						<div class="flex items-center relative">
+							<Tooltip
+								content={webConfig.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL
+									? $i18n.t(
+											'Inject the entire content as context for comprehensive processing, this is recommended for complex queries.'
+										)
+									: $i18n.t(
+											'Default to segmented retrieval for focused and relevant content extraction, this is recommended for most cases.'
+										)}
+							>
+								<Switch bind:state={webConfig.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL} />
+							</Tooltip>
+						</div>
 					</div>

-					<div class="mb-2.5 flex w-full flex-col">
-						<div class="  text-xs font-medium mb-1">
-							{$i18n.t('Domain Filter List')}
+					<div class="  mb-2.5 flex w-full justify-between">
+						<div class=" self-center text-xs font-medium">
+							{$i18n.t('Trust Proxy Environment')}
+						</div>
+						<div class="flex items-center relative">
+							<Tooltip
+								content={webConfig.WEB_SEARCH_TRUST_ENV
+									? $i18n.t(
+											'Use proxy designated by http_proxy and https_proxy environment variables to fetch page contents.'
+										)
+									: $i18n.t('Use no proxy to fetch page contents.')}
+							>
+								<Switch bind:state={webConfig.WEB_SEARCH_TRUST_ENV} />
+							</Tooltip>
 						</div>
-
-						<input
-							class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300dark:bg-gray-850 outline-hidden"
-							placeholder={$i18n.t(
-								'Enter domains separated by commas (e.g., example.com,site.org)'
-							)}
-							bind:value={webConfig.search.domain_filter_list}
-						/>
 					</div>
+				</div>

+				<div class="mb-3">
 					<div class=" mb-2.5 text-base font-medium">{$i18n.t('Loader')}</div>

 					<hr class=" border-gray-100 dark:border-gray-850 my-2" />
@@ -492,11 +522,11 @@
 						<div class="flex items-center relative">
 							<select
 								class="dark:bg-gray-900 w-fit pr-8 rounded-sm px-2 p-1 text-xs bg-transparent outline-hidden text-right"
-								bind:value={webConfig.loader.engine}
+								bind:value={webConfig.WEB_LOADER_ENGINE}
 								placeholder={$i18n.t('Select a engine')}
 								required
 							>
-								<option disabled selected value="">{$i18n.t('Select a engine')}</option>
+								<option value="">{$i18n.t('Default')}</option>
 								{#each webLoaderEngines as engine}
 									<option value={engine}>{engine}</option>
 								{/each}
@@ -504,113 +534,118 @@
 						</div>
 					</div>

-					{#if webConfig.loader.engine !== ''}
-						{#if webConfig.loader.engine === 'playwright'}
-							<div class="mb-2.5 flex w-full flex-col">
-								<div>
-									<div class=" self-center text-xs font-medium mb-1">
-										{$i18n.t('Playwright WebSocket URL')}
-									</div>
-
-									<div class="flex w-full">
-										<div class="flex-1">
-											<input
-												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
-												type="text"
-												placeholder={$i18n.t('Enter Playwright WebSocket URL')}
-												bind:value={webConfig.loader.playwright_ws_uri}
-												autocomplete="off"
-											/>
-										</div>
-									</div>
+					{#if webConfig.WEB_LOADER_ENGINE === '' || webConfig.WEB_LOADER_ENGINE === 'safe_web'}
+						<div class="  mb-2.5 flex w-full justify-between">
+							<div class=" self-center text-xs font-medium">
+								{$i18n.t('Verify SSL Certificate')}
+							</div>
+							<div class="flex items-center relative">
+								<Switch bind:state={webConfig.ENABLE_WEB_LOADER_SSL_VERIFICATION} />
+							</div>
+						</div>
+					{:else if webConfig.WEB_LOADER_ENGINE === 'playwright'}
+						<div class="mb-2.5 flex w-full flex-col">
+							<div>
+								<div class=" self-center text-xs font-medium mb-1">
+									{$i18n.t('Playwright WebSocket URL')}
 								</div>

-								<div class="mt-2">
-									<div class=" self-center text-xs font-medium mb-1">
-										{$i18n.t('Playwright Timeout (ms)')}
-									</div>
-
-									<div class="flex w-full">
-										<div class="flex-1">
-											<input
-												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
-												placeholder={$i18n.t('Enter Playwright Timeout (ms)')}
-												bind:value={webConfig.loader.playwright_timeout}
-												autocomplete="off"
-											/>
-										</div>
+								<div class="flex w-full">
+									<div class="flex-1">
+										<input
+											class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
+											type="text"
+											placeholder={$i18n.t('Enter Playwright WebSocket URL')}
+											bind:value={webConfig.PLAYWRIGHT_WS_URL}
+											autocomplete="off"
+										/>
 									</div>
 								</div>
 							</div>
-						{:else if webConfig.loader.engine === 'firecrawl'}
-							<div class="mb-2.5 flex w-full flex-col">
-								<div>
-									<div class=" self-center text-xs font-medium mb-1">
-										{$i18n.t('Firecrawl API Base URL')}
-									</div>

-									<div class="flex w-full">
-										<div class="flex-1">
-											<input
-												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
-												type="text"
-												placeholder={$i18n.t('Enter Firecrawl API Base URL')}
-												bind:value={webConfig.loader.firecrawl_api_base_url}
-												autocomplete="off"
-											/>
-										</div>
-									</div>
+							<div class="mt-2">
+								<div class=" self-center text-xs font-medium mb-1">
+									{$i18n.t('Playwright Timeout (ms)')}
 								</div>

+								<div class="flex w-full">
+									<div class="flex-1">
+										<input
+											class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
+											placeholder={$i18n.t('Enter Playwright Timeout')}
+											bind:value={webConfig.PLAYWRIGHT_TIMEOUT}
+											autocomplete="off"
+										/>
+									</div>
+								</div>
+							</div>
+						</div>
+					{:else if webConfig.WEB_LOADER_ENGINE === 'firecrawl'}
+						<div class="mb-2.5 flex w-full flex-col">
+							<div>
+								<div class=" self-center text-xs font-medium mb-1">
+									{$i18n.t('Firecrawl API Base URL')}
+								</div>
+
+								<div class="flex w-full">
+									<div class="flex-1">
+										<input
+											class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
+											type="text"
+											placeholder={$i18n.t('Enter Firecrawl API Base URL')}
+											bind:value={webConfig.FIRECRAWL_API_BASE_URL}
+											autocomplete="off"
+										/>
+									</div>
+								</div>
+							</div>
+
+							<div class="mt-2">
+								<div class=" self-center text-xs font-medium mb-1">
+									{$i18n.t('Firecrawl API Key')}
+								</div>
+
+								<SensitiveInput
+									placeholder={$i18n.t('Enter Firecrawl API Key')}
+									bind:value={webConfig.FIRECRAWL_API_KEY}
+								/>
+							</div>
+						</div>
+					{:else if webConfig.WEB_LOADER_ENGINE === 'tavily'}
+						<div class="mb-2.5 flex w-full flex-col">
+							<div>
+								<div class=" self-center text-xs font-medium mb-1">
+									{$i18n.t('Tavily Extract Depth')}
+								</div>
+
+								<div class="flex w-full">
+									<div class="flex-1">
+										<input
+											class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
+											type="text"
+											placeholder={$i18n.t('Enter Tavily Extract Depth')}
+											bind:value={webConfig.TAVILY_EXTRACT_DEPTH}
+											autocomplete="off"
+										/>
+									</div>
+								</div>
+							</div>
+
+							{#if webConfig.WEB_SEARCH_ENGINE !== 'tavily'}
 								<div class="mt-2">
 									<div class=" self-center text-xs font-medium mb-1">
-										{$i18n.t('Firecrawl API Key')}
+										{$i18n.t('Tavily API Key')}
 									</div>

 									<SensitiveInput
-										placeholder={$i18n.t('Enter Firecrawl API Key')}
-										bind:value={webConfig.loader.firecrawl_api_key}
+										placeholder={$i18n.t('Enter Tavily API Key')}
+										bind:value={webConfig.TAVILY_API_KEY}
 									/>
 								</div>
-							</div>
-						{:else if webConfig.loader.engine === 'tavily'}
-							<div class="mb-2.5 flex w-full flex-col">
-								<div>
-									<div class=" self-center text-xs font-medium mb-1">
-										{$i18n.t('Tavily Extract Depth')}
-									</div>
-
-									<div class="flex w-full">
-										<div class="flex-1">
-											<input
-												class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
-												type="text"
-												placeholder={$i18n.t('Enter Tavily Extract Depth')}
-												bind:value={webConfig.loader.tavily_extract_depth}
-												autocomplete="off"
-											/>
-										</div>
-									</div>
-								</div>
-
-								{#if webConfig.search.engine !== 'tavily'}
-									<div class="mt-2">
-										<div class=" self-center text-xs font-medium mb-1">
-											{$i18n.t('Tavily API Key')}
-										</div>
-
-										<SensitiveInput
-											placeholder={$i18n.t('Enter Tavily API Key')}
-											bind:value={tavily_api_key}
-										/>
-									</div>
-								{/if}
-							</div>
-						{/if}
+							{/if}
+						</div>
 					{/if}

-					<hr class=" border-gray-100 dark:border-gray-850 my-2" />
-
 					<div class="  mb-2.5 flex w-full justify-between">
 						<div class=" self-center text-xs font-medium">
 							{$i18n.t('Youtube Language')}
@@ -620,7 +655,7 @@
 								class="flex-1 w-full rounded-lg text-sm bg-transparent outline-hidden"
 								type="text"
 								placeholder={$i18n.t('Enter language codes')}
-								bind:value={youtube_language}
+								bind:value={webConfig.YOUTUBE_LOADER_LANGUAGE}
 								autocomplete="off"
 							/>
 						</div>
@@ -632,63 +667,14 @@
 						</div>
 						<div class="flex items-center relative">
 							<input
-								class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
+								class="flex-1 w-full rounded-lg text-sm bg-transparent outline-hidden"
 								type="text"
 								placeholder={$i18n.t('Enter proxy URL (e.g. https://user:password@host:port)')}
-								bind:value={webConfig.loader.youtube.proxy_url}
+								bind:value={webConfig.YOUTUBE_LOADER_PROXY_URL}
 								autocomplete="off"
 							/>
 						</div>
 					</div>
-
-					<hr class=" border-gray-100 dark:border-gray-850 my-2" />
-
-					<div class="  mb-2.5 flex w-full justify-between">
-						<div class=" self-center text-xs font-medium">
-							{$i18n.t('Bypass SSL verification for Websites')}
-						</div>
-						<div class="flex items-center relative">
-							<Switch bind:state={bypass_ssl_verification} />
-						</div>
-					</div>
-
-					<div class="  mb-2.5 flex w-full justify-between">
-						<div class=" self-center text-xs font-medium">
-							{$i18n.t('Trust Proxy Environment')}
-						</div>
-						<div class="flex items-center relative">
-							<Tooltip
-								content={webConfig.loader.trust_env
-									? $i18n.t(
-											'Use proxy designated by http_proxy and https_proxy environment variables to fetch page contents.'
-										)
-									: $i18n.t('Use no proxy to fetch page contents.')}
-							>
-								<Switch bind:state={webConfig.loader.trust_env} />
-							</Tooltip>
-						</div>
-					</div>
-
-					<div class="  mb-2.5 flex w-full justify-between">
-						<div class=" self-center text-xs font-medium">
-							<Tooltip content={$i18n.t('Full Context Mode')} placement="top-start">
-								{$i18n.t('Bypass Embedding and Retrieval')}
-							</Tooltip>
-						</div>
-						<div class="flex items-center relative">
-							<Tooltip
-								content={webConfig.loader.bypass_embedding_and_retrieval
-									? $i18n.t(
-											'Inject the entire content as context for comprehensive processing, this is recommended for complex queries.'
-										)
-									: $i18n.t(
-											'Default to segmented retrieval for focused and relevant content extraction, this is recommended for most cases.'
-										)}
-							>
-								<Switch bind:state={webConfig.loader.bypass_embedding_and_retrieval} />
-							</Tooltip>
-						</div>
-					</div>
 				</div>
 			</div>
 		{/if}
--- a/src/lib/utils/rag/index.ts
+++ b/src/lib/utils/rag/index.ts
@@ -1,24 +0,0 @@
-import { getRAGTemplate } from '$lib/apis/retrieval';
-
-export const RAGTemplate = async (token: string, context: string, query: string) => {
-	let template = await getRAGTemplate(token).catch(() => {
-		return `Use the following context as your learned knowledge, inside <context></context> XML tags.
-		<context>
-		  [context]
-		</context>
-		
-		When answer to user:
-		- If you don't know, just say that you don't know.
-		- If you don't know when you are not sure, ask for clarification.
-		Avoid mentioning that you obtained the information from the context.
-		And answer according to the language of the user's question.
-				
-		Given the context information, answer the query.
-		Query: [query]`;
-	});
-
-	template = template.replace(/\[context\]/g, context);
-	template = template.replace(/\[query\]/g, query);
-
-	return template;
-};