diff --git a/backend/apps/rag/main.py b/backend/apps/rag/main.py index bbd33618f..8d1ea27e1 100644 --- a/backend/apps/rag/main.py +++ b/backend/apps/rag/main.py @@ -313,6 +313,7 @@ async def get_rag_config(user=Depends(get_admin_user)): "chunk_size": app.state.CHUNK_SIZE, "chunk_overlap": app.state.CHUNK_OVERLAP, }, + "web_loader_ssl_verification": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION, } @@ -322,15 +323,34 @@ class ChunkParamUpdateForm(BaseModel): class ConfigUpdateForm(BaseModel): - pdf_extract_images: bool - chunk: ChunkParamUpdateForm + pdf_extract_images: Optional[bool] = None + chunk: Optional[ChunkParamUpdateForm] = None + web_loader_ssl_verification: Optional[bool] = None @app.post("/config/update") async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_user)): - app.state.PDF_EXTRACT_IMAGES = form_data.pdf_extract_images - app.state.CHUNK_SIZE = form_data.chunk.chunk_size - app.state.CHUNK_OVERLAP = form_data.chunk.chunk_overlap + app.state.PDF_EXTRACT_IMAGES = ( + form_data.pdf_extract_images + if form_data.pdf_extract_images != None + else app.state.PDF_EXTRACT_IMAGES + ) + + app.state.CHUNK_SIZE = ( + form_data.chunk.chunk_size if form_data.chunk != None else app.state.CHUNK_SIZE + ) + + app.state.CHUNK_OVERLAP = ( + form_data.chunk.chunk_overlap + if form_data.chunk != None + else app.state.CHUNK_OVERLAP + ) + + app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = ( + form_data.web_loader_ssl_verification + if form_data.web_loader_ssl_verification != None + else app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION + ) return { "status": True, @@ -339,6 +359,7 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_ "chunk_size": app.state.CHUNK_SIZE, "chunk_overlap": app.state.CHUNK_OVERLAP, }, + "web_loader_ssl_verification": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION, } @@ -490,10 +511,9 @@ def store_youtube_video(form_data: UrlForm, user=Depends(get_current_user)): def store_web(form_data: UrlForm, user=Depends(get_current_user)): # "https://www.gutenberg.org/files/1727/1727-h/1727-h.htm" try: - loader = get_web_loader(form_data.url) - loader.requests_kwargs = { - "verify": app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION - } + loader = get_web_loader( + form_data.url, verify_ssl=app.state.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION + ) data = loader.load() collection_name = form_data.collection_name @@ -514,7 +534,7 @@ def store_web(form_data: UrlForm, user=Depends(get_current_user)): ) -def get_web_loader(url: str): +def get_web_loader(url: str, verify_ssl: bool = True): # Check if the URL is valid if isinstance(validators.url(url), validators.ValidationError): raise ValueError(ERROR_MESSAGES.INVALID_URL) @@ -531,7 +551,7 @@ def get_web_loader(url: str): for ip in ipv6_addresses: if validators.ipv6(ip, private=True): raise ValueError(ERROR_MESSAGES.INVALID_URL) - return WebBaseLoader(url) + return WebBaseLoader(url, verify_ssl=verify_ssl) def resolve_hostname(hostname): diff --git a/backend/config.py b/backend/config.py index 9963db44a..17091e4ac 100644 --- a/backend/config.py +++ b/backend/config.py @@ -456,7 +456,7 @@ ENABLE_RAG_HYBRID_SEARCH = ( ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = ( - os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "").lower() == "true" + os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "True").lower() == "true" ) RAG_EMBEDDING_ENGINE = os.environ.get("RAG_EMBEDDING_ENGINE", "") diff --git a/src/lib/apis/rag/index.ts b/src/lib/apis/rag/index.ts index a9d163f87..ccf166dab 100644 --- a/src/lib/apis/rag/index.ts +++ b/src/lib/apis/rag/index.ts @@ -33,8 +33,9 @@ type ChunkConfigForm = { }; type RAGConfigForm = { - pdf_extract_images: boolean; - chunk: ChunkConfigForm; + pdf_extract_images?: boolean; + chunk?: ChunkConfigForm; + web_loader_ssl_verification?: boolean; }; export const updateRAGConfig = async (token: string, payload: RAGConfigForm) => { diff --git a/src/lib/components/documents/Settings/WebLoader.svelte b/src/lib/components/documents/Settings/WebLoader.svelte deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/lib/components/documents/Settings/WebParams.svelte b/src/lib/components/documents/Settings/WebParams.svelte new file mode 100644 index 000000000..bb78686d1 --- /dev/null +++ b/src/lib/components/documents/Settings/WebParams.svelte @@ -0,0 +1,74 @@ + + +
{ + submitHandler(); + saveHandler(); + }} +> +
+
+
+ {$i18n.t('Retrieval Augmented Generation Settings')} +
+ +
+
+
+ {$i18n.t('Bypass SSL verification for Websites')} +
+ + +
+
+
+
+
+ +
+
diff --git a/src/lib/components/documents/SettingsModal.svelte b/src/lib/components/documents/SettingsModal.svelte index 63e7dcf0d..cff059866 100644 --- a/src/lib/components/documents/SettingsModal.svelte +++ b/src/lib/components/documents/SettingsModal.svelte @@ -4,6 +4,8 @@ import General from './Settings/General.svelte'; import ChunkParams from './Settings/ChunkParams.svelte'; import QueryParams from './Settings/QueryParams.svelte'; + import WebParams from './Settings/WebParams.svelte'; + import { toast } from 'svelte-sonner'; const i18n = getContext('i18n'); @@ -123,14 +125,56 @@
{$i18n.t('Query Params')}
+ +
{#if selectedTab === 'general'} - {}} /> + { + toast.success($i18n.t('Settings saved successfully!')); + }} + /> {:else if selectedTab === 'chunk'} - {}} /> + { + toast.success($i18n.t('Settings saved successfully!')); + }} + /> {:else if selectedTab === 'query'} - {}} /> + { + toast.success($i18n.t('Settings saved successfully!')); + }} + /> + {:else if selectedTab === 'web'} + { + toast.success($i18n.t('Settings saved successfully!')); + }} + /> {/if}