From 9ed1a31575c1b4dfec1628ac79846d3847a2e030 Mon Sep 17 00:00:00 2001 From: Jun Siang Cheah Date: Sun, 12 May 2024 15:19:07 +0800 Subject: [PATCH] fix: continue with failures when bulk loading urls with WebBaseLoader --- backend/apps/rag/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/apps/rag/main.py b/backend/apps/rag/main.py index 03604d34f..e63d06208 100644 --- a/backend/apps/rag/main.py +++ b/backend/apps/rag/main.py @@ -584,6 +584,7 @@ def get_web_loader(url: Union[str, Sequence[str]], verify_ssl: bool = True): url, verify_ssl=verify_ssl, requests_per_second=RAG_WEB_SEARCH_CONCURRENT_REQUESTS, + continue_on_failure=True, )