UniqAI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-02-24 06:06:14 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

fix: Filter out invalid RAG web URLs (continued)

Browse Source
This commit is contained in:
Rory 2025-02-03 18:18:49 -06:00
parent 121a13d4ed
commit 3db6b4352f
1 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
backend/open_webui/retrieval/web/utils.py
View File

@ -42,6 +42,15 @@ def validate_url(url: Union[str, Sequence[str]]):
else: else:
return False return False
def safe_validate_urls(url: Sequence[str]) -> Sequence[str]:
valid_urls = []
for u in url:
try:
if validate_url(u):
valid_urls.append(u)
except ValueError:
continue
return valid_urls
def resolve_hostname(hostname): def resolve_hostname(hostname):
# Get address information # Get address information
@ -86,11 +95,11 @@ def get_web_loader(
verify_ssl: bool = True, verify_ssl: bool = True,
requests_per_second: int = 2, requests_per_second: int = 2,
): ):
# Check if the URL is valid # Check if the URLs are valid
if not validate_url(urls): safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
raise ValueError(ERROR_MESSAGES.INVALID_URL)
return SafeWebBaseLoader( return SafeWebBaseLoader(
urls, safe_urls,
verify_ssl=verify_ssl, verify_ssl=verify_ssl,
requests_per_second=requests_per_second, requests_per_second=requests_per_second,
continue_on_failure=True, continue_on_failure=True,