From b50dcb18626e28f29ecd3f587354a2b25e822d31 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Wed, 7 May 2025 22:25:18 +0400 Subject: [PATCH] refac: remove duplicate urls --- backend/open_webui/routers/retrieval.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index f75b03483..bac75e905 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -1601,6 +1601,10 @@ async def process_web_search( doc.metadata.get("source") for doc in docs if doc.metadata.get("source") ] # only keep URLs + # Remove duplicates + urls = list(dict.fromkeys(urls)) + log.debug(f"urls: {urls}") + if request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: return { "status": True,