From a506a1a61ed05ad2d06a3fb0d716c6b0684359c8 Mon Sep 17 00:00:00 2001
From: Jan Kessler <jakessle@uni-mainz.de>
Date: Sun, 6 Apr 2025 20:31:12 +0200
Subject: [PATCH] only keep URLs as sources for which the content could
 actually be retrieved

---
 backend/open_webui/routers/retrieval.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py
index 6f71e11d3..7c010a16a 100644
--- a/backend/open_webui/routers/retrieval.py
+++ b/backend/open_webui/routers/retrieval.py
@@ -1478,6 +1478,7 @@ async def process_web_search(
             trust_env=request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
         )
         docs = await loader.aload()
+        urls = [doc.metadata["source"] for doc in docs] # only keep URLs which could be retrieved
 
         if request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL:
             return {