From 4e8b3906821a9a10f4fd0038373291dff41b65cf Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Tue, 28 Jan 2025 23:03:15 -0600
Subject: [PATCH 01/11] Add RAG_WEB_LOADER + Playwright mode + improve
 stability of search

---
 backend/open_webui/config.py              |   5 +
 backend/open_webui/main.py                |   2 +
 backend/open_webui/retrieval/web/main.py  |   4 +
 backend/open_webui/retrieval/web/utils.py | 179 +++++++++++++++++++---
 backend/open_webui/routers/retrieval.py   |  21 ++-
 backend/open_webui/utils/middleware.py    |  27 ++--
 backend/requirements.txt                  |   2 +-
 backend/start.sh                          |   9 ++
 backend/start_windows.bat                 |   9 ++
 pyproject.toml                            |   1 +
 10 files changed, 220 insertions(+), 39 deletions(-)

diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py
index c37b831de..3cec6edd7 100644
--- a/backend/open_webui/config.py
+++ b/backend/open_webui/config.py
@@ -1712,6 +1712,11 @@ RAG_WEB_SEARCH_CONCURRENT_REQUESTS = PersistentConfig(
     int(os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")),
 )
 
+RAG_WEB_LOADER = PersistentConfig(
+    "RAG_WEB_LOADER",
+    "rag.web.loader",
+    os.environ.get("RAG_WEB_LOADER", "safe_web")
+)
 
 ####################################
 # Images
diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py
index 00270aabc..985624d81 100644
--- a/backend/open_webui/main.py
+++ b/backend/open_webui/main.py
@@ -129,6 +129,7 @@ from open_webui.config import (
     AUDIO_TTS_VOICE,
     AUDIO_TTS_AZURE_SPEECH_REGION,
     AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
+    RAG_WEB_LOADER,
     WHISPER_MODEL,
     WHISPER_MODEL_AUTO_UPDATE,
     WHISPER_MODEL_DIR,
@@ -526,6 +527,7 @@ app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_K
 
 app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
 app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
+app.state.config.RAG_WEB_LOADER = RAG_WEB_LOADER
 
 app.state.EMBEDDING_FUNCTION = None
 app.state.ef = None
diff --git a/backend/open_webui/retrieval/web/main.py b/backend/open_webui/retrieval/web/main.py
index 1af8a70aa..28a749e7d 100644
--- a/backend/open_webui/retrieval/web/main.py
+++ b/backend/open_webui/retrieval/web/main.py
@@ -1,3 +1,5 @@
+import validators
+
 from typing import Optional
 from urllib.parse import urlparse
 
@@ -10,6 +12,8 @@ def get_filtered_results(results, filter_list):
     filtered_results = []
     for result in results:
         url = result.get("url") or result.get("link", "")
+        if not validators.url(url):
+            continue
         domain = urlparse(url).netloc
         if any(domain.endswith(filtered_domain) for filtered_domain in filter_list):
             filtered_results.append(result)
diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index a322bbbfc..bdc626749 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -1,16 +1,21 @@
+import asyncio
+from datetime import datetime, time, timedelta
 import socket
+import ssl
 import urllib.parse
+import certifi
 import validators
-from typing import Union, Sequence, Iterator
+from typing import AsyncIterator, Dict, List, Optional, Union, Sequence, Iterator
 
 from langchain_community.document_loaders import (
     WebBaseLoader,
+    PlaywrightURLLoader
 )
 from langchain_core.documents import Document
 
 
 from open_webui.constants import ERROR_MESSAGES
-from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH
+from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, RAG_WEB_LOADER
 from open_webui.env import SRC_LOG_LEVELS
 
 import logging
@@ -42,6 +47,15 @@ def validate_url(url: Union[str, Sequence[str]]):
     else:
         return False
 
+def safe_validate_urls(url: Sequence[str]) -> Sequence[str]:
+    valid_urls = []
+    for u in url:
+        try:
+            if validate_url(u):
+                valid_urls.append(u)
+        except ValueError:
+            continue
+    return valid_urls
 
 def resolve_hostname(hostname):
     # Get address information
@@ -53,6 +67,131 @@ def resolve_hostname(hostname):
 
     return ipv4_addresses, ipv6_addresses
 
+def extract_metadata(soup, url):
+    metadata = {
+        "source": url
+    }
+    if title := soup.find("title"):
+        metadata["title"] = title.get_text()
+    if description := soup.find("meta", attrs={"name": "description"}):
+        metadata["description"] = description.get(
+            "content", "No description found."
+        )
+    if html := soup.find("html"):
+        metadata["language"] = html.get("lang", "No language found.")
+    return metadata
+
+class SafePlaywrightURLLoader(PlaywrightURLLoader):
+    """Load HTML pages safely with Playwright, supporting SSL verification and rate limiting.
+    
+    Attributes:
+        urls (List[str]): List of URLs to load.
+        verify_ssl (bool): If True, verify SSL certificates.
+        requests_per_second (Optional[float]): Number of requests per second to limit to.
+        continue_on_failure (bool): If True, continue loading other URLs on failure.
+        headless (bool): If True, the browser will run in headless mode.
+    """
+
+    def __init__(
+        self,
+        urls: List[str],
+        verify_ssl: bool = True,
+        requests_per_second: Optional[float] = None,
+        continue_on_failure: bool = True,
+        headless: bool = True,
+        remove_selectors: Optional[List[str]] = None,
+        proxy: Optional[Dict[str, str]] = None
+    ):
+        """Initialize with additional safety parameters."""
+        super().__init__(
+            urls=urls,
+            continue_on_failure=continue_on_failure,
+            headless=headless,
+            remove_selectors=remove_selectors,
+            proxy=proxy
+        )
+        self.verify_ssl = verify_ssl
+        self.requests_per_second = requests_per_second
+        self.last_request_time = None
+
+    def _verify_ssl_cert(self, url: str) -> bool:
+        """Verify SSL certificate for the given URL."""
+        if not url.startswith("https://"):
+            return True
+            
+        try:
+            hostname = url.split("://")[-1].split("/")[0]
+            context = ssl.create_default_context(cafile=certifi.where())
+            with context.wrap_socket(ssl.socket(), server_hostname=hostname) as s:
+                s.connect((hostname, 443))
+            return True
+        except ssl.SSLError:
+            return False
+        except Exception as e:
+            log.warning(f"SSL verification failed for {url}: {str(e)}")
+            return False
+
+    async def _wait_for_rate_limit(self):
+        """Wait to respect the rate limit if specified."""
+        if self.requests_per_second and self.last_request_time:
+            min_interval = timedelta(seconds=1.0 / self.requests_per_second)
+            time_since_last = datetime.now() - self.last_request_time
+            if time_since_last < min_interval:
+                await asyncio.sleep((min_interval - time_since_last).total_seconds())
+        self.last_request_time = datetime.now()
+
+    def _sync_wait_for_rate_limit(self):
+        """Synchronous version of rate limit wait."""
+        if self.requests_per_second and self.last_request_time:
+            min_interval = timedelta(seconds=1.0 / self.requests_per_second)
+            time_since_last = datetime.now() - self.last_request_time
+            if time_since_last < min_interval:
+                time.sleep((min_interval - time_since_last).total_seconds())
+        self.last_request_time = datetime.now()
+
+    async def _safe_process_url(self, url: str) -> bool:
+        """Perform safety checks before processing a URL."""
+        if self.verify_ssl and not self._verify_ssl_cert(url):
+            raise ValueError(f"SSL certificate verification failed for {url}")
+        await self._wait_for_rate_limit()
+        return True
+
+    def _safe_process_url_sync(self, url: str) -> bool:
+        """Synchronous version of safety checks."""
+        if self.verify_ssl and not self._verify_ssl_cert(url):
+            raise ValueError(f"SSL certificate verification failed for {url}")
+        self._sync_wait_for_rate_limit()
+        return True
+
+    async def alazy_load(self) -> AsyncIterator[Document]:
+        """Safely load URLs asynchronously."""
+        parent_iterator = super().alazy_load()
+        
+        async for document in parent_iterator:
+            url = document.metadata["source"]
+            try:
+                await self._safe_process_url(url)
+                yield document
+            except Exception as e:
+                if self.continue_on_failure:
+                    log.error(f"Error processing {url}, exception: {e}")
+                    continue
+                raise e
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Safely load URLs synchronously."""
+        parent_iterator = super().lazy_load()
+        
+        for document in parent_iterator:
+            url = document.metadata["source"]
+            try:
+                self._safe_process_url_sync(url)
+                yield document
+            except Exception as e:
+                if self.continue_on_failure:
+                    log.error(f"Error processing {url}, exception: {e}")
+                    continue
+                raise e
 
 class SafeWebBaseLoader(WebBaseLoader):
     """WebBaseLoader with enhanced error handling for URLs."""
@@ -65,15 +204,7 @@ class SafeWebBaseLoader(WebBaseLoader):
                 text = soup.get_text(**self.bs_get_text_kwargs)
 
                 # Build metadata
-                metadata = {"source": path}
-                if title := soup.find("title"):
-                    metadata["title"] = title.get_text()
-                if description := soup.find("meta", attrs={"name": "description"}):
-                    metadata["description"] = description.get(
-                        "content", "No description found."
-                    )
-                if html := soup.find("html"):
-                    metadata["language"] = html.get("lang", "No language found.")
+                metadata = extract_metadata(soup, path)
 
                 yield Document(page_content=text, metadata=metadata)
             except Exception as e:
@@ -87,11 +218,21 @@ def get_web_loader(
     requests_per_second: int = 2,
 ):
     # Check if the URL is valid
-    if not validate_url(urls):
-        raise ValueError(ERROR_MESSAGES.INVALID_URL)
-    return SafeWebBaseLoader(
-        urls,
-        verify_ssl=verify_ssl,
-        requests_per_second=requests_per_second,
-        continue_on_failure=True,
-    )
+    safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
+
+    if RAG_WEB_LOADER.value == "chromium":
+        log.info("Using SafePlaywrightURLLoader")
+        return SafePlaywrightURLLoader(
+            safe_urls,
+            verify_ssl=verify_ssl,
+            requests_per_second=requests_per_second,
+            continue_on_failure=True,
+        )
+    else:
+        log.info("Using SafeWebBaseLoader")
+        return SafeWebBaseLoader(
+            safe_urls,
+            verify_ssl=verify_ssl,
+            requests_per_second=requests_per_second,
+            continue_on_failure=True,
+        )
diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py
index 2cffd9ead..e65a76050 100644
--- a/backend/open_webui/routers/retrieval.py
+++ b/backend/open_webui/routers/retrieval.py
@@ -1238,9 +1238,11 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
 
 
 @router.post("/process/web/search")
-def process_web_search(
-    request: Request, form_data: SearchForm, user=Depends(get_verified_user)
+async def process_web_search(
+    request: Request, form_data: SearchForm, extra_params: dict, user=Depends(get_verified_user)
 ):
+    event_emitter = extra_params["__event_emitter__"]
+    
     try:
         logging.info(
             f"trying to web search with {request.app.state.config.RAG_WEB_SEARCH_ENGINE, form_data.query}"
@@ -1258,6 +1260,18 @@ def process_web_search(
 
     log.debug(f"web_results: {web_results}")
 
+    await event_emitter(
+        {
+            "type": "status",
+            "data": {
+                "action": "web_search",
+                "description": "Loading {{count}} sites...",
+                "urls": [result.link for result in web_results],
+                "done": False
+            },
+        }
+    )
+
     try:
         collection_name = form_data.collection_name
         if collection_name == "" or collection_name is None:
@@ -1271,7 +1285,8 @@ def process_web_search(
             verify_ssl=request.app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
             requests_per_second=request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
         )
-        docs = loader.load()
+        docs = [doc async for doc in loader.alazy_load()]
+        # docs = loader.load()
         save_docs_to_vector_db(request, docs, collection_name, overwrite=True)
 
         return {
diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py
index 6b2329be1..27e499e0c 100644
--- a/backend/open_webui/utils/middleware.py
+++ b/backend/open_webui/utils/middleware.py
@@ -419,21 +419,16 @@ async def chat_web_search_handler(
 
     try:
 
-        # Offload process_web_search to a separate thread
-        loop = asyncio.get_running_loop()
-        with ThreadPoolExecutor() as executor:
-            results = await loop.run_in_executor(
-                executor,
-                lambda: process_web_search(
-                    request,
-                    SearchForm(
-                        **{
-                            "query": searchQuery,
-                        }
-                    ),
-                    user,
-                ),
-            )
+        results = await process_web_search(
+            request,
+            SearchForm(
+                **{
+                    "query": searchQuery,
+                }
+            ),
+            extra_params=extra_params,
+            user=user
+        )
 
         if results:
             await event_emitter(
@@ -441,7 +436,7 @@ async def chat_web_search_handler(
                     "type": "status",
                     "data": {
                         "action": "web_search",
-                        "description": "Searched {{count}} sites",
+                        "description": "Loaded {{count}} sites",
                         "query": searchQuery,
                         "urls": results["filenames"],
                         "done": True,
diff --git a/backend/requirements.txt b/backend/requirements.txt
index eecb9c4a5..0dd7b1a8a 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -46,7 +46,7 @@ chromadb==0.6.2
 pymilvus==2.5.0
 qdrant-client~=1.12.0
 opensearch-py==2.7.1
-
+playwright==1.49.1
 
 transformers
 sentence-transformers==3.3.1
diff --git a/backend/start.sh b/backend/start.sh
index a945acb62..ce56b1867 100755
--- a/backend/start.sh
+++ b/backend/start.sh
@@ -3,6 +3,15 @@
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 cd "$SCRIPT_DIR" || exit
 
+# Add conditional Playwright browser installation
+if [[ "${RAG_WEB_LOADER,,}" == "chromium" ]]; then
+    echo "Installing Playwright browsers..."
+    playwright install chromium
+    playwright install-deps chromium
+
+    python -c "import nltk; nltk.download('punkt_tab')"
+fi
+
 KEY_FILE=.webui_secret_key
 
 PORT="${PORT:-8080}"
diff --git a/backend/start_windows.bat b/backend/start_windows.bat
index 3e8c6b97c..3b6446258 100644
--- a/backend/start_windows.bat
+++ b/backend/start_windows.bat
@@ -6,6 +6,15 @@ SETLOCAL ENABLEDELAYEDEXPANSION
 SET "SCRIPT_DIR=%~dp0"
 cd /d "%SCRIPT_DIR%" || exit /b
 
+:: Add conditional Playwright browser installation
+IF /I "%RAG_WEB_LOADER%" == "chromium" (
+    echo Installing Playwright browsers...
+    playwright install chromium
+    playwright install-deps chromium
+
+    python -c "import nltk; nltk.download('punkt_tab')"
+)
+
 SET "KEY_FILE=.webui_secret_key"
 IF "%PORT%"=="" SET PORT=8080
 IF "%HOST%"=="" SET HOST=0.0.0.0
diff --git a/pyproject.toml b/pyproject.toml
index edd01db8f..c8ec0f497 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -53,6 +53,7 @@ dependencies = [
     "pymilvus==2.5.0",
     "qdrant-client~=1.12.0",
     "opensearch-py==2.7.1",
+    "playwright==1.49.1",
 
     "transformers",
     "sentence-transformers==3.3.1",

From 2452e271cddccf0c835ae17f4505471eb41a4313 Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Thu, 30 Jan 2025 20:31:31 -0600
Subject: [PATCH 02/11] Refine RAG_WEB_LOADER

---
 backend/open_webui/retrieval/web/utils.py | 34 +++++++++++------------
 backend/start.sh                          |  2 +-
 backend/start_windows.bat                 |  2 +-
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index bdc626749..3c0c34074 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -5,6 +5,7 @@ import ssl
 import urllib.parse
 import certifi
 import validators
+from collections import defaultdict
 from typing import AsyncIterator, Dict, List, Optional, Union, Sequence, Iterator
 
 from langchain_community.document_loaders import (
@@ -211,28 +212,27 @@ class SafeWebBaseLoader(WebBaseLoader):
                 # Log the error and continue with the next URL
                 log.error(f"Error loading {path}: {e}")
 
+RAG_WEB_LOADERS = defaultdict(lambda: SafeWebBaseLoader)
+RAG_WEB_LOADERS["playwright"] = SafePlaywrightURLLoader
+RAG_WEB_LOADERS["safe_web"] = SafeWebBaseLoader
 
 def get_web_loader(
     urls: Union[str, Sequence[str]],
     verify_ssl: bool = True,
     requests_per_second: int = 2,
 ):
-    # Check if the URL is valid
+    # Check if the URLs are valid
     safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
 
-    if RAG_WEB_LOADER.value == "chromium":
-        log.info("Using SafePlaywrightURLLoader")
-        return SafePlaywrightURLLoader(
-            safe_urls,
-            verify_ssl=verify_ssl,
-            requests_per_second=requests_per_second,
-            continue_on_failure=True,
-        )
-    else:
-        log.info("Using SafeWebBaseLoader")
-        return SafeWebBaseLoader(
-            safe_urls,
-            verify_ssl=verify_ssl,
-            requests_per_second=requests_per_second,
-            continue_on_failure=True,
-        )
+    # Get the appropriate WebLoader based on the configuration
+    WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value]
+    web_loader = WebLoaderClass(
+        safe_urls,
+        verify_ssl=verify_ssl,
+        requests_per_second=requests_per_second,
+        continue_on_failure=True,
+    )
+
+    log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
+
+    return web_loader
\ No newline at end of file
diff --git a/backend/start.sh b/backend/start.sh
index ce56b1867..2501f413f 100755
--- a/backend/start.sh
+++ b/backend/start.sh
@@ -4,7 +4,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 cd "$SCRIPT_DIR" || exit
 
 # Add conditional Playwright browser installation
-if [[ "${RAG_WEB_LOADER,,}" == "chromium" ]]; then
+if [[ "${RAG_WEB_LOADER,,}" == "playwright" ]]; then
     echo "Installing Playwright browsers..."
     playwright install chromium
     playwright install-deps chromium
diff --git a/backend/start_windows.bat b/backend/start_windows.bat
index 3b6446258..0f2792cc0 100644
--- a/backend/start_windows.bat
+++ b/backend/start_windows.bat
@@ -7,7 +7,7 @@ SET "SCRIPT_DIR=%~dp0"
 cd /d "%SCRIPT_DIR%" || exit /b
 
 :: Add conditional Playwright browser installation
-IF /I "%RAG_WEB_LOADER%" == "chromium" (
+IF /I "%RAG_WEB_LOADER%" == "playwright" (
     echo Installing Playwright browsers...
     playwright install chromium
     playwright install-deps chromium

From 77ae73e659e6fea6da34c3ea913edb3dc4f037a9 Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Thu, 30 Jan 2025 23:18:11 -0600
Subject: [PATCH 03/11] Adjust search event messages + translations

---
 backend/open_webui/routers/retrieval.py                 | 2 +-
 backend/open_webui/utils/middleware.py                  | 2 +-
 src/lib/components/chat/Messages/ResponseMessage.svelte | 1 +
 src/lib/i18n/locales/ar-BH/translation.json             | 1 +
 src/lib/i18n/locales/bg-BG/translation.json             | 1 +
 src/lib/i18n/locales/bn-BD/translation.json             | 1 +
 src/lib/i18n/locales/ca-ES/translation.json             | 1 +
 src/lib/i18n/locales/ceb-PH/translation.json            | 1 +
 src/lib/i18n/locales/cs-CZ/translation.json             | 1 +
 src/lib/i18n/locales/da-DK/translation.json             | 1 +
 src/lib/i18n/locales/de-DE/translation.json             | 1 +
 src/lib/i18n/locales/dg-DG/translation.json             | 1 +
 src/lib/i18n/locales/el-GR/translation.json             | 1 +
 src/lib/i18n/locales/en-GB/translation.json             | 1 +
 src/lib/i18n/locales/en-US/translation.json             | 1 +
 src/lib/i18n/locales/es-ES/translation.json             | 1 +
 src/lib/i18n/locales/eu-ES/translation.json             | 1 +
 src/lib/i18n/locales/fa-IR/translation.json             | 1 +
 src/lib/i18n/locales/fi-FI/translation.json             | 1 +
 src/lib/i18n/locales/fr-CA/translation.json             | 1 +
 src/lib/i18n/locales/fr-FR/translation.json             | 1 +
 src/lib/i18n/locales/he-IL/translation.json             | 1 +
 src/lib/i18n/locales/hi-IN/translation.json             | 1 +
 src/lib/i18n/locales/hr-HR/translation.json             | 1 +
 src/lib/i18n/locales/hu-HU/translation.json             | 1 +
 src/lib/i18n/locales/id-ID/translation.json             | 1 +
 src/lib/i18n/locales/ie-GA/translation.json             | 1 +
 src/lib/i18n/locales/it-IT/translation.json             | 1 +
 src/lib/i18n/locales/ja-JP/translation.json             | 1 +
 src/lib/i18n/locales/ka-GE/translation.json             | 1 +
 src/lib/i18n/locales/ko-KR/translation.json             | 1 +
 src/lib/i18n/locales/lt-LT/translation.json             | 1 +
 src/lib/i18n/locales/ms-MY/translation.json             | 1 +
 src/lib/i18n/locales/nb-NO/translation.json             | 1 +
 src/lib/i18n/locales/nl-NL/translation.json             | 1 +
 src/lib/i18n/locales/pa-IN/translation.json             | 1 +
 src/lib/i18n/locales/pl-PL/translation.json             | 1 +
 src/lib/i18n/locales/pt-BR/translation.json             | 1 +
 src/lib/i18n/locales/pt-PT/translation.json             | 1 +
 src/lib/i18n/locales/ro-RO/translation.json             | 1 +
 src/lib/i18n/locales/ru-RU/translation.json             | 1 +
 src/lib/i18n/locales/sk-SK/translation.json             | 1 +
 src/lib/i18n/locales/sr-RS/translation.json             | 1 +
 src/lib/i18n/locales/sv-SE/translation.json             | 1 +
 src/lib/i18n/locales/th-TH/translation.json             | 1 +
 src/lib/i18n/locales/tk-TW/translation.json             | 1 +
 src/lib/i18n/locales/tr-TR/translation.json             | 1 +
 src/lib/i18n/locales/uk-UA/translation.json             | 1 +
 src/lib/i18n/locales/ur-PK/translation.json             | 1 +
 src/lib/i18n/locales/vi-VN/translation.json             | 1 +
 src/lib/i18n/locales/zh-CN/translation.json             | 1 +
 src/lib/i18n/locales/zh-TW/translation.json             | 1 +
 52 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py
index e65a76050..507698084 100644
--- a/backend/open_webui/routers/retrieval.py
+++ b/backend/open_webui/routers/retrieval.py
@@ -1265,7 +1265,7 @@ async def process_web_search(
             "type": "status",
             "data": {
                 "action": "web_search",
-                "description": "Loading {{count}} sites...",
+                "description": "Loading {{count}} sites",
                 "urls": [result.link for result in web_results],
                 "done": False
             },
diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py
index dde34bec8..2a68d8d0a 100644
--- a/backend/open_webui/utils/middleware.py
+++ b/backend/open_webui/utils/middleware.py
@@ -436,7 +436,7 @@ async def chat_web_search_handler(
                     "type": "status",
                     "data": {
                         "action": "web_search",
-                        "description": "Loaded {{count}} sites",
+                        "description": "Searched {{count}} sites",
                         "query": searchQuery,
                         "urls": results["filenames"],
                         "done": True,
diff --git a/src/lib/components/chat/Messages/ResponseMessage.svelte b/src/lib/components/chat/Messages/ResponseMessage.svelte
index d6b31e6a0..9952f0f8e 100644
--- a/src/lib/components/chat/Messages/ResponseMessage.svelte
+++ b/src/lib/components/chat/Messages/ResponseMessage.svelte
@@ -580,6 +580,7 @@
 													: ''} text-gray-500 dark:text-gray-500 text-base line-clamp-1 text-wrap"
 											>
 												<!-- $i18n.t(`Searching "{{searchQuery}}"`) -->
+												<!-- $i18n.t(`Loading {{count}} sites`) -->
 												{#if status?.description.includes('{{searchQuery}}')}
 													{$i18n.t(status?.description, {
 														searchQuery: status?.query
diff --git a/src/lib/i18n/locales/ar-BH/translation.json b/src/lib/i18n/locales/ar-BH/translation.json
index d115f4227..f2f9d25f2 100644
--- a/src/lib/i18n/locales/ar-BH/translation.json
+++ b/src/lib/i18n/locales/ar-BH/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "يمكن أن تصدر بعض الأخطاء. لذلك يجب التحقق من المعلومات المهمة",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/bg-BG/translation.json b/src/lib/i18n/locales/bg-BG/translation.json
index f2eea288a..a4d06d389 100644
--- a/src/lib/i18n/locales/bg-BG/translation.json
+++ b/src/lib/i18n/locales/bg-BG/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs могат да правят грешки. Проверете важните данни.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/bn-BD/translation.json b/src/lib/i18n/locales/bn-BD/translation.json
index 18a270eb2..5ef08af65 100644
--- a/src/lib/i18n/locales/bn-BD/translation.json
+++ b/src/lib/i18n/locales/bn-BD/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM ভুল করতে পারে। গুরুত্বপূর্ণ তথ্য যাচাই করে নিন।",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ca-ES/translation.json b/src/lib/i18n/locales/ca-ES/translation.json
index 3eeef48d4..db3589a2d 100644
--- a/src/lib/i18n/locales/ca-ES/translation.json
+++ b/src/lib/i18n/locales/ca-ES/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Escoltant...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "Els models de llenguatge poden cometre errors. Verifica la informació important.",
+	"Loading {{count}} sites": "",
 	"Local": "Local",
 	"Local Models": "Models locals",
 	"Lost": "Perdut",
diff --git a/src/lib/i18n/locales/ceb-PH/translation.json b/src/lib/i18n/locales/ceb-PH/translation.json
index 9029fbae0..23a7c01ba 100644
--- a/src/lib/i18n/locales/ceb-PH/translation.json
+++ b/src/lib/i18n/locales/ceb-PH/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Ang mga LLM mahimong masayop. ",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/cs-CZ/translation.json b/src/lib/i18n/locales/cs-CZ/translation.json
index d07b3b6ec..6eefeb323 100644
--- a/src/lib/i18n/locales/cs-CZ/translation.json
+++ b/src/lib/i18n/locales/cs-CZ/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Poslouchání...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM mohou dělat chyby. Ověřte si důležité informace.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokální modely",
 	"Lost": "Ztracený",
diff --git a/src/lib/i18n/locales/da-DK/translation.json b/src/lib/i18n/locales/da-DK/translation.json
index dca76c74d..4f7cbd00d 100644
--- a/src/lib/i18n/locales/da-DK/translation.json
+++ b/src/lib/i18n/locales/da-DK/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Lytter...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM'er kan lave fejl. Bekræft vigtige oplysninger.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokale modeller",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/de-DE/translation.json b/src/lib/i18n/locales/de-DE/translation.json
index f96e6bf08..bf934cedf 100644
--- a/src/lib/i18n/locales/de-DE/translation.json
+++ b/src/lib/i18n/locales/de-DE/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Höre zu...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs können Fehler machen. Überprüfe wichtige Informationen.",
+	"Loading {{count}} sites": "",
 	"Local": "Lokal",
 	"Local Models": "Lokale Modelle",
 	"Lost": "Verloren",
diff --git a/src/lib/i18n/locales/dg-DG/translation.json b/src/lib/i18n/locales/dg-DG/translation.json
index aaca0565b..bfe1b7602 100644
--- a/src/lib/i18n/locales/dg-DG/translation.json
+++ b/src/lib/i18n/locales/dg-DG/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs can make borks. Verify important info.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/el-GR/translation.json b/src/lib/i18n/locales/el-GR/translation.json
index 8058615a0..dcc608769 100644
--- a/src/lib/i18n/locales/el-GR/translation.json
+++ b/src/lib/i18n/locales/el-GR/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Ακούγεται...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Τα LLM μπορούν να κάνουν λάθη. Επαληθεύστε σημαντικές πληροφορίες.",
+	"Loading {{count}} sites": "",
 	"Local": "Τοπικό",
 	"Local Models": "Τοπικά Μοντέλα",
 	"Lost": "Χαμένος",
diff --git a/src/lib/i18n/locales/en-GB/translation.json b/src/lib/i18n/locales/en-GB/translation.json
index 3075e7c0f..108c1d9e8 100644
--- a/src/lib/i18n/locales/en-GB/translation.json
+++ b/src/lib/i18n/locales/en-GB/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/en-US/translation.json b/src/lib/i18n/locales/en-US/translation.json
index 3075e7c0f..108c1d9e8 100644
--- a/src/lib/i18n/locales/en-US/translation.json
+++ b/src/lib/i18n/locales/en-US/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/es-ES/translation.json b/src/lib/i18n/locales/es-ES/translation.json
index d748943ca..4a1d42166 100644
--- a/src/lib/i18n/locales/es-ES/translation.json
+++ b/src/lib/i18n/locales/es-ES/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Escuchando...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Los LLM pueden cometer errores. Verifica la información importante.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modelos locales",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/eu-ES/translation.json b/src/lib/i18n/locales/eu-ES/translation.json
index 1e5da410f..334fc11f6 100644
--- a/src/lib/i18n/locales/eu-ES/translation.json
+++ b/src/lib/i18n/locales/eu-ES/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Entzuten...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMek akatsak egin ditzakete. Egiaztatu informazio garrantzitsua.",
+	"Loading {{count}} sites": "",
 	"Local": "Lokala",
 	"Local Models": "Modelo lokalak",
 	"Lost": "Galduta",
diff --git a/src/lib/i18n/locales/fa-IR/translation.json b/src/lib/i18n/locales/fa-IR/translation.json
index cd647b8a7..512bb0dbb 100644
--- a/src/lib/i18n/locales/fa-IR/translation.json
+++ b/src/lib/i18n/locales/fa-IR/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "مدل\u200cهای زبانی بزرگ می\u200cتوانند اشتباه کنند. اطلاعات مهم را راستی\u200cآزمایی کنید.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/fi-FI/translation.json b/src/lib/i18n/locales/fi-FI/translation.json
index 399da492f..bbc0a2d9a 100644
--- a/src/lib/i18n/locales/fi-FI/translation.json
+++ b/src/lib/i18n/locales/fi-FI/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Kuuntelee...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Kielimallit voivat tehdä virheitä. Tarkista tärkeät tiedot.",
+	"Loading {{count}} sites": "",
 	"Local": "Paikallinen",
 	"Local Models": "Paikalliset mallit",
 	"Lost": "Mennyt",
diff --git a/src/lib/i18n/locales/fr-CA/translation.json b/src/lib/i18n/locales/fr-CA/translation.json
index 051780d71..3f16e3a61 100644
--- a/src/lib/i18n/locales/fr-CA/translation.json
+++ b/src/lib/i18n/locales/fr-CA/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "En train d'écouter...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Les LLM peuvent faire des erreurs. Vérifiez les informations importantes.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modèles locaux",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/fr-FR/translation.json b/src/lib/i18n/locales/fr-FR/translation.json
index 62bdd8139..e8fe232d1 100644
--- a/src/lib/i18n/locales/fr-FR/translation.json
+++ b/src/lib/i18n/locales/fr-FR/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Écoute en cours...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Les LLM peuvent faire des erreurs. Vérifiez les informations importantes.",
+	"Loading {{count}} sites": "",
 	"Local": "Local",
 	"Local Models": "Modèles locaux",
 	"Lost": "Perdu",
diff --git a/src/lib/i18n/locales/he-IL/translation.json b/src/lib/i18n/locales/he-IL/translation.json
index 17756cbb0..d54937e0f 100644
--- a/src/lib/i18n/locales/he-IL/translation.json
+++ b/src/lib/i18n/locales/he-IL/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "מודלים בשפה טבעית יכולים לטעות. אמת מידע חשוב.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/hi-IN/translation.json b/src/lib/i18n/locales/hi-IN/translation.json
index ea039a59d..9cd2a5899 100644
--- a/src/lib/i18n/locales/hi-IN/translation.json
+++ b/src/lib/i18n/locales/hi-IN/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "एलएलएम गलतियाँ कर सकते हैं। महत्वपूर्ण जानकारी सत्यापित करें.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/hr-HR/translation.json b/src/lib/i18n/locales/hr-HR/translation.json
index c9e293389..72826f0d9 100644
--- a/src/lib/i18n/locales/hr-HR/translation.json
+++ b/src/lib/i18n/locales/hr-HR/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Slušam...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM-ovi mogu pogriješiti. Provjerite važne informacije.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokalni modeli",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/hu-HU/translation.json b/src/lib/i18n/locales/hu-HU/translation.json
index 0b9e15140..7b4e38819 100644
--- a/src/lib/i18n/locales/hu-HU/translation.json
+++ b/src/lib/i18n/locales/hu-HU/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Hallgatás...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Az LLM-ek hibázhatnak. Ellenőrizze a fontos információkat.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Helyi modellek",
 	"Lost": "Elveszett",
diff --git a/src/lib/i18n/locales/id-ID/translation.json b/src/lib/i18n/locales/id-ID/translation.json
index 9c4d477a0..4d9a9e8a1 100644
--- a/src/lib/i18n/locales/id-ID/translation.json
+++ b/src/lib/i18n/locales/id-ID/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Mendengarkan",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM dapat membuat kesalahan. Verifikasi informasi penting.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Model Lokal",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ie-GA/translation.json b/src/lib/i18n/locales/ie-GA/translation.json
index a07d36065..21e6005ed 100644
--- a/src/lib/i18n/locales/ie-GA/translation.json
+++ b/src/lib/i18n/locales/ie-GA/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Éisteacht...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Is féidir le LLManna botúin a dhéanamh. Fíoraigh faisnéis thábhachtach.",
+	"Loading {{count}} sites": "",
 	"Local": "Áitiúil",
 	"Local Models": "Múnlaí Áitiúla",
 	"Lost": "Cailleadh",
diff --git a/src/lib/i18n/locales/it-IT/translation.json b/src/lib/i18n/locales/it-IT/translation.json
index 2fa94bace..97b0a5e44 100644
--- a/src/lib/i18n/locales/it-IT/translation.json
+++ b/src/lib/i18n/locales/it-IT/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Gli LLM possono commettere errori. Verifica le informazioni importanti.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ja-JP/translation.json b/src/lib/i18n/locales/ja-JP/translation.json
index 492acce1b..ca1f5d1ff 100644
--- a/src/lib/i18n/locales/ja-JP/translation.json
+++ b/src/lib/i18n/locales/ja-JP/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM は間違いを犯す可能性があります。重要な情報を検証してください。",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "ローカルモデル",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ka-GE/translation.json b/src/lib/i18n/locales/ka-GE/translation.json
index b26aa6c73..2d93798e5 100644
--- a/src/lib/i18n/locales/ka-GE/translation.json
+++ b/src/lib/i18n/locales/ka-GE/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "შესაძლოა LLM-ებმა შეცდომები დაუშვან. გადაამოწმეთ მნიშვნელოვანი ინფორმაცია.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ko-KR/translation.json b/src/lib/i18n/locales/ko-KR/translation.json
index ff33f8b96..c34103efa 100644
--- a/src/lib/i18n/locales/ko-KR/translation.json
+++ b/src/lib/i18n/locales/ko-KR/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "듣는 중...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM에 오류가 있을 수 있습니다. 중요한 정보는 확인이 필요합니다.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "로컬 모델",
 	"Lost": "패배",
diff --git a/src/lib/i18n/locales/lt-LT/translation.json b/src/lib/i18n/locales/lt-LT/translation.json
index b562d3f7c..e21ed8f80 100644
--- a/src/lib/i18n/locales/lt-LT/translation.json
+++ b/src/lib/i18n/locales/lt-LT/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Klausoma...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Dideli kalbos modeliai gali klysti. Patikrinkite atsakymų teisingumą.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokalūs modeliai",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ms-MY/translation.json b/src/lib/i18n/locales/ms-MY/translation.json
index 950a065af..a37f71376 100644
--- a/src/lib/i18n/locales/ms-MY/translation.json
+++ b/src/lib/i18n/locales/ms-MY/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Mendengar...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM boleh membuat kesilapan. Sahkan maklumat penting",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Model Tempatan",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/nb-NO/translation.json b/src/lib/i18n/locales/nb-NO/translation.json
index 1ce3a9aa9..9619ad9e1 100644
--- a/src/lib/i18n/locales/nb-NO/translation.json
+++ b/src/lib/i18n/locales/nb-NO/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Lytter ...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Språkmodeller kan gjøre feil. Kontroller viktige opplysninger.",
+	"Loading {{count}} sites": "",
 	"Local": "Lokal",
 	"Local Models": "Lokale modeller",
 	"Lost": "Tapt",
diff --git a/src/lib/i18n/locales/nl-NL/translation.json b/src/lib/i18n/locales/nl-NL/translation.json
index 68764d8a8..dd82213ed 100644
--- a/src/lib/i18n/locales/nl-NL/translation.json
+++ b/src/lib/i18n/locales/nl-NL/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Aan het luisteren...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs kunnen fouten maken. Verifieer belangrijke informatie.",
+	"Loading {{count}} sites": "",
 	"Local": "Lokaal",
 	"Local Models": "Lokale modellen",
 	"Lost": "Verloren",
diff --git a/src/lib/i18n/locales/pa-IN/translation.json b/src/lib/i18n/locales/pa-IN/translation.json
index 58e8bf339..59826ad7c 100644
--- a/src/lib/i18n/locales/pa-IN/translation.json
+++ b/src/lib/i18n/locales/pa-IN/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs ਗਲਤੀਆਂ ਕਰ ਸਕਦੇ ਹਨ। ਮਹੱਤਵਪੂਰਨ ਜਾਣਕਾਰੀ ਦੀ ਪੁਸ਼ਟੀ ਕਰੋ।",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/pl-PL/translation.json b/src/lib/i18n/locales/pl-PL/translation.json
index 64168bfe3..aaf9c2c8c 100644
--- a/src/lib/i18n/locales/pl-PL/translation.json
+++ b/src/lib/i18n/locales/pl-PL/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMy mogą popełniać błędy. Zweryfikuj ważne informacje.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/pt-BR/translation.json b/src/lib/i18n/locales/pt-BR/translation.json
index 4c416f409..3773f4314 100644
--- a/src/lib/i18n/locales/pt-BR/translation.json
+++ b/src/lib/i18n/locales/pt-BR/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Escutando...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs podem cometer erros. Verifique informações importantes.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modelos Locais",
 	"Lost": "Perdeu",
diff --git a/src/lib/i18n/locales/pt-PT/translation.json b/src/lib/i18n/locales/pt-PT/translation.json
index 092669629..864fa90d5 100644
--- a/src/lib/i18n/locales/pt-PT/translation.json
+++ b/src/lib/i18n/locales/pt-PT/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "A escutar...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs podem cometer erros. Verifique informações importantes.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modelos Locais",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ro-RO/translation.json b/src/lib/i18n/locales/ro-RO/translation.json
index 01994c631..0de81364b 100644
--- a/src/lib/i18n/locales/ro-RO/translation.json
+++ b/src/lib/i18n/locales/ro-RO/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Ascult...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM-urile pot face greșeli. Verificați informațiile importante.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modele Locale",
 	"Lost": "Pierdut",
diff --git a/src/lib/i18n/locales/ru-RU/translation.json b/src/lib/i18n/locales/ru-RU/translation.json
index b4d6aa916..34ef2a98a 100644
--- a/src/lib/i18n/locales/ru-RU/translation.json
+++ b/src/lib/i18n/locales/ru-RU/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Слушаю...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs могут допускать ошибки. Проверяйте важную информацию.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Локальные модели",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/sk-SK/translation.json b/src/lib/i18n/locales/sk-SK/translation.json
index 29d8622ea..f720bfed8 100644
--- a/src/lib/i18n/locales/sk-SK/translation.json
+++ b/src/lib/i18n/locales/sk-SK/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Počúvanie...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM môžu robiť chyby. Overte si dôležité informácie.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokálne modely",
 	"Lost": "Stratený",
diff --git a/src/lib/i18n/locales/sr-RS/translation.json b/src/lib/i18n/locales/sr-RS/translation.json
index 8dc255655..a6d13b4dd 100644
--- a/src/lib/i18n/locales/sr-RS/translation.json
+++ b/src/lib/i18n/locales/sr-RS/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "ВЈМ-ови (LLM-ови) могу правити грешке. Проверите важне податке.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "Пораза",
diff --git a/src/lib/i18n/locales/sv-SE/translation.json b/src/lib/i18n/locales/sv-SE/translation.json
index 0abc82341..ae936e3ee 100644
--- a/src/lib/i18n/locales/sv-SE/translation.json
+++ b/src/lib/i18n/locales/sv-SE/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Lyssnar...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM:er kan göra misstag. Granska viktig information.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokala modeller",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/th-TH/translation.json b/src/lib/i18n/locales/th-TH/translation.json
index f1f23478b..a57eaa3b4 100644
--- a/src/lib/i18n/locales/th-TH/translation.json
+++ b/src/lib/i18n/locales/th-TH/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "กำลังฟัง...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs สามารถทำผิดพลาดได้ ตรวจสอบข้อมูลสำคัญ",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "โมเดลท้องถิ่น",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/tk-TW/translation.json b/src/lib/i18n/locales/tk-TW/translation.json
index 3075e7c0f..108c1d9e8 100644
--- a/src/lib/i18n/locales/tk-TW/translation.json
+++ b/src/lib/i18n/locales/tk-TW/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/tr-TR/translation.json b/src/lib/i18n/locales/tr-TR/translation.json
index e6d05dfe1..892db13e6 100644
--- a/src/lib/i18n/locales/tr-TR/translation.json
+++ b/src/lib/i18n/locales/tr-TR/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Dinleniyor...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM'ler hata yapabilir. Önemli bilgileri doğrulayın.",
+	"Loading {{count}} sites": "",
 	"Local": "Yerel",
 	"Local Models": "Yerel Modeller",
 	"Lost": "Kayıp",
diff --git a/src/lib/i18n/locales/uk-UA/translation.json b/src/lib/i18n/locales/uk-UA/translation.json
index 449958c2b..c971d0049 100644
--- a/src/lib/i18n/locales/uk-UA/translation.json
+++ b/src/lib/i18n/locales/uk-UA/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Слухаю...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs можуть помилятися. Перевірте важливу інформацію.",
+	"Loading {{count}} sites": "",
 	"Local": "Локальний",
 	"Local Models": "Локальні моделі",
 	"Lost": "Втрачене",
diff --git a/src/lib/i18n/locales/ur-PK/translation.json b/src/lib/i18n/locales/ur-PK/translation.json
index 437d943ae..31662f695 100644
--- a/src/lib/i18n/locales/ur-PK/translation.json
+++ b/src/lib/i18n/locales/ur-PK/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "سن رہے ہیں...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "ایل ایل ایم غلطیاں کر سکتے ہیں اہم معلومات کی تصدیق کریں",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "مقامی ماڈلز",
 	"Lost": "گم شدہ",
diff --git a/src/lib/i18n/locales/vi-VN/translation.json b/src/lib/i18n/locales/vi-VN/translation.json
index c786b67a7..f6f8db552 100644
--- a/src/lib/i18n/locales/vi-VN/translation.json
+++ b/src/lib/i18n/locales/vi-VN/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "Đang nghe...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Hệ thống có thể tạo ra nội dung không chính xác hoặc sai. Hãy kiểm chứng kỹ lưỡng thông tin trước khi tiếp nhận và sử dụng.",
+	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/zh-CN/translation.json b/src/lib/i18n/locales/zh-CN/translation.json
index da26f9cc9..9d8149471 100644
--- a/src/lib/i18n/locales/zh-CN/translation.json
+++ b/src/lib/i18n/locales/zh-CN/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "正在倾听...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "大语言模型可能会生成误导性错误信息，请对关键信息加以验证。",
+	"Loading {{count}} sites": "",
 	"Local": "本地",
 	"Local Models": "本地模型",
 	"Lost": "落败",
diff --git a/src/lib/i18n/locales/zh-TW/translation.json b/src/lib/i18n/locales/zh-TW/translation.json
index 81db88766..fb22677fa 100644
--- a/src/lib/i18n/locales/zh-TW/translation.json
+++ b/src/lib/i18n/locales/zh-TW/translation.json
@@ -561,6 +561,7 @@
 	"Listening...": "正在聆聽...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "大型語言模型可能會出錯。請驗證重要資訊。",
+	"Loading {{count}} sites": "",
 	"Local": "本機",
 	"Local Models": "本機模型",
 	"Lost": "已遺失",

From a84e488a4ea681c580a2b9cca22fe176f8c0014c Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Sat, 1 Feb 2025 22:58:28 -0600
Subject: [PATCH 04/11] Fix playwright in docker by updating unstructured

---
 backend/open_webui/retrieval/web/utils.py | 6 +++---
 backend/requirements.txt                  | 2 +-
 pyproject.toml                            | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index 3c0c34074..0568c795c 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -175,7 +175,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
                 yield document
             except Exception as e:
                 if self.continue_on_failure:
-                    log.error(f"Error processing {url}, exception: {e}")
+                    log.exception(e, "Error loading %s", url)
                     continue
                 raise e
 
@@ -190,7 +190,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
                 yield document
             except Exception as e:
                 if self.continue_on_failure:
-                    log.error(f"Error processing {url}, exception: {e}")
+                    log.exception(e, "Error loading %s", url)
                     continue
                 raise e
 
@@ -210,7 +210,7 @@ class SafeWebBaseLoader(WebBaseLoader):
                 yield Document(page_content=text, metadata=metadata)
             except Exception as e:
                 # Log the error and continue with the next URL
-                log.error(f"Error loading {path}: {e}")
+                log.exception(e, "Error loading %s", path)
 
 RAG_WEB_LOADERS = defaultdict(lambda: SafeWebBaseLoader)
 RAG_WEB_LOADERS["playwright"] = SafePlaywrightURLLoader
diff --git a/backend/requirements.txt b/backend/requirements.txt
index bb124bf11..cf5cb4a2f 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -60,7 +60,7 @@ fpdf2==2.8.2
 pymdown-extensions==10.11.2
 docx2txt==0.8
 python-pptx==1.0.0
-unstructured==0.15.9
+unstructured==0.16.17
 nltk==3.9.1
 Markdown==3.7
 pypandoc==1.13
diff --git a/pyproject.toml b/pyproject.toml
index 41c79ddb8..6e7f607b3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -66,7 +66,7 @@ dependencies = [
     "pymdown-extensions==10.11.2",
     "docx2txt==0.8",
     "python-pptx==1.0.0",
-    "unstructured==0.15.9",
+    "unstructured==0.16.17",
     "nltk==3.9.1",
     "Markdown==3.7",
     "pypandoc==1.13",

From 8da33721d563754becd0d03bf86605441e0bd9e3 Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Sun, 2 Feb 2025 17:58:09 -0600
Subject: [PATCH 05/11] Support PLAYWRIGHT_WS_URI

---
 backend/open_webui/config.py              |   6 ++
 backend/open_webui/main.py                |   2 +
 backend/open_webui/retrieval/web/utils.py | 121 ++++++++++++++--------
 backend/start.sh                          |   8 +-
 backend/start_windows.bat                 |   8 +-
 5 files changed, 97 insertions(+), 48 deletions(-)

diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py
index 278f50663..80e1e7ab2 100644
--- a/backend/open_webui/config.py
+++ b/backend/open_webui/config.py
@@ -1724,6 +1724,12 @@ RAG_WEB_LOADER = PersistentConfig(
     os.environ.get("RAG_WEB_LOADER", "safe_web")
 )
 
+PLAYWRIGHT_WS_URI = PersistentConfig(
+    "PLAYWRIGHT_WS_URI",
+    "rag.web.loader.playwright.ws.uri",
+    os.environ.get("PLAYWRIGHT_WS_URI", None)
+)
+
 ####################################
 # Images
 ####################################
diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py
index c5dfad047..fd8a4c957 100644
--- a/backend/open_webui/main.py
+++ b/backend/open_webui/main.py
@@ -129,6 +129,7 @@ from open_webui.config import (
     AUDIO_TTS_VOICE,
     AUDIO_TTS_AZURE_SPEECH_REGION,
     AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
+    PLAYWRIGHT_WS_URI,
     RAG_WEB_LOADER,
     WHISPER_MODEL,
     WHISPER_MODEL_AUTO_UPDATE,
@@ -528,6 +529,7 @@ app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_K
 app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
 app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
 app.state.config.RAG_WEB_LOADER = RAG_WEB_LOADER
+app.state.config.PLAYWRIGHT_WS_URI = PLAYWRIGHT_WS_URI
 
 app.state.EMBEDDING_FUNCTION = None
 app.state.ef = None
diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index 0568c795c..3c77402c3 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -16,7 +16,7 @@ from langchain_core.documents import Document
 
 
 from open_webui.constants import ERROR_MESSAGES
-from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, RAG_WEB_LOADER
+from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER
 from open_webui.env import SRC_LOG_LEVELS
 
 import logging
@@ -83,7 +83,7 @@ def extract_metadata(soup, url):
     return metadata
 
 class SafePlaywrightURLLoader(PlaywrightURLLoader):
-    """Load HTML pages safely with Playwright, supporting SSL verification and rate limiting.
+    """Load HTML pages safely with Playwright, supporting SSL verification, rate limiting, and remote browser connection.
     
     Attributes:
         urls (List[str]): List of URLs to load.
@@ -91,6 +91,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
         requests_per_second (Optional[float]): Number of requests per second to limit to.
         continue_on_failure (bool): If True, continue loading other URLs on failure.
         headless (bool): If True, the browser will run in headless mode.
+        playwright_ws_url (Optional[str]): WebSocket endpoint URI for remote browser connection.
     """
 
     def __init__(
@@ -101,19 +102,80 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
         continue_on_failure: bool = True,
         headless: bool = True,
         remove_selectors: Optional[List[str]] = None,
-        proxy: Optional[Dict[str, str]] = None
+        proxy: Optional[Dict[str, str]] = None,
+        playwright_ws_url: Optional[str] = None
     ):
-        """Initialize with additional safety parameters."""
+        """Initialize with additional safety parameters and remote browser support."""
+        # We'll set headless to False if using playwright_ws_url since it's handled by the remote browser
         super().__init__(
             urls=urls,
             continue_on_failure=continue_on_failure,
-            headless=headless,
+            headless=headless if playwright_ws_url is None else False,
             remove_selectors=remove_selectors,
             proxy=proxy
         )
         self.verify_ssl = verify_ssl
         self.requests_per_second = requests_per_second
         self.last_request_time = None
+        self.playwright_ws_url = playwright_ws_url
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Safely load URLs synchronously with support for remote browser."""
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as p:
+            # Use remote browser if ws_endpoint is provided, otherwise use local browser
+            if self.playwright_ws_url:
+                browser = p.chromium.connect(self.playwright_ws_url)
+            else:
+                browser = p.chromium.launch(headless=self.headless, proxy=self.proxy)
+
+            for url in self.urls:
+                try:
+                    self._safe_process_url_sync(url)
+                    page = browser.new_page()
+                    response = page.goto(url)
+                    if response is None:
+                        raise ValueError(f"page.goto() returned None for url {url}")
+
+                    text = self.evaluator.evaluate(page, browser, response)
+                    metadata = {"source": url}
+                    yield Document(page_content=text, metadata=metadata)
+                except Exception as e:
+                    if self.continue_on_failure:
+                        log.exception(e, "Error loading %s", url)
+                        continue
+                    raise e
+            browser.close()
+
+    async def alazy_load(self) -> AsyncIterator[Document]:
+        """Safely load URLs asynchronously with support for remote browser."""
+        from playwright.async_api import async_playwright
+
+        async with async_playwright() as p:
+            # Use remote browser if ws_endpoint is provided, otherwise use local browser
+            if self.playwright_ws_url:
+                browser = await p.chromium.connect(self.playwright_ws_url)
+            else:
+                browser = await p.chromium.launch(headless=self.headless, proxy=self.proxy)
+
+            for url in self.urls:
+                try:
+                    await self._safe_process_url(url)
+                    page = await browser.new_page()
+                    response = await page.goto(url)
+                    if response is None:
+                        raise ValueError(f"page.goto() returned None for url {url}")
+
+                    text = await self.evaluator.evaluate_async(page, browser, response)
+                    metadata = {"source": url}
+                    yield Document(page_content=text, metadata=metadata)
+                except Exception as e:
+                    if self.continue_on_failure:
+                        log.exception(e, "Error loading %s", url)
+                        continue
+                    raise e
+            await browser.close()
 
     def _verify_ssl_cert(self, url: str) -> bool:
         """Verify SSL certificate for the given URL."""
@@ -164,36 +226,6 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
         self._sync_wait_for_rate_limit()
         return True
 
-    async def alazy_load(self) -> AsyncIterator[Document]:
-        """Safely load URLs asynchronously."""
-        parent_iterator = super().alazy_load()
-        
-        async for document in parent_iterator:
-            url = document.metadata["source"]
-            try:
-                await self._safe_process_url(url)
-                yield document
-            except Exception as e:
-                if self.continue_on_failure:
-                    log.exception(e, "Error loading %s", url)
-                    continue
-                raise e
-
-    def lazy_load(self) -> Iterator[Document]:
-        """Safely load URLs synchronously."""
-        parent_iterator = super().lazy_load()
-        
-        for document in parent_iterator:
-            url = document.metadata["source"]
-            try:
-                self._safe_process_url_sync(url)
-                yield document
-            except Exception as e:
-                if self.continue_on_failure:
-                    log.exception(e, "Error loading %s", url)
-                    continue
-                raise e
-
 class SafeWebBaseLoader(WebBaseLoader):
     """WebBaseLoader with enhanced error handling for URLs."""
 
@@ -224,14 +256,19 @@ def get_web_loader(
     # Check if the URLs are valid
     safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
 
-    # Get the appropriate WebLoader based on the configuration
+    web_loader_args = {
+        "urls": safe_urls,
+        "verify_ssl": verify_ssl,
+        "requests_per_second": requests_per_second,
+        "continue_on_failure": True
+    }
+
+    if PLAYWRIGHT_WS_URI.value:
+        web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value
+
+    # Create the appropriate WebLoader based on the configuration
     WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value]
-    web_loader = WebLoaderClass(
-        safe_urls,
-        verify_ssl=verify_ssl,
-        requests_per_second=requests_per_second,
-        continue_on_failure=True,
-    )
+    web_loader = WebLoaderClass(**web_loader_args)
 
     log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
 
diff --git a/backend/start.sh b/backend/start.sh
index 2501f413f..3b08cf549 100755
--- a/backend/start.sh
+++ b/backend/start.sh
@@ -5,9 +5,11 @@ cd "$SCRIPT_DIR" || exit
 
 # Add conditional Playwright browser installation
 if [[ "${RAG_WEB_LOADER,,}" == "playwright" ]]; then
-    echo "Installing Playwright browsers..."
-    playwright install chromium
-    playwright install-deps chromium
+    if [[ -z "${PLAYWRIGHT_WS_URI}" ]]; then
+        echo "Installing Playwright browsers..."
+        playwright install chromium
+        playwright install-deps chromium
+    fi
 
     python -c "import nltk; nltk.download('punkt_tab')"
 fi
diff --git a/backend/start_windows.bat b/backend/start_windows.bat
index 0f2792cc0..036e1f721 100644
--- a/backend/start_windows.bat
+++ b/backend/start_windows.bat
@@ -8,9 +8,11 @@ cd /d "%SCRIPT_DIR%" || exit /b
 
 :: Add conditional Playwright browser installation
 IF /I "%RAG_WEB_LOADER%" == "playwright" (
-    echo Installing Playwright browsers...
-    playwright install chromium
-    playwright install-deps chromium
+    IF "%PLAYWRIGHT_WS_URI%" == "" (
+        echo Installing Playwright browsers...
+        playwright install chromium
+        playwright install-deps chromium
+    )
 
     python -c "import nltk; nltk.download('punkt_tab')"
 )

From c3df481b22d8bc13a7deb045e94b0bcf4235224e Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Sun, 2 Feb 2025 19:44:40 -0600
Subject: [PATCH 06/11] Introduce docker-compose.playwright.yaml + run-compose
 update

---
 backend/requirements.txt       |  2 +-
 docker-compose.playwright.yaml | 10 ++++++++++
 run-compose.sh                 |  9 +++++++++
 3 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 docker-compose.playwright.yaml

diff --git a/backend/requirements.txt b/backend/requirements.txt
index cf5cb4a2f..b08c17677 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -46,7 +46,7 @@ chromadb==0.6.2
 pymilvus==2.5.0
 qdrant-client~=1.12.0
 opensearch-py==2.7.1
-playwright==1.49.1
+playwright==1.49.1 # Caution: version must match docker-compose.playwright.yaml
 
 transformers
 sentence-transformers==3.3.1
diff --git a/docker-compose.playwright.yaml b/docker-compose.playwright.yaml
new file mode 100644
index 000000000..0a4bb3f76
--- /dev/null
+++ b/docker-compose.playwright.yaml
@@ -0,0 +1,10 @@
+services:
+  playwright:
+    image: mcr.microsoft.com/playwright:v1.49.1-noble # Version must match requirements.txt
+    container_name: playwright
+    command: npx -y playwright@1.49.1 run-server --port 3000 --host 0.0.0.0
+
+  open-webui:
+    environment:
+      - 'RAG_WEB_LOADER=playwright'
+      - 'PLAYWRIGHT_WS_URI=ws://playwright:3000'
\ No newline at end of file
diff --git a/run-compose.sh b/run-compose.sh
index 21574e959..4fafedc6f 100755
--- a/run-compose.sh
+++ b/run-compose.sh
@@ -74,6 +74,7 @@ usage() {
     echo "  --enable-api[port=PORT]    Enable API and expose it on the specified port."
     echo "  --webui[port=PORT]         Set the port for the web user interface."
     echo "  --data[folder=PATH]        Bind mount for ollama data folder (by default will create the 'ollama' volume)."
+    echo "  --playwright               Enable Playwright support for web scraping."
     echo "  --build                    Build the docker image before running the compose project."
     echo "  --drop                     Drop the compose project."
     echo "  -q, --quiet                Run script in headless mode."
@@ -100,6 +101,7 @@ webui_port=3000
 headless=false
 build_image=false
 kill_compose=false
+enable_playwright=false
 
 # Function to extract value from the parameter
 extract_value() {
@@ -129,6 +131,9 @@ while [[ $# -gt 0 ]]; do
             value=$(extract_value "$key")
             data_dir=${value:-"./ollama-data"}
             ;;
+        --playwright)
+            enable_playwright=true
+            ;;
         --drop)
             kill_compose=true
             ;;
@@ -182,6 +187,9 @@ else
         DEFAULT_COMPOSE_COMMAND+=" -f docker-compose.data.yaml"
         export OLLAMA_DATA_DIR=$data_dir # Set OLLAMA_DATA_DIR environment variable
     fi
+    if [[ $enable_playwright == true ]]; then
+        DEFAULT_COMPOSE_COMMAND+=" -f docker-compose.playwright.yaml"
+    fi
     if [[ -n $webui_port ]]; then
         export OPEN_WEBUI_PORT=$webui_port # Set OPEN_WEBUI_PORT environment variable
     fi
@@ -201,6 +209,7 @@ echo -e "   ${GREEN}${BOLD}GPU Count:${NC} ${OLLAMA_GPU_COUNT:-Not Enabled}"
 echo -e "   ${GREEN}${BOLD}WebAPI Port:${NC} ${OLLAMA_WEBAPI_PORT:-Not Enabled}"
 echo -e "   ${GREEN}${BOLD}Data Folder:${NC} ${data_dir:-Using ollama volume}"
 echo -e "   ${GREEN}${BOLD}WebUI Port:${NC} $webui_port"
+echo -e "   ${GREEN}${BOLD}Playwright:${NC} ${enable_playwright:-false}"
 echo
 
 if [[ $headless == true ]]; then

From 1b581b714f6749e51bf17c49434976a0c57900c6 Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Mon, 3 Feb 2025 18:47:26 -0600
Subject: [PATCH 07/11] Moving code out of playwright branch

---
 backend/open_webui/retrieval/web/main.py      |  4 ----
 backend/open_webui/retrieval/web/utils.py     | 19 +++++--------------
 backend/open_webui/routers/retrieval.py       | 16 +---------------
 backend/open_webui/utils/middleware.py        |  1 -
 .../chat/Messages/ResponseMessage.svelte      |  1 -
 src/lib/i18n/locales/ar-BH/translation.json   |  1 -
 src/lib/i18n/locales/bg-BG/translation.json   |  1 -
 src/lib/i18n/locales/bn-BD/translation.json   |  1 -
 src/lib/i18n/locales/ca-ES/translation.json   |  1 -
 src/lib/i18n/locales/ceb-PH/translation.json  |  1 -
 src/lib/i18n/locales/cs-CZ/translation.json   |  1 -
 src/lib/i18n/locales/da-DK/translation.json   |  1 -
 src/lib/i18n/locales/de-DE/translation.json   |  1 -
 src/lib/i18n/locales/dg-DG/translation.json   |  1 -
 src/lib/i18n/locales/el-GR/translation.json   |  1 -
 src/lib/i18n/locales/en-GB/translation.json   |  1 -
 src/lib/i18n/locales/en-US/translation.json   |  1 -
 src/lib/i18n/locales/es-ES/translation.json   |  1 -
 src/lib/i18n/locales/eu-ES/translation.json   |  1 -
 src/lib/i18n/locales/fa-IR/translation.json   |  1 -
 src/lib/i18n/locales/fi-FI/translation.json   |  1 -
 src/lib/i18n/locales/fr-CA/translation.json   |  1 -
 src/lib/i18n/locales/fr-FR/translation.json   |  1 -
 src/lib/i18n/locales/he-IL/translation.json   |  1 -
 src/lib/i18n/locales/hi-IN/translation.json   |  1 -
 src/lib/i18n/locales/hr-HR/translation.json   |  1 -
 src/lib/i18n/locales/hu-HU/translation.json   |  1 -
 src/lib/i18n/locales/id-ID/translation.json   |  1 -
 src/lib/i18n/locales/ie-GA/translation.json   |  1 -
 src/lib/i18n/locales/it-IT/translation.json   |  1 -
 src/lib/i18n/locales/ja-JP/translation.json   |  1 -
 src/lib/i18n/locales/ka-GE/translation.json   |  1 -
 src/lib/i18n/locales/ko-KR/translation.json   |  1 -
 src/lib/i18n/locales/lt-LT/translation.json   |  1 -
 src/lib/i18n/locales/ms-MY/translation.json   |  1 -
 src/lib/i18n/locales/nb-NO/translation.json   |  1 -
 src/lib/i18n/locales/nl-NL/translation.json   |  1 -
 src/lib/i18n/locales/pa-IN/translation.json   |  1 -
 src/lib/i18n/locales/pl-PL/translation.json   |  1 -
 src/lib/i18n/locales/pt-BR/translation.json   |  1 -
 src/lib/i18n/locales/pt-PT/translation.json   |  1 -
 src/lib/i18n/locales/ro-RO/translation.json   |  1 -
 src/lib/i18n/locales/ru-RU/translation.json   |  1 -
 src/lib/i18n/locales/sk-SK/translation.json   |  1 -
 src/lib/i18n/locales/sr-RS/translation.json   |  1 -
 src/lib/i18n/locales/sv-SE/translation.json   |  1 -
 src/lib/i18n/locales/th-TH/translation.json   |  1 -
 src/lib/i18n/locales/tk-TW/translation.json   |  1 -
 src/lib/i18n/locales/tr-TR/translation.json   |  1 -
 src/lib/i18n/locales/uk-UA/translation.json   |  1 -
 src/lib/i18n/locales/ur-PK/translation.json   |  1 -
 src/lib/i18n/locales/vi-VN/translation.json   |  1 -
 src/lib/i18n/locales/zh-CN/translation.json   |  1 -
 src/lib/i18n/locales/zh-TW/translation.json   |  1 -
 54 files changed, 6 insertions(+), 84 deletions(-)

diff --git a/backend/open_webui/retrieval/web/main.py b/backend/open_webui/retrieval/web/main.py
index 28a749e7d..1af8a70aa 100644
--- a/backend/open_webui/retrieval/web/main.py
+++ b/backend/open_webui/retrieval/web/main.py
@@ -1,5 +1,3 @@
-import validators
-
 from typing import Optional
 from urllib.parse import urlparse
 
@@ -12,8 +10,6 @@ def get_filtered_results(results, filter_list):
     filtered_results = []
     for result in results:
         url = result.get("url") or result.get("link", "")
-        if not validators.url(url):
-            continue
         domain = urlparse(url).netloc
         if any(domain.endswith(filtered_domain) for filtered_domain in filter_list):
             filtered_results.append(result)
diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index 3c77402c3..ddbdc6004 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -48,16 +48,6 @@ def validate_url(url: Union[str, Sequence[str]]):
     else:
         return False
 
-def safe_validate_urls(url: Sequence[str]) -> Sequence[str]:
-    valid_urls = []
-    for u in url:
-        try:
-            if validate_url(u):
-                valid_urls.append(u)
-        except ValueError:
-            continue
-    return valid_urls
-
 def resolve_hostname(hostname):
     # Get address information
     addr_info = socket.getaddrinfo(hostname, None)
@@ -253,11 +243,12 @@ def get_web_loader(
     verify_ssl: bool = True,
     requests_per_second: int = 2,
 ):
-    # Check if the URLs are valid
-    safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
+    # Check if the URL is valid
+    if not validate_url(urls):
+        raise ValueError(ERROR_MESSAGES.INVALID_URL)
 
     web_loader_args = {
-        "urls": safe_urls,
+        "urls": urls,
         "verify_ssl": verify_ssl,
         "requests_per_second": requests_per_second,
         "continue_on_failure": True
@@ -270,6 +261,6 @@ def get_web_loader(
     WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value]
     web_loader = WebLoaderClass(**web_loader_args)
 
-    log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
+    log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(urls))
 
     return web_loader
\ No newline at end of file
diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py
index 507698084..65fa12ab2 100644
--- a/backend/open_webui/routers/retrieval.py
+++ b/backend/open_webui/routers/retrieval.py
@@ -1239,10 +1239,8 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
 
 @router.post("/process/web/search")
 async def process_web_search(
-    request: Request, form_data: SearchForm, extra_params: dict, user=Depends(get_verified_user)
+    request: Request, form_data: SearchForm, user=Depends(get_verified_user)
 ):
-    event_emitter = extra_params["__event_emitter__"]
-    
     try:
         logging.info(
             f"trying to web search with {request.app.state.config.RAG_WEB_SEARCH_ENGINE, form_data.query}"
@@ -1260,18 +1258,6 @@ async def process_web_search(
 
     log.debug(f"web_results: {web_results}")
 
-    await event_emitter(
-        {
-            "type": "status",
-            "data": {
-                "action": "web_search",
-                "description": "Loading {{count}} sites",
-                "urls": [result.link for result in web_results],
-                "done": False
-            },
-        }
-    )
-
     try:
         collection_name = form_data.collection_name
         if collection_name == "" or collection_name is None:
diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py
index 961e57b9e..77b820cd9 100644
--- a/backend/open_webui/utils/middleware.py
+++ b/backend/open_webui/utils/middleware.py
@@ -443,7 +443,6 @@ async def chat_web_search_handler(
                     "query": searchQuery,
                 }
             ),
-            extra_params=extra_params,
             user=user
         )
 
diff --git a/src/lib/components/chat/Messages/ResponseMessage.svelte b/src/lib/components/chat/Messages/ResponseMessage.svelte
index 479180ae8..f6a4b0bc0 100644
--- a/src/lib/components/chat/Messages/ResponseMessage.svelte
+++ b/src/lib/components/chat/Messages/ResponseMessage.svelte
@@ -585,7 +585,6 @@
 													: ''} text-gray-500 dark:text-gray-500 text-base line-clamp-1 text-wrap"
 											>
 												<!-- $i18n.t(`Searching "{{searchQuery}}"`) -->
-												<!-- $i18n.t(`Loading {{count}} sites`) -->
 												{#if status?.description.includes('{{searchQuery}}')}
 													{$i18n.t(status?.description, {
 														searchQuery: status?.query
diff --git a/src/lib/i18n/locales/ar-BH/translation.json b/src/lib/i18n/locales/ar-BH/translation.json
index 8149c9fe6..98a4e557b 100644
--- a/src/lib/i18n/locales/ar-BH/translation.json
+++ b/src/lib/i18n/locales/ar-BH/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "يمكن أن تصدر بعض الأخطاء. لذلك يجب التحقق من المعلومات المهمة",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/bg-BG/translation.json b/src/lib/i18n/locales/bg-BG/translation.json
index e85f7bd53..e12120e77 100644
--- a/src/lib/i18n/locales/bg-BG/translation.json
+++ b/src/lib/i18n/locales/bg-BG/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs могат да правят грешки. Проверете важните данни.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/bn-BD/translation.json b/src/lib/i18n/locales/bn-BD/translation.json
index cfd4ccd0c..9aba7a61d 100644
--- a/src/lib/i18n/locales/bn-BD/translation.json
+++ b/src/lib/i18n/locales/bn-BD/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM ভুল করতে পারে। গুরুত্বপূর্ণ তথ্য যাচাই করে নিন।",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ca-ES/translation.json b/src/lib/i18n/locales/ca-ES/translation.json
index 6ebd4a790..6de03ddda 100644
--- a/src/lib/i18n/locales/ca-ES/translation.json
+++ b/src/lib/i18n/locales/ca-ES/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Escoltant...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "Els models de llenguatge poden cometre errors. Verifica la informació important.",
-	"Loading {{count}} sites": "",
 	"Local": "Local",
 	"Local Models": "Models locals",
 	"Lost": "Perdut",
diff --git a/src/lib/i18n/locales/ceb-PH/translation.json b/src/lib/i18n/locales/ceb-PH/translation.json
index 3ca5045f0..8d85343a1 100644
--- a/src/lib/i18n/locales/ceb-PH/translation.json
+++ b/src/lib/i18n/locales/ceb-PH/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Ang mga LLM mahimong masayop. ",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/cs-CZ/translation.json b/src/lib/i18n/locales/cs-CZ/translation.json
index 76d4b9d39..fe7c4cef3 100644
--- a/src/lib/i18n/locales/cs-CZ/translation.json
+++ b/src/lib/i18n/locales/cs-CZ/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Poslouchání...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM mohou dělat chyby. Ověřte si důležité informace.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokální modely",
 	"Lost": "Ztracený",
diff --git a/src/lib/i18n/locales/da-DK/translation.json b/src/lib/i18n/locales/da-DK/translation.json
index 79c959542..a358dc25b 100644
--- a/src/lib/i18n/locales/da-DK/translation.json
+++ b/src/lib/i18n/locales/da-DK/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Lytter...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM'er kan lave fejl. Bekræft vigtige oplysninger.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokale modeller",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/de-DE/translation.json b/src/lib/i18n/locales/de-DE/translation.json
index f8d09d322..64ec0776d 100644
--- a/src/lib/i18n/locales/de-DE/translation.json
+++ b/src/lib/i18n/locales/de-DE/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Höre zu...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs können Fehler machen. Überprüfe wichtige Informationen.",
-	"Loading {{count}} sites": "",
 	"Local": "Lokal",
 	"Local Models": "Lokale Modelle",
 	"Lost": "Verloren",
diff --git a/src/lib/i18n/locales/dg-DG/translation.json b/src/lib/i18n/locales/dg-DG/translation.json
index 9439b4b37..6733ae692 100644
--- a/src/lib/i18n/locales/dg-DG/translation.json
+++ b/src/lib/i18n/locales/dg-DG/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs can make borks. Verify important info.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/el-GR/translation.json b/src/lib/i18n/locales/el-GR/translation.json
index 7e531fc94..c44018fe0 100644
--- a/src/lib/i18n/locales/el-GR/translation.json
+++ b/src/lib/i18n/locales/el-GR/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Ακούγεται...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Τα LLM μπορούν να κάνουν λάθη. Επαληθεύστε σημαντικές πληροφορίες.",
-	"Loading {{count}} sites": "",
 	"Local": "Τοπικό",
 	"Local Models": "Τοπικά Μοντέλα",
 	"Lost": "Χαμένος",
diff --git a/src/lib/i18n/locales/en-GB/translation.json b/src/lib/i18n/locales/en-GB/translation.json
index da563d920..8d8c14864 100644
--- a/src/lib/i18n/locales/en-GB/translation.json
+++ b/src/lib/i18n/locales/en-GB/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/en-US/translation.json b/src/lib/i18n/locales/en-US/translation.json
index da563d920..8d8c14864 100644
--- a/src/lib/i18n/locales/en-US/translation.json
+++ b/src/lib/i18n/locales/en-US/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/es-ES/translation.json b/src/lib/i18n/locales/es-ES/translation.json
index 4a0162a53..1b75f4727 100644
--- a/src/lib/i18n/locales/es-ES/translation.json
+++ b/src/lib/i18n/locales/es-ES/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Escuchando...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Los LLM pueden cometer errores. Verifica la información importante.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modelos locales",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/eu-ES/translation.json b/src/lib/i18n/locales/eu-ES/translation.json
index 1bbb0fbe3..47c5df84e 100644
--- a/src/lib/i18n/locales/eu-ES/translation.json
+++ b/src/lib/i18n/locales/eu-ES/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Entzuten...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMek akatsak egin ditzakete. Egiaztatu informazio garrantzitsua.",
-	"Loading {{count}} sites": "",
 	"Local": "Lokala",
 	"Local Models": "Modelo lokalak",
 	"Lost": "Galduta",
diff --git a/src/lib/i18n/locales/fa-IR/translation.json b/src/lib/i18n/locales/fa-IR/translation.json
index 80a646e4f..75609ffdf 100644
--- a/src/lib/i18n/locales/fa-IR/translation.json
+++ b/src/lib/i18n/locales/fa-IR/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "مدل\u200cهای زبانی بزرگ می\u200cتوانند اشتباه کنند. اطلاعات مهم را راستی\u200cآزمایی کنید.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/fi-FI/translation.json b/src/lib/i18n/locales/fi-FI/translation.json
index bd60c5789..e9f7f5653 100644
--- a/src/lib/i18n/locales/fi-FI/translation.json
+++ b/src/lib/i18n/locales/fi-FI/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Kuuntelee...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Kielimallit voivat tehdä virheitä. Tarkista tärkeät tiedot.",
-	"Loading {{count}} sites": "",
 	"Local": "Paikallinen",
 	"Local Models": "Paikalliset mallit",
 	"Lost": "Mennyt",
diff --git a/src/lib/i18n/locales/fr-CA/translation.json b/src/lib/i18n/locales/fr-CA/translation.json
index 6f507c133..a6abf8908 100644
--- a/src/lib/i18n/locales/fr-CA/translation.json
+++ b/src/lib/i18n/locales/fr-CA/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "En train d'écouter...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Les LLM peuvent faire des erreurs. Vérifiez les informations importantes.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modèles locaux",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/fr-FR/translation.json b/src/lib/i18n/locales/fr-FR/translation.json
index 491a86f88..d188ff917 100644
--- a/src/lib/i18n/locales/fr-FR/translation.json
+++ b/src/lib/i18n/locales/fr-FR/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Écoute en cours...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "Les LLM peuvent faire des erreurs. Vérifiez les informations importantes.",
-	"Loading {{count}} sites": "",
 	"Local": "Local",
 	"Local Models": "Modèles locaux",
 	"Lost": "Perdu",
diff --git a/src/lib/i18n/locales/he-IL/translation.json b/src/lib/i18n/locales/he-IL/translation.json
index 06599d9d0..695fd575e 100644
--- a/src/lib/i18n/locales/he-IL/translation.json
+++ b/src/lib/i18n/locales/he-IL/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "מודלים בשפה טבעית יכולים לטעות. אמת מידע חשוב.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/hi-IN/translation.json b/src/lib/i18n/locales/hi-IN/translation.json
index 70d4584e1..e98ca6e0e 100644
--- a/src/lib/i18n/locales/hi-IN/translation.json
+++ b/src/lib/i18n/locales/hi-IN/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "एलएलएम गलतियाँ कर सकते हैं। महत्वपूर्ण जानकारी सत्यापित करें.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/hr-HR/translation.json b/src/lib/i18n/locales/hr-HR/translation.json
index b4a27a781..61c6f7601 100644
--- a/src/lib/i18n/locales/hr-HR/translation.json
+++ b/src/lib/i18n/locales/hr-HR/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Slušam...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM-ovi mogu pogriješiti. Provjerite važne informacije.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokalni modeli",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/hu-HU/translation.json b/src/lib/i18n/locales/hu-HU/translation.json
index 764cf2db5..14818eeb1 100644
--- a/src/lib/i18n/locales/hu-HU/translation.json
+++ b/src/lib/i18n/locales/hu-HU/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Hallgatás...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Az LLM-ek hibázhatnak. Ellenőrizze a fontos információkat.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Helyi modellek",
 	"Lost": "Elveszett",
diff --git a/src/lib/i18n/locales/id-ID/translation.json b/src/lib/i18n/locales/id-ID/translation.json
index e47acd089..6a61ea717 100644
--- a/src/lib/i18n/locales/id-ID/translation.json
+++ b/src/lib/i18n/locales/id-ID/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Mendengarkan",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM dapat membuat kesalahan. Verifikasi informasi penting.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Model Lokal",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ie-GA/translation.json b/src/lib/i18n/locales/ie-GA/translation.json
index 1e114503b..4f320ba7f 100644
--- a/src/lib/i18n/locales/ie-GA/translation.json
+++ b/src/lib/i18n/locales/ie-GA/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Éisteacht...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Is féidir le LLManna botúin a dhéanamh. Fíoraigh faisnéis thábhachtach.",
-	"Loading {{count}} sites": "",
 	"Local": "Áitiúil",
 	"Local Models": "Múnlaí Áitiúla",
 	"Lost": "Cailleadh",
diff --git a/src/lib/i18n/locales/it-IT/translation.json b/src/lib/i18n/locales/it-IT/translation.json
index 08954b502..2770d9c9c 100644
--- a/src/lib/i18n/locales/it-IT/translation.json
+++ b/src/lib/i18n/locales/it-IT/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Gli LLM possono commettere errori. Verifica le informazioni importanti.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ja-JP/translation.json b/src/lib/i18n/locales/ja-JP/translation.json
index e529a6092..baa80bc2b 100644
--- a/src/lib/i18n/locales/ja-JP/translation.json
+++ b/src/lib/i18n/locales/ja-JP/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM は間違いを犯す可能性があります。重要な情報を検証してください。",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "ローカルモデル",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ka-GE/translation.json b/src/lib/i18n/locales/ka-GE/translation.json
index ffb1eb9fc..edac4d43f 100644
--- a/src/lib/i18n/locales/ka-GE/translation.json
+++ b/src/lib/i18n/locales/ka-GE/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "შესაძლოა LLM-ებმა შეცდომები დაუშვან. გადაამოწმეთ მნიშვნელოვანი ინფორმაცია.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ko-KR/translation.json b/src/lib/i18n/locales/ko-KR/translation.json
index 56703424f..43de6554d 100644
--- a/src/lib/i18n/locales/ko-KR/translation.json
+++ b/src/lib/i18n/locales/ko-KR/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "듣는 중...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM에 오류가 있을 수 있습니다. 중요한 정보는 확인이 필요합니다.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "로컬 모델",
 	"Lost": "패배",
diff --git a/src/lib/i18n/locales/lt-LT/translation.json b/src/lib/i18n/locales/lt-LT/translation.json
index 09f8ed02b..5a644032e 100644
--- a/src/lib/i18n/locales/lt-LT/translation.json
+++ b/src/lib/i18n/locales/lt-LT/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Klausoma...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Dideli kalbos modeliai gali klysti. Patikrinkite atsakymų teisingumą.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokalūs modeliai",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ms-MY/translation.json b/src/lib/i18n/locales/ms-MY/translation.json
index 31f29d4b6..236d0bbc5 100644
--- a/src/lib/i18n/locales/ms-MY/translation.json
+++ b/src/lib/i18n/locales/ms-MY/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Mendengar...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM boleh membuat kesilapan. Sahkan maklumat penting",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Model Tempatan",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/nb-NO/translation.json b/src/lib/i18n/locales/nb-NO/translation.json
index 352044fee..5121d297e 100644
--- a/src/lib/i18n/locales/nb-NO/translation.json
+++ b/src/lib/i18n/locales/nb-NO/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Lytter ...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Språkmodeller kan gjøre feil. Kontroller viktige opplysninger.",
-	"Loading {{count}} sites": "",
 	"Local": "Lokal",
 	"Local Models": "Lokale modeller",
 	"Lost": "Tapt",
diff --git a/src/lib/i18n/locales/nl-NL/translation.json b/src/lib/i18n/locales/nl-NL/translation.json
index e05391048..78dfd0a54 100644
--- a/src/lib/i18n/locales/nl-NL/translation.json
+++ b/src/lib/i18n/locales/nl-NL/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Aan het luisteren...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs kunnen fouten maken. Verifieer belangrijke informatie.",
-	"Loading {{count}} sites": "",
 	"Local": "Lokaal",
 	"Local Models": "Lokale modellen",
 	"Lost": "Verloren",
diff --git a/src/lib/i18n/locales/pa-IN/translation.json b/src/lib/i18n/locales/pa-IN/translation.json
index 408be6142..cae4f25c6 100644
--- a/src/lib/i18n/locales/pa-IN/translation.json
+++ b/src/lib/i18n/locales/pa-IN/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs ਗਲਤੀਆਂ ਕਰ ਸਕਦੇ ਹਨ। ਮਹੱਤਵਪੂਰਨ ਜਾਣਕਾਰੀ ਦੀ ਪੁਸ਼ਟੀ ਕਰੋ।",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/pl-PL/translation.json b/src/lib/i18n/locales/pl-PL/translation.json
index 2f38deaf7..caebf0272 100644
--- a/src/lib/i18n/locales/pl-PL/translation.json
+++ b/src/lib/i18n/locales/pl-PL/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMy mogą popełniać błędy. Zweryfikuj ważne informacje.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/pt-BR/translation.json b/src/lib/i18n/locales/pt-BR/translation.json
index fa1187e85..b89592035 100644
--- a/src/lib/i18n/locales/pt-BR/translation.json
+++ b/src/lib/i18n/locales/pt-BR/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Escutando...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs podem cometer erros. Verifique informações importantes.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modelos Locais",
 	"Lost": "Perdeu",
diff --git a/src/lib/i18n/locales/pt-PT/translation.json b/src/lib/i18n/locales/pt-PT/translation.json
index 0863ed7da..18e2b137a 100644
--- a/src/lib/i18n/locales/pt-PT/translation.json
+++ b/src/lib/i18n/locales/pt-PT/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "A escutar...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs podem cometer erros. Verifique informações importantes.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modelos Locais",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/ro-RO/translation.json b/src/lib/i18n/locales/ro-RO/translation.json
index 1a3905d4f..2d6c02277 100644
--- a/src/lib/i18n/locales/ro-RO/translation.json
+++ b/src/lib/i18n/locales/ro-RO/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Ascult...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM-urile pot face greșeli. Verificați informațiile importante.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Modele Locale",
 	"Lost": "Pierdut",
diff --git a/src/lib/i18n/locales/ru-RU/translation.json b/src/lib/i18n/locales/ru-RU/translation.json
index ca6d12bae..f0fc3c8e1 100644
--- a/src/lib/i18n/locales/ru-RU/translation.json
+++ b/src/lib/i18n/locales/ru-RU/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Слушаю...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs могут допускать ошибки. Проверяйте важную информацию.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Локальные модели",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/sk-SK/translation.json b/src/lib/i18n/locales/sk-SK/translation.json
index d4bf1ea35..8e4742fbb 100644
--- a/src/lib/i18n/locales/sk-SK/translation.json
+++ b/src/lib/i18n/locales/sk-SK/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Počúvanie...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM môžu robiť chyby. Overte si dôležité informácie.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokálne modely",
 	"Lost": "Stratený",
diff --git a/src/lib/i18n/locales/sr-RS/translation.json b/src/lib/i18n/locales/sr-RS/translation.json
index c6bdaf359..a16b78328 100644
--- a/src/lib/i18n/locales/sr-RS/translation.json
+++ b/src/lib/i18n/locales/sr-RS/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Слушам...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "ВЈМ-ови (LLM-ови) могу правити грешке. Проверите важне податке.",
-	"Loading {{count}} sites": "",
 	"Local": "Локално",
 	"Local Models": "Локални модели",
 	"Lost": "Пораза",
diff --git a/src/lib/i18n/locales/sv-SE/translation.json b/src/lib/i18n/locales/sv-SE/translation.json
index c8ea56901..a2d103b23 100644
--- a/src/lib/i18n/locales/sv-SE/translation.json
+++ b/src/lib/i18n/locales/sv-SE/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Lyssnar...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM:er kan göra misstag. Granska viktig information.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "Lokala modeller",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/th-TH/translation.json b/src/lib/i18n/locales/th-TH/translation.json
index 8f41d77d6..8b2e59199 100644
--- a/src/lib/i18n/locales/th-TH/translation.json
+++ b/src/lib/i18n/locales/th-TH/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "กำลังฟัง...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLMs สามารถทำผิดพลาดได้ ตรวจสอบข้อมูลสำคัญ",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "โมเดลท้องถิ่น",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/tk-TW/translation.json b/src/lib/i18n/locales/tk-TW/translation.json
index da563d920..8d8c14864 100644
--- a/src/lib/i18n/locales/tk-TW/translation.json
+++ b/src/lib/i18n/locales/tk-TW/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/tr-TR/translation.json b/src/lib/i18n/locales/tr-TR/translation.json
index 6eb6bcb7c..8195df8d0 100644
--- a/src/lib/i18n/locales/tr-TR/translation.json
+++ b/src/lib/i18n/locales/tr-TR/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Dinleniyor...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "LLM'ler hata yapabilir. Önemli bilgileri doğrulayın.",
-	"Loading {{count}} sites": "",
 	"Local": "Yerel",
 	"Local Models": "Yerel Modeller",
 	"Lost": "Kayıp",
diff --git a/src/lib/i18n/locales/uk-UA/translation.json b/src/lib/i18n/locales/uk-UA/translation.json
index 1bd9a15bd..2aa92ae15 100644
--- a/src/lib/i18n/locales/uk-UA/translation.json
+++ b/src/lib/i18n/locales/uk-UA/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Слухаю...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "LLMs можуть помилятися. Перевірте важливу інформацію.",
-	"Loading {{count}} sites": "",
 	"Local": "Локальний",
 	"Local Models": "Локальні моделі",
 	"Lost": "Втрачене",
diff --git a/src/lib/i18n/locales/ur-PK/translation.json b/src/lib/i18n/locales/ur-PK/translation.json
index 806598987..f733eda3d 100644
--- a/src/lib/i18n/locales/ur-PK/translation.json
+++ b/src/lib/i18n/locales/ur-PK/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "سن رہے ہیں...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "ایل ایل ایم غلطیاں کر سکتے ہیں اہم معلومات کی تصدیق کریں",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "مقامی ماڈلز",
 	"Lost": "گم شدہ",
diff --git a/src/lib/i18n/locales/vi-VN/translation.json b/src/lib/i18n/locales/vi-VN/translation.json
index 9a6e89610..93e98680b 100644
--- a/src/lib/i18n/locales/vi-VN/translation.json
+++ b/src/lib/i18n/locales/vi-VN/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "Đang nghe...",
 	"Llama.cpp": "",
 	"LLMs can make mistakes. Verify important information.": "Hệ thống có thể tạo ra nội dung không chính xác hoặc sai. Hãy kiểm chứng kỹ lưỡng thông tin trước khi tiếp nhận và sử dụng.",
-	"Loading {{count}} sites": "",
 	"Local": "",
 	"Local Models": "",
 	"Lost": "",
diff --git a/src/lib/i18n/locales/zh-CN/translation.json b/src/lib/i18n/locales/zh-CN/translation.json
index b8fbd3617..f32067afb 100644
--- a/src/lib/i18n/locales/zh-CN/translation.json
+++ b/src/lib/i18n/locales/zh-CN/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "正在倾听...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "大语言模型可能会生成误导性错误信息，请对关键信息加以验证。",
-	"Loading {{count}} sites": "",
 	"Local": "本地",
 	"Local Models": "本地模型",
 	"Lost": "落败",
diff --git a/src/lib/i18n/locales/zh-TW/translation.json b/src/lib/i18n/locales/zh-TW/translation.json
index c838cad4e..78dcddaff 100644
--- a/src/lib/i18n/locales/zh-TW/translation.json
+++ b/src/lib/i18n/locales/zh-TW/translation.json
@@ -565,7 +565,6 @@
 	"Listening...": "正在聆聽...",
 	"Llama.cpp": "Llama.cpp",
 	"LLMs can make mistakes. Verify important information.": "大型語言模型可能會出錯。請驗證重要資訊。",
-	"Loading {{count}} sites": "",
 	"Local": "本機",
 	"Local Models": "本機模型",
 	"Lost": "已遺失",

From aa2b764d743f348f6d380aa43fd2184e49ab6745 Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Fri, 14 Feb 2025 22:32:45 -0600
Subject: [PATCH 08/11] Finalize incomplete merge to update playwright branch

Introduced feature parity for trust_env
---
 backend/open_webui/retrieval/web/utils.py | 59 ++++++++++++++---------
 1 file changed, 37 insertions(+), 22 deletions(-)

diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index 25ca5aef5..b9fbbee3a 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -1,30 +1,33 @@
 import asyncio
-from datetime import datetime, time, timedelta
+import logging
 import socket
 import ssl
-import aiohttp
-import asyncio
 import urllib.parse
+import urllib.request
+from collections import defaultdict
+from datetime import datetime, time, timedelta
+from typing import (
+    Any,
+    AsyncIterator,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Sequence,
+    Union
+)
+import aiohttp
 import certifi
 import validators
-from collections import defaultdict
-from typing import AsyncIterator, Dict, List, Optional, Union, Sequence, Iterator
-from typing import Any, AsyncIterator, Dict, Iterator, List, Sequence, Union
-
-
 from langchain_community.document_loaders import (
-    WebBaseLoader,
-    PlaywrightURLLoader
+    PlaywrightURLLoader,
+    WebBaseLoader
 )
 from langchain_core.documents import Document
-
-
 from open_webui.constants import ERROR_MESSAGES
 from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER
 from open_webui.env import SRC_LOG_LEVELS
 
-import logging
-
 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["RAG"])
 
@@ -91,18 +94,20 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
     """Load HTML pages safely with Playwright, supporting SSL verification, rate limiting, and remote browser connection.
     
     Attributes:
-        urls (List[str]): List of URLs to load.
+        web_paths (List[str]): List of URLs to load.
         verify_ssl (bool): If True, verify SSL certificates.
         requests_per_second (Optional[float]): Number of requests per second to limit to.
         continue_on_failure (bool): If True, continue loading other URLs on failure.
         headless (bool): If True, the browser will run in headless mode.
         playwright_ws_url (Optional[str]): WebSocket endpoint URI for remote browser connection.
+        trust_env (bool): If True, use proxy settings from environment variables.
     """
 
     def __init__(
         self,
-        urls: List[str],
+        web_paths: List[str],
         verify_ssl: bool = True,
+        trust_env: bool = False,
         requests_per_second: Optional[float] = None,
         continue_on_failure: bool = True,
         headless: bool = True,
@@ -111,9 +116,20 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
         playwright_ws_url: Optional[str] = None
     ):
         """Initialize with additional safety parameters and remote browser support."""
+
+        proxy_server = proxy.get('server') if proxy else None
+        if trust_env and not proxy_server:
+            env_proxies = urllib.request.getproxies()
+            env_proxy_server = env_proxies.get('https') or env_proxies.get('http')
+            if env_proxy_server:
+                if proxy:
+                    proxy['server'] = env_proxy_server
+                else:
+                    proxy = { 'server': env_proxy_server }
+
         # We'll set headless to False if using playwright_ws_url since it's handled by the remote browser
         super().__init__(
-            urls=urls,
+            urls=web_paths,
             continue_on_failure=continue_on_failure,
             headless=headless if playwright_ws_url is None else False,
             remove_selectors=remove_selectors,
@@ -123,6 +139,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
         self.requests_per_second = requests_per_second
         self.last_request_time = None
         self.playwright_ws_url = playwright_ws_url
+        self.trust_env = trust_env
 
     def lazy_load(self) -> Iterator[Document]:
         """Safely load URLs synchronously with support for remote browser."""
@@ -347,14 +364,12 @@ def get_web_loader(
     # Check if the URLs are valid
     safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
 
-
     web_loader_args = {
-        web_path=safe_urls,
-        "urls": safe_urls,
+        "web_paths": safe_urls,
         "verify_ssl": verify_ssl,
         "requests_per_second": requests_per_second,
         "continue_on_failure": True,
-        trust_env=trust_env
+        "trust_env": trust_env
     }
 
     if PLAYWRIGHT_WS_URI.value:
@@ -364,6 +379,6 @@ def get_web_loader(
     WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value]
     web_loader = WebLoaderClass(**web_loader_args)
 
-    log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(urls))
+    log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
 
     return web_loader
\ No newline at end of file

From b1bab2ece88618a34deb228b72c44b8981d7db8c Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Fri, 14 Feb 2025 22:43:46 -0600
Subject: [PATCH 09/11] Remove duplicate loader.alazy_load line from merge

---
 backend/open_webui/routers/retrieval.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py
index e79e414b1..84a49e4ca 100644
--- a/backend/open_webui/routers/retrieval.py
+++ b/backend/open_webui/routers/retrieval.py
@@ -1348,8 +1348,6 @@ async def process_web_search(
             requests_per_second=request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
             trust_env=request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
         )
-        docs = [doc async for doc in loader.alazy_load()]
-        # docs = loader.load()
         docs = await loader.aload()
         await run_in_threadpool(
             save_docs_to_vector_db,

From 8e9b00a017ddd5995fe2d5f2dd4dd7dcf3955efb Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Fri, 14 Feb 2025 22:48:15 -0600
Subject: [PATCH 10/11] Fix docstring

---
 backend/open_webui/retrieval/web/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index b9fbbee3a..0718b6b85 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -96,11 +96,12 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader):
     Attributes:
         web_paths (List[str]): List of URLs to load.
         verify_ssl (bool): If True, verify SSL certificates.
+        trust_env (bool): If True, use proxy settings from environment variables.
         requests_per_second (Optional[float]): Number of requests per second to limit to.
         continue_on_failure (bool): If True, continue loading other URLs on failure.
         headless (bool): If True, the browser will run in headless mode.
+        proxy (dict): Proxy override settings for the Playwright session.
         playwright_ws_url (Optional[str]): WebSocket endpoint URI for remote browser connection.
-        trust_env (bool): If True, use proxy settings from environment variables.
     """
 
     def __init__(

From bc82f48ebfa7dd1ea334efd31b9c73de2cc456c0 Mon Sep 17 00:00:00 2001
From: Rory <16675082+roryeckel@users.noreply.github.com>
Date: Mon, 17 Feb 2025 21:43:32 -0600
Subject: [PATCH 11/11] refac: RAG_WEB_LOADER -> RAG_WEB_LOADER_ENGINE

---
 backend/open_webui/config.py              | 10 +++++-----
 backend/open_webui/main.py                |  4 ++--
 backend/open_webui/retrieval/web/utils.py | 12 ++++++------
 backend/start.sh                          |  2 +-
 backend/start_windows.bat                 |  2 +-
 docker-compose.playwright.yaml            |  2 +-
 6 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py
index c926759ca..0c796eb49 100644
--- a/backend/open_webui/config.py
+++ b/backend/open_webui/config.py
@@ -1853,10 +1853,10 @@ RAG_WEB_SEARCH_CONCURRENT_REQUESTS = PersistentConfig(
     int(os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")),
 )
 
-RAG_WEB_LOADER = PersistentConfig(
-    "RAG_WEB_LOADER",
-    "rag.web.loader",
-    os.environ.get("RAG_WEB_LOADER", "safe_web")
+RAG_WEB_LOADER_ENGINE = PersistentConfig(
+    "RAG_WEB_LOADER_ENGINE",
+    "rag.web.loader.engine",
+    os.environ.get("RAG_WEB_LOADER_ENGINE", "safe_web")
 )
 
 RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig(
@@ -1867,7 +1867,7 @@ RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig(
 
 PLAYWRIGHT_WS_URI = PersistentConfig(
     "PLAYWRIGHT_WS_URI",
-    "rag.web.loader.playwright.ws.uri",
+    "rag.web.loader.engine.playwright.ws.uri",
     os.environ.get("PLAYWRIGHT_WS_URI", None)
 )
 
diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py
index 5035119a0..0f6f91608 100644
--- a/backend/open_webui/main.py
+++ b/backend/open_webui/main.py
@@ -140,7 +140,7 @@ from open_webui.config import (
     AUDIO_TTS_AZURE_SPEECH_REGION,
     AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
     PLAYWRIGHT_WS_URI,
-    RAG_WEB_LOADER,
+    RAG_WEB_LOADER_ENGINE,
     WHISPER_MODEL,
     DEEPGRAM_API_KEY,
     WHISPER_MODEL_AUTO_UPDATE,
@@ -561,7 +561,7 @@ app.state.config.EXA_API_KEY = EXA_API_KEY
 
 app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
 app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
-app.state.config.RAG_WEB_LOADER = RAG_WEB_LOADER
+app.state.config.RAG_WEB_LOADER_ENGINE = RAG_WEB_LOADER_ENGINE
 app.state.config.RAG_WEB_SEARCH_TRUST_ENV = RAG_WEB_SEARCH_TRUST_ENV
 app.state.config.PLAYWRIGHT_WS_URI = PLAYWRIGHT_WS_URI
 
diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index 0718b6b85..dcf1728e9 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -25,7 +25,7 @@ from langchain_community.document_loaders import (
 )
 from langchain_core.documents import Document
 from open_webui.constants import ERROR_MESSAGES
-from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER
+from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER_ENGINE
 from open_webui.env import SRC_LOG_LEVELS
 
 log = logging.getLogger(__name__)
@@ -352,9 +352,9 @@ class SafeWebBaseLoader(WebBaseLoader):
         """Load data into Document objects."""
         return [document async for document in self.alazy_load()]
 
-RAG_WEB_LOADERS = defaultdict(lambda: SafeWebBaseLoader)
-RAG_WEB_LOADERS["playwright"] = SafePlaywrightURLLoader
-RAG_WEB_LOADERS["safe_web"] = SafeWebBaseLoader
+RAG_WEB_LOADER_ENGINES = defaultdict(lambda: SafeWebBaseLoader)
+RAG_WEB_LOADER_ENGINES["playwright"] = SafePlaywrightURLLoader
+RAG_WEB_LOADER_ENGINES["safe_web"] = SafeWebBaseLoader
 
 def get_web_loader(
     urls: Union[str, Sequence[str]],
@@ -377,9 +377,9 @@ def get_web_loader(
         web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value
 
     # Create the appropriate WebLoader based on the configuration
-    WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value]
+    WebLoaderClass = RAG_WEB_LOADER_ENGINES[RAG_WEB_LOADER_ENGINE.value]
     web_loader = WebLoaderClass(**web_loader_args)
 
-    log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
+    log.debug("Using RAG_WEB_LOADER_ENGINE %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
 
     return web_loader
\ No newline at end of file
diff --git a/backend/start.sh b/backend/start.sh
index 3b08cf549..671c22ff7 100755
--- a/backend/start.sh
+++ b/backend/start.sh
@@ -4,7 +4,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 cd "$SCRIPT_DIR" || exit
 
 # Add conditional Playwright browser installation
-if [[ "${RAG_WEB_LOADER,,}" == "playwright" ]]; then
+if [[ "${RAG_WEB_LOADER_ENGINE,,}" == "playwright" ]]; then
     if [[ -z "${PLAYWRIGHT_WS_URI}" ]]; then
         echo "Installing Playwright browsers..."
         playwright install chromium
diff --git a/backend/start_windows.bat b/backend/start_windows.bat
index 036e1f721..7049cd1b3 100644
--- a/backend/start_windows.bat
+++ b/backend/start_windows.bat
@@ -7,7 +7,7 @@ SET "SCRIPT_DIR=%~dp0"
 cd /d "%SCRIPT_DIR%" || exit /b
 
 :: Add conditional Playwright browser installation
-IF /I "%RAG_WEB_LOADER%" == "playwright" (
+IF /I "%RAG_WEB_LOADER_ENGINE%" == "playwright" (
     IF "%PLAYWRIGHT_WS_URI%" == "" (
         echo Installing Playwright browsers...
         playwright install chromium
diff --git a/docker-compose.playwright.yaml b/docker-compose.playwright.yaml
index 0a4bb3f76..fe570bed0 100644
--- a/docker-compose.playwright.yaml
+++ b/docker-compose.playwright.yaml
@@ -6,5 +6,5 @@ services:
 
   open-webui:
     environment:
-      - 'RAG_WEB_LOADER=playwright'
+      - 'RAG_WEB_LOADER_ENGINE=playwright'
       - 'PLAYWRIGHT_WS_URI=ws://playwright:3000'
\ No newline at end of file