OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-06-26 18:26:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

perf: set shorter timeout for playwright and make it configurable

Browse Source
This commit is contained in:
genjuro
2025-03-20 14:58:38 +08:00
parent 70550e41fc
commit 07098c6352
3 changed files with 22 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
backend/open_webui/retrieval/web/utils.py
View File

@@ -29,6 +29,7 @@ from open_webui.constants import ERROR_MESSAGES
from open_webui.config import (
ENABLE_RAG_LOCAL_WEB_FETCH,
PLAYWRIGHT_WS_URI,
PLAYWRIGHT_GOTO_TIMEOUT,
RAG_WEB_LOADER_ENGINE,
FIRECRAWL_API_BASE_URL,
FIRECRAWL_API_KEY,
@@ -376,6 +377,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
headless (bool): If True, the browser will run in headless mode.
proxy (dict): Proxy override settings for the Playwright session.
playwright_ws_url (Optional[str]): WebSocket endpoint URI for remote browser connection.
playwright_goto_timeout (Optional[int]): Maximum operation time in milliseconds.
"""
def __init__(
@@ -389,6 +391,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
remove_selectors: Optional[List[str]] = None,
proxy: Optional[Dict[str, str]] = None,
playwright_ws_url: Optional[str] = None,
playwright_goto_timeout: Optional[int] = 10000,
):
"""Initialize with additional safety parameters and remote browser support."""
@@ -415,6 +418,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
self.last_request_time = None
self.playwright_ws_url = playwright_ws_url
self.trust_env = trust_env
self.playwright_goto_timeout = playwright_goto_timeout
def lazy_load(self) -> Iterator[Document]:
"""Safely load URLs synchronously with support for remote browser."""
@@ -431,7 +435,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
try:
self._safe_process_url_sync(url)
page = browser.new_page()
response = page.goto(url)
response = page.goto(url, timeout=self.playwright_goto_timeout)
if response is None:
raise ValueError(f"page.goto() returned None for url {url}")
@@ -462,7 +466,9 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
try:
await self._safe_process_url(url)
page = await browser.new_page()
response = await page.goto(url)
response = await page.goto(
url, timeout=self.playwright_goto_timeout
)
if response is None:
raise ValueError(f"page.goto() returned None for url {url}")
@@ -604,8 +610,12 @@ def get_web_loader(
"trust_env": trust_env,
}
if PLAYWRIGHT_WS_URI.value:
web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value
if RAG_WEB_LOADER_ENGINE.value == "playwright":
web_loader_args["playwright_goto_timeout"] = (
PLAYWRIGHT_GOTO_TIMEOUT.value * 1000
)
if PLAYWRIGHT_WS_URI.value:
web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value
if RAG_WEB_LOADER_ENGINE.value == "firecrawl":
web_loader_args["api_key"] = FIRECRAWL_API_KEY.value