OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-06-26 18:26:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refac: RAG_WEB_LOADER -> RAG_WEB_LOADER_ENGINE

Browse Source
This commit is contained in:
Rory
2025-02-17 21:43:32 -06:00
parent 66c2acc08d
commit bc82f48ebf
6 changed files with 16 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/open_webui/config.py
View File

@@ -1853,10 +1853,10 @@ RAG_WEB_SEARCH_CONCURRENT_REQUESTS = PersistentConfig(
int(os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")),
)
RAG_WEB_LOADER = PersistentConfig(
"RAG_WEB_LOADER",
"rag.web.loader",
os.environ.get("RAG_WEB_LOADER", "safe_web")
RAG_WEB_LOADER_ENGINE = PersistentConfig(
"RAG_WEB_LOADER_ENGINE",
"rag.web.loader.engine",
os.environ.get("RAG_WEB_LOADER_ENGINE", "safe_web")
)
RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig(
@@ -1867,7 +1867,7 @@ RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig(
PLAYWRIGHT_WS_URI = PersistentConfig(
"PLAYWRIGHT_WS_URI",
"rag.web.loader.playwright.ws.uri",
"rag.web.loader.engine.playwright.ws.uri",
os.environ.get("PLAYWRIGHT_WS_URI", None)
)

4
backend/open_webui/main.py
View File

@@ -140,7 +140,7 @@ from open_webui.config import (
AUDIO_TTS_AZURE_SPEECH_REGION,
AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
PLAYWRIGHT_WS_URI,
RAG_WEB_LOADER,
RAG_WEB_LOADER_ENGINE,
WHISPER_MODEL,
DEEPGRAM_API_KEY,
WHISPER_MODEL_AUTO_UPDATE,
@@ -561,7 +561,7 @@ app.state.config.EXA_API_KEY = EXA_API_KEY
app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
app.state.config.RAG_WEB_LOADER = RAG_WEB_LOADER
app.state.config.RAG_WEB_LOADER_ENGINE = RAG_WEB_LOADER_ENGINE
app.state.config.RAG_WEB_SEARCH_TRUST_ENV = RAG_WEB_SEARCH_TRUST_ENV
app.state.config.PLAYWRIGHT_WS_URI = PLAYWRIGHT_WS_URI

12
backend/open_webui/retrieval/web/utils.py
View File

@@ -25,7 +25,7 @@ from langchain_community.document_loaders import (
)
from langchain_core.documents import Document
from open_webui.constants import ERROR_MESSAGES
from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER
from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER_ENGINE
from open_webui.env import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
@@ -352,9 +352,9 @@ class SafeWebBaseLoader(WebBaseLoader):
"""Load data into Document objects."""
return [document async for document in self.alazy_load()]
RAG_WEB_LOADERS = defaultdict(lambda: SafeWebBaseLoader)
RAG_WEB_LOADERS["playwright"] = SafePlaywrightURLLoader
RAG_WEB_LOADERS["safe_web"] = SafeWebBaseLoader
RAG_WEB_LOADER_ENGINES = defaultdict(lambda: SafeWebBaseLoader)
RAG_WEB_LOADER_ENGINES["playwright"] = SafePlaywrightURLLoader
RAG_WEB_LOADER_ENGINES["safe_web"] = SafeWebBaseLoader
def get_web_loader(
urls: Union[str, Sequence[str]],
@@ -377,9 +377,9 @@ def get_web_loader(
web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value
# Create the appropriate WebLoader based on the configuration
WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value]
WebLoaderClass = RAG_WEB_LOADER_ENGINES[RAG_WEB_LOADER_ENGINE.value]
web_loader = WebLoaderClass(**web_loader_args)
log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
log.debug("Using RAG_WEB_LOADER_ENGINE %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
return web_loader