UniqAI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-04-28 02:01:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refac: RAG_WEB_LOADER -> RAG_WEB_LOADER_ENGINE

Browse Source
This commit is contained in:
Rory 2025-02-17 21:43:32 -06:00
parent 66c2acc08d
commit bc82f48ebf
6 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/open_webui/config.py
View File

@ -1853,10 +1853,10 @@ RAG_WEB_SEARCH_CONCURRENT_REQUESTS = PersistentConfig(
int(os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")), int(os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")),
) )
RAG_WEB_LOADER = PersistentConfig( RAG_WEB_LOADER_ENGINE = PersistentConfig(
"RAG_WEB_LOADER", "RAG_WEB_LOADER_ENGINE",
"rag.web.loader", "rag.web.loader.engine",
os.environ.get("RAG_WEB_LOADER", "safe_web") os.environ.get("RAG_WEB_LOADER_ENGINE", "safe_web")
) )
RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig( RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig(
@ -1867,7 +1867,7 @@ RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig(
PLAYWRIGHT_WS_URI = PersistentConfig( PLAYWRIGHT_WS_URI = PersistentConfig(
"PLAYWRIGHT_WS_URI", "PLAYWRIGHT_WS_URI",
"rag.web.loader.playwright.ws.uri", "rag.web.loader.engine.playwright.ws.uri",
os.environ.get("PLAYWRIGHT_WS_URI", None) os.environ.get("PLAYWRIGHT_WS_URI", None)
) )

4
backend/open_webui/main.py
View File

@ -140,7 +140,7 @@ from open_webui.config import (
AUDIO_TTS_AZURE_SPEECH_REGION, AUDIO_TTS_AZURE_SPEECH_REGION,
AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT, AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
PLAYWRIGHT_WS_URI, PLAYWRIGHT_WS_URI,
RAG_WEB_LOADER, RAG_WEB_LOADER_ENGINE,
WHISPER_MODEL, WHISPER_MODEL,
DEEPGRAM_API_KEY, DEEPGRAM_API_KEY,
WHISPER_MODEL_AUTO_UPDATE, WHISPER_MODEL_AUTO_UPDATE,
@ -561,7 +561,7 @@ app.state.config.EXA_API_KEY = EXA_API_KEY
app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
app.state.config.RAG_WEB_LOADER = RAG_WEB_LOADER app.state.config.RAG_WEB_LOADER_ENGINE = RAG_WEB_LOADER_ENGINE
app.state.config.RAG_WEB_SEARCH_TRUST_ENV = RAG_WEB_SEARCH_TRUST_ENV app.state.config.RAG_WEB_SEARCH_TRUST_ENV = RAG_WEB_SEARCH_TRUST_ENV
app.state.config.PLAYWRIGHT_WS_URI = PLAYWRIGHT_WS_URI app.state.config.PLAYWRIGHT_WS_URI = PLAYWRIGHT_WS_URI

12
backend/open_webui/retrieval/web/utils.py
View File

@ -25,7 +25,7 @@ from langchain_community.document_loaders import (
) )
from langchain_core.documents import Document from langchain_core.documents import Document
from open_webui.constants import ERROR_MESSAGES from open_webui.constants import ERROR_MESSAGES
from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER from open_webui.config import ENABLE_RAG_LOCAL_WEB_FETCH, PLAYWRIGHT_WS_URI, RAG_WEB_LOADER_ENGINE
from open_webui.env import SRC_LOG_LEVELS from open_webui.env import SRC_LOG_LEVELS
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -352,9 +352,9 @@ class SafeWebBaseLoader(WebBaseLoader):
"""Load data into Document objects.""" """Load data into Document objects."""
return [document async for document in self.alazy_load()] return [document async for document in self.alazy_load()]
RAG_WEB_LOADERS = defaultdict(lambda: SafeWebBaseLoader) RAG_WEB_LOADER_ENGINES = defaultdict(lambda: SafeWebBaseLoader)
RAG_WEB_LOADERS["playwright"] = SafePlaywrightURLLoader RAG_WEB_LOADER_ENGINES["playwright"] = SafePlaywrightURLLoader
RAG_WEB_LOADERS["safe_web"] = SafeWebBaseLoader RAG_WEB_LOADER_ENGINES["safe_web"] = SafeWebBaseLoader
def get_web_loader( def get_web_loader(
urls: Union[str, Sequence[str]], urls: Union[str, Sequence[str]],
@ -377,9 +377,9 @@ def get_web_loader(
web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value
# Create the appropriate WebLoader based on the configuration # Create the appropriate WebLoader based on the configuration
WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value] WebLoaderClass = RAG_WEB_LOADER_ENGINES[RAG_WEB_LOADER_ENGINE.value]
web_loader = WebLoaderClass(**web_loader_args) web_loader = WebLoaderClass(**web_loader_args)
log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls)) log.debug("Using RAG_WEB_LOADER_ENGINE %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
return web_loader return web_loader

2
backend/start.sh
View File

@ -4,7 +4,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
cd "$SCRIPT_DIR" || exit cd "$SCRIPT_DIR" || exit
# Add conditional Playwright browser installation # Add conditional Playwright browser installation
if [[ "${RAG_WEB_LOADER,,}" == "playwright" ]]; then if [[ "${RAG_WEB_LOADER_ENGINE,,}" == "playwright" ]]; then
if [[ -z "${PLAYWRIGHT_WS_URI}" ]]; then if [[ -z "${PLAYWRIGHT_WS_URI}" ]]; then
echo "Installing Playwright browsers..." echo "Installing Playwright browsers..."
playwright install chromium playwright install chromium

2
backend/start_windows.bat
View File

@ -7,7 +7,7 @@ SET "SCRIPT_DIR=%~dp0"
cd /d "%SCRIPT_DIR%" || exit /b cd /d "%SCRIPT_DIR%" || exit /b
:: Add conditional Playwright browser installation :: Add conditional Playwright browser installation
IF /I "%RAG_WEB_LOADER%" == "playwright" ( IF /I "%RAG_WEB_LOADER_ENGINE%" == "playwright" (
IF "%PLAYWRIGHT_WS_URI%" == "" ( IF "%PLAYWRIGHT_WS_URI%" == "" (
echo Installing Playwright browsers... echo Installing Playwright browsers...
playwright install chromium playwright install chromium

2
docker-compose.playwright.yaml
View File

@ -6,5 +6,5 @@ services:
open-webui: open-webui:
environment: environment:
- 'RAG_WEB_LOADER=playwright' - 'RAG_WEB_LOADER_ENGINE=playwright'
- 'PLAYWRIGHT_WS_URI=ws://playwright:3000' - 'PLAYWRIGHT_WS_URI=ws://playwright:3000'