OpenWebUI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2025-06-09 07:56:42 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

refac: web/rag config

Browse Source
This commit is contained in:
Timothy Jaeryang Baek 2025-04-12 16:33:36 -07:00
parent c3497da5dd
commit 48a23ce3fe
11 changed files with 1367 additions and 1530 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
backend/open_webui/config.py
View File

@ -201,7 +201,10 @@ def save_config(config):
T = TypeVar("T") T = TypeVar("T")
ENABLE_PERSISTENT_CONFIG = os.environ.get("ENABLE_PERSISTENT_CONFIG", "True").lower() == "true" ENABLE_PERSISTENT_CONFIG = (
os.environ.get("ENABLE_PERSISTENT_CONFIG", "True").lower() == "true"
)
class PersistentConfig(Generic[T]): class PersistentConfig(Generic[T]):
def __init__(self, env_name: str, config_path: str, env_value: T): def __init__(self, env_name: str, config_path: str, env_value: T):
@ -612,10 +615,16 @@ def load_oauth_providers():
"scope": OAUTH_SCOPES.value, "scope": OAUTH_SCOPES.value,
} }
if OAUTH_CODE_CHALLENGE_METHOD.value and OAUTH_CODE_CHALLENGE_METHOD.value == "S256": if (
OAUTH_CODE_CHALLENGE_METHOD.value
and OAUTH_CODE_CHALLENGE_METHOD.value == "S256"
):
client_kwargs["code_challenge_method"] = "S256" client_kwargs["code_challenge_method"] = "S256"
elif OAUTH_CODE_CHALLENGE_METHOD.value: elif OAUTH_CODE_CHALLENGE_METHOD.value:
raise Exception('Code challenge methods other than "%s" not supported. Given: "%s"' % ("S256", OAUTH_CODE_CHALLENGE_METHOD.value)) raise Exception(
'Code challenge methods other than "%s" not supported. Given: "%s"'
% ("S256", OAUTH_CODE_CHALLENGE_METHOD.value)
)
client.register( client.register(
name="oidc", name="oidc",
@ -1820,12 +1829,6 @@ RAG_FILE_MAX_SIZE = PersistentConfig(
), ),
) )
ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = PersistentConfig(
"ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION",
"rag.enable_web_loader_ssl_verification",
os.environ.get("ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION", "True").lower() == "true",
)
RAG_EMBEDDING_ENGINE = PersistentConfig( RAG_EMBEDDING_ENGINE = PersistentConfig(
"RAG_EMBEDDING_ENGINE", "RAG_EMBEDDING_ENGINE",
"rag.embedding_engine", "rag.embedding_engine",
@ -1990,16 +1993,20 @@ YOUTUBE_LOADER_PROXY_URL = PersistentConfig(
) )
ENABLE_RAG_WEB_SEARCH = PersistentConfig( ####################################
"ENABLE_RAG_WEB_SEARCH", # Web Search (RAG)
####################################
ENABLE_WEB_SEARCH = PersistentConfig(
"ENABLE_WEB_SEARCH",
"rag.web.search.enable", "rag.web.search.enable",
os.getenv("ENABLE_RAG_WEB_SEARCH", "False").lower() == "true", os.getenv("ENABLE_WEB_SEARCH", "False").lower() == "true",
) )
RAG_WEB_SEARCH_ENGINE = PersistentConfig( WEB_SEARCH_ENGINE = PersistentConfig(
"RAG_WEB_SEARCH_ENGINE", "WEB_SEARCH_ENGINE",
"rag.web.search.engine", "rag.web.search.engine",
os.getenv("RAG_WEB_SEARCH_ENGINE", ""), os.getenv("WEB_SEARCH_ENGINE", ""),
) )
BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = PersistentConfig( BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = PersistentConfig(
@ -2008,10 +2015,18 @@ BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = PersistentConfig(
os.getenv("BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL", "False").lower() == "true", os.getenv("BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL", "False").lower() == "true",
) )
WEB_SEARCH_RESULT_COUNT = PersistentConfig(
"WEB_SEARCH_RESULT_COUNT",
"rag.web.search.result_count",
int(os.getenv("WEB_SEARCH_RESULT_COUNT", "3")),
)
# You can provide a list of your own websites to filter after performing a web search. # You can provide a list of your own websites to filter after performing a web search.
# This ensures the highest level of safety and reliability of the information sources. # This ensures the highest level of safety and reliability of the information sources.
RAG_WEB_SEARCH_DOMAIN_FILTER_LIST = PersistentConfig( WEB_SEARCH_DOMAIN_FILTER_LIST = PersistentConfig(
"RAG_WEB_SEARCH_DOMAIN_FILTER_LIST", "WEB_SEARCH_DOMAIN_FILTER_LIST",
"rag.web.search.domain.filter_list", "rag.web.search.domain.filter_list",
[ [
# "wikipedia.com", # "wikipedia.com",
@ -2020,6 +2035,30 @@ RAG_WEB_SEARCH_DOMAIN_FILTER_LIST = PersistentConfig(
], ],
) )
WEB_SEARCH_CONCURRENT_REQUESTS = PersistentConfig(
"WEB_SEARCH_CONCURRENT_REQUESTS",
"rag.web.search.concurrent_requests",
int(os.getenv("WEB_SEARCH_CONCURRENT_REQUESTS", "10")),
)
WEB_LOADER_ENGINE = PersistentConfig(
"WEB_LOADER_ENGINE",
"rag.web.loader.engine",
os.environ.get("WEB_LOADER_ENGINE", ""),
)
ENABLE_WEB_LOADER_SSL_VERIFICATION = PersistentConfig(
"ENABLE_WEB_LOADER_SSL_VERIFICATION",
"rag.web.loader.ssl_verification",
os.environ.get("ENABLE_WEB_LOADER_SSL_VERIFICATION", "True").lower() == "true",
)
WEB_SEARCH_TRUST_ENV = PersistentConfig(
"WEB_SEARCH_TRUST_ENV",
"rag.web.search.trust_env",
os.getenv("WEB_SEARCH_TRUST_ENV", "False").lower() == "true",
)
SEARXNG_QUERY_URL = PersistentConfig( SEARXNG_QUERY_URL = PersistentConfig(
"SEARXNG_QUERY_URL", "SEARXNG_QUERY_URL",
@ -2155,34 +2194,22 @@ SOUGOU_API_SK = PersistentConfig(
os.getenv("SOUGOU_API_SK", ""), os.getenv("SOUGOU_API_SK", ""),
) )
RAG_WEB_SEARCH_RESULT_COUNT = PersistentConfig( TAVILY_API_KEY = PersistentConfig(
"RAG_WEB_SEARCH_RESULT_COUNT", "TAVILY_API_KEY",
"rag.web.search.result_count", "rag.web.search.tavily_api_key",
int(os.getenv("RAG_WEB_SEARCH_RESULT_COUNT", "3")), os.getenv("TAVILY_API_KEY", ""),
) )
RAG_WEB_SEARCH_CONCURRENT_REQUESTS = PersistentConfig( TAVILY_EXTRACT_DEPTH = PersistentConfig(
"RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "TAVILY_EXTRACT_DEPTH",
"rag.web.search.concurrent_requests", "rag.web.search.tavily_extract_depth",
int(os.getenv("RAG_WEB_SEARCH_CONCURRENT_REQUESTS", "10")), os.getenv("TAVILY_EXTRACT_DEPTH", "basic"),
) )
RAG_WEB_LOADER_ENGINE = PersistentConfig( PLAYWRIGHT_WS_URL = PersistentConfig(
"RAG_WEB_LOADER_ENGINE", "PLAYWRIGHT_WS_URL",
"rag.web.loader.engine", "rag.web.loader.PLAYWRIGHT_WS_URL",
os.environ.get("RAG_WEB_LOADER_ENGINE", "safe_web"), os.environ.get("PLAYWRIGHT_WS_URL", ""),
)
RAG_WEB_SEARCH_TRUST_ENV = PersistentConfig(
"RAG_WEB_SEARCH_TRUST_ENV",
"rag.web.search.trust_env",
os.getenv("RAG_WEB_SEARCH_TRUST_ENV", "False").lower() == "true",
)
PLAYWRIGHT_WS_URI = PersistentConfig(
"PLAYWRIGHT_WS_URI",
"rag.web.loader.playwright_ws_uri",
os.environ.get("PLAYWRIGHT_WS_URI", ""),
) )
PLAYWRIGHT_TIMEOUT = PersistentConfig( PLAYWRIGHT_TIMEOUT = PersistentConfig(
@ -2203,17 +2230,6 @@ FIRECRAWL_API_BASE_URL = PersistentConfig(
os.environ.get("FIRECRAWL_API_BASE_URL", "https://api.firecrawl.dev"), os.environ.get("FIRECRAWL_API_BASE_URL", "https://api.firecrawl.dev"),
) )
TAVILY_API_KEY = PersistentConfig(
"TAVILY_API_KEY",
"rag.web.loader.tavily_api_key",
os.getenv("TAVILY_API_KEY", ""),
)
TAVILY_EXTRACT_DEPTH = PersistentConfig(
"TAVILY_EXTRACT_DEPTH",
"rag.web.loader.tavily_extract_depth",
os.getenv("TAVILY_EXTRACT_DEPTH", "basic"),
)
#################################### ####################################
# Images # Images

41
backend/open_webui/main.py
View File

@ -160,11 +160,11 @@ from open_webui.config import (
AUDIO_TTS_VOICE, AUDIO_TTS_VOICE,
AUDIO_TTS_AZURE_SPEECH_REGION, AUDIO_TTS_AZURE_SPEECH_REGION,
AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT, AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
PLAYWRIGHT_WS_URI, PLAYWRIGHT_WS_URL,
PLAYWRIGHT_TIMEOUT, PLAYWRIGHT_TIMEOUT,
FIRECRAWL_API_BASE_URL, FIRECRAWL_API_BASE_URL,
FIRECRAWL_API_KEY, FIRECRAWL_API_KEY,
RAG_WEB_LOADER_ENGINE, WEB_LOADER_ENGINE,
WHISPER_MODEL, WHISPER_MODEL,
DEEPGRAM_API_KEY, DEEPGRAM_API_KEY,
WHISPER_MODEL_AUTO_UPDATE, WHISPER_MODEL_AUTO_UPDATE,
@ -205,12 +205,13 @@ from open_webui.config import (
YOUTUBE_LOADER_LANGUAGE, YOUTUBE_LOADER_LANGUAGE,
YOUTUBE_LOADER_PROXY_URL, YOUTUBE_LOADER_PROXY_URL,
# Retrieval (Web Search) # Retrieval (Web Search)
RAG_WEB_SEARCH_ENGINE, ENABLE_WEB_SEARCH,
WEB_SEARCH_ENGINE,
BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL, BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
RAG_WEB_SEARCH_RESULT_COUNT, WEB_SEARCH_RESULT_COUNT,
RAG_WEB_SEARCH_CONCURRENT_REQUESTS, WEB_SEARCH_CONCURRENT_REQUESTS,
RAG_WEB_SEARCH_TRUST_ENV, WEB_SEARCH_TRUST_ENV,
RAG_WEB_SEARCH_DOMAIN_FILTER_LIST, WEB_SEARCH_DOMAIN_FILTER_LIST,
JINA_API_KEY, JINA_API_KEY,
SEARCHAPI_API_KEY, SEARCHAPI_API_KEY,
SEARCHAPI_ENGINE, SEARCHAPI_ENGINE,
@ -240,8 +241,7 @@ from open_webui.config import (
ONEDRIVE_CLIENT_ID, ONEDRIVE_CLIENT_ID,
ENABLE_RAG_HYBRID_SEARCH, ENABLE_RAG_HYBRID_SEARCH,
ENABLE_RAG_LOCAL_WEB_FETCH, ENABLE_RAG_LOCAL_WEB_FETCH,
ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION, ENABLE_WEB_LOADER_SSL_VERIFICATION,
ENABLE_RAG_WEB_SEARCH,
ENABLE_GOOGLE_DRIVE_INTEGRATION, ENABLE_GOOGLE_DRIVE_INTEGRATION,
ENABLE_ONEDRIVE_INTEGRATION, ENABLE_ONEDRIVE_INTEGRATION,
UPLOAD_DIR, UPLOAD_DIR,
@ -594,9 +594,7 @@ app.state.config.FILE_MAX_COUNT = RAG_FILE_MAX_COUNT
app.state.config.RAG_FULL_CONTEXT = RAG_FULL_CONTEXT app.state.config.RAG_FULL_CONTEXT = RAG_FULL_CONTEXT
app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL = BYPASS_EMBEDDING_AND_RETRIEVAL app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL = BYPASS_EMBEDDING_AND_RETRIEVAL
app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH
app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION = ( app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION = ENABLE_WEB_LOADER_SSL_VERIFICATION
ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
)
app.state.config.CONTENT_EXTRACTION_ENGINE = CONTENT_EXTRACTION_ENGINE app.state.config.CONTENT_EXTRACTION_ENGINE = CONTENT_EXTRACTION_ENGINE
app.state.config.TIKA_SERVER_URL = TIKA_SERVER_URL app.state.config.TIKA_SERVER_URL = TIKA_SERVER_URL
@ -629,12 +627,16 @@ app.state.config.YOUTUBE_LOADER_LANGUAGE = YOUTUBE_LOADER_LANGUAGE
app.state.config.YOUTUBE_LOADER_PROXY_URL = YOUTUBE_LOADER_PROXY_URL app.state.config.YOUTUBE_LOADER_PROXY_URL = YOUTUBE_LOADER_PROXY_URL
app.state.config.ENABLE_RAG_WEB_SEARCH = ENABLE_RAG_WEB_SEARCH app.state.config.ENABLE_WEB_SEARCH = ENABLE_WEB_SEARCH
app.state.config.RAG_WEB_SEARCH_ENGINE = RAG_WEB_SEARCH_ENGINE app.state.config.WEB_SEARCH_ENGINE = WEB_SEARCH_ENGINE
app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST = WEB_SEARCH_DOMAIN_FILTER_LIST
app.state.config.WEB_SEARCH_RESULT_COUNT = WEB_SEARCH_RESULT_COUNT
app.state.config.WEB_SEARCH_CONCURRENT_REQUESTS = WEB_SEARCH_CONCURRENT_REQUESTS
app.state.config.WEB_LOADER_ENGINE = WEB_LOADER_ENGINE
app.state.config.WEB_SEARCH_TRUST_ENV = WEB_SEARCH_TRUST_ENV
app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = ( app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = (
BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL
) )
app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST = RAG_WEB_SEARCH_DOMAIN_FILTER_LIST
app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ENABLE_GOOGLE_DRIVE_INTEGRATION app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ENABLE_GOOGLE_DRIVE_INTEGRATION
app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION
@ -662,11 +664,8 @@ app.state.config.PERPLEXITY_API_KEY = PERPLEXITY_API_KEY
app.state.config.SOUGOU_API_SID = SOUGOU_API_SID app.state.config.SOUGOU_API_SID = SOUGOU_API_SID
app.state.config.SOUGOU_API_SK = SOUGOU_API_SK app.state.config.SOUGOU_API_SK = SOUGOU_API_SK
app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS app.state.config.PLAYWRIGHT_WS_URL = PLAYWRIGHT_WS_URL
app.state.config.RAG_WEB_LOADER_ENGINE = RAG_WEB_LOADER_ENGINE
app.state.config.RAG_WEB_SEARCH_TRUST_ENV = RAG_WEB_SEARCH_TRUST_ENV
app.state.config.PLAYWRIGHT_WS_URI = PLAYWRIGHT_WS_URI
app.state.config.PLAYWRIGHT_TIMEOUT = PLAYWRIGHT_TIMEOUT app.state.config.PLAYWRIGHT_TIMEOUT = PLAYWRIGHT_TIMEOUT
app.state.config.FIRECRAWL_API_BASE_URL = FIRECRAWL_API_BASE_URL app.state.config.FIRECRAWL_API_BASE_URL = FIRECRAWL_API_BASE_URL
app.state.config.FIRECRAWL_API_KEY = FIRECRAWL_API_KEY app.state.config.FIRECRAWL_API_KEY = FIRECRAWL_API_KEY
@ -1261,7 +1260,7 @@ async def get_app_config(request: Request):
{ {
"enable_direct_connections": app.state.config.ENABLE_DIRECT_CONNECTIONS, "enable_direct_connections": app.state.config.ENABLE_DIRECT_CONNECTIONS,
"enable_channels": app.state.config.ENABLE_CHANNELS, "enable_channels": app.state.config.ENABLE_CHANNELS,
"enable_web_search": app.state.config.ENABLE_RAG_WEB_SEARCH, "enable_web_search": app.state.config.ENABLE_WEB_SEARCH,
"enable_code_execution": app.state.config.ENABLE_CODE_EXECUTION, "enable_code_execution": app.state.config.ENABLE_CODE_EXECUTION,
"enable_code_interpreter": app.state.config.ENABLE_CODE_INTERPRETER, "enable_code_interpreter": app.state.config.ENABLE_CODE_INTERPRETER,
"enable_image_generation": app.state.config.ENABLE_IMAGE_GENERATION, "enable_image_generation": app.state.config.ENABLE_IMAGE_GENERATION,

48
backend/open_webui/retrieval/web/utils.py
View File

@ -28,9 +28,9 @@ from open_webui.retrieval.loaders.tavily import TavilyLoader
from open_webui.constants import ERROR_MESSAGES from open_webui.constants import ERROR_MESSAGES
from open_webui.config import ( from open_webui.config import (
ENABLE_RAG_LOCAL_WEB_FETCH, ENABLE_RAG_LOCAL_WEB_FETCH,
PLAYWRIGHT_WS_URI, PLAYWRIGHT_WS_URL,
PLAYWRIGHT_TIMEOUT, PLAYWRIGHT_TIMEOUT,
RAG_WEB_LOADER_ENGINE, WEB_LOADER_ENGINE,
FIRECRAWL_API_BASE_URL, FIRECRAWL_API_BASE_URL,
FIRECRAWL_API_KEY, FIRECRAWL_API_KEY,
TAVILY_API_KEY, TAVILY_API_KEY,
@ -584,13 +584,6 @@ class SafeWebBaseLoader(WebBaseLoader):
return [document async for document in self.alazy_load()] return [document async for document in self.alazy_load()]
RAG_WEB_LOADER_ENGINES = defaultdict(lambda: SafeWebBaseLoader)
RAG_WEB_LOADER_ENGINES["playwright"] = SafePlaywrightURLLoader
RAG_WEB_LOADER_ENGINES["safe_web"] = SafeWebBaseLoader
RAG_WEB_LOADER_ENGINES["firecrawl"] = SafeFireCrawlLoader
RAG_WEB_LOADER_ENGINES["tavily"] = SafeTavilyLoader
def get_web_loader( def get_web_loader(
urls: Union[str, Sequence[str]], urls: Union[str, Sequence[str]],
verify_ssl: bool = True, verify_ssl: bool = True,
@ -608,27 +601,36 @@ def get_web_loader(
"trust_env": trust_env, "trust_env": trust_env,
} }
if RAG_WEB_LOADER_ENGINE.value == "playwright": if WEB_LOADER_ENGINE.value == "" or WEB_LOADER_ENGINE.value == "safe_web":
WebLoaderClass = SafeWebBaseLoader
if WEB_LOADER_ENGINE.value == "playwright":
WebLoaderClass = SafePlaywrightURLLoader
web_loader_args["playwright_timeout"] = PLAYWRIGHT_TIMEOUT.value * 1000 web_loader_args["playwright_timeout"] = PLAYWRIGHT_TIMEOUT.value * 1000
if PLAYWRIGHT_WS_URI.value: if PLAYWRIGHT_WS_URL.value:
web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URI.value web_loader_args["playwright_ws_url"] = PLAYWRIGHT_WS_URL.value
if RAG_WEB_LOADER_ENGINE.value == "firecrawl": if WEB_LOADER_ENGINE.value == "firecrawl":
WebLoaderClass = SafeFireCrawlLoader
web_loader_args["api_key"] = FIRECRAWL_API_KEY.value web_loader_args["api_key"] = FIRECRAWL_API_KEY.value
web_loader_args["api_url"] = FIRECRAWL_API_BASE_URL.value web_loader_args["api_url"] = FIRECRAWL_API_BASE_URL.value
if RAG_WEB_LOADER_ENGINE.value == "tavily": if WEB_LOADER_ENGINE.value == "tavily":
WebLoaderClass = SafeTavilyLoader
web_loader_args["api_key"] = TAVILY_API_KEY.value web_loader_args["api_key"] = TAVILY_API_KEY.value
web_loader_args["extract_depth"] = TAVILY_EXTRACT_DEPTH.value web_loader_args["extract_depth"] = TAVILY_EXTRACT_DEPTH.value
# Create the appropriate WebLoader based on the configuration if WebLoaderClass:
WebLoaderClass = RAG_WEB_LOADER_ENGINES[RAG_WEB_LOADER_ENGINE.value] web_loader = WebLoaderClass(**web_loader_args)
web_loader = WebLoaderClass(**web_loader_args)
log.debug( log.debug(
"Using RAG_WEB_LOADER_ENGINE %s for %s URLs", "Using WEB_LOADER_ENGINE %s for %s URLs",
web_loader.__class__.__name__, web_loader.__class__.__name__,
len(safe_urls), len(safe_urls),
) )
return web_loader return web_loader
else:
raise ValueError(
f"Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE.value}. "
"Please set it to 'safe_web', 'playwright', 'firecrawl', or 'tavily'."
)

873
backend/open_webui/routers/retrieval.py
View File

File diff suppressed because it is too large Load Diff

4
backend/start.sh
View File

@ -4,8 +4,8 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
cd "$SCRIPT_DIR" || exit cd "$SCRIPT_DIR" || exit
# Add conditional Playwright browser installation # Add conditional Playwright browser installation
if [[ "${RAG_WEB_LOADER_ENGINE,,}" == "playwright" ]]; then if [[ "${WEB_LOADER_ENGINE,,}" == "playwright" ]]; then
if [[ -z "${PLAYWRIGHT_WS_URI}" ]]; then if [[ -z "${PLAYWRIGHT_WS_URL}" ]]; then
echo "Installing Playwright browsers..." echo "Installing Playwright browsers..."
playwright install chromium playwright install chromium
playwright install-deps chromium playwright install-deps chromium

4
backend/start_windows.bat
View File

@ -7,8 +7,8 @@ SET "SCRIPT_DIR=%~dp0"
cd /d "%SCRIPT_DIR%" || exit /b cd /d "%SCRIPT_DIR%" || exit /b
:: Add conditional Playwright browser installation :: Add conditional Playwright browser installation
IF /I "%RAG_WEB_LOADER_ENGINE%" == "playwright" ( IF /I "%WEB_LOADER_ENGINE%" == "playwright" (
IF "%PLAYWRIGHT_WS_URI%" == "" ( IF "%PLAYWRIGHT_WS_URL%" == "" (
echo Installing Playwright browsers... echo Installing Playwright browsers...
playwright install chromium playwright install chromium
playwright install-deps chromium playwright install-deps chromium

4
docker-compose.playwright.yaml
View File

@ -6,5 +6,5 @@ services:
open-webui: open-webui:
environment: environment:
- 'RAG_WEB_LOADER_ENGINE=playwright' - 'WEB_LOADER_ENGINE=playwright'
- 'PLAYWRIGHT_WS_URI=ws://playwright:3000' - 'PLAYWRIGHT_WS_URL=ws://playwright:3000'

33
src/lib/apis/retrieval/index.ts
View File

@ -50,9 +50,9 @@ type YoutubeConfigForm = {
}; };
type RAGConfigForm = { type RAGConfigForm = {
pdf_extract_images?: boolean; PDF_EXTRACT_IMAGES?: boolean;
enable_google_drive_integration?: boolean; ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
enable_onedrive_integration?: boolean; ENABLE_ONEDRIVE_INTEGRATION?: boolean;
chunk?: ChunkConfigForm; chunk?: ChunkConfigForm;
content_extraction?: ContentExtractConfigForm; content_extraction?: ContentExtractConfigForm;
web_loader_ssl_verification?: boolean; web_loader_ssl_verification?: boolean;
@ -89,33 +89,6 @@ export const updateRAGConfig = async (token: string, payload: RAGConfigForm) =>
return res; return res;
}; };
export const getRAGTemplate = async (token: string) => {
let error = null;
const res = await fetch(`${RETRIEVAL_API_BASE_URL}/template`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
console.log(err);
error = err.detail;
return null;
});
if (error) {
throw error;
}
return res?.template ?? '';
};
export const getQuerySettings = async (token: string) => { export const getQuerySettings = async (token: string) => {
let error = null; let error = null;

1206
src/lib/components/admin/Settings/Documents.svelte
View File

File diff suppressed because it is too large Load Diff

540
src/lib/components/admin/Settings/WebSearch.svelte
View File

@ -12,12 +12,6 @@
export let saveHandler: Function; export let saveHandler: Function;
let webConfig = null;
let bypass_ssl_verification = null;
let tavily_api_key = null;
let youtube_language = null;
let webSearchEngines = [ let webSearchEngines = [
'searxng', 'searxng',
'google_pse', 'google_pse',
@ -38,33 +32,25 @@
'perplexity', 'perplexity',
'sougou' 'sougou'
]; ];
let webLoaderEngines = ['safe_web', 'playwright', 'firecrawl', 'tavily']; let webLoaderEngines = ['playwright', 'firecrawl', 'tavily'];
let webConfig = null;
const submitHandler = async () => { const submitHandler = async () => {
// Convert domain filter string to array before sending // Convert domain filter string to array before sending
if (webConfig.search.domain_filter_list) { if (webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST) {
webConfig.search.domain_filter_list = webConfig.search.domain_filter_list webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST = webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST.split(',')
.split(',')
.map((domain) => domain.trim()) .map((domain) => domain.trim())
.filter((domain) => domain.length > 0); .filter((domain) => domain.length > 0);
} else { } else {
webConfig.search.domain_filter_list = []; webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST = [];
} }
// Set the enable_ssl_verification flag based on the switch state
webConfig.loader.enable_ssl_verification = !bypass_ssl_verification;
// Set shared tavily_api_key
webConfig.search.tavily_api_key = tavily_api_key;
webConfig.loader.tavily_api_key = tavily_api_key;
webConfig.loader.youtube.language = youtube_language.split(',').map((lang) => lang.trim());
const res = await updateRAGConfig(localStorage.token, { const res = await updateRAGConfig(localStorage.token, {
web: webConfig web: webConfig
}); });
webConfig.search.domain_filter_list = webConfig.search.domain_filter_list.join(', '); webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST = webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST.join(', ');
youtube_language = webConfig.loader.youtube.language.join(', ');
}; };
onMount(async () => { onMount(async () => {
@ -72,13 +58,14 @@
if (res) { if (res) {
webConfig = res.web; webConfig = res.web;
// Convert array back to comma-separated string for display // Convert array back to comma-separated string for display
if (webConfig?.search?.domain_filter_list) { if (webConfig?.WEB_SEARCH_DOMAIN_FILTER_LIST) {
webConfig.search.domain_filter_list = webConfig.search.domain_filter_list.join(', '); webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST =
webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST.join(', ');
} }
bypass_ssl_verification = !webConfig.loader.enable_ssl_verification;
tavily_api_key = webConfig.search.tavily_api_key || webConfig.loader.tavily_api_key; webConfig.YOUTUBE_LOADER_LANGUAGE = webConfig.YOUTUBE_LOADER_LANGUAGE.join(',');
youtube_language = webConfig.loader.youtube.language.join(', ');
} }
}); });
</script> </script>
@ -100,10 +87,10 @@
<div class=" mb-2.5 flex w-full justify-between"> <div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium"> <div class=" self-center text-xs font-medium">
{$i18n.t('Enable Web Search')} {$i18n.t('Web Search')}
</div> </div>
<div class="flex items-center relative"> <div class="flex items-center relative">
<Switch bind:state={webConfig.ENABLE_RAG_WEB_SEARCH} /> <Switch bind:state={webConfig.ENABLE_WEB_SEARCH} />
</div> </div>
</div> </div>
@ -114,7 +101,7 @@
<div class="flex items-center relative"> <div class="flex items-center relative">
<select <select
class="dark:bg-gray-900 w-fit pr-8 rounded-sm px-2 p-1 text-xs bg-transparent outline-hidden text-right" class="dark:bg-gray-900 w-fit pr-8 rounded-sm px-2 p-1 text-xs bg-transparent outline-hidden text-right"
bind:value={webConfig.search.engine} bind:value={webConfig.WEB_SEARCH_ENGINE}
placeholder={$i18n.t('Select a engine')} placeholder={$i18n.t('Select a engine')}
required required
> >
@ -126,8 +113,8 @@
</div> </div>
</div> </div>
{#if webConfig.search.engine !== ''} {#if webConfig.WEB_SEARCH_ENGINE !== ''}
{#if webConfig.search.engine === 'searxng'} {#if webConfig.WEB_SEARCH_ENGINE === 'searxng'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -140,14 +127,14 @@
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text" type="text"
placeholder={$i18n.t('Enter Searxng Query URL')} placeholder={$i18n.t('Enter Searxng Query URL')}
bind:value={webConfig.search.searxng_query_url} bind:value={webConfig.SEARXNG_QUERY_URL}
autocomplete="off" autocomplete="off"
/> />
</div> </div>
</div> </div>
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'google_pse'} {:else if webConfig.WEB_SEARCH_ENGINE === 'google_pse'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -156,7 +143,7 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Google PSE API Key')} placeholder={$i18n.t('Enter Google PSE API Key')}
bind:value={webConfig.search.google_pse_api_key} bind:value={webConfig.GOOGLE_PSE_API_KEY}
/> />
</div> </div>
<div class="mt-1.5"> <div class="mt-1.5">
@ -170,14 +157,14 @@
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text" type="text"
placeholder={$i18n.t('Enter Google PSE Engine Id')} placeholder={$i18n.t('Enter Google PSE Engine Id')}
bind:value={webConfig.search.google_pse_engine_id} bind:value={webConfig.GOOGLE_PSE_ENGINE_ID}
autocomplete="off" autocomplete="off"
/> />
</div> </div>
</div> </div>
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'brave'} {:else if webConfig.WEB_SEARCH_ENGINE === 'brave'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -186,11 +173,11 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Brave Search API Key')} placeholder={$i18n.t('Enter Brave Search API Key')}
bind:value={webConfig.search.brave_search_api_key} bind:value={webConfig.BRAVE_SEARCH_API_KEY}
/> />
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'kagi'} {:else if webConfig.WEB_SEARCH_ENGINE === 'kagi'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -199,11 +186,12 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Kagi Search API Key')} placeholder={$i18n.t('Enter Kagi Search API Key')}
bind:value={webConfig.search.kagi_search_api_key} bind:value={webConfig.KAGI_SEARCH_API_KEY}
/> />
</div> </div>
.
</div> </div>
{:else if webConfig.search.engine === 'mojeek'} {:else if webConfig.WEB_SEARCH_ENGINE === 'mojeek'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -212,11 +200,11 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Mojeek Search API Key')} placeholder={$i18n.t('Enter Mojeek Search API Key')}
bind:value={webConfig.search.mojeek_search_api_key} bind:value={webConfig.MOJEEK_SEARCH_API_KEY}
/> />
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'bocha'} {:else if webConfig.WEB_SEARCH_ENGINE === 'bocha'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -225,11 +213,11 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Bocha Search API Key')} placeholder={$i18n.t('Enter Bocha Search API Key')}
bind:value={webConfig.search.bocha_search_api_key} bind:value={webConfig.BOCHA_SEARCH_API_KEY}
/> />
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'serpstack'} {:else if webConfig.WEB_SEARCH_ENGINE === 'serpstack'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -238,11 +226,11 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Serpstack API Key')} placeholder={$i18n.t('Enter Serpstack API Key')}
bind:value={webConfig.search.serpstack_api_key} bind:value={webConfig.SERPSTACK_API_KEY}
/> />
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'serper'} {:else if webConfig.WEB_SEARCH_ENGINE === 'serper'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -251,11 +239,11 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Serper API Key')} placeholder={$i18n.t('Enter Serper API Key')}
bind:value={webConfig.search.serper_api_key} bind:value={webConfig.SERPER_API_KEY}
/> />
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'serply'} {:else if webConfig.WEB_SEARCH_ENGINE === 'serply'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -264,11 +252,24 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Serply API Key')} placeholder={$i18n.t('Enter Serply API Key')}
bind:value={webConfig.search.serply_api_key} bind:value={webConfig.SERPLY_API_KEY}
/> />
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'searchapi'} {:else if webConfig.WEB_SEARCH_ENGINE === 'tavily'}
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Tavily API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Tavily API Key')}
bind:value={webConfig.TAVILY_API_KEY}
/>
</div>
</div>
{:else if webConfig.WEB_SEARCH_ENGINE === 'searchapi'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -277,7 +278,7 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter SearchApi API Key')} placeholder={$i18n.t('Enter SearchApi API Key')}
bind:value={webConfig.search.searchapi_api_key} bind:value={webConfig.SEARCHAPI_API_KEY}
/> />
</div> </div>
<div class="mt-1.5"> <div class="mt-1.5">
@ -291,14 +292,14 @@
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text" type="text"
placeholder={$i18n.t('Enter SearchApi Engine')} placeholder={$i18n.t('Enter SearchApi Engine')}
bind:value={webConfig.search.searchapi_engine} bind:value={webConfig.SEARCHAPI_ENGINE}
autocomplete="off" autocomplete="off"
/> />
</div> </div>
</div> </div>
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'serpapi'} {:else if webConfig.WEB_SEARCH_ENGINE === 'serpapi'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -307,7 +308,7 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter SerpApi API Key')} placeholder={$i18n.t('Enter SerpApi API Key')}
bind:value={webConfig.search.serpapi_api_key} bind:value={webConfig.SERPAPI_API_KEY}
/> />
</div> </div>
<div class="mt-1.5"> <div class="mt-1.5">
@ -321,27 +322,14 @@
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text" type="text"
placeholder={$i18n.t('Enter SerpApi Engine')} placeholder={$i18n.t('Enter SerpApi Engine')}
bind:value={webConfig.search.serpapi_engine} bind:value={webConfig.SERPAPI_ENGINE}
autocomplete="off" autocomplete="off"
/> />
</div> </div>
</div> </div>
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'tavily'} {:else if webConfig.WEB_SEARCH_ENGINE === 'jina'}
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Tavily API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Tavily API Key')}
bind:value={tavily_api_key}
/>
</div>
</div>
{:else if webConfig.search.engine === 'jina'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -350,35 +338,11 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Jina API Key')} placeholder={$i18n.t('Enter Jina API Key')}
bind:value={webConfig.search.jina_api_key} bind:value={webConfig.JINA_API_KEY}
/> />
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'exa'} {:else if webConfig.WEB_SEARCH_ENGINE === 'bing'}
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Exa API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Exa API Key')}
bind:value={webConfig.search.exa_api_key}
/>
</div>
</div>
{:else if webConfig.search.engine === 'perplexity'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Perplexity API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Perplexity API Key')}
bind:value={webConfig.search.perplexity_api_key}
/>
</div>
{:else if webConfig.search.engine === 'bing'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -391,7 +355,7 @@
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text" type="text"
placeholder={$i18n.t('Enter Bing Search V7 Endpoint')} placeholder={$i18n.t('Enter Bing Search V7 Endpoint')}
bind:value={webConfig.search.bing_search_v7_endpoint} bind:value={webConfig.BING_SEARCH_V7_ENDPOINT}
autocomplete="off" autocomplete="off"
/> />
</div> </div>
@ -405,11 +369,35 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Bing Search V7 Subscription Key')} placeholder={$i18n.t('Enter Bing Search V7 Subscription Key')}
bind:value={webConfig.search.bing_search_v7_subscription_key} bind:value={webConfig.BING_SEARCH_V7_SUBSCRIPTION_KEY}
/> />
</div> </div>
</div> </div>
{:else if webConfig.search.engine === 'sougou'} {:else if webConfig.WEB_SEARCH_ENGINE === 'exa'}
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Exa API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Exa API Key')}
bind:value={webConfig.EXA_API_KEY}
/>
</div>
</div>
{:else if webConfig.WEB_SEARCH_ENGINE === 'perplexity'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Perplexity API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Perplexity API Key')}
bind:value={webConfig.PERPLEXITY_API_KEY}
/>
</div>
{:else if webConfig.WEB_SEARCH_ENGINE === 'sougou'}
<div class="mb-2.5 flex w-full flex-col"> <div class="mb-2.5 flex w-full flex-col">
<div> <div>
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
@ -418,7 +406,7 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Sougou Search API sID')} placeholder={$i18n.t('Enter Sougou Search API sID')}
bind:value={webConfig.search.sougou_api_sid} bind:value={webConfig.SOUGOU_API_SID}
/> />
</div> </div>
</div> </div>
@ -430,57 +418,99 @@
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Sougou Search API SK')} placeholder={$i18n.t('Enter Sougou Search API SK')}
bind:value={webConfig.search.sougou_api_sk} bind:value={webConfig.SOUGOU_API_SK}
/> />
</div> </div>
</div> </div>
{/if} {/if}
{/if} {/if}
<div class="mb-2.5 flex w-full flex-col"> {#if webConfig.ENABLE_WEB_SEARCH}
<div class="flex gap-2"> <div class="mb-2.5 flex w-full flex-col">
<div class="w-full"> <div class="flex gap-2">
<div class=" self-center text-xs font-medium mb-1"> <div class="w-full">
{$i18n.t('Search Result Count')} <div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Search Result Count')}
</div>
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
placeholder={$i18n.t('Search Result Count')}
bind:value={webConfig.WEB_SEARCH_RESULT_COUNT}
required
/>
</div> </div>
<input <div class="w-full">
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" <div class=" self-center text-xs font-medium mb-1">
placeholder={$i18n.t('Search Result Count')} {$i18n.t('Concurrent Requests')}
bind:value={webConfig.search.result_count} </div>
required
/>
</div>
<div class="w-full"> <input
<div class=" self-center text-xs font-medium mb-1"> class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
{$i18n.t('Concurrent Requests')} placeholder={$i18n.t('Concurrent Requests')}
bind:value={webConfig.WEB_SEARCH_CONCURRENT_REQUESTS}
required
/>
</div> </div>
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
placeholder={$i18n.t('Concurrent Requests')}
bind:value={webConfig.search.concurrent_requests}
required
/>
</div> </div>
</div> </div>
<div class="mb-2.5 flex w-full flex-col">
<div class=" text-xs font-medium mb-1">
{$i18n.t('Domain Filter List')}
</div>
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
placeholder={$i18n.t(
'Enter domains separated by commas (e.g., example.com,site.org)'
)}
bind:value={webConfig.WEB_SEARCH_DOMAIN_FILTER_LIST}
/>
</div>
{/if}
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
<Tooltip content={$i18n.t('Full Context Mode')} placement="top-start">
{$i18n.t('Bypass Embedding and Retrieval')}
</Tooltip>
</div>
<div class="flex items-center relative">
<Tooltip
content={webConfig.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL
? $i18n.t(
'Inject the entire content as context for comprehensive processing, this is recommended for complex queries.'
)
: $i18n.t(
'Default to segmented retrieval for focused and relevant content extraction, this is recommended for most cases.'
)}
>
<Switch bind:state={webConfig.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL} />
</Tooltip>
</div>
</div> </div>
<div class="mb-2.5 flex w-full flex-col"> <div class=" mb-2.5 flex w-full justify-between">
<div class=" text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium">
{$i18n.t('Domain Filter List')} {$i18n.t('Trust Proxy Environment')}
</div>
<div class="flex items-center relative">
<Tooltip
content={webConfig.WEB_SEARCH_TRUST_ENV
? $i18n.t(
'Use proxy designated by http_proxy and https_proxy environment variables to fetch page contents.'
)
: $i18n.t('Use no proxy to fetch page contents.')}
>
<Switch bind:state={webConfig.WEB_SEARCH_TRUST_ENV} />
</Tooltip>
</div> </div>
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300dark:bg-gray-850 outline-hidden"
placeholder={$i18n.t(
'Enter domains separated by commas (e.g., example.com,site.org)'
)}
bind:value={webConfig.search.domain_filter_list}
/>
</div> </div>
</div>
<div class="mb-3">
<div class=" mb-2.5 text-base font-medium">{$i18n.t('Loader')}</div> <div class=" mb-2.5 text-base font-medium">{$i18n.t('Loader')}</div>
<hr class=" border-gray-100 dark:border-gray-850 my-2" /> <hr class=" border-gray-100 dark:border-gray-850 my-2" />
@ -492,11 +522,11 @@
<div class="flex items-center relative"> <div class="flex items-center relative">
<select <select
class="dark:bg-gray-900 w-fit pr-8 rounded-sm px-2 p-1 text-xs bg-transparent outline-hidden text-right" class="dark:bg-gray-900 w-fit pr-8 rounded-sm px-2 p-1 text-xs bg-transparent outline-hidden text-right"
bind:value={webConfig.loader.engine} bind:value={webConfig.WEB_LOADER_ENGINE}
placeholder={$i18n.t('Select a engine')} placeholder={$i18n.t('Select a engine')}
required required
> >
<option disabled selected value="">{$i18n.t('Select a engine')}</option> <option value="">{$i18n.t('Default')}</option>
{#each webLoaderEngines as engine} {#each webLoaderEngines as engine}
<option value={engine}>{engine}</option> <option value={engine}>{engine}</option>
{/each} {/each}
@ -504,113 +534,118 @@
</div> </div>
</div> </div>
{#if webConfig.loader.engine !== ''} {#if webConfig.WEB_LOADER_ENGINE === '' || webConfig.WEB_LOADER_ENGINE === 'safe_web'}
{#if webConfig.loader.engine === 'playwright'} <div class=" mb-2.5 flex w-full justify-between">
<div class="mb-2.5 flex w-full flex-col"> <div class=" self-center text-xs font-medium">
<div> {$i18n.t('Verify SSL Certificate')}
<div class=" self-center text-xs font-medium mb-1"> </div>
{$i18n.t('Playwright WebSocket URL')} <div class="flex items-center relative">
</div> <Switch bind:state={webConfig.ENABLE_WEB_LOADER_SSL_VERIFICATION} />
</div>
<div class="flex w-full"> </div>
<div class="flex-1"> {:else if webConfig.WEB_LOADER_ENGINE === 'playwright'}
<input <div class="mb-2.5 flex w-full flex-col">
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" <div>
type="text" <div class=" self-center text-xs font-medium mb-1">
placeholder={$i18n.t('Enter Playwright WebSocket URL')} {$i18n.t('Playwright WebSocket URL')}
bind:value={webConfig.loader.playwright_ws_uri}
autocomplete="off"
/>
</div>
</div>
</div> </div>
<div class="mt-2"> <div class="flex w-full">
<div class=" self-center text-xs font-medium mb-1"> <div class="flex-1">
{$i18n.t('Playwright Timeout (ms)')} <input
</div> class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
<div class="flex w-full"> placeholder={$i18n.t('Enter Playwright WebSocket URL')}
<div class="flex-1"> bind:value={webConfig.PLAYWRIGHT_WS_URL}
<input autocomplete="off"
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" />
placeholder={$i18n.t('Enter Playwright Timeout (ms)')}
bind:value={webConfig.loader.playwright_timeout}
autocomplete="off"
/>
</div>
</div> </div>
</div> </div>
</div> </div>
{:else if webConfig.loader.engine === 'firecrawl'}
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Firecrawl API Base URL')}
</div>
<div class="flex w-full"> <div class="mt-2">
<div class="flex-1"> <div class=" self-center text-xs font-medium mb-1">
<input {$i18n.t('Playwright Timeout (ms)')}
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
placeholder={$i18n.t('Enter Firecrawl API Base URL')}
bind:value={webConfig.loader.firecrawl_api_base_url}
autocomplete="off"
/>
</div>
</div>
</div> </div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
placeholder={$i18n.t('Enter Playwright Timeout')}
bind:value={webConfig.PLAYWRIGHT_TIMEOUT}
autocomplete="off"
/>
</div>
</div>
</div>
</div>
{:else if webConfig.WEB_LOADER_ENGINE === 'firecrawl'}
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Firecrawl API Base URL')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
placeholder={$i18n.t('Enter Firecrawl API Base URL')}
bind:value={webConfig.FIRECRAWL_API_BASE_URL}
autocomplete="off"
/>
</div>
</div>
</div>
<div class="mt-2">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Firecrawl API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Firecrawl API Key')}
bind:value={webConfig.FIRECRAWL_API_KEY}
/>
</div>
</div>
{:else if webConfig.WEB_LOADER_ENGINE === 'tavily'}
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Tavily Extract Depth')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
placeholder={$i18n.t('Enter Tavily Extract Depth')}
bind:value={webConfig.TAVILY_EXTRACT_DEPTH}
autocomplete="off"
/>
</div>
</div>
</div>
{#if webConfig.WEB_SEARCH_ENGINE !== 'tavily'}
<div class="mt-2"> <div class="mt-2">
<div class=" self-center text-xs font-medium mb-1"> <div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Firecrawl API Key')} {$i18n.t('Tavily API Key')}
</div> </div>
<SensitiveInput <SensitiveInput
placeholder={$i18n.t('Enter Firecrawl API Key')} placeholder={$i18n.t('Enter Tavily API Key')}
bind:value={webConfig.loader.firecrawl_api_key} bind:value={webConfig.TAVILY_API_KEY}
/> />
</div> </div>
</div> {/if}
{:else if webConfig.loader.engine === 'tavily'} </div>
<div class="mb-2.5 flex w-full flex-col">
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Tavily Extract Depth')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="text"
placeholder={$i18n.t('Enter Tavily Extract Depth')}
bind:value={webConfig.loader.tavily_extract_depth}
autocomplete="off"
/>
</div>
</div>
</div>
{#if webConfig.search.engine !== 'tavily'}
<div class="mt-2">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Tavily API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Tavily API Key')}
bind:value={tavily_api_key}
/>
</div>
{/if}
</div>
{/if}
{/if} {/if}
<hr class=" border-gray-100 dark:border-gray-850 my-2" />
<div class=" mb-2.5 flex w-full justify-between"> <div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium"> <div class=" self-center text-xs font-medium">
{$i18n.t('Youtube Language')} {$i18n.t('Youtube Language')}
@ -620,7 +655,7 @@
class="flex-1 w-full rounded-lg text-sm bg-transparent outline-hidden" class="flex-1 w-full rounded-lg text-sm bg-transparent outline-hidden"
type="text" type="text"
placeholder={$i18n.t('Enter language codes')} placeholder={$i18n.t('Enter language codes')}
bind:value={youtube_language} bind:value={webConfig.YOUTUBE_LOADER_LANGUAGE}
autocomplete="off" autocomplete="off"
/> />
</div> </div>
@ -632,63 +667,14 @@
</div> </div>
<div class="flex items-center relative"> <div class="flex items-center relative">
<input <input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden" class="flex-1 w-full rounded-lg text-sm bg-transparent outline-hidden"
type="text" type="text"
placeholder={$i18n.t('Enter proxy URL (e.g. https://user:password@host:port)')} placeholder={$i18n.t('Enter proxy URL (e.g. https://user:password@host:port)')}
bind:value={webConfig.loader.youtube.proxy_url} bind:value={webConfig.YOUTUBE_LOADER_PROXY_URL}
autocomplete="off" autocomplete="off"
/> />
</div> </div>
</div> </div>
<hr class=" border-gray-100 dark:border-gray-850 my-2" />
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Bypass SSL verification for Websites')}
</div>
<div class="flex items-center relative">
<Switch bind:state={bypass_ssl_verification} />
</div>
</div>
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Trust Proxy Environment')}
</div>
<div class="flex items-center relative">
<Tooltip
content={webConfig.loader.trust_env
? $i18n.t(
'Use proxy designated by http_proxy and https_proxy environment variables to fetch page contents.'
)
: $i18n.t('Use no proxy to fetch page contents.')}
>
<Switch bind:state={webConfig.loader.trust_env} />
</Tooltip>
</div>
</div>
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
<Tooltip content={$i18n.t('Full Context Mode')} placement="top-start">
{$i18n.t('Bypass Embedding and Retrieval')}
</Tooltip>
</div>
<div class="flex items-center relative">
<Tooltip
content={webConfig.loader.bypass_embedding_and_retrieval
? $i18n.t(
'Inject the entire content as context for comprehensive processing, this is recommended for complex queries.'
)
: $i18n.t(
'Default to segmented retrieval for focused and relevant content extraction, this is recommended for most cases.'
)}
>
<Switch bind:state={webConfig.loader.bypass_embedding_and_retrieval} />
</Tooltip>
</div>
</div>
</div> </div>
</div> </div>
{/if} {/if}

24
src/lib/utils/rag/index.ts
View File

@ -1,24 +0,0 @@
import { getRAGTemplate } from '$lib/apis/retrieval';
export const RAGTemplate = async (token: string, context: string, query: string) => {
let template = await getRAGTemplate(token).catch(() => {
return `Use the following context as your learned knowledge, inside <context></context> XML tags.
<context>
[context]
</context>
When answer to user:
- If you don't know, just say that you don't know.
- If you don't know when you are not sure, ask for clarification.
Avoid mentioning that you obtained the information from the context.
And answer according to the language of the user's question.
Given the context information, answer the query.
Query: [query]`;
});
template = template.replace(/\[context\]/g, context);
template = template.replace(/\[query\]/g, query);
return template;
};