mirror of
https://github.com/open-webui/open-webui
synced 2025-06-25 17:57:20 +00:00
Add yacy config for user/pass, automatically add yacy json api path
This commit is contained in:
parent
0f73b96616
commit
240d91d38d
backend/open_webui
src/lib/components/admin/Settings
@ -2093,6 +2093,18 @@ YACY_QUERY_URL = PersistentConfig(
|
|||||||
os.getenv("YACY_QUERY_URL", ""),
|
os.getenv("YACY_QUERY_URL", ""),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
YACY_USERNAME = PersistentConfig(
|
||||||
|
"YACY_USERNAME",
|
||||||
|
"rag.web.search.yacy_username",
|
||||||
|
os.getenv("YACY_USERNAME", ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
YACY_PASSWORD = PersistentConfig(
|
||||||
|
"YACY_PASSWORD",
|
||||||
|
"rag.web.search.yacy_password",
|
||||||
|
os.getenv("YACY_PASSWORD", ""),
|
||||||
|
)
|
||||||
|
|
||||||
GOOGLE_PSE_API_KEY = PersistentConfig(
|
GOOGLE_PSE_API_KEY = PersistentConfig(
|
||||||
"GOOGLE_PSE_API_KEY",
|
"GOOGLE_PSE_API_KEY",
|
||||||
"rag.web.search.google_pse_api_key",
|
"rag.web.search.google_pse_api_key",
|
||||||
|
@ -220,6 +220,8 @@ from open_webui.config import (
|
|||||||
SERPAPI_ENGINE,
|
SERPAPI_ENGINE,
|
||||||
SEARXNG_QUERY_URL,
|
SEARXNG_QUERY_URL,
|
||||||
YACY_QUERY_URL,
|
YACY_QUERY_URL,
|
||||||
|
YACY_USERNAME,
|
||||||
|
YACY_PASSWORD,
|
||||||
SERPER_API_KEY,
|
SERPER_API_KEY,
|
||||||
SERPLY_API_KEY,
|
SERPLY_API_KEY,
|
||||||
SERPSTACK_API_KEY,
|
SERPSTACK_API_KEY,
|
||||||
@ -648,6 +650,8 @@ app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ENABLE_GOOGLE_DRIVE_INTEGRATI
|
|||||||
app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION
|
app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION
|
||||||
app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL
|
app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL
|
||||||
app.state.config.YACY_QUERY_URL = YACY_QUERY_URL
|
app.state.config.YACY_QUERY_URL = YACY_QUERY_URL
|
||||||
|
app.state.config.YACY_USERNAME = YACY_USERNAME
|
||||||
|
app.state.config.YACY_PASSWORD = YACY_PASSWORD
|
||||||
app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY
|
app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY
|
||||||
app.state.config.GOOGLE_PSE_ENGINE_ID = GOOGLE_PSE_ENGINE_ID
|
app.state.config.GOOGLE_PSE_ENGINE_ID = GOOGLE_PSE_ENGINE_ID
|
||||||
app.state.config.BRAVE_SEARCH_API_KEY = BRAVE_SEARCH_API_KEY
|
app.state.config.BRAVE_SEARCH_API_KEY = BRAVE_SEARCH_API_KEY
|
||||||
|
@ -12,6 +12,8 @@ log.setLevel(SRC_LOG_LEVELS["RAG"])
|
|||||||
|
|
||||||
def search_yacy(
|
def search_yacy(
|
||||||
query_url: str,
|
query_url: str,
|
||||||
|
username: Optional[str],
|
||||||
|
password: Optional[str],
|
||||||
query: str,
|
query: str,
|
||||||
count: int,
|
count: int,
|
||||||
filter_list: Optional[list[str]] = None,
|
filter_list: Optional[list[str]] = None,
|
||||||
@ -46,6 +48,11 @@ def search_yacy(
|
|||||||
time_range = kwargs.get("time_range", "")
|
time_range = kwargs.get("time_range", "")
|
||||||
categories = "".join(kwargs.get("categories", []))
|
categories = "".join(kwargs.get("categories", []))
|
||||||
|
|
||||||
|
# Use authentication if either username or password is set
|
||||||
|
yacy_auth = None
|
||||||
|
if username or password:
|
||||||
|
yacy_auth = HTTPDigestAuth(username, password)
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
"query": query,
|
"query": query,
|
||||||
"resource": "global",
|
"resource": "global",
|
||||||
@ -60,16 +67,16 @@ def search_yacy(
|
|||||||
# "image_proxy": 0,
|
# "image_proxy": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Legacy query format
|
# Check if provided a json API URL
|
||||||
if "<query>" in query_url:
|
if not query_url.endswith("yacysearch.json"):
|
||||||
# Strip all query parameters from the URL
|
# Strip all query parameters from the URL
|
||||||
query_url = query_url.split("?")[0]
|
query_url = query_url.rstrip('/') + "/yacysearch.json"
|
||||||
|
|
||||||
log.debug(f"searching {query_url}")
|
log.debug(f"searching {query_url}")
|
||||||
|
|
||||||
response = requests.get(
|
response = requests.get(
|
||||||
query_url,
|
query_url,
|
||||||
auth=HTTPDigestAuth('admin', 'yacy'),
|
auth=yacy_auth,
|
||||||
headers={
|
headers={
|
||||||
"User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot",
|
"User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot",
|
||||||
"Accept": "text/html",
|
"Accept": "text/html",
|
||||||
|
@ -391,6 +391,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
"BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
|
"BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
|
||||||
"SEARXNG_QUERY_URL": request.app.state.config.SEARXNG_QUERY_URL,
|
"SEARXNG_QUERY_URL": request.app.state.config.SEARXNG_QUERY_URL,
|
||||||
"YACY_QUERY_URL": request.app.state.config.YACY_QUERY_URL,
|
"YACY_QUERY_URL": request.app.state.config.YACY_QUERY_URL,
|
||||||
|
"YACY_USERNAME": request.app.state.config.YACY_USERNAME,
|
||||||
|
"YACY_PASSWORD": request.app.state.config.YACY_PASSWORD,
|
||||||
"GOOGLE_PSE_API_KEY": request.app.state.config.GOOGLE_PSE_API_KEY,
|
"GOOGLE_PSE_API_KEY": request.app.state.config.GOOGLE_PSE_API_KEY,
|
||||||
"GOOGLE_PSE_ENGINE_ID": request.app.state.config.GOOGLE_PSE_ENGINE_ID,
|
"GOOGLE_PSE_ENGINE_ID": request.app.state.config.GOOGLE_PSE_ENGINE_ID,
|
||||||
"BRAVE_SEARCH_API_KEY": request.app.state.config.BRAVE_SEARCH_API_KEY,
|
"BRAVE_SEARCH_API_KEY": request.app.state.config.BRAVE_SEARCH_API_KEY,
|
||||||
@ -437,6 +439,8 @@ class WebConfig(BaseModel):
|
|||||||
BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: Optional[bool] = None
|
BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: Optional[bool] = None
|
||||||
SEARXNG_QUERY_URL: Optional[str] = None
|
SEARXNG_QUERY_URL: Optional[str] = None
|
||||||
YACY_QUERY_URL: Optional[str] = None
|
YACY_QUERY_URL: Optional[str] = None
|
||||||
|
YACY_USERNAME: Optional[str] = None
|
||||||
|
YACY_PASSWORD: Optional[str] = None
|
||||||
GOOGLE_PSE_API_KEY: Optional[str] = None
|
GOOGLE_PSE_API_KEY: Optional[str] = None
|
||||||
GOOGLE_PSE_ENGINE_ID: Optional[str] = None
|
GOOGLE_PSE_ENGINE_ID: Optional[str] = None
|
||||||
BRAVE_SEARCH_API_KEY: Optional[str] = None
|
BRAVE_SEARCH_API_KEY: Optional[str] = None
|
||||||
@ -655,6 +659,8 @@ async def update_rag_config(
|
|||||||
)
|
)
|
||||||
request.app.state.config.SEARXNG_QUERY_URL = form_data.web.SEARXNG_QUERY_URL
|
request.app.state.config.SEARXNG_QUERY_URL = form_data.web.SEARXNG_QUERY_URL
|
||||||
request.app.state.config.YACY_QUERY_URL = form_data.web.YACY_QUERY_URL
|
request.app.state.config.YACY_QUERY_URL = form_data.web.YACY_QUERY_URL
|
||||||
|
request.app.state.config.YACY_USERNAME = form_data.web.YACY_USERNAME
|
||||||
|
request.app.state.config.YACY_PASSWORD = form_data.web.YACY_PASSWORD
|
||||||
request.app.state.config.GOOGLE_PSE_API_KEY = form_data.web.GOOGLE_PSE_API_KEY
|
request.app.state.config.GOOGLE_PSE_API_KEY = form_data.web.GOOGLE_PSE_API_KEY
|
||||||
request.app.state.config.GOOGLE_PSE_ENGINE_ID = (
|
request.app.state.config.GOOGLE_PSE_ENGINE_ID = (
|
||||||
form_data.web.GOOGLE_PSE_ENGINE_ID
|
form_data.web.GOOGLE_PSE_ENGINE_ID
|
||||||
@ -754,6 +760,8 @@ async def update_rag_config(
|
|||||||
"BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
|
"BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL": request.app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL,
|
||||||
"SEARXNG_QUERY_URL": request.app.state.config.SEARXNG_QUERY_URL,
|
"SEARXNG_QUERY_URL": request.app.state.config.SEARXNG_QUERY_URL,
|
||||||
"YACY_QUERY_URL": request.app.state.config.YACY_QUERY_URL,
|
"YACY_QUERY_URL": request.app.state.config.YACY_QUERY_URL,
|
||||||
|
"YACY_USERNAME": request.app.state.config.YACY_USERNAME,
|
||||||
|
"YACY_PASSWORD": request.app.state.config.YACY_PASSWORD,
|
||||||
"GOOGLE_PSE_API_KEY": request.app.state.config.GOOGLE_PSE_API_KEY,
|
"GOOGLE_PSE_API_KEY": request.app.state.config.GOOGLE_PSE_API_KEY,
|
||||||
"GOOGLE_PSE_ENGINE_ID": request.app.state.config.GOOGLE_PSE_ENGINE_ID,
|
"GOOGLE_PSE_ENGINE_ID": request.app.state.config.GOOGLE_PSE_ENGINE_ID,
|
||||||
"BRAVE_SEARCH_API_KEY": request.app.state.config.BRAVE_SEARCH_API_KEY,
|
"BRAVE_SEARCH_API_KEY": request.app.state.config.BRAVE_SEARCH_API_KEY,
|
||||||
@ -1271,7 +1279,7 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
|
|||||||
"""Search the web using a search engine and return the results as a list of SearchResult objects.
|
"""Search the web using a search engine and return the results as a list of SearchResult objects.
|
||||||
Will look for a search engine API key in environment variables in the following order:
|
Will look for a search engine API key in environment variables in the following order:
|
||||||
- SEARXNG_QUERY_URL
|
- SEARXNG_QUERY_URL
|
||||||
- YACY_QUERY_URL
|
- YACY_QUERY_URL + YACY_USERNAME + YACY_PASSWORD
|
||||||
- GOOGLE_PSE_API_KEY + GOOGLE_PSE_ENGINE_ID
|
- GOOGLE_PSE_API_KEY + GOOGLE_PSE_ENGINE_ID
|
||||||
- BRAVE_SEARCH_API_KEY
|
- BRAVE_SEARCH_API_KEY
|
||||||
- KAGI_SEARCH_API_KEY
|
- KAGI_SEARCH_API_KEY
|
||||||
@ -1305,6 +1313,8 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
|
|||||||
if request.app.state.config.YACY_QUERY_URL:
|
if request.app.state.config.YACY_QUERY_URL:
|
||||||
return search_yacy(
|
return search_yacy(
|
||||||
request.app.state.config.YACY_QUERY_URL,
|
request.app.state.config.YACY_QUERY_URL,
|
||||||
|
request.app.state.config.YACY_USERNAME,
|
||||||
|
request.app.state.config.YACY_PASSWORD,
|
||||||
query,
|
query,
|
||||||
request.app.state.config.WEB_SEARCH_RESULT_COUNT,
|
request.app.state.config.WEB_SEARCH_RESULT_COUNT,
|
||||||
request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
|
@ -148,7 +148,7 @@
|
|||||||
<div class="mb-2.5 flex w-full flex-col">
|
<div class="mb-2.5 flex w-full flex-col">
|
||||||
<div>
|
<div>
|
||||||
<div class=" self-center text-xs font-medium mb-1">
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
{$i18n.t('Yacy Query URL')}
|
{$i18n.t('Yacy Instance URL')}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="flex w-full">
|
<div class="flex w-full">
|
||||||
@ -156,7 +156,7 @@
|
|||||||
<input
|
<input
|
||||||
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||||
type="text"
|
type="text"
|
||||||
placeholder={$i18n.t('Enter Yacy Query URL')}
|
placeholder={$i18n.t('Enter Yacy URL (e.g. http://yacy.example.com:8090)')}
|
||||||
bind:value={webConfig.YACY_QUERY_URL}
|
bind:value={webConfig.YACY_QUERY_URL}
|
||||||
autocomplete="off"
|
autocomplete="off"
|
||||||
/>
|
/>
|
||||||
@ -164,6 +164,33 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="mb-2.5 flex w-full flex-col">
|
||||||
|
<div class="flex gap-2">
|
||||||
|
<div class="w-full">
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Yacy Username')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<input
|
||||||
|
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||||
|
placeholder={$i18n.t('Enter Yacy Username')}
|
||||||
|
bind:value={webConfig.YACY_USERNAME}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="w-full">
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Yacy Password')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<SensitiveInput
|
||||||
|
placeholder={$i18n.t('Enter Yacy Password')}
|
||||||
|
bind:value={webConfig.YACY_PASSWORD}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
{:else if webConfig.WEB_SEARCH_ENGINE === 'google_pse'}
|
{:else if webConfig.WEB_SEARCH_ENGINE === 'google_pse'}
|
||||||
<div class="mb-2.5 flex w-full flex-col">
|
<div class="mb-2.5 flex w-full flex-col">
|
||||||
<div>
|
<div>
|
||||||
|
Loading…
Reference in New Issue
Block a user