UniqAI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2024-11-24 13:07:25 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

add mojeek as websearch option

Browse Source
This commit is contained in:
Igor Brai 2024-10-29 16:45:38 +02:00
parent 09935d191f
commit b7d730e244
4 changed files with 77 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
backend/open_webui/apps/retrieval/main.py
View File

@ -28,6 +28,7 @@ from open_webui.apps.retrieval.loaders.main import Loader
from open_webui.apps.retrieval.web.main import SearchResult
from open_webui.apps.retrieval.web.utils import get_web_loader
from open_webui.apps.retrieval.web.brave import search_brave
from open_webui.apps.retrieval.web.mojeek import search_mojeek
from open_webui.apps.retrieval.web.duckduckgo import search_duckduckgo
from open_webui.apps.retrieval.web.google_pse import search_google_pse
from open_webui.apps.retrieval.web.jina_search import search_jina
@ -51,6 +52,7 @@ from open_webui.apps.retrieval.utils import (
from open_webui.apps.webui.models.files import Files
from open_webui.config import (
BRAVE_SEARCH_API_KEY,
MOJEEK_SEARCH_API_KEY,
TIKTOKEN_ENCODING_NAME,
RAG_TEXT_SPLITTER,
CHUNK_OVERLAP,
@ -164,6 +166,7 @@ app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL
app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY
app.state.config.GOOGLE_PSE_ENGINE_ID = GOOGLE_PSE_ENGINE_ID
app.state.config.BRAVE_SEARCH_API_KEY = BRAVE_SEARCH_API_KEY
app.state.config.MOJEEK_SEARCH_API_KEY = MOJEEK_SEARCH_API_KEY
app.state.config.SERPSTACK_API_KEY = SERPSTACK_API_KEY
app.state.config.SERPSTACK_HTTPS = SERPSTACK_HTTPS
app.state.config.SERPER_API_KEY = SERPER_API_KEY
@ -419,6 +422,7 @@ async def get_rag_config(user=Depends(get_admin_user)):
"google_pse_api_key": app.state.config.GOOGLE_PSE_API_KEY,
"google_pse_engine_id": app.state.config.GOOGLE_PSE_ENGINE_ID,
"brave_search_api_key": app.state.config.BRAVE_SEARCH_API_KEY,
"mojeek_search_api_key": app.state.config.MOJEEK_SEARCH_API_KEY,
"serpstack_api_key": app.state.config.SERPSTACK_API_KEY,
"serpstack_https": app.state.config.SERPSTACK_HTTPS,
"serper_api_key": app.state.config.SERPER_API_KEY,
@ -461,6 +465,7 @@ class WebSearchConfig(BaseModel):
google_pse_api_key: Optional[str] = None
google_pse_engine_id: Optional[str] = None
brave_search_api_key: Optional[str] = None
mojeek_search_api_key: Optional[str] = None
serpstack_api_key: Optional[str] = None
serpstack_https: Optional[bool] = None
serper_api_key: Optional[str] = None
@ -527,6 +532,7 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_
app.state.config.BRAVE_SEARCH_API_KEY = (
form_data.web.search.brave_search_api_key
)
app.state.config.MOJEEK_SEARCH_API_KEY = form_data.web.search.mojeek_search_api_key
app.state.config.SERPSTACK_API_KEY = form_data.web.search.serpstack_api_key
app.state.config.SERPSTACK_HTTPS = form_data.web.search.serpstack_https
app.state.config.SERPER_API_KEY = form_data.web.search.serper_api_key
@ -568,6 +574,7 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_
"google_pse_api_key": app.state.config.GOOGLE_PSE_API_KEY,
"google_pse_engine_id": app.state.config.GOOGLE_PSE_ENGINE_ID,
"brave_search_api_key": app.state.config.BRAVE_SEARCH_API_KEY,
"mojeek_search_api_key": app.state.config.MOJEEK_SEARCH_API_KEY,
"serpstack_api_key": app.state.config.SERPSTACK_API_KEY,
"serpstack_https": app.state.config.SERPSTACK_HTTPS,
"serper_api_key": app.state.config.SERPER_API_KEY,
@ -1028,6 +1035,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]:
- SEARXNG_QUERY_URL
- GOOGLE_PSE_API_KEY + GOOGLE_PSE_ENGINE_ID
- BRAVE_SEARCH_API_KEY
- MOJEEK_SEARCH_API_KEY
- SERPSTACK_API_KEY
- SERPER_API_KEY
- SERPLY_API_KEY
@ -1074,6 +1082,16 @@ def search_web(engine: str, query: str) -> list[SearchResult]:
)
else:
raise Exception("No BRAVE_SEARCH_API_KEY found in environment variables")
elif engine == "mojeek":
if app.state.config.MOJEEK_SEARCH_API_KEY:
return search_mojeek(
app.state.config.MOJEEK_SEARCH_API_KEY,
query,
app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
)
else:
raise Exception("No MOJEEK_SEARCH_API_KEY found in environment variables")
elif engine == "serpstack":
if app.state.config.SERPSTACK_API_KEY:
return search_serpstack(

41
backend/open_webui/apps/retrieval/web/mojeek.py Normal file
View File

@ -0,0 +1,41 @@
import logging
from typing import Optional
import requests
from open_webui.apps.retrieval.web.main import SearchResult, get_filtered_results
from open_webui.env import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_mojeek(
api_key: str, query: str, count: int, filter_list: Optional[list[str]] = None
) -> list[SearchResult]:
"""Search using Mojeek's Search API and return the results as a list of SearchResult objects.
Args:
api_key (str): A Mojeek Search API key
query (str): The query to search for
"""
url = "https://api.mojeek.com/search"
headers = {
"Accept": "application/json",
}
params = {"q": query, "api_key": api_key, 'fmt': 'json', 't': count}
response = requests.get(url, headers=headers, params=params)
response.raise_for_status()
json_response = response.json()
results = json_response.get("response", {}).get("results", [])
print(results)
if filter_list:
results = get_filtered_results(results, filter_list)
return [
SearchResult(
link=result["url"], title=result.get("title"), snippet=result.get("desc")
)
for result in results
]

6
backend/open_webui/config.py
View File

@ -1186,6 +1186,12 @@ BRAVE_SEARCH_API_KEY = PersistentConfig(
os.getenv("BRAVE_SEARCH_API_KEY", ""),
)
MOJEEK_SEARCH_API_KEY = PersistentConfig(
"MOJEEK_SEARCH_API_KEY",
"rag.web.search.MOJEEK_search_api_key",
os.getenv("MOJEEK_SEARCH_API_KEY", ""),
)
SERPSTACK_API_KEY = PersistentConfig(
"SERPSTACK_API_KEY",
"rag.web.search.serpstack_api_key",

12
src/lib/components/admin/Settings/WebSearch.svelte
View File

@ -16,6 +16,7 @@
'searxng',
'google_pse',
'brave',
'mojeek',
'serpstack',
'serper',
'serply',
@ -150,6 +151,17 @@
bind:value={webConfig.search.brave_search_api_key}
/>
</div>
{:else if webConfig.search.engine === 'mojeek'}
<div>
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Mojeek Search API Key')}
</div>
<SensitiveInput
placeholder={$i18n.t('Enter Mojeek Search API Key')}
bind:value={webConfig.search.mojeek_search_api_key}
/>
</div>
{:else if webConfig.search.engine === 'serpstack'}
<div>
<div class=" self-center text-xs font-medium mb-1">