From b7d730e244a6c08612a1b327a3aad3a6700721f3 Mon Sep 17 00:00:00 2001 From: Igor Brai Date: Tue, 29 Oct 2024 16:45:38 +0200 Subject: [PATCH] add mojeek as websearch option --- backend/open_webui/apps/retrieval/main.py | 18 ++++++++ .../open_webui/apps/retrieval/web/mojeek.py | 41 +++++++++++++++++++ backend/open_webui/config.py | 6 +++ .../admin/Settings/WebSearch.svelte | 12 ++++++ 4 files changed, 77 insertions(+) create mode 100644 backend/open_webui/apps/retrieval/web/mojeek.py diff --git a/backend/open_webui/apps/retrieval/main.py b/backend/open_webui/apps/retrieval/main.py index e67d1df23..fa7bb5caa 100644 --- a/backend/open_webui/apps/retrieval/main.py +++ b/backend/open_webui/apps/retrieval/main.py @@ -28,6 +28,7 @@ from open_webui.apps.retrieval.loaders.main import Loader from open_webui.apps.retrieval.web.main import SearchResult from open_webui.apps.retrieval.web.utils import get_web_loader from open_webui.apps.retrieval.web.brave import search_brave +from open_webui.apps.retrieval.web.mojeek import search_mojeek from open_webui.apps.retrieval.web.duckduckgo import search_duckduckgo from open_webui.apps.retrieval.web.google_pse import search_google_pse from open_webui.apps.retrieval.web.jina_search import search_jina @@ -51,6 +52,7 @@ from open_webui.apps.retrieval.utils import ( from open_webui.apps.webui.models.files import Files from open_webui.config import ( BRAVE_SEARCH_API_KEY, + MOJEEK_SEARCH_API_KEY, TIKTOKEN_ENCODING_NAME, RAG_TEXT_SPLITTER, CHUNK_OVERLAP, @@ -164,6 +166,7 @@ app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY app.state.config.GOOGLE_PSE_ENGINE_ID = GOOGLE_PSE_ENGINE_ID app.state.config.BRAVE_SEARCH_API_KEY = BRAVE_SEARCH_API_KEY +app.state.config.MOJEEK_SEARCH_API_KEY = MOJEEK_SEARCH_API_KEY app.state.config.SERPSTACK_API_KEY = SERPSTACK_API_KEY app.state.config.SERPSTACK_HTTPS = SERPSTACK_HTTPS app.state.config.SERPER_API_KEY = SERPER_API_KEY @@ -419,6 +422,7 @@ async def get_rag_config(user=Depends(get_admin_user)): "google_pse_api_key": app.state.config.GOOGLE_PSE_API_KEY, "google_pse_engine_id": app.state.config.GOOGLE_PSE_ENGINE_ID, "brave_search_api_key": app.state.config.BRAVE_SEARCH_API_KEY, + "mojeek_search_api_key": app.state.config.MOJEEK_SEARCH_API_KEY, "serpstack_api_key": app.state.config.SERPSTACK_API_KEY, "serpstack_https": app.state.config.SERPSTACK_HTTPS, "serper_api_key": app.state.config.SERPER_API_KEY, @@ -461,6 +465,7 @@ class WebSearchConfig(BaseModel): google_pse_api_key: Optional[str] = None google_pse_engine_id: Optional[str] = None brave_search_api_key: Optional[str] = None + mojeek_search_api_key: Optional[str] = None serpstack_api_key: Optional[str] = None serpstack_https: Optional[bool] = None serper_api_key: Optional[str] = None @@ -527,6 +532,7 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_ app.state.config.BRAVE_SEARCH_API_KEY = ( form_data.web.search.brave_search_api_key ) + app.state.config.MOJEEK_SEARCH_API_KEY = form_data.web.search.mojeek_search_api_key app.state.config.SERPSTACK_API_KEY = form_data.web.search.serpstack_api_key app.state.config.SERPSTACK_HTTPS = form_data.web.search.serpstack_https app.state.config.SERPER_API_KEY = form_data.web.search.serper_api_key @@ -568,6 +574,7 @@ async def update_rag_config(form_data: ConfigUpdateForm, user=Depends(get_admin_ "google_pse_api_key": app.state.config.GOOGLE_PSE_API_KEY, "google_pse_engine_id": app.state.config.GOOGLE_PSE_ENGINE_ID, "brave_search_api_key": app.state.config.BRAVE_SEARCH_API_KEY, + "mojeek_search_api_key": app.state.config.MOJEEK_SEARCH_API_KEY, "serpstack_api_key": app.state.config.SERPSTACK_API_KEY, "serpstack_https": app.state.config.SERPSTACK_HTTPS, "serper_api_key": app.state.config.SERPER_API_KEY, @@ -1028,6 +1035,7 @@ def search_web(engine: str, query: str) -> list[SearchResult]: - SEARXNG_QUERY_URL - GOOGLE_PSE_API_KEY + GOOGLE_PSE_ENGINE_ID - BRAVE_SEARCH_API_KEY + - MOJEEK_SEARCH_API_KEY - SERPSTACK_API_KEY - SERPER_API_KEY - SERPLY_API_KEY @@ -1074,6 +1082,16 @@ def search_web(engine: str, query: str) -> list[SearchResult]: ) else: raise Exception("No BRAVE_SEARCH_API_KEY found in environment variables") + elif engine == "mojeek": + if app.state.config.MOJEEK_SEARCH_API_KEY: + return search_mojeek( + app.state.config.MOJEEK_SEARCH_API_KEY, + query, + app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST, + ) + else: + raise Exception("No MOJEEK_SEARCH_API_KEY found in environment variables") elif engine == "serpstack": if app.state.config.SERPSTACK_API_KEY: return search_serpstack( diff --git a/backend/open_webui/apps/retrieval/web/mojeek.py b/backend/open_webui/apps/retrieval/web/mojeek.py new file mode 100644 index 000000000..46a6526c7 --- /dev/null +++ b/backend/open_webui/apps/retrieval/web/mojeek.py @@ -0,0 +1,41 @@ +import logging +from typing import Optional + +import requests +from open_webui.apps.retrieval.web.main import SearchResult, get_filtered_results +from open_webui.env import SRC_LOG_LEVELS + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + + +def search_mojeek( + api_key: str, query: str, count: int, filter_list: Optional[list[str]] = None +) -> list[SearchResult]: + """Search using Mojeek's Search API and return the results as a list of SearchResult objects. + + Args: + api_key (str): A Mojeek Search API key + query (str): The query to search for + """ + url = "https://api.mojeek.com/search" + headers = { + "Accept": "application/json", + } + params = {"q": query, "api_key": api_key, 'fmt': 'json', 't': count} + + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() + json_response = response.json() + results = json_response.get("response", {}).get("results", []) + print(results) + if filter_list: + results = get_filtered_results(results, filter_list) + + + return [ + SearchResult( + link=result["url"], title=result.get("title"), snippet=result.get("desc") + ) + for result in results + ] \ No newline at end of file diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 9d1bd72d8..9340f4bf2 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1186,6 +1186,12 @@ BRAVE_SEARCH_API_KEY = PersistentConfig( os.getenv("BRAVE_SEARCH_API_KEY", ""), ) +MOJEEK_SEARCH_API_KEY = PersistentConfig( + "MOJEEK_SEARCH_API_KEY", + "rag.web.search.MOJEEK_search_api_key", + os.getenv("MOJEEK_SEARCH_API_KEY", ""), +) + SERPSTACK_API_KEY = PersistentConfig( "SERPSTACK_API_KEY", "rag.web.search.serpstack_api_key", diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 69f1f9104..efbb8e0d2 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -16,6 +16,7 @@ 'searxng', 'google_pse', 'brave', + 'mojeek', 'serpstack', 'serper', 'serply', @@ -150,6 +151,17 @@ bind:value={webConfig.search.brave_search_api_key} /> + {:else if webConfig.search.engine === 'mojeek'} +
+
+ {$i18n.t('Mojeek Search API Key')} +
+ + +
{:else if webConfig.search.engine === 'serpstack'}