diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index e087110fe..adfdcfec8 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1809,6 +1809,18 @@ SEARCHAPI_ENGINE = PersistentConfig( os.getenv("SEARCHAPI_ENGINE", ""), ) +SERPAPI_API_KEY = PersistentConfig( + "SERPAPI_API_KEY", + "rag.web.search.serpapi_api_key", + os.getenv("SERPAPI_API_KEY", ""), +) + +SERPAPI_ENGINE = PersistentConfig( + "SERPAPI_ENGINE", + "rag.web.search.serpapi_engine", + os.getenv("SERPAPI_ENGINE", ""), +) + BING_SEARCH_V7_ENDPOINT = PersistentConfig( "BING_SEARCH_V7_ENDPOINT", "rag.web.search.bing_search_v7_endpoint", diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index a69221d78..a36323151 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -179,6 +179,8 @@ from open_webui.config import ( JINA_API_KEY, SEARCHAPI_API_KEY, SEARCHAPI_ENGINE, + SERPAPI_API_KEY, + SERPAPI_ENGINE, SEARXNG_QUERY_URL, SERPER_API_KEY, SERPLY_API_KEY, @@ -547,6 +549,8 @@ app.state.config.SERPLY_API_KEY = SERPLY_API_KEY app.state.config.TAVILY_API_KEY = TAVILY_API_KEY app.state.config.SEARCHAPI_API_KEY = SEARCHAPI_API_KEY app.state.config.SEARCHAPI_ENGINE = SEARCHAPI_ENGINE +app.state.config.SERPAPI_API_KEY = SERPAPI_API_KEY +app.state.config.SERPAPI_ENGINE = SERPAPI_ENGINE app.state.config.JINA_API_KEY = JINA_API_KEY app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY diff --git a/backend/open_webui/retrieval/web/serpapi.py b/backend/open_webui/retrieval/web/serpapi.py new file mode 100644 index 000000000..028b6bcfe --- /dev/null +++ b/backend/open_webui/retrieval/web/serpapi.py @@ -0,0 +1,48 @@ +import logging +from typing import Optional +from urllib.parse import urlencode + +import requests +from open_webui.retrieval.web.main import SearchResult, get_filtered_results +from open_webui.env import SRC_LOG_LEVELS + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + + +def search_serpapi( + api_key: str, + engine: str, + query: str, + count: int, + filter_list: Optional[list[str]] = None, +) -> list[SearchResult]: + """Search using serpapi.com's API and return the results as a list of SearchResult objects. + + Args: + api_key (str): A serpapi.com API key + query (str): The query to search for + """ + url = "https://serpapi.com/search" + + engine = engine or "google" + + payload = {"engine": engine, "q": query, "api_key": api_key} + + url = f"{url}?{urlencode(payload)}" + response = requests.request("GET", url) + + json_response = response.json() + log.info(f"results from serpapi search: {json_response}") + + results = sorted( + json_response.get("organic_results", []), key=lambda x: x.get("position", 0) + ) + if filter_list: + results = get_filtered_results(results, filter_list) + return [ + SearchResult( + link=result["link"], title=result["title"], snippet=result["snippet"] + ) + for result in results[:count] + ] diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index baabaf2c9..91cf27e0c 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -50,6 +50,7 @@ from open_webui.retrieval.web.duckduckgo import search_duckduckgo from open_webui.retrieval.web.google_pse import search_google_pse from open_webui.retrieval.web.jina_search import search_jina from open_webui.retrieval.web.searchapi import search_searchapi +from open_webui.retrieval.web.serpapi import search_serpapi from open_webui.retrieval.web.searxng import search_searxng from open_webui.retrieval.web.serper import search_serper from open_webui.retrieval.web.serply import search_serply @@ -388,6 +389,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): "tavily_api_key": request.app.state.config.TAVILY_API_KEY, "searchapi_api_key": request.app.state.config.SEARCHAPI_API_KEY, "searchapi_engine": request.app.state.config.SEARCHAPI_ENGINE, + "serpapi_api_key": request.app.state.config.SERPAPI_API_KEY, + "serpapi_engine": request.app.state.config.SERPAPI_ENGINE, "jina_api_key": request.app.state.config.JINA_API_KEY, "bing_search_v7_endpoint": request.app.state.config.BING_SEARCH_V7_ENDPOINT, "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY, @@ -439,6 +442,8 @@ class WebSearchConfig(BaseModel): tavily_api_key: Optional[str] = None searchapi_api_key: Optional[str] = None searchapi_engine: Optional[str] = None + serpapi_api_key: Optional[str] = None + serpapi_engine: Optional[str] = None jina_api_key: Optional[str] = None bing_search_v7_endpoint: Optional[str] = None bing_search_v7_subscription_key: Optional[str] = None @@ -545,6 +550,13 @@ async def update_rag_config( form_data.web.search.searchapi_engine ) + request.app.state.config.SERPAPI_API_KEY = ( + form_data.web.search.serpapi_api_key + ) + request.app.state.config.SERPAPI_ENGINE = ( + form_data.web.search.serpapi_engine + ) + request.app.state.config.JINA_API_KEY = form_data.web.search.jina_api_key request.app.state.config.BING_SEARCH_V7_ENDPOINT = ( form_data.web.search.bing_search_v7_endpoint @@ -604,6 +616,8 @@ async def update_rag_config( "serply_api_key": request.app.state.config.SERPLY_API_KEY, "serachapi_api_key": request.app.state.config.SEARCHAPI_API_KEY, "searchapi_engine": request.app.state.config.SEARCHAPI_ENGINE, + "serpapi_api_key": request.app.state.config.SERPAPI_API_KEY, + "serpapi_engine": request.app.state.config.SERPAPI_ENGINE, "tavily_api_key": request.app.state.config.TAVILY_API_KEY, "jina_api_key": request.app.state.config.JINA_API_KEY, "bing_search_v7_endpoint": request.app.state.config.BING_SEARCH_V7_ENDPOINT, @@ -1131,6 +1145,7 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]: - TAVILY_API_KEY - EXA_API_KEY - SEARCHAPI_API_KEY + SEARCHAPI_ENGINE (by default `google`) + - SERPAPI_API_KEY + SERPAPI_ENGINE (by default `google`) Args: query (str): The query to search for """ @@ -1259,6 +1274,17 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]: ) else: raise Exception("No SEARCHAPI_API_KEY found in environment variables") + elif engine == "serpapi": + if request.app.state.config.SERPAPI_API_KEY: + return search_serpapi( + request.app.state.config.SERPAPI_API_KEY, + request.app.state.config.SERPAPI_ENGINE, + query, + request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT, + request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST, + ) + else: + raise Exception("No SERPAPI_API_KEY found in environment variables") elif engine == "jina": return search_jina( request.app.state.config.JINA_API_KEY, diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 6943a367e..5940a0310 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -23,6 +23,7 @@ 'serper', 'serply', 'searchapi', + 'serpapi', 'duckduckgo', 'tavily', 'jina', @@ -268,6 +269,34 @@ + {:else if webConfig.search.engine === 'serpapi'} +
+
+ {$i18n.t('SerpApi API Key')} +
+ + +
+
+
+ {$i18n.t('SerpApi Engine')} +
+ +
+
+ +
+
+
{:else if webConfig.search.engine === 'tavily'}