mirror of
https://github.com/open-webui/open-webui
synced 2025-05-29 17:52:48 +00:00
feat: implement Exa search engine integration
This commit is contained in:
parent
70116e7251
commit
2bb6b49f11
@ -1741,6 +1741,11 @@ BING_SEARCH_V7_SUBSCRIPTION_KEY = PersistentConfig(
|
|||||||
os.environ.get("BING_SEARCH_V7_SUBSCRIPTION_KEY", ""),
|
os.environ.get("BING_SEARCH_V7_SUBSCRIPTION_KEY", ""),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
EXA_API_KEY = PersistentConfig(
|
||||||
|
"EXA_API_KEY",
|
||||||
|
"rag.web.search.exa_api_key",
|
||||||
|
os.getenv("EXA_API_KEY", ""),
|
||||||
|
)
|
||||||
|
|
||||||
RAG_WEB_SEARCH_RESULT_COUNT = PersistentConfig(
|
RAG_WEB_SEARCH_RESULT_COUNT = PersistentConfig(
|
||||||
"RAG_WEB_SEARCH_RESULT_COUNT",
|
"RAG_WEB_SEARCH_RESULT_COUNT",
|
||||||
|
@ -177,6 +177,7 @@ from open_webui.config import (
|
|||||||
BING_SEARCH_V7_ENDPOINT,
|
BING_SEARCH_V7_ENDPOINT,
|
||||||
BING_SEARCH_V7_SUBSCRIPTION_KEY,
|
BING_SEARCH_V7_SUBSCRIPTION_KEY,
|
||||||
BRAVE_SEARCH_API_KEY,
|
BRAVE_SEARCH_API_KEY,
|
||||||
|
EXA_API_KEY,
|
||||||
KAGI_SEARCH_API_KEY,
|
KAGI_SEARCH_API_KEY,
|
||||||
MOJEEK_SEARCH_API_KEY,
|
MOJEEK_SEARCH_API_KEY,
|
||||||
GOOGLE_PSE_API_KEY,
|
GOOGLE_PSE_API_KEY,
|
||||||
@ -523,6 +524,7 @@ app.state.config.SEARCHAPI_ENGINE = SEARCHAPI_ENGINE
|
|||||||
app.state.config.JINA_API_KEY = JINA_API_KEY
|
app.state.config.JINA_API_KEY = JINA_API_KEY
|
||||||
app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT
|
app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT
|
||||||
app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY
|
app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY
|
||||||
|
app.state.config.EXA_API_KEY = EXA_API_KEY
|
||||||
|
|
||||||
app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
|
app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
|
||||||
app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
|
app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
|
||||||
|
74
backend/open_webui/retrieval/web/exa.py
Normal file
74
backend/open_webui/retrieval/web/exa.py
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
import logging
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from open_webui.env import SRC_LOG_LEVELS
|
||||||
|
from open_webui.retrieval.web.main import SearchResult
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
log.setLevel(SRC_LOG_LEVELS["RAG"])
|
||||||
|
|
||||||
|
EXA_API_BASE = "https://api.exa.ai"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ExaResult:
|
||||||
|
url: str
|
||||||
|
title: str
|
||||||
|
text: str
|
||||||
|
|
||||||
|
|
||||||
|
def search_exa(
|
||||||
|
api_key: str,
|
||||||
|
query: str,
|
||||||
|
count: int,
|
||||||
|
filter_list: Optional[list[str]] = None,
|
||||||
|
) -> list[SearchResult]:
|
||||||
|
"""Search using Exa Search API and return the results as a list of SearchResult objects.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
api_key (str): A Exa Search API key
|
||||||
|
query (str): The query to search for
|
||||||
|
count (int): Number of results to return
|
||||||
|
filter_list (Optional[list[str]]): List of domains to filter results by
|
||||||
|
"""
|
||||||
|
log.info(f"Searching with Exa for query: {query}")
|
||||||
|
|
||||||
|
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"query": query,
|
||||||
|
"numResults": count or 5,
|
||||||
|
"includeDomains": filter_list,
|
||||||
|
"contents": {"text": True, "highlights": True},
|
||||||
|
"type": "auto", # Use the auto search type (keyword or neural)
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = requests.post(f"{EXA_API_BASE}/search", headers=headers, json=payload)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for result in data["results"]:
|
||||||
|
results.append(
|
||||||
|
ExaResult(
|
||||||
|
url=result["url"],
|
||||||
|
title=result["title"],
|
||||||
|
text=result["text"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info(f"Found {len(results)} results")
|
||||||
|
return [
|
||||||
|
SearchResult(
|
||||||
|
link=result.url,
|
||||||
|
title=result.title,
|
||||||
|
snippet=result.text,
|
||||||
|
)
|
||||||
|
for result in results
|
||||||
|
]
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"Error searching Exa: {e}")
|
||||||
|
return []
|
@ -55,6 +55,7 @@ from open_webui.retrieval.web.serply import search_serply
|
|||||||
from open_webui.retrieval.web.serpstack import search_serpstack
|
from open_webui.retrieval.web.serpstack import search_serpstack
|
||||||
from open_webui.retrieval.web.tavily import search_tavily
|
from open_webui.retrieval.web.tavily import search_tavily
|
||||||
from open_webui.retrieval.web.bing import search_bing
|
from open_webui.retrieval.web.bing import search_bing
|
||||||
|
from open_webui.retrieval.web.exa import search_exa
|
||||||
|
|
||||||
|
|
||||||
from open_webui.retrieval.utils import (
|
from open_webui.retrieval.utils import (
|
||||||
@ -388,6 +389,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
"jina_api_key": request.app.state.config.JINA_API_KEY,
|
"jina_api_key": request.app.state.config.JINA_API_KEY,
|
||||||
"bing_search_v7_endpoint": request.app.state.config.BING_SEARCH_V7_ENDPOINT,
|
"bing_search_v7_endpoint": request.app.state.config.BING_SEARCH_V7_ENDPOINT,
|
||||||
"bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
|
"bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
|
||||||
|
"exa_api_key": request.app.state.config.EXA_API_KEY,
|
||||||
"result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
|
"result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
|
||||||
"concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
|
"concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
|
||||||
},
|
},
|
||||||
@ -436,6 +438,7 @@ class WebSearchConfig(BaseModel):
|
|||||||
jina_api_key: Optional[str] = None
|
jina_api_key: Optional[str] = None
|
||||||
bing_search_v7_endpoint: Optional[str] = None
|
bing_search_v7_endpoint: Optional[str] = None
|
||||||
bing_search_v7_subscription_key: Optional[str] = None
|
bing_search_v7_subscription_key: Optional[str] = None
|
||||||
|
exa_api_key: Optional[str] = None
|
||||||
result_count: Optional[int] = None
|
result_count: Optional[int] = None
|
||||||
concurrent_requests: Optional[int] = None
|
concurrent_requests: Optional[int] = None
|
||||||
|
|
||||||
@ -542,6 +545,8 @@ async def update_rag_config(
|
|||||||
form_data.web.search.bing_search_v7_subscription_key
|
form_data.web.search.bing_search_v7_subscription_key
|
||||||
)
|
)
|
||||||
|
|
||||||
|
request.app.state.config.EXA_API_KEY = form_data.web.search.exa_api_key
|
||||||
|
|
||||||
request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = (
|
request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = (
|
||||||
form_data.web.search.result_count
|
form_data.web.search.result_count
|
||||||
)
|
)
|
||||||
@ -591,6 +596,7 @@ async def update_rag_config(
|
|||||||
"jina_api_key": request.app.state.config.JINA_API_KEY,
|
"jina_api_key": request.app.state.config.JINA_API_KEY,
|
||||||
"bing_search_v7_endpoint": request.app.state.config.BING_SEARCH_V7_ENDPOINT,
|
"bing_search_v7_endpoint": request.app.state.config.BING_SEARCH_V7_ENDPOINT,
|
||||||
"bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
|
"bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
|
||||||
|
"exa_api_key": request.app.state.config.EXA_API_KEY,
|
||||||
"result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
|
"result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
|
||||||
"concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
|
"concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
|
||||||
},
|
},
|
||||||
@ -1099,6 +1105,7 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
|
|||||||
- SERPER_API_KEY
|
- SERPER_API_KEY
|
||||||
- SERPLY_API_KEY
|
- SERPLY_API_KEY
|
||||||
- TAVILY_API_KEY
|
- TAVILY_API_KEY
|
||||||
|
- EXA_API_KEY
|
||||||
- SEARCHAPI_API_KEY + SEARCHAPI_ENGINE (by default `google`)
|
- SEARCHAPI_API_KEY + SEARCHAPI_ENGINE (by default `google`)
|
||||||
Args:
|
Args:
|
||||||
query (str): The query to search for
|
query (str): The query to search for
|
||||||
@ -1233,6 +1240,13 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
|
|||||||
request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
|
request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
|
||||||
request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
|
request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
)
|
)
|
||||||
|
elif engine == "exa":
|
||||||
|
return search_exa(
|
||||||
|
request.app.state.config.EXA_API_KEY,
|
||||||
|
query,
|
||||||
|
request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
|
||||||
|
request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
raise Exception("No search engine API key found in environment variables")
|
raise Exception("No search engine API key found in environment variables")
|
||||||
|
|
||||||
|
@ -25,7 +25,8 @@
|
|||||||
'duckduckgo',
|
'duckduckgo',
|
||||||
'tavily',
|
'tavily',
|
||||||
'jina',
|
'jina',
|
||||||
'bing'
|
'bing',
|
||||||
|
'exa'
|
||||||
];
|
];
|
||||||
|
|
||||||
let youtubeLanguage = 'en';
|
let youtubeLanguage = 'en';
|
||||||
@ -261,6 +262,17 @@
|
|||||||
bind:value={webConfig.search.jina_api_key}
|
bind:value={webConfig.search.jina_api_key}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
{:else if webConfig.search.engine === 'exa'}
|
||||||
|
<div>
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Exa API Key')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<SensitiveInput
|
||||||
|
placeholder={$i18n.t('Enter Exa API Key')}
|
||||||
|
bind:value={webConfig.search.exa_api_key}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
{:else if webConfig.search.engine === 'bing'}
|
{:else if webConfig.search.engine === 'bing'}
|
||||||
<div>
|
<div>
|
||||||
<div class=" self-center text-xs font-medium mb-1">
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
Loading…
Reference in New Issue
Block a user