Merge pull request #12683 from Youggls/dev

feat: Add sougou web search engine.

backend/open_webui/config.py

@@ -2142,6 +2142,18 @@ PERPLEXITY_API_KEY = PersistentConfig(
     os.getenv("PERPLEXITY_API_KEY", ""),
 )
 
+SOUGOU_API_SID = PersistentConfig(
+    "SOUGOU_API_SID",
+    "rag.web.search.sougou_api_sid",
+    os.getenv("SOUGOU_API_SID", ""),
+)
+
+SOUGOU_API_SK = PersistentConfig(
+    "SOUGOU_API_SK",
+    "rag.web.search.sougou_api_sk",
+    os.getenv("SOUGOU_API_SK", ""),
+)
+
 RAG_WEB_SEARCH_RESULT_COUNT = PersistentConfig(
     "RAG_WEB_SEARCH_RESULT_COUNT",
     "rag.web.search.result_count",

backend/open_webui/main.py

@@ -225,6 +225,8 @@ from open_webui.config import (
     BRAVE_SEARCH_API_KEY,
     EXA_API_KEY,
     PERPLEXITY_API_KEY,
+    SOUGOU_API_SID,
+    SOUGOU_API_SK,
     KAGI_SEARCH_API_KEY,
     MOJEEK_SEARCH_API_KEY,
     BOCHA_SEARCH_API_KEY,
@@ -652,6 +654,8 @@ app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT
 app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY
 app.state.config.EXA_API_KEY = EXA_API_KEY
 app.state.config.PERPLEXITY_API_KEY = PERPLEXITY_API_KEY
+app.state.config.SOUGOU_API_SID = SOUGOU_API_SID
+app.state.config.SOUGOU_API_SK = SOUGOU_API_SK
 
 app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
 app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS

backend/open_webui/retrieval/web/sougou.py

@@ -0,0 +1,48 @@
+import logging
+import json
+from typing import Optional, List
+
+from tencentcloud.common.common_client import CommonClient
+from tencentcloud.common import credential
+from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
+from tencentcloud.common.profile.client_profile import ClientProfile
+from tencentcloud.common.profile.http_profile import HttpProfile
+
+from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.env import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_sougou(
+    sougou_api_sid: str,
+    sougou_api_sk: str,
+    query: str,
+    count: int,
+    filter_list: Optional[List[str]] = None,
+) -> List[SearchResult]:
+    try:
+        cred = credential.Credential(sougou_api_sid, sougou_api_sk)
+        http_profile = HttpProfile()
+        http_profile.endpoint = "tms.tencentcloudapi.com"
+        client_profile = ClientProfile()
+        client_profile.http_profile = http_profile
+        params = json.dumps({"Query": query, 'Cnt': 20})
+        common_client = CommonClient("tms", "2020-12-29", cred, "", profile=client_profile)
+        results = [
+            json.loads(page) for page in common_client.call_json("SearchPro", json.loads(params))["Response"]["Pages"]
+        ]
+        sorted_results = sorted(results, key=lambda x: x.get("scour", 0.0), reverse=True)
+        if filter_list:
+            sorted_results = get_filtered_results(sorted_results, filter_list)
+
+        return [
+            SearchResult(
+                link=result.get("url"), title=result.get("title"), snippet=result.get("passage")
+            )
+            for result in sorted_results[:count]
+        ]
+    except TencentCloudSDKException as err:
+        log.error(f"Error in Sougou search: {err}")
+        return []

backend/open_webui/routers/retrieval.py

@@ -60,6 +60,7 @@ from open_webui.retrieval.web.tavily import search_tavily
 from open_webui.retrieval.web.bing import search_bing
 from open_webui.retrieval.web.exa import search_exa
 from open_webui.retrieval.web.perplexity import search_perplexity
+from open_webui.retrieval.web.sougou import search_sougou
 
 from open_webui.retrieval.utils import (
     get_embedding_function,
@@ -411,6 +412,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
                 "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
                 "exa_api_key": request.app.state.config.EXA_API_KEY,
                 "perplexity_api_key": request.app.state.config.PERPLEXITY_API_KEY,
+                "sougou_api_sid": request.app.state.config.SOUGOU_API_SID,
+                "sougou_api_sk": request.app.state.config.SOUGOU_API_SK,
                 "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
                 "trust_env": request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
                 "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
@@ -478,6 +481,8 @@ class WebSearchConfig(BaseModel):
     bing_search_v7_subscription_key: Optional[str] = None
     exa_api_key: Optional[str] = None
     perplexity_api_key: Optional[str] = None
+    sougou_api_sid: Optional[str] = None
+    sougou_api_sk: Optional[str] = None
     result_count: Optional[int] = None
     concurrent_requests: Optional[int] = None
     trust_env: Optional[bool] = None
@@ -640,6 +645,12 @@ async def update_rag_config(
         request.app.state.config.PERPLEXITY_API_KEY = (
             form_data.web.search.perplexity_api_key
         )
+        request.app.state.config.SOUGOU_API_SID = (
+            form_data.web.search.sougou_api_sid
+        )
+        request.app.state.config.SOUGOU_API_SK = (
+            form_data.web.search.sougou_api_sk
+        )
 
         request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = (
             form_data.web.search.result_count
@@ -712,6 +723,8 @@ async def update_rag_config(
                 "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
                 "exa_api_key": request.app.state.config.EXA_API_KEY,
                 "perplexity_api_key": request.app.state.config.PERPLEXITY_API_KEY,
+                "sougou_api_sid": request.app.state.config.SOUGOU_API_SID,
+                "sougou_api_sk": request.app.state.config.SOUGOU_API_SK,
                 "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
                 "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
                 "trust_env": request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
@@ -1267,6 +1280,7 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
     - TAVILY_API_KEY
     - EXA_API_KEY
     - PERPLEXITY_API_KEY
+    - SOUGOU_API_SID + SOUGOU_API_SK
     - SEARCHAPI_API_KEY + SEARCHAPI_ENGINE (by default `google`)
     - SERPAPI_API_KEY + SERPAPI_ENGINE (by default `google`)
     Args:
@@ -1438,6 +1452,17 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
             request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
             request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
         )
+    elif engine == 'sougou':
+        if request.app.state.config.SOUGOU_API_SID and request.app.state.config.SOUGOU_API_SK:
+            return search_sougou(
+                request.app.state.config.SOUGOU_API_SID,
+                request.app.state.config.SOUGOU_API_SK,
+                query,
+                request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
+                request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
+            )
+        else:
+            raise Exception("No SOUGOU_API_SID or SOUGOU_API_SK found in environment variables")
     else:
         raise Exception("No search engine API key found in environment variables")
 

backend/requirements.txt

@@ -123,6 +123,9 @@ ldap3==2.9.1
 ## Firecrawl
 firecrawl-py==1.12.0
 
+# Sougou API SDK(Tencentcloud SDK)
+tencentcloud-sdk-python==3.0.1336
+
 ## Trace
 opentelemetry-api==1.31.1
 opentelemetry-sdk==1.31.1

pyproject.toml

@@ -125,6 +125,8 @@ dependencies = [
 
     "firecrawl-py==1.12.0",
 
+    "tencentcloud-sdk-python==3.0.1336",
+
     "gcp-storage-emulator>=2024.8.3",
 ]
 readme = "README.md"

src/lib/components/admin/Settings/WebSearch.svelte

@@ -30,7 +30,8 @@
 		'jina',
 		'bing',
 		'exa',
-		'perplexity'
+		'perplexity',
+		'sougou'
 	];
 
 	let youtubeLanguage = 'en';
@@ -404,6 +405,31 @@
 									/>
 								</div>
 							</div>
+							{:else if webConfig.search.engine === 'sougou'}
+								<div class="mb-2.5 flex w-full flex-col">
+									<div>
+										<div class=" self-center text-xs font-medium mb-1">
+											{$i18n.t('Sougou Search API sID')}
+										</div>
+	
+										<SensitiveInput
+											placeholder={$i18n.t('Enter Sougou Search API sID')}
+											bind:value={webConfig.search.sougou_api_sid}
+										/>
+									</div>
+								</div>
+								<div class="mb-2.5 flex w-full flex-col">
+									<div>
+										<div class=" self-center text-xs font-medium mb-1">
+											{$i18n.t('Sougou Search API SK')}
+										</div>
+	
+										<SensitiveInput
+											placeholder={$i18n.t('Enter Sougou Search API SK')}
+											bind:value={webConfig.search.sougou_api_sk}
+										/>
+									</div>
+								</div>
 						{/if}
 					{/if}
 

src/lib/i18n/locales/en-US/translation.json

@@ -424,6 +424,8 @@
 	"Enter Mojeek Search API Key": "",
 	"Enter Number of Steps (e.g. 50)": "",
 	"Enter Perplexity API Key": "",
+	"Enter Sougou Search API sID": "",
+	"Enter Sougou Search API SK": "",
 	"Enter proxy URL (e.g. https://user:password@host:port)": "",
 	"Enter reasoning effort": "",
 	"Enter Sampler (e.g. Euler a)": "",
@@ -822,6 +824,8 @@
 	"Permission denied when accessing microphone: {{error}}": "",
 	"Permissions": "",
 	"Perplexity API Key": "",
+	"Sougou Search API sID": "",
+	"Sougou Search API SK": "",
 	"Personalization": "",
 	"Pin": "",
 	"Pinned": "",

src/lib/i18n/locales/zh-CN/translation.json

@@ -424,6 +424,8 @@
 	"Enter Mojeek Search API Key": "输入 Mojeek Search API 密钥",
 	"Enter Number of Steps (e.g. 50)": "输入步骤数 (Steps) (例如：50)",
 	"Enter Perplexity API Key": "输入 Perplexity API 密钥",
+	"Enter Sougou Search API sID": "输入搜狗搜索 API 的 Secret ID",
+	"Enter Sougou Search API SK": "输入搜狗搜索 API 的 Secret Key",
 	"Enter proxy URL (e.g. https://user:password@host:port)": "输入代理 URL (例如：https://用户名:密码@主机名:端口）",
 	"Enter reasoning effort": "设置推理努力",
 	"Enter Sampler (e.g. Euler a)": "输入 Sampler (例如：Euler a)",
@@ -822,6 +824,8 @@
 	"Permission denied when accessing microphone: {{error}}": "申请麦克风权限被拒绝：{{error}}",
 	"Permissions": "权限",
 	"Perplexity API Key": "Perplexity API 密钥",
+	"Sougou Search API sID": "搜狗搜索 API 的 Secret ID",
+	"Sougou Search API SK": "搜狗搜索 API 的 Secret Key",
 	"Personalization": "个性化",
 	"Pin": "置顶",
 	"Pinned": "已置顶",