Merge pull request #12683 from Youggls/dev

feat: Add sougou web search engine.
2025-05-19 12:51:35 +00:00 · 2025-04-10 08:45:15 -07:00 · 2025-04-10 08:45:15 -07:00 · 1865e29bfc
commit 1865e29bfc
parent f7dc6474ee 3e2a6df1fb
9 changed files with 129 additions and 1 deletions
--- a/backend/open_webui/config.py
+++ b/backend/open_webui/config.py
@ -2142,6 +2142,18 @@ PERPLEXITY_API_KEY = PersistentConfig(
    os.getenv("PERPLEXITY_API_KEY", ""),
 )

+SOUGOU_API_SID = PersistentConfig(
+    "SOUGOU_API_SID",
+    "rag.web.search.sougou_api_sid",
+    os.getenv("SOUGOU_API_SID", ""),
+)
+
+SOUGOU_API_SK = PersistentConfig(
+    "SOUGOU_API_SK",
+    "rag.web.search.sougou_api_sk",
+    os.getenv("SOUGOU_API_SK", ""),
+)
+
 RAG_WEB_SEARCH_RESULT_COUNT = PersistentConfig(
    "RAG_WEB_SEARCH_RESULT_COUNT",
    "rag.web.search.result_count",
--- a/backend/open_webui/main.py
+++ b/backend/open_webui/main.py
@ -225,6 +225,8 @@ from open_webui.config import (
    BRAVE_SEARCH_API_KEY,
    EXA_API_KEY,
    PERPLEXITY_API_KEY,
+    SOUGOU_API_SID,
+    SOUGOU_API_SK,
    KAGI_SEARCH_API_KEY,
    MOJEEK_SEARCH_API_KEY,
    BOCHA_SEARCH_API_KEY,
@ -652,6 +654,8 @@ app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT
 app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY
 app.state.config.EXA_API_KEY = EXA_API_KEY
 app.state.config.PERPLEXITY_API_KEY = PERPLEXITY_API_KEY
+app.state.config.SOUGOU_API_SID = SOUGOU_API_SID
+app.state.config.SOUGOU_API_SK = SOUGOU_API_SK

 app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = RAG_WEB_SEARCH_RESULT_COUNT
 app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS = RAG_WEB_SEARCH_CONCURRENT_REQUESTS
--- a/backend/open_webui/retrieval/web/sougou.py
+++ b/backend/open_webui/retrieval/web/sougou.py
@ -0,0 +1,48 @@
+import logging
+import json
+from typing import Optional, List
+
+from tencentcloud.common.common_client import CommonClient
+from tencentcloud.common import credential
+from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
+from tencentcloud.common.profile.client_profile import ClientProfile
+from tencentcloud.common.profile.http_profile import HttpProfile
+
+from open_webui.retrieval.web.main import SearchResult, get_filtered_results
+from open_webui.env import SRC_LOG_LEVELS
+
+log = logging.getLogger(__name__)
+log.setLevel(SRC_LOG_LEVELS["RAG"])
+
+
+def search_sougou(
+    sougou_api_sid: str,
+    sougou_api_sk: str,
+    query: str,
+    count: int,
+    filter_list: Optional[List[str]] = None,
+) -> List[SearchResult]:
+    try:
+        cred = credential.Credential(sougou_api_sid, sougou_api_sk)
+        http_profile = HttpProfile()
+        http_profile.endpoint = "tms.tencentcloudapi.com"
+        client_profile = ClientProfile()
+        client_profile.http_profile = http_profile
+        params = json.dumps({"Query": query, 'Cnt': 20})
+        common_client = CommonClient("tms", "2020-12-29", cred, "", profile=client_profile)
+        results = [
+            json.loads(page) for page in common_client.call_json("SearchPro", json.loads(params))["Response"]["Pages"]
+        ]
+        sorted_results = sorted(results, key=lambda x: x.get("scour", 0.0), reverse=True)
+        if filter_list:
+            sorted_results = get_filtered_results(sorted_results, filter_list)
+
+        return [
+            SearchResult(
+                link=result.get("url"), title=result.get("title"), snippet=result.get("passage")
+            )
+            for result in sorted_results[:count]
+        ]
+    except TencentCloudSDKException as err:
+        log.error(f"Error in Sougou search: {err}")
+        return []
--- a/backend/open_webui/routers/retrieval.py
+++ b/backend/open_webui/routers/retrieval.py
@ -60,6 +60,7 @@ from open_webui.retrieval.web.tavily import search_tavily
 from open_webui.retrieval.web.bing import search_bing
 from open_webui.retrieval.web.exa import search_exa
 from open_webui.retrieval.web.perplexity import search_perplexity
+from open_webui.retrieval.web.sougou import search_sougou

 from open_webui.retrieval.utils import (
    get_embedding_function,
@ -411,6 +412,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
                "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
                "exa_api_key": request.app.state.config.EXA_API_KEY,
                "perplexity_api_key": request.app.state.config.PERPLEXITY_API_KEY,
+                "sougou_api_sid": request.app.state.config.SOUGOU_API_SID,
+                "sougou_api_sk": request.app.state.config.SOUGOU_API_SK,
                "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
                "trust_env": request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
                "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
@ -478,6 +481,8 @@ class WebSearchConfig(BaseModel):
    bing_search_v7_subscription_key: Optional[str] = None
    exa_api_key: Optional[str] = None
    perplexity_api_key: Optional[str] = None
+    sougou_api_sid: Optional[str] = None
+    sougou_api_sk: Optional[str] = None
    result_count: Optional[int] = None
    concurrent_requests: Optional[int] = None
    trust_env: Optional[bool] = None
@ -640,6 +645,12 @@ async def update_rag_config(
        request.app.state.config.PERPLEXITY_API_KEY = (
            form_data.web.search.perplexity_api_key
        )
+        request.app.state.config.SOUGOU_API_SID = (
+            form_data.web.search.sougou_api_sid
+        )
+        request.app.state.config.SOUGOU_API_SK = (
+            form_data.web.search.sougou_api_sk
+        )

        request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = (
            form_data.web.search.result_count
@ -712,6 +723,8 @@ async def update_rag_config(
                "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
                "exa_api_key": request.app.state.config.EXA_API_KEY,
                "perplexity_api_key": request.app.state.config.PERPLEXITY_API_KEY,
+                "sougou_api_sid": request.app.state.config.SOUGOU_API_SID,
+                "sougou_api_sk": request.app.state.config.SOUGOU_API_SK,
                "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
                "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
                "trust_env": request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
@ -1267,6 +1280,7 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
    - TAVILY_API_KEY
    - EXA_API_KEY
    - PERPLEXITY_API_KEY
+    - SOUGOU_API_SID + SOUGOU_API_SK
    - SEARCHAPI_API_KEY + SEARCHAPI_ENGINE (by default `google`)
    - SERPAPI_API_KEY + SERPAPI_ENGINE (by default `google`)
    Args:
@ -1438,6 +1452,17 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
            request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
            request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
        )
+    elif engine == 'sougou':
+        if request.app.state.config.SOUGOU_API_SID and request.app.state.config.SOUGOU_API_SK:
+            return search_sougou(
+                request.app.state.config.SOUGOU_API_SID,
+                request.app.state.config.SOUGOU_API_SK,
+                query,
+                request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
+                request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
+            )
+        else:
+            raise Exception("No SOUGOU_API_SID or SOUGOU_API_SK found in environment variables")
    else:
        raise Exception("No search engine API key found in environment variables")

--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@ -123,6 +123,9 @@ ldap3==2.9.1
 ## Firecrawl
 firecrawl-py==1.12.0

+# Sougou API SDK(Tencentcloud SDK)
+tencentcloud-sdk-python==3.0.1336
+
 ## Trace
 opentelemetry-api==1.31.1
 opentelemetry-sdk==1.31.1
--- a/pyproject.toml
+++ b/pyproject.toml
@ -125,6 +125,8 @@ dependencies = [

    "firecrawl-py==1.12.0",

+    "tencentcloud-sdk-python==3.0.1336",
+
    "gcp-storage-emulator>=2024.8.3",
 ]
 readme = "README.md"
--- a/src/lib/components/admin/Settings/WebSearch.svelte
+++ b/src/lib/components/admin/Settings/WebSearch.svelte
@ -30,7 +30,8 @@
 		'jina',
 		'bing',
 		'exa',
-		'perplexity'
+		'perplexity',
+		'sougou'
 	];

 	let youtubeLanguage = 'en';
@ -404,6 +405,31 @@
 									/>
 								</div>
 							</div>
+							{:else if webConfig.search.engine === 'sougou'}
+								<div class="mb-2.5 flex w-full flex-col">
+									<div>
+										<div class=" self-center text-xs font-medium mb-1">
+											{$i18n.t('Sougou Search API sID')}
+										</div>
+	
+										<SensitiveInput
+											placeholder={$i18n.t('Enter Sougou Search API sID')}
+											bind:value={webConfig.search.sougou_api_sid}
+										/>
+									</div>
+								</div>
+								<div class="mb-2.5 flex w-full flex-col">
+									<div>
+										<div class=" self-center text-xs font-medium mb-1">
+											{$i18n.t('Sougou Search API SK')}
+										</div>
+	
+										<SensitiveInput
+											placeholder={$i18n.t('Enter Sougou Search API SK')}
+											bind:value={webConfig.search.sougou_api_sk}
+										/>
+									</div>
+								</div>
 						{/if}
 					{/if}

--- a/src/lib/i18n/locales/en-US/translation.json
+++ b/src/lib/i18n/locales/en-US/translation.json
@ -424,6 +424,8 @@
 	"Enter Mojeek Search API Key": "",
 	"Enter Number of Steps (e.g. 50)": "",
 	"Enter Perplexity API Key": "",
+	"Enter Sougou Search API sID": "",
+	"Enter Sougou Search API SK": "",
 	"Enter proxy URL (e.g. https://user:password@host:port)": "",
 	"Enter reasoning effort": "",
 	"Enter Sampler (e.g. Euler a)": "",
@ -822,6 +824,8 @@
 	"Permission denied when accessing microphone: {{error}}": "",
 	"Permissions": "",
 	"Perplexity API Key": "",
+	"Sougou Search API sID": "",
+	"Sougou Search API SK": "",
 	"Personalization": "",
 	"Pin": "",
 	"Pinned": "",
--- a/src/lib/i18n/locales/zh-CN/translation.json
+++ b/src/lib/i18n/locales/zh-CN/translation.json
@ -424,6 +424,8 @@
 	"Enter Mojeek Search API Key": "输入 Mojeek Search API 密钥",
 	"Enter Number of Steps (e.g. 50)": "输入步骤数 (Steps) (例如：50)",
 	"Enter Perplexity API Key": "输入 Perplexity API 密钥",
+	"Enter Sougou Search API sID": "输入搜狗搜索 API 的 Secret ID",
+	"Enter Sougou Search API SK": "输入搜狗搜索 API 的 Secret Key",
 	"Enter proxy URL (e.g. https://user:password@host:port)": "输入代理 URL (例如：https://用户名:密码@主机名:端口）",
 	"Enter reasoning effort": "设置推理努力",
 	"Enter Sampler (e.g. Euler a)": "输入 Sampler (例如：Euler a)",
@ -822,6 +824,8 @@
 	"Permission denied when accessing microphone: {{error}}": "申请麦克风权限被拒绝：{{error}}",
 	"Permissions": "权限",
 	"Perplexity API Key": "Perplexity API 密钥",
+	"Sougou Search API sID": "搜狗搜索 API 的 Secret ID",
+	"Sougou Search API SK": "搜狗搜索 API 的 Secret Key",
 	"Personalization": "个性化",
 	"Pin": "置顶",
 	"Pinned": "已置顶",