From 7e761a69a7dc272a6d8afd73de64660cda281895 Mon Sep 17 00:00:00 2001
From: Lyuboslav Petrov <petrov.lyuboslav@gmail.com>
Date: Mon, 3 Jun 2024 12:44:46 +0300
Subject: [PATCH 1/2] FIX searxng URL construction using params for arg passing

Accept additional parameters such as language, time_range, and categories to tailor the search results.
Raise an exception if a request error occurs during the search process.
Use params argument to construct the query string
Sort by relevance
Expand docstring
---
 backend/apps/rag/search/searxng.py | 63 +++++++++++++++++++++++-------
 1 file changed, 49 insertions(+), 14 deletions(-)

diff --git a/backend/apps/rag/search/searxng.py b/backend/apps/rag/search/searxng.py
index 477d95c3f..a1d07d920 100644
--- a/backend/apps/rag/search/searxng.py
+++ b/backend/apps/rag/search/searxng.py
@@ -1,7 +1,8 @@
 import logging
-
 import requests
 
+from typing import List
+
 from apps.rag.search.main import SearchResult
 from config import SRC_LOG_LEVELS
 
@@ -9,20 +10,52 @@ log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["RAG"])
 
 
-def search_searxng(query_url: str, query: str, count: int) -> list[SearchResult]:
-    """Search a SearXNG instance for a query and return the results as a list of SearchResult objects.
+def search_searxng(query_url: str, query: str, count: int, **kwargs) -> List[SearchResult]:
+    """
+    Search a SearXNG instance for a given query and return the results as a list of SearchResult objects.
+    
+    The function allows passing additional parameters such as language or time_range to tailor the search result.
 
     Args:
-        query_url (str): The URL of the SearXNG instance to search. Must contain "<query>" as a placeholder
-        query (str): The query to search for
+        query_url (str): The base URL of the SearXNG server with a placeholder for the query "<query>".
+        query (str): The search term or question to find in the SearXNG database.
+        count (int): The maximum number of results to retrieve from the search.
+        
+    Keyword Args:
+        language (str): Language filter for the search results; e.g., "en-US". Defaults to an empty string.
+        time_range (str): Time range for filtering results by date; e.g., "2023-04-05..today" or "all-time". Defaults to ''.
+        categories: (Optional[List[str]]): Specific categories within which the search should be performed, defaulting to an empty string if not provided.
+    
+    Returns:
+        List[SearchResult]: A list of SearchResults sorted by relevance score in descending order.
+        
+    Raise:
+        requests.exceptions.RequestException: If a request error occurs during the search process.
     """
-    url = query_url.replace("<query>", query)
-    if "&format=json" not in url:
-        url += "&format=json"
-    log.debug(f"searching {url}")
+    
+    # Default values for optional parameters are provided as empty strings or None when not specified.
+    language = kwargs.get('language', 'en-US')
+    time_range = kwargs.get('time_range', '')
+    categories = ''.join(kwargs.get('categories', []))
 
-    r = requests.get(
-        url,
+    params = {
+        "q": query,
+        "format": "json",
+        "pageno": 1,
+        "results_per_page": count,
+        'language': language,
+        'time_range': time_range,
+        'engines': '',
+        'categories': categories,
+        'theme': 'simple',
+        'image_proxy': 0
+
+    }
+
+    log.info(f"searching {query_url}")
+
+    response = requests.get(
+        query_url,
         headers={
             "User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot",
             "Accept": "text/html",
@@ -30,15 +63,17 @@ def search_searxng(query_url: str, query: str, count: int) -> list[SearchResult]
             "Accept-Language": "en-US,en;q=0.5",
             "Connection": "keep-alive",
         },
+        params=params,
     )
-    r.raise_for_status()
 
-    json_response = r.json()
+    response.raise_for_status()  # Raise an exception for HTTP errors.
+
+    json_response = response.json()
     results = json_response.get("results", [])
     sorted_results = sorted(results, key=lambda x: x.get("score", 0), reverse=True)
     return [
         SearchResult(
             link=result["url"], title=result.get("title"), snippet=result.get("content")
         )
-        for result in sorted_results[:count]
+        for result in sorted_results
     ]

From 08443b3c550592d849af670abfad1f2f68a0341f Mon Sep 17 00:00:00 2001
From: Lyuboslav Petrov <petrov.lyuboslav@gmail.com>
Date: Mon, 3 Jun 2024 12:48:40 +0300
Subject: [PATCH 2/2] Revert log level to debug

---
 backend/apps/rag/search/searxng.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/apps/rag/search/searxng.py b/backend/apps/rag/search/searxng.py
index a1d07d920..24f9985a9 100644
--- a/backend/apps/rag/search/searxng.py
+++ b/backend/apps/rag/search/searxng.py
@@ -52,7 +52,7 @@ def search_searxng(query_url: str, query: str, count: int, **kwargs) -> List[Sea
 
     }
 
-    log.info(f"searching {query_url}")
+    log.debug(f"searching {query_url}")
 
     response = requests.get(
         query_url,