UniqAI/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui synced 2024-11-16 13:40:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: backend implementation of various search APIs

Browse Source
This commit is contained in:
Jun Siang Cheah 2024-05-06 12:27:46 +08:00
parent 594f2f264f
commit 501ff7a98b
13 changed files with 2987 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
backend/apps/rag/search/brave.py Normal file
View File

@ -0,0 +1,37 @@
import logging
import requests
from apps.rag.search.main import SearchResult
from config import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_brave(api_key: str, query: str) -> list[SearchResult]:
"""Search using Brave's Search API and return the results as a list of SearchResult objects.
Args:
api_key (str): A Brave Search API key
query (str): The query to search for
"""
url = "https://api.search.brave.com/res/v1/web/search"
headers = {
"Accept": "application/json",
"Accept-Encoding": "gzip",
"X-Subscription-Token": api_key,
}
params = {"q": query, "count": 5}
response = requests.get(url, headers=headers, params=params)
response.raise_for_status()
json_response = response.json()
results = json_response.get("web", {}).get("results", [])
return [
SearchResult(
link=result["url"], title=result.get("title"), snippet=result.get("snippet")
)
for result in results[:5]
]

43
backend/apps/rag/search/google_pse.py Normal file
View File

@ -0,0 +1,43 @@
import json
import logging
import requests
from apps.rag.search.main import SearchResult
from config import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_google_pse(
api_key: str, search_engine_id: str, query: str
) -> list[SearchResult]:
"""Search using Google's Programmable Search Engine API and return the results as a list of SearchResult objects.
Args:
api_key (str): A Programmable Search Engine API key
search_engine_id (str): A Programmable Search Engine ID
query (str): The query to search for
"""
url = "https://www.googleapis.com/customsearch/v1"
headers = {"Content-Type": "application/json"}
params = {
"cx": search_engine_id,
"q": query,
"key": api_key,
"num": 5,
}
response = requests.request("POST", url, headers=headers, params=params)
response.raise_for_status()
json_response = response.json()
results = json_response.get("items", [])
return [
SearchResult(
link=result["url"], title=result.get("title"), snippet=result.get("snippet")
)
for result in results
]

9
backend/apps/rag/search/main.py Normal file
View File

@ -0,0 +1,9 @@
from typing import Optional
from pydantic import BaseModel
class SearchResult(BaseModel):
link: str
title: Optional[str]
snippet: Optional[str]

44
backend/apps/rag/search/searxng.py Normal file
View File

@ -0,0 +1,44 @@
import logging
import requests
from apps.rag.search.main import SearchResult
from config import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_searxng(query_url: str, query: str) -> list[SearchResult]:
"""Search a SearXNG instance for a query and return the results as a list of SearchResult objects.
Args:
query_url (str): The URL of the SearXNG instance to search. Must contain "<query>" as a placeholder
query (str): The query to search for
"""
url = query_url.replace("<query>", query)
if "&format=json" not in url:
url += "&format=json"
log.debug(f"searching {url}")
r = requests.get(
url,
headers={
"User-Agent": "Open WebUI (https://github.com/open-webui/open-webui) RAG Bot",
"Accept": "text/html",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "en-US,en;q=0.5",
"Connection": "keep-alive",
},
)
r.raise_for_status()
json_response = r.json()
results = json_response.get("results", [])
sorted_results = sorted(results, key=lambda x: x.get("score", 0), reverse=True)
return [
SearchResult(
link=result["url"], title=result.get("title"), snippet=result.get("content")
)
for result in sorted_results[:5]
]

39
backend/apps/rag/search/serper.py Normal file
View File

@ -0,0 +1,39 @@
import json
import logging
import requests
from apps.rag.search.main import SearchResult
from config import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_serper(api_key: str, query: str) -> list[SearchResult]:
"""Search using serper.dev's API and return the results as a list of SearchResult objects.
Args:
api_key (str): A serper.dev API key
query (str): The query to search for
"""
url = "https://google.serper.dev/search"
payload = json.dumps({"q": query})
headers = {"X-API-KEY": api_key, "Content-Type": "application/json"}
response = requests.request("POST", url, headers=headers, data=payload)
response.raise_for_status()
json_response = response.json()
results = sorted(
json_response.get("organic", []), key=lambda x: x.get("position", 0)
)
return [
SearchResult(
link=result["url"],
title=result.get("title"),
snippet=result.get("description"),
)
for result in results
]

43
backend/apps/rag/search/serpstack.py Normal file
View File

@ -0,0 +1,43 @@
import json
import logging
import requests
from apps.rag.search.main import SearchResult
from config import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_serpstack(
api_key: str, query: str, https_enabled: bool = True
) -> list[SearchResult]:
"""Search using serpstack.com's and return the results as a list of SearchResult objects.
Args:
api_key (str): A serpstack.com API key
query (str): The query to search for
https_enabled (bool): Whether to use HTTPS or HTTP for the API request
"""
url = f"{'https' if https_enabled else 'http'}://api.serpstack.com/search"
headers = {"Content-Type": "application/json"}
params = {
"access_key": api_key,
"query": query,
}
response = requests.request("POST", url, headers=headers, params=params)
response.raise_for_status()
json_response = response.json()
results = sorted(
json_response.get("organic_results", []), key=lambda x: x.get("position", 0)
)
return [
SearchResult(
link=result["url"], title=result.get("title"), snippet=result.get("snippet")
)
for result in results[:5]
]

1000
backend/apps/rag/search/testdata/brave.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff

442
backend/apps/rag/search/testdata/google_pse.json vendored Normal file
View File

@ -0,0 +1,442 @@
{
"kind": "customsearch#search",
"url": {
"type": "application/json",
"template": "https://www.googleapis.com/customsearch/v1?q={searchTerms}&num={count?}&start={startIndex?}&lr={language?}&safe={safe?}&cx={cx?}&sort={sort?}&filter={filter?}&gl={gl?}&cr={cr?}&googlehost={googleHost?}&c2coff={disableCnTwTranslation?}&hq={hq?}&hl={hl?}&siteSearch={siteSearch?}&siteSearchFilter={siteSearchFilter?}&exactTerms={exactTerms?}&excludeTerms={excludeTerms?}&linkSite={linkSite?}&orTerms={orTerms?}&dateRestrict={dateRestrict?}&lowRange={lowRange?}&highRange={highRange?}&searchType={searchType}&fileType={fileType?}&rights={rights?}&imgSize={imgSize?}&imgType={imgType?}&imgColorType={imgColorType?}&imgDominantColor={imgDominantColor?}&alt=json"
},
"queries": {
"request": [
{
"title": "Google Custom Search - lectures",
"totalResults": "2450000000",
"searchTerms": "lectures",
"count": 10,
"startIndex": 1,
"inputEncoding": "utf8",
"outputEncoding": "utf8",
"safe": "off",
"cx": "0473ef98502d44e18"
}
],
"nextPage": [
{
"title": "Google Custom Search - lectures",
"totalResults": "2450000000",
"searchTerms": "lectures",
"count": 10,
"startIndex": 11,
"inputEncoding": "utf8",
"outputEncoding": "utf8",
"safe": "off",
"cx": "0473ef98502d44e18"
}
]
},
"context": {
"title": "LLM Search"
},
"searchInformation": {
"searchTime": 0.445959,
"formattedSearchTime": "0.45",
"totalResults": "2450000000",
"formattedTotalResults": "2,450,000,000"
},
"items": [
{
"kind": "customsearch#result",
"title": "The Feynman Lectures on Physics",
"htmlTitle": "The Feynman \u003cb\u003eLectures\u003c/b\u003e on Physics",
"link": "https://www.feynmanlectures.caltech.edu/",
"displayLink": "www.feynmanlectures.caltech.edu",
"snippet": "This edition has been designed for ease of reading on devices of any size or shape; text, figures and equations can all be zoomed without degradation.",
"htmlSnippet": "This edition has been designed for ease of reading on devices of any size or shape; text, figures and equations can all be zoomed without degradation.",
"cacheId": "CyXMWYWs9UEJ",
"formattedUrl": "https://www.feynmanlectures.caltech.edu/",
"htmlFormattedUrl": "https://www.feynman\u003cb\u003electures\u003c/b\u003e.caltech.edu/",
"pagemap": {
"metatags": [
{
"viewport": "width=device-width, initial-scale=1.0"
}
]
}
},
{
"kind": "customsearch#result",
"title": "Video Lectures",
"htmlTitle": "Video \u003cb\u003eLectures\u003c/b\u003e",
"link": "https://www.reddit.com/r/lectures/",
"displayLink": "www.reddit.com",
"snippet": "r/lectures: This subreddit is all about video lectures, talks and interesting public speeches. The topics include mathematics, physics, computer…",
"htmlSnippet": "r/\u003cb\u003electures\u003c/b\u003e: This subreddit is all about video \u003cb\u003electures\u003c/b\u003e, talks and interesting public speeches. The topics include mathematics, physics, computer…",
"formattedUrl": "https://www.reddit.com/r/lectures/",
"htmlFormattedUrl": "https://www.reddit.com/r/\u003cb\u003electures\u003c/b\u003e/",
"pagemap": {
"cse_thumbnail": [
{
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTZtOjhfkgUKQbL3DZxe5F6OVsgeDNffleObjJ7n9RllKQTSsimax7VIaY&s",
"width": "192",
"height": "192"
}
],
"metatags": [
{
"og:image": "https://www.redditstatic.com/shreddit/assets/favicon/192x192.png",
"theme-color": "#000000",
"og:image:width": "256",
"og:type": "website",
"twitter:card": "summary",
"twitter:title": "r/lectures",
"og:site_name": "Reddit",
"og:title": "r/lectures",
"og:image:height": "256",
"bingbot": "noarchive",
"msapplication-navbutton-color": "#000000",
"og:description": "This subreddit is all about video lectures, talks and interesting public speeches.\n\nThe topics include mathematics, physics, computer science, programming, engineering, biology, medicine, economics, politics, social sciences, and any other subjects!",
"twitter:image": "https://www.redditstatic.com/shreddit/assets/favicon/192x192.png",
"apple-mobile-web-app-status-bar-style": "black",
"twitter:site": "@reddit",
"viewport": "width=device-width, initial-scale=1, viewport-fit=cover",
"apple-mobile-web-app-capable": "yes",
"og:ttl": "600",
"og:url": "https://www.reddit.com/r/lectures/"
}
],
"cse_image": [
{
"src": "https://www.redditstatic.com/shreddit/assets/favicon/192x192.png"
}
]
}
},
{
"kind": "customsearch#result",
"title": "Lectures & Discussions | Flint Institute of Arts",
"htmlTitle": "\u003cb\u003eLectures\u003c/b\u003e &amp; Discussions | Flint Institute of Arts",
"link": "https://flintarts.org/events/lectures",
"displayLink": "flintarts.org",
"snippet": "It will trace the intricate relationship between jewelry, attire, and the expression of personal identity, social hierarchy, and spiritual belief systems that ...",
"htmlSnippet": "It will trace the intricate relationship between jewelry, attire, and the expression of personal identity, social hierarchy, and spiritual belief systems that&nbsp;...",
"cacheId": "jvpb9DxrfxoJ",
"formattedUrl": "https://flintarts.org/events/lectures",
"htmlFormattedUrl": "https://flintarts.org/events/\u003cb\u003electures\u003c/b\u003e",
"pagemap": {
"cse_thumbnail": [
{
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS23tMtAeNhJbOWdGxShYsmnyzFdzOC9Hb7lRykA9Pw72z1IlKTkjTdZw&s",
"width": "447",
"height": "113"
}
],
"metatags": [
{
"og:image": "https://flintarts.org/uploads/images/page-headers/_headerImage/nightshot.jpg",
"og:type": "website",
"viewport": "width=device-width, initial-scale=1",
"og:title": "Lectures & Discussions | Flint Institute of Arts",
"og:description": "The Flint Institute of Arts is the second largest art museum in Michigan and one of the largest museum art schools in the nation."
}
],
"cse_image": [
{
"src": "https://flintarts.org/uploads/images/page-headers/_headerImage/nightshot.jpg"
}
]
}
},
{
"kind": "customsearch#result",
"title": "Mandel Lectures | Mandel Center for the Humanities ... - Waltham",
"htmlTitle": "Mandel \u003cb\u003eLectures\u003c/b\u003e | Mandel Center for the Humanities ... - Waltham",
"link": "https://www.brandeis.edu/mandel-center-humanities/mandel-lectures.html",
"displayLink": "www.brandeis.edu",
"snippet": "Past Lectures · Lecture 1: \"Invisible Music: The Sonic Idea of Black Revolution From Captivity to Reconstruction\" · Lecture 2: \"Solidarity in Sound: Grassroots ...",
"htmlSnippet": "Past \u003cb\u003eLectures\u003c/b\u003e &middot; \u003cb\u003eLecture\u003c/b\u003e 1: &quot;Invisible Music: The Sonic Idea of Black Revolution From Captivity to Reconstruction&quot; &middot; \u003cb\u003eLecture\u003c/b\u003e 2: &quot;Solidarity in Sound: Grassroots&nbsp;...",
"cacheId": "cQLOZr0kgEEJ",
"formattedUrl": "https://www.brandeis.edu/mandel-center-humanities/mandel-lectures.html",
"htmlFormattedUrl": "https://www.brandeis.edu/mandel-center-humanities/mandel-\u003cb\u003electures\u003c/b\u003e.html",
"pagemap": {
"cse_thumbnail": [
{
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQWlU7bcJ5pIHk7RBCk2QKE-48ejF7hyPV0pr-20_cBt2BGdfKtiYXBuyw&s",
"width": "275",
"height": "183"
}
],
"metatags": [
{
"og:image": "https://www.brandeis.edu/mandel-center-humanities/events/events-images/mlhzumba",
"twitter:card": "summary_large_image",
"viewport": "width=device-width,initial-scale=1,minimum-scale=1",
"og:title": "Mandel Lectures in the Humanities",
"og:url": "https://www.brandeis.edu/mandel-center-humanities/mandel-lectures.html",
"og:description": "Annual Lecture Series",
"twitter:image": "https://www.brandeis.edu/mandel-center-humanities/events/events-images/mlhzumba"
}
],
"cse_image": [
{
"src": "https://www.brandeis.edu/mandel-center-humanities/events/events-images/mlhzumba"
}
]
}
},
{
"kind": "customsearch#result",
"title": "Brian Douglas - YouTube",
"htmlTitle": "Brian Douglas - YouTube",
"link": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"displayLink": "www.youtube.com",
"snippet": "Welcome to Control Systems Lectures! This collection of videos is intended to supplement a first year controls class, not replace it.",
"htmlSnippet": "Welcome to Control Systems \u003cb\u003eLectures\u003c/b\u003e! This collection of videos is intended to supplement a first year controls class, not replace it.",
"cacheId": "NEROyBHolL0J",
"formattedUrl": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"htmlFormattedUrl": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"pagemap": {
"hcard": [
{
"fn": "Brian Douglas",
"url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg"
}
],
"cse_thumbnail": [
{
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcR7G0CeCBz_wVTZgjnhEr2QbiKP7f3uYzKitZYn74Mi32cDmVxvsegJoLI&s",
"width": "225",
"height": "225"
}
],
"imageobject": [
{
"width": "900",
"url": "https://yt3.googleusercontent.com/ytc/AIdro_nLo68wetImbwGUYP3stve_iKmAEccjhqB-q4o79xdInN4=s900-c-k-c0x00ffffff-no-rj",
"height": "900"
}
],
"person": [
{
"name": "Brian Douglas",
"url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg"
}
],
"metatags": [
{
"apple-itunes-app": "app-id=544007664, app-argument=https://m.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg?referring_app=com.apple.mobilesafari-smartbanner, affiliate-data=ct=smart_app_banner_polymer&pt=9008",
"og:image": "https://yt3.googleusercontent.com/ytc/AIdro_nLo68wetImbwGUYP3stve_iKmAEccjhqB-q4o79xdInN4=s900-c-k-c0x00ffffff-no-rj",
"twitter:app:url:iphone": "vnd.youtube://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"twitter:app:id:googleplay": "com.google.android.youtube",
"theme-color": "rgb(255, 255, 255)",
"og:image:width": "900",
"twitter:card": "summary",
"og:site_name": "YouTube",
"twitter:url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"twitter:app:url:ipad": "vnd.youtube://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"al:android:package": "com.google.android.youtube",
"twitter:app:name:googleplay": "YouTube",
"al:ios:url": "vnd.youtube://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"twitter:app:id:iphone": "544007664",
"og:description": "Welcome to Control Systems Lectures! This collection of videos is intended to supplement a first year controls class, not replace it. My goal is to take specific concepts in controls and expand on them in order to provide an intuitive understanding which will ultimately make you a better controls engineer. \n\nI'm glad you made it to my channel and I hope you find it useful.\n\nShoot me a message at controlsystemlectures@gmail.com, leave a comment or question and I'll get back to you if I can. Don't forget to subscribe!\n \nTwitter: @BrianBDouglas for engineering tweets and announcement of new videos.\nWebpage: http://engineeringmedia.com\n\nHere is the hardware/software I use: http://www.youtube.com/watch?v=m-M5_mIyHe4\n\nHere's a list of my favorite references: http://bit.ly/2skvmWd\n\n--Brian",
"al:ios:app_store_id": "544007664",
"twitter:image": "https://yt3.googleusercontent.com/ytc/AIdro_nLo68wetImbwGUYP3stve_iKmAEccjhqB-q4o79xdInN4=s900-c-k-c0x00ffffff-no-rj",
"twitter:site": "@youtube",
"og:type": "profile",
"twitter:title": "Brian Douglas",
"al:ios:app_name": "YouTube",
"og:title": "Brian Douglas",
"og:image:height": "900",
"twitter:app:id:ipad": "544007664",
"al:web:url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg?feature=applinks",
"al:android:url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg?feature=applinks",
"fb:app_id": "87741124305",
"twitter:app:url:googleplay": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"twitter:app:name:ipad": "YouTube",
"viewport": "width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no,",
"twitter:description": "Welcome to Control Systems Lectures! This collection of videos is intended to supplement a first year controls class, not replace it. My goal is to take specific concepts in controls and expand on them in order to provide an intuitive understanding which will ultimately make you a better controls engineer. \n\nI'm glad you made it to my channel and I hope you find it useful.\n\nShoot me a message at controlsystemlectures@gmail.com, leave a comment or question and I'll get back to you if I can. Don't forget to subscribe!\n \nTwitter: @BrianBDouglas for engineering tweets and announcement of new videos.\nWebpage: http://engineeringmedia.com\n\nHere is the hardware/software I use: http://www.youtube.com/watch?v=m-M5_mIyHe4\n\nHere's a list of my favorite references: http://bit.ly/2skvmWd\n\n--Brian",
"og:url": "https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg",
"al:android:app_name": "YouTube",
"twitter:app:name:iphone": "YouTube"
}
],
"cse_image": [
{
"src": "https://yt3.googleusercontent.com/ytc/AIdro_nLo68wetImbwGUYP3stve_iKmAEccjhqB-q4o79xdInN4=s900-c-k-c0x00ffffff-no-rj"
}
]
}
},
{
"kind": "customsearch#result",
"title": "Lecture - Wikipedia",
"htmlTitle": "\u003cb\u003eLecture\u003c/b\u003e - Wikipedia",
"link": "https://en.wikipedia.org/wiki/Lecture",
"displayLink": "en.wikipedia.org",
"snippet": "Lecture ... For the academic rank, see Lecturer. A lecture (from Latin: lēctūra 'reading') is an oral presentation intended to present information or teach people ...",
"htmlSnippet": "\u003cb\u003eLecture\u003c/b\u003e ... For the academic rank, see \u003cb\u003eLecturer\u003c/b\u003e. A \u003cb\u003electure\u003c/b\u003e (from Latin: lēctūra &#39;reading&#39;) is an oral presentation intended to present information or teach people&nbsp;...",
"cacheId": "d9Pjta02fmgJ",
"formattedUrl": "https://en.wikipedia.org/wiki/Lecture",
"htmlFormattedUrl": "https://en.wikipedia.org/wiki/Lecture",
"pagemap": {
"metatags": [
{
"referrer": "origin",
"og:image": "https://upload.wikimedia.org/wikipedia/commons/thumb/2/26/ADFA_Lecture_Theatres.jpg/1200px-ADFA_Lecture_Theatres.jpg",
"theme-color": "#eaecf0",
"og:image:width": "1200",
"og:type": "website",
"viewport": "width=device-width, initial-scale=1.0, user-scalable=yes, minimum-scale=0.25, maximum-scale=5.0",
"og:title": "Lecture - Wikipedia",
"og:image:height": "799",
"format-detection": "telephone=no"
}
]
}
},
{
"kind": "customsearch#result",
"title": "Mount Wilson Observatory | Lectures",
"htmlTitle": "Mount Wilson Observatory | \u003cb\u003eLectures\u003c/b\u003e",
"link": "https://www.mtwilson.edu/lectures/",
"displayLink": "www.mtwilson.edu",
"snippet": "Talks & Telescopes: August 24, 2024 – Panel: The Triumph of Hubble ... Compelling talks followed by picnicking and convivial stargazing through both the big ...",
"htmlSnippet": "Talks &amp; Telescopes: August 24, 2024 – Panel: The Triumph of Hubble ... Compelling talks followed by picnicking and convivial stargazing through both the big&nbsp;...",
"cacheId": "wdXI0azqx5UJ",
"formattedUrl": "https://www.mtwilson.edu/lectures/",
"htmlFormattedUrl": "https://www.mtwilson.edu/\u003cb\u003electures\u003c/b\u003e/",
"pagemap": {
"metatags": [
{
"viewport": "width=device-width,initial-scale=1,user-scalable=no"
}
],
"webpage": [
{
"image": "http://www.mtwilson.edu/wp-content/uploads/2016/09/Logo.jpg",
"url": "https://www.facebook.com/WilsonObs"
}
]
}
},
{
"kind": "customsearch#result",
"title": "Lectures | NBER",
"htmlTitle": "\u003cb\u003eLectures\u003c/b\u003e | NBER",
"link": "https://www.nber.org/research/lectures",
"displayLink": "www.nber.org",
"snippet": "Results 1 - 50 of 354 ... Among featured events at the NBER Summer Institute are the Martin Feldstein Lecture, which examines a current issue involving economic ...",
"htmlSnippet": "Results 1 - 50 of 354 \u003cb\u003e...\u003c/b\u003e Among featured events at the NBER Summer Institute are the Martin Feldstein \u003cb\u003eLecture\u003c/b\u003e, which examines a current issue involving economic&nbsp;...",
"cacheId": "CvvP3U3nb44J",
"formattedUrl": "https://www.nber.org/research/lectures",
"htmlFormattedUrl": "https://www.nber.org/research/\u003cb\u003electures\u003c/b\u003e",
"pagemap": {
"cse_thumbnail": [
{
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTmeViEZyV1YmFEFLhcA6WdgAG3v3RV6tB93ncyxSJ5JPst_p2aWrL7D1k&s",
"width": "310",
"height": "163"
}
],
"metatags": [
{
"og:image": "https://www.nber.org/sites/default/files/2022-06/NBER-FB-Share-Tile-1200.jpg",
"og:site_name": "NBER",
"handheldfriendly": "true",
"viewport": "width=device-width, initial-scale=1.0",
"og:title": "Lectures",
"mobileoptimized": "width",
"og:url": "https://www.nber.org/research/lectures"
}
],
"cse_image": [
{
"src": "https://www.nber.org/sites/default/files/2022-06/NBER-FB-Share-Tile-1200.jpg"
}
]
}
},
{
"kind": "customsearch#result",
"title": "STUDENTS CANNOT ACCESS RECORDED LECTURES ... - Solved",
"htmlTitle": "STUDENTS CANNOT ACCESS RECORDED LECTURES ... - Solved",
"link": "https://community.canvaslms.com/t5/Canvas-Question-Forum/STUDENTS-CANNOT-ACCESS-RECORDED-LECTURES/td-p/190358",
"displayLink": "community.canvaslms.com",
"snippet": "Mar 19, 2020 ... I believe the issue is that students were not invited. Are you trying to capture your screen? If not, there is an option to just record your web ...",
"htmlSnippet": "Mar 19, 2020 \u003cb\u003e...\u003c/b\u003e I believe the issue is that students were not invited. Are you trying to capture your screen? If not, there is an option to just record your web&nbsp;...",
"cacheId": "wqrynQXX61sJ",
"formattedUrl": "https://community.canvaslms.com/t5/Canvas...LECTURES/td-p/190358",
"htmlFormattedUrl": "https://community.canvaslms.com/t5/Canvas...\u003cb\u003eLECTURES\u003c/b\u003e/td-p/190358",
"pagemap": {
"cse_thumbnail": [
{
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRUqXau3N8LfKgSD7OJOvV7xzGarLKRU-ckWXy1ZQ1p4CLPsedvLKmLMhk&s",
"width": "310",
"height": "163"
}
],
"metatags": [
{
"og:image": "https://community.canvaslms.com/html/@6A1FDD4D5FF35E4BBB4083A1022FA0DB/assets/CommunityPreview23.png",
"og:type": "article",
"article:section": "Canvas Question Forum",
"article:published_time": "2020-03-19T15:50:03.409Z",
"og:site_name": "Instructure Community",
"article:modified_time": "2020-03-19T13:55:53-07:00",
"viewport": "width=device-width, initial-scale=1.0, user-scalable=yes",
"og:title": "STUDENTS CANNOT ACCESS RECORDED LECTURES",
"og:url": "https://community.canvaslms.com/t5/Canvas-Question-Forum/STUDENTS-CANNOT-ACCESS-RECORDED-LECTURES/m-p/190358#M93667",
"og:description": "I can access and see my recorded lectures but my students can't. They have an error message when they try to open the recorded presentation or notes.",
"article:author": "https://community.canvaslms.com/t5/user/viewprofilepage/user-id/794287",
"twitter:image": "https://community.canvaslms.com/html/@6A1FDD4D5FF35E4BBB4083A1022FA0DB/assets/CommunityPreview23.png"
}
],
"cse_image": [
{
"src": "https://community.canvaslms.com/html/@6A1FDD4D5FF35E4BBB4083A1022FA0DB/assets/CommunityPreview23.png"
}
]
}
},
{
"kind": "customsearch#result",
"title": "Public Lecture Series - Sam Fox School of Design & Visual Arts",
"htmlTitle": "Public \u003cb\u003eLecture\u003c/b\u003e Series - Sam Fox School of Design &amp; Visual Arts",
"link": "https://samfoxschool.wustl.edu/calendar/series/2-public-lecture-series",
"displayLink": "samfoxschool.wustl.edu",
"snippet": "The Sam Fox School's Spring 2024 Public Lecture Series highlights design and art as catalysts for change. Renowned speakers will delve into themes like ...",
"htmlSnippet": "The Sam Fox School&#39;s Spring 2024 Public \u003cb\u003eLecture\u003c/b\u003e Series highlights design and art as catalysts for change. Renowned speakers will delve into themes like&nbsp;...",
"cacheId": "B-cgQG0j6tUJ",
"formattedUrl": "https://samfoxschool.wustl.edu/calendar/series/2-public-lecture-series",
"htmlFormattedUrl": "https://samfoxschool.wustl.edu/calendar/series/2-public-lecture-series",
"pagemap": {
"cse_thumbnail": [
{
"src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQSmHaGianm-64m-qauYjkPK_Q0JKWe-7yom4m1ogFYTmpWArA7k6dmk0sR&s",
"width": "307",
"height": "164"
}
],
"website": [
{
"name": "Public Lecture Series - Sam Fox School of Design & Visual Arts — Washington University in St. Louis"
}
],
"metatags": [
{
"og:image": "https://dvsp0hlm0xrn3.cloudfront.net/assets/default_og_image-44e73dee4b9d1e2c6a6295901371270c8ec5899eaed48ee8167a9b12f1b0f8b3.jpg",
"og:type": "website",
"og:site_name": "Sam Fox School of Design & Visual Arts — Washington University in St. Louis",
"viewport": "width=device-width, initial-scale=1.0",
"og:title": "Public Lecture Series - Sam Fox School of Design & Visual Arts — Washington University in St. Louis",
"csrf-token": "jBQsfZGY3RH8NVs0-KVDBYB-2N2kib4UYZHYdrShfTdLkvzfSvGeOaMrRKTRdYBPRKzdcGIuP7zwm9etqX_uvg",
"csrf-param": "authenticity_token",
"og:description": "The Sam Fox School's Spring 2024 Public Lecture Series highlights design and art as catalysts for change. Renowned speakers will delve into themes like social equity, resilient cities, and the impact of emerging technologies on contemporary life. Speakers include artists, architects, designers, and critics of the highest caliber, widely recognized for their research-based practices and multidisciplinary approaches to their fields."
}
],
"cse_image": [
{
"src": "https://dvsp0hlm0xrn3.cloudfront.net/assets/default_og_image-44e73dee4b9d1e2c6a6295901371270c8ec5899eaed48ee8167a9b12f1b0f8b3.jpg"
}
]
}
}
]
}

805
backend/apps/rag/search/testdata/searxng.json vendored Normal file
View File

@ -0,0 +1,805 @@
{
"query": "python",
"number_of_results": 116000000,
"results": [
{
"url": "https://www.python.org/",
"title": "Welcome to Python.org",
"content": "Python is a versatile and powerful language that lets you work quickly and integrate systems more effectively. Learn how to get started, download the latest version, access documentation, find jobs, and join the Python community.",
"engine": "bing",
"parsed_url": [
"https",
"www.python.org",
"/",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing",
"qwant",
"duckduckgo"
],
"positions": [
1,
1,
1
],
"score": 9.0,
"category": "general"
},
{
"url": "https://wiki.nerdvpn.de/wiki/Python_(programming_language)",
"title": "Python (programming language) - Wikipedia",
"content": "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected. It supports multiple programming paradigms, including structured (particularly procedural), object-oriented and functional programming.",
"engine": "bing",
"parsed_url": [
"https",
"wiki.nerdvpn.de",
"/wiki/Python_(programming_language)",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing",
"qwant",
"duckduckgo"
],
"positions": [
4,
3,
2
],
"score": 3.25,
"category": "general"
},
{
"url": "https://docs.python.org/3/tutorial/index.html",
"title": "The Python Tutorial \u2014 Python 3.12.3 documentation",
"content": "3 days ago \u00b7 Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python\u2019s elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development in many \u2026",
"engine": "bing",
"parsed_url": [
"https",
"docs.python.org",
"/3/tutorial/index.html",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing",
"qwant",
"duckduckgo"
],
"positions": [
5,
5,
3
],
"score": 2.2,
"category": "general"
},
{
"url": "https://www.python.org/downloads/",
"title": "Download Python | Python.org",
"content": "Python is a popular programming language for various purposes. Find the latest version of Python for different operating systems, download release notes, and learn about the development process.",
"engine": "bing",
"parsed_url": [
"https",
"www.python.org",
"/downloads/",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing",
"duckduckgo"
],
"positions": [
2,
2
],
"score": 2.0,
"category": "general"
},
{
"url": "https://www.python.org/about/gettingstarted/",
"title": "Python For Beginners | Python.org",
"content": "Learn the basics of Python, a popular and easy-to-use programming language, from installing it to using it for various purposes. Find out how to access online documentation, tutorials, books, code samples, and more resources to help you get started with Python.",
"engine": "bing",
"parsed_url": [
"https",
"www.python.org",
"/about/gettingstarted/",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing",
"qwant",
"duckduckgo"
],
"positions": [
9,
4,
4
],
"score": 1.8333333333333333,
"category": "general"
},
{
"url": "https://www.python.org/shell/",
"title": "Welcome to Python.org",
"content": "Python is a versatile and easy-to-use programming language that lets you work quickly. Learn more about Python, download the latest version, access documentation, find jobs, and join the community.",
"engine": "bing",
"parsed_url": [
"https",
"www.python.org",
"/shell/",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing",
"qwant",
"duckduckgo"
],
"positions": [
3,
10,
8
],
"score": 1.675,
"category": "general"
},
{
"url": "https://realpython.com/",
"title": "Python Tutorials \u2013 Real Python",
"content": "Real Python offers comprehensive and up-to-date tutorials, books, and courses for Python developers of all skill levels. Whether you want to learn Python basics, web development, data science, machine learning, or more, you can find clear and practical guides and code examples here.",
"engine": "bing",
"parsed_url": [
"https",
"realpython.com",
"/",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing",
"qwant",
"duckduckgo"
],
"positions": [
6,
6,
5
],
"score": 1.6,
"category": "general"
},
{
"url": "https://wiki.nerdvpn.de/wiki/Python",
"title": "Python",
"content": "Topics referred to by the same term",
"engine": "wikipedia",
"parsed_url": [
"https",
"wiki.nerdvpn.de",
"/wiki/Python",
"",
"",
""
],
"template": "default.html",
"engines": [
"wikipedia"
],
"positions": [
1
],
"score": 1.0,
"category": "general"
},
{
"title": "Online Python - IDE, Editor, Compiler, Interpreter",
"content": "Online Python IDE is a free online tool that lets you write, execute, and share Python code in the web browser. Learn about Python, its features, and its popularity as a general-purpose programming language for web development, data science, and more.",
"url": "https://www.online-python.com/",
"engine": "duckduckgo",
"parsed_url": [
"https",
"www.online-python.com",
"/",
"",
"",
""
],
"template": "default.html",
"engines": [
"qwant",
"duckduckgo"
],
"positions": [
8,
6
],
"score": 0.5833333333333333,
"category": "general"
},
{
"url": "https://micropython.org/",
"title": "MicroPython - Python for microcontrollers",
"content": "MicroPython is a full Python compiler and runtime that runs on the bare-metal. You get an interactive prompt (the REPL) to execute commands immediately, along ...",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"micropython.org",
"/",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
1
],
"score": 1.0,
"category": "general"
},
{
"url": "https://dictionary.cambridge.org/uk/dictionary/english/python",
"title": "PYTHON | \u0417\u043d\u0430\u0447\u0435\u043d\u043d\u044f \u0432 \u0430\u043d\u0433\u043b\u0456\u0439\u0441\u044c\u043a\u0456\u0439 \u043c\u043e\u0432\u0456 - Cambridge Dictionary",
"content": "Apr 17, 2024 \u2014 \u0412\u0438\u0437\u043d\u0430\u0447\u0435\u043d\u043d\u044f PYTHON: 1. a very large snake that kills animals for food by wrapping itself around them and crushing them\u2026. \u0414\u0456\u0437\u043d\u0430\u0439\u0442\u0435\u0441\u044f \u0431\u0456\u043b\u044c\u0448\u0435.",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"dictionary.cambridge.org",
"/uk/dictionary/english/python",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
2
],
"score": 0.5,
"category": "general"
},
{
"url": "https://www.codetoday.co.uk/code",
"title": "Web-based Python Editor (with Turtle graphics)",
"content": "Quick way of starting to write Python code, including drawing with Turtle, provided by CodeToday using Trinket.io Ideal for young children to start ...",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"www.codetoday.co.uk",
"/code",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
3
],
"score": 0.3333333333333333,
"category": "general"
},
{
"url": "https://snapcraft.io/docs/python-plugin",
"title": "The python plugin | Snapcraft documentation",
"content": "The python plugin can be used by either Python 2 or Python 3 based parts using a setup.py script for building the project, or using a package published to ...",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"snapcraft.io",
"/docs/python-plugin",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
4
],
"score": 0.25,
"category": "general"
},
{
"url": "https://www.developer-tech.com/categories/developer-languages/developer-languages-python/",
"title": "Latest Python Developer News",
"content": "Python's status as the primary language for AI and machine learning projects, from its extensive data-handling capabilities to its flexibility and ...",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"www.developer-tech.com",
"/categories/developer-languages/developer-languages-python/",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
5
],
"score": 0.2,
"category": "general"
},
{
"url": "https://subjectguides.york.ac.uk/coding/python",
"title": "Coding: a Practical Guide - Python - Subject Guides",
"content": "Python is a coding language used for a wide range of things, including working with data, building systems and software, and even creating games.",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"subjectguides.york.ac.uk",
"/coding/python",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
6
],
"score": 0.16666666666666666,
"category": "general"
},
{
"url": "https://hub.salford.ac.uk/psytech/python/getting-started-python/",
"title": "Getting Started - Python - Salford PsyTech Home - The Hub",
"content": "Python in itself is a very friendly programming language, when we get to grips with writing code, once you grasp the logic, it will become very intuitive.",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"hub.salford.ac.uk",
"/psytech/python/getting-started-python/",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
7
],
"score": 0.14285714285714285,
"category": "general"
},
{
"url": "https://snapcraft.io/docs/python-apps",
"title": "Python apps | Snapcraft documentation",
"content": "Snapcraft can be used to package and distribute Python applications in a way that enables convenient installation by users. The process of creating a snap ...",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"snapcraft.io",
"/docs/python-apps",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
8
],
"score": 0.125,
"category": "general"
},
{
"url": "https://anvil.works/",
"title": "Anvil | Build Web Apps with Nothing but Python",
"content": "Anvil is a free Python-based drag-and-drop web app builder.\u200eSign Up \u00b7 \u200eSign in \u00b7 \u200ePricing \u00b7 \u200eForum",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"anvil.works",
"/",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
9
],
"score": 0.1111111111111111,
"category": "general"
},
{
"url": "https://docs.python.org/",
"title": "Python 3.12.3 documentation",
"content": "3 days ago \u00b7 This is the official documentation for Python 3.12.3. Documentation sections: What's new in Python 3.12? Or all \"What's new\" documents since Python 2.0. Tutorial. Start here: a tour of Python's syntax and features. Library reference. Standard library and builtins. Language reference.",
"engine": "bing",
"parsed_url": [
"https",
"docs.python.org",
"/",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing",
"duckduckgo"
],
"positions": [
7,
13
],
"score": 0.43956043956043955,
"category": "general"
},
{
"title": "How to Use Python: Your First Steps - Real Python",
"content": "Learn the basics of Python syntax, installation, error handling, and more in this tutorial. You'll also code your first Python program and test your knowledge with a quiz.",
"url": "https://realpython.com/python-first-steps/",
"engine": "duckduckgo",
"parsed_url": [
"https",
"realpython.com",
"/python-first-steps/",
"",
"",
""
],
"template": "default.html",
"engines": [
"qwant",
"duckduckgo"
],
"positions": [
14,
7
],
"score": 0.42857142857142855,
"category": "general"
},
{
"title": "The Python Tutorial \u2014 Python 3.11.8 documentation",
"content": "This tutorial introduces the reader informally to the basic concepts and features of the Python language and system. It helps to have a Python interpreter handy for hands-on experience, but all examples are self-contained, so the tutorial can be read off-line as well. For a description of standard objects and modules, see The Python Standard ...",
"url": "https://docs.python.org/3.11/tutorial/",
"engine": "duckduckgo",
"parsed_url": [
"https",
"docs.python.org",
"/3.11/tutorial/",
"",
"",
""
],
"template": "default.html",
"engines": [
"duckduckgo"
],
"positions": [
7
],
"score": 0.14285714285714285,
"category": "general"
},
{
"url": "https://realpython.com/python-introduction/",
"title": "Introduction to Python 3 \u2013 Real Python",
"content": "Python programming language, including a brief history of the development of Python and reasons why you might select Python as your language of choice.",
"engine": "bing",
"parsed_url": [
"https",
"realpython.com",
"/python-introduction/",
"",
"",
""
],
"template": "default.html",
"engines": [
"bing"
],
"positions": [
8
],
"score": 0.125,
"category": "general"
},
{
"title": "Our Documentation | Python.org",
"content": "Find online or download Python's documentation, tutorials, and guides for beginners and advanced users. Learn how to port from Python 2 to Python 3, contribute to Python, and access Python videos and books.",
"url": "https://www.python.org/doc/",
"engine": "duckduckgo",
"parsed_url": [
"https",
"www.python.org",
"/doc/",
"",
"",
""
],
"template": "default.html",
"engines": [
"duckduckgo"
],
"positions": [
9
],
"score": 0.1111111111111111,
"category": "general"
},
{
"title": "Welcome to Python.org",
"url": "http://www.get-python.org/shell/",
"content": "The mission of the Python Software Foundation is to promote, protect, and advance the Python programming language, and to support and facilitate the growth of a diverse and international community of Python programmers. Learn more. Become a Member Donate to the PSF.",
"engine": "qwant",
"parsed_url": [
"http",
"www.get-python.org",
"/shell/",
"",
"",
""
],
"template": "default.html",
"engines": [
"qwant"
],
"positions": [
9
],
"score": 0.1111111111111111,
"category": "general"
},
{
"title": "About Python\u2122 | Python.org",
"content": "Python is a powerful, fast, and versatile programming language that runs on various platforms and is easy to learn. Learn how to get started, explore the applications, and join the community of Python programmers and users.",
"url": "https://www.python.org/about/",
"engine": "duckduckgo",
"parsed_url": [
"https",
"www.python.org",
"/about/",
"",
"",
""
],
"template": "default.html",
"engines": [
"duckduckgo"
],
"positions": [
11
],
"score": 0.09090909090909091,
"category": "general"
},
{
"title": "Online Python Compiler (Interpreter) - Programiz",
"content": "Write and run Python code using this online tool. You can use Python Shell like IDLE, and take inputs from the user in our Python compiler.",
"url": "https://www.programiz.com/python-programming/online-compiler/",
"engine": "duckduckgo",
"parsed_url": [
"https",
"www.programiz.com",
"/python-programming/online-compiler/",
"",
"",
""
],
"template": "default.html",
"engines": [
"duckduckgo"
],
"positions": [
12
],
"score": 0.08333333333333333,
"category": "general"
},
{
"title": "Welcome to Python.org",
"content": "Python is a versatile and powerful language that lets you work quickly and integrate systems more effectively. Download the latest version, read the documentation, find jobs, events, success stories, and more on Python.org.",
"url": "https://www.python.org/?downloads",
"engine": "duckduckgo",
"parsed_url": [
"https",
"www.python.org",
"/",
"",
"downloads",
""
],
"template": "default.html",
"engines": [
"duckduckgo"
],
"positions": [
15
],
"score": 0.06666666666666667,
"category": "general"
},
{
"url": "https://www.matillion.com/blog/the-importance-of-python-and-its-growing-influence-on-data-productivty-a-matillion-perspective",
"title": "The Importance of Python and its Growing Influence on ...",
"content": "Jan 30, 2024 \u2014 The synergy of low-code functionality with Python's versatility empowers data professionals to orchestrate complex transformations seamlessly.",
"img_src": null,
"engine": "google",
"parsed_url": [
"https",
"www.matillion.com",
"/blog/the-importance-of-python-and-its-growing-influence-on-data-productivty-a-matillion-perspective",
"",
"",
""
],
"template": "default.html",
"engines": [
"google"
],
"positions": [
10
],
"score": 0.1,
"category": "general"
},
{
"title": "BeginnersGuide - Python Wiki",
"content": "This is the program that reads Python programs and carries out their instructions; you need it before you can do any Python programming. Mac and Linux distributions may include an outdated version of Python (Python 2), but you should install an updated one (Python 3). See BeginnersGuide/Download for instructions to download the correct version ...",
"url": "https://wiki.python.org/moin/BeginnersGuide",
"engine": "duckduckgo",
"parsed_url": [
"https",
"wiki.python.org",
"/moin/BeginnersGuide",
"",
"",
""
],
"template": "default.html",
"engines": [
"duckduckgo"
],
"positions": [
16
],
"score": 0.0625,
"category": "general"
},
{
"title": "Learn Python - Free Interactive Python Tutorial",
"content": "Learn Python from scratch or improve your skills with this website that offers tutorials, exercises, tests and certification. Explore topics such as basics, data science, advanced features and more with DataCamp.",
"url": "https://www.learnpython.org/",
"engine": "duckduckgo",
"parsed_url": [
"https",
"www.learnpython.org",
"/",
"",
"",
""
],
"template": "default.html",
"engines": [
"duckduckgo"
],
"positions": [
17
],
"score": 0.058823529411764705,
"category": "general"
}
],
"answers": [],
"corrections": [],
"infoboxes": [
{
"infobox": "Python",
"id": "https://en.wikipedia.org/wiki/Python_(programming_language)",
"content": "general-purpose programming language",
"img_src": "https://upload.wikimedia.org/wikipedia/commons/thumb/6/6f/.PY_file_recreation.png/500px-.PY_file_recreation.png",
"urls": [
{
"title": "Official website",
"url": "https://www.python.org/",
"official": true
},
{
"title": "Wikipedia (en)",
"url": "https://en.wikipedia.org/wiki/Python_(programming_language)"
},
{
"title": "Wikidata",
"url": "http://www.wikidata.org/entity/Q28865"
}
],
"attributes": [
{
"label": "Inception",
"value": "Wednesday, February 20, 1991",
"entity": "P571"
},
{
"label": "Developer",
"value": "Python Software Foundation, Guido van Rossum",
"entity": "P178"
},
{
"label": "Copyright license",
"value": "Python Software Foundation License",
"entity": "P275"
},
{
"label": "Programmed in",
"value": "C, Python",
"entity": "P277"
},
{
"label": "Software version identifier",
"value": "3.12.3, 3.13.0a6",
"entity": "P348"
}
],
"engine": "wikidata",
"engines": [
"wikidata"
]
}
],
"suggestions": [
"python turtle",
"micro python tutorial",
"python docs",
"python compiler",
"snapcraft python",
"micropython vs python",
"python online",
"python download"
],
"unresponsive_engines": []
}

190
backend/apps/rag/search/testdata/serper.json vendored Normal file
View File

@ -0,0 +1,190 @@
{
"searchParameters": {
"q": "apple inc",
"gl": "us",
"hl": "en",
"autocorrect": true,
"page": 1,
"type": "search"
},
"knowledgeGraph": {
"title": "Apple",
"type": "Technology company",
"website": "http://www.apple.com/",
"imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQwGQRv5TjjkycpctY66mOg_e2-npacrmjAb6_jAWhzlzkFE3OTjxyzbA&s=0",
"description": "Apple Inc. is an American multinational technology company specializing in consumer electronics, software and online services headquartered in Cupertino, California, United States.",
"descriptionSource": "Wikipedia",
"descriptionLink": "https://en.wikipedia.org/wiki/Apple_Inc.",
"attributes": {
"Headquarters": "Cupertino, CA",
"CEO": "Tim Cook (Aug 24, 2011–)",
"Founded": "April 1, 1976, Los Altos, CA",
"Sales": "1 (800) 692-7753",
"Products": "iPhone, Apple Watch, iPad, and more",
"Founders": "Steve Jobs, Steve Wozniak, and Ronald Wayne",
"Subsidiaries": "Apple Store, Beats Electronics, Beddit, and more"
}
},
"organic": [
{
"title": "Apple",
"link": "https://www.apple.com/",
"snippet": "Discover the innovative world of Apple and shop everything iPhone, iPad, Apple Watch, Mac, and Apple TV, plus explore accessories, entertainment, ...",
"sitelinks": [
{
"title": "Support",
"link": "https://support.apple.com/"
},
{
"title": "iPhone",
"link": "https://www.apple.com/iphone/"
},
{
"title": "Apple makes business better.",
"link": "https://www.apple.com/business/"
},
{
"title": "Mac",
"link": "https://www.apple.com/mac/"
}
],
"position": 1
},
{
"title": "Apple Inc. - Wikipedia",
"link": "https://en.wikipedia.org/wiki/Apple_Inc.",
"snippet": "Apple Inc. is an American multinational technology company specializing in consumer electronics, software and online services headquartered in Cupertino, ...",
"attributes": {
"Products": "AirPods; Apple Watch; iPad; iPhone; Mac",
"Founders": "Steve Jobs; Steve Wozniak; Ronald Wayne",
"Founded": "April 1, 1976; 46 years ago in Los Altos, California, U.S",
"Industry": "Consumer electronics; Software services; Online services"
},
"sitelinks": [
{
"title": "History",
"link": "https://en.wikipedia.org/wiki/History_of_Apple_Inc."
},
{
"title": "Timeline of Apple Inc. products",
"link": "https://en.wikipedia.org/wiki/Timeline_of_Apple_Inc._products"
},
{
"title": "List of software by Apple Inc.",
"link": "https://en.wikipedia.org/wiki/List_of_software_by_Apple_Inc."
},
{
"title": "Apple Store",
"link": "https://en.wikipedia.org/wiki/Apple_Store"
}
],
"position": 2
},
{
"title": "Apple Inc. | History, Products, Headquarters, & Facts | Britannica",
"link": "https://www.britannica.com/topic/Apple-Inc",
"snippet": "Apple Inc., formerly Apple Computer, Inc., American manufacturer of personal computers, smartphones, tablet computers, computer peripherals, ...",
"date": "Aug 31, 2022",
"attributes": {
"Related People": "Steve Jobs Steve Wozniak Jony Ive Tim Cook Angela Ahrendts",
"Date": "1976 - present",
"Areas Of Involvement": "peripheral device"
},
"position": 3
},
{
"title": "AAPL: Apple Inc Stock Price Quote - NASDAQ GS - Bloomberg.com",
"link": "https://www.bloomberg.com/quote/AAPL:US",
"snippet": "Stock analysis for Apple Inc (AAPL:NASDAQ GS) including stock price, stock chart, company news, key statistics, fundamentals and company profile.",
"position": 4
},
{
"title": "Apple Inc. (AAPL) Company Profile & Facts - Yahoo Finance",
"link": "https://finance.yahoo.com/quote/AAPL/profile/",
"snippet": "Apple Inc. designs, manufactures, and markets smartphones, personal computers, tablets, wearables, and accessories worldwide. It also sells various related ...",
"position": 5
},
{
"title": "AAPL | Apple Inc. Stock Price & News - WSJ",
"link": "https://www.wsj.com/market-data/quotes/AAPL",
"snippet": "Apple, Inc. engages in the design, manufacture, and sale of smartphones, personal computers, tablets, wearables and accessories, and other varieties of ...",
"position": 6
},
{
"title": "Apple Inc Company Profile - Apple Inc Overview - GlobalData",
"link": "https://www.globaldata.com/company-profile/apple-inc/",
"snippet": "Apple Inc (Apple) designs, manufactures, and markets smartphones, tablets, personal computers (PCs), portable and wearable devices. The company also offers ...",
"position": 7
},
{
"title": "Apple Inc (AAPL) Stock Price & News - Google Finance",
"link": "https://www.google.com/finance/quote/AAPL:NASDAQ?hl=en",
"snippet": "Get the latest Apple Inc (AAPL) real-time quote, historical performance, charts, and other financial information to help you make more informed trading and ...",
"position": 8
}
],
"peopleAlsoAsk": [
{
"question": "What does Apple Inc mean?",
"snippet": "Apple Inc., formerly Apple Computer, Inc., American manufacturer of personal\ncomputers, smartphones, tablet computers, computer peripherals, and computer\nsoftware. It was the first successful personal computer company and the\npopularizer of the graphical user interface.\nAug 31, 2022",
"title": "Apple Inc. | History, Products, Headquarters, & Facts | Britannica",
"link": "https://www.britannica.com/topic/Apple-Inc"
},
{
"question": "Is Apple and Apple Inc same?",
"snippet": "Apple was founded as Apple Computer Company on April 1, 1976, by Steve Jobs,\nSteve Wozniak and Ronald Wayne to develop and sell Wozniak's Apple I personal\ncomputer. It was incorporated by Jobs and Wozniak as Apple Computer, Inc.",
"title": "Apple Inc. - Wikipedia",
"link": "https://en.wikipedia.org/wiki/Apple_Inc."
},
{
"question": "Who owns Apple Inc?",
"snippet": "Apple Inc. is owned by two main institutional investors (Vanguard Group and\nBlackRock, Inc). While its major individual shareholders comprise people like\nArt Levinson, Tim Cook, Bruce Sewell, Al Gore, Johny Sroujli, and others.",
"title": "Who Owns Apple In 2022? - FourWeekMBA",
"link": "https://fourweekmba.com/who-owns-apple/"
},
{
"question": "What products does Apple Inc offer?",
"snippet": "APPLE FOOTER\nStore.\nMac.\niPad.\niPhone.\nWatch.\nAirPods.\nTV & Home.\nAirTag.",
"title": "More items...",
"link": "https://www.apple.com/business/"
}
],
"relatedSearches": [
{
"query": "Who invented the iPhone"
},
{
"query": "Apple Inc competitors"
},
{
"query": "Apple iPad"
},
{
"query": "iPhones"
},
{
"query": "Apple Inc us"
},
{
"query": "Apple company history"
},
{
"query": "Apple Store"
},
{
"query": "Apple customer service"
},
{
"query": "Apple Watch"
},
{
"query": "Apple Inc Industry"
},
{
"query": "Apple Inc registered address"
},
{
"query": "Apple Inc Bloomberg"
}
]
}

278
backend/apps/rag/search/testdata/serpstack.json vendored Normal file
View File

@ -0,0 +1,278 @@
{
"request": {
"success": true,
"total_time_taken": 3.4,
"processed_timestamp": 1714968442,
"search_url": "http://www.google.com/search?q=mcdonalds\u0026gl=us\u0026hl=en\u0026safe=0\u0026num=10"
},
"search_parameters": {
"engine": "google",
"type": "web",
"device": "desktop",
"auto_location": "1",
"google_domain": "google.com",
"gl": "us",
"hl": "en",
"safe": "0",
"news_type": "all",
"exclude_autocorrected_results": "0",
"images_color": "any",
"page": "1",
"num": "10",
"output": "json",
"csv_fields": "search_parameters.query,organic_results.position,organic_results.title,organic_results.url,organic_results.domain",
"query": "mcdonalds",
"action": "search",
"access_key": "aac48e007e15c532bb94ffb34532a4b2",
"error": {}
},
"search_information": {
"total_results": 1170000000,
"time_taken_displayed": 0.49,
"detected_location": {},
"did_you_mean": {},
"no_results_for_original_query": false,
"showing_results_for": {}
},
"organic_results": [
{
"position": 1,
"title": "Our Full McDonald\u0027s Food Menu",
"snippet": "",
"prerender": false,
"cached_page_url": {},
"related_pages_url": {},
"url": "https://www.mcdonalds.com/us/en-us/full-menu.html",
"domain": "www.mcdonalds.com",
"displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a full-menu"
},
{
"position": 2,
"title": "McDonald\u0027s",
"snippet": "McDonald\u0027s is the world\u0027s largest fast food restaurant chain, serving over 69 million customers daily in over 100 countries in more than 40,000 outlets as of\u00a0...",
"prerender": false,
"cached_page_url": {},
"related_pages_url": {},
"url": "https://en.wikipedia.org/wiki/McDonald%27s",
"domain": "en.wikipedia.org",
"displayed_url": "https://en.wikipedia.org \u203a wiki \u203a McDonald\u0027s"
},
{
"position": 3,
"title": "Restaurants Near Me: Nearby McDonald\u0027s Locations",
"snippet": "",
"prerender": false,
"cached_page_url": {},
"related_pages_url": {},
"url": "https://www.mcdonalds.com/us/en-us/restaurant-locator.html",
"domain": "www.mcdonalds.com",
"displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a restaurant-locator"
},
{
"position": 4,
"title": "Download the McDonald\u0027s App: Deals, Promotions \u0026 ...",
"snippet": "Download the McDonald\u0027s app for Mobile Order \u0026 Pay, exclusive deals and coupons, menu information and special promotions.",
"prerender": false,
"cached_page_url": {},
"related_pages_url": {},
"url": "https://www.mcdonalds.com/us/en-us/download-app.html",
"domain": "www.mcdonalds.com",
"displayed_url": "https://www.mcdonalds.com \u203a en-us \u203a download-app"
},
{
"position": 5,
"title": "McDonald\u0027s Restaurant Careers in the US",
"snippet": "McDonald\u0027s restaurant jobs are one-of-a-kind \u2013 just like you. Restaurants are hiring across all levels, from Crew team to Management. Apply today!",
"prerender": false,
"cached_page_url": {},
"related_pages_url": {},
"url": "https://jobs.mchire.com/",
"domain": "jobs.mchire.com",
"displayed_url": "https://jobs.mchire.com"
}
],
"inline_images": [
{
"image_url": "https://serpstack-assets.apilayer.net/2418910010831954152.png",
"title": ""
}
],
"local_results": [
{
"position": 1,
"title": "McDonald\u0027s",
"coordinates": {
"latitude": 0,
"longitude": 0
},
"address": "",
"rating": 0,
"reviews": 0,
"type": "",
"price": {},
"url": 0
},
{
"position": 2,
"title": "McDonald\u0027s",
"coordinates": {
"latitude": 0,
"longitude": 0
},
"address": "",
"rating": 0,
"reviews": 0,
"type": "",
"price": {},
"url": 0
},
{
"position": 3,
"title": "McDonald\u0027s",
"coordinates": {
"latitude": 0,
"longitude": 0
},
"address": "",
"rating": 0,
"reviews": 0,
"type": "",
"price": {},
"url": 0
}
],
"top_stories": [
{
"block_position": 1,
"title": "Menu nutrition",
"url": "/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=mcdonald%27s+double+quarter+pounder+with+cheese\u0026stick=H4sIAAAAAAAAAONgFuLUz9U3ME-vLDBX4tVP1zc0TCsuNE0ytjTTUs5OttJPy89P0c9NzSuNLyjKL8tMSS2yAvNS80qKMlOLF7Hq5ian5Ocl5qSoFyuk5Jcm5aQqFJYmFpWkFikU5JfmATUolGeWZCgkZ6SmFqcCAM4ilJtxAAAA\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4Qri56BAh0EAM",
"source": "",
"uploaded": "",
"uploaded_utc": "2024-05-06T04:07:22.082Z"
},
{
"block_position": 2,
"title": "Profiles",
"url": "https://www.instagram.com/McDonalds",
"source": "",
"uploaded": "",
"uploaded_utc": "2024-05-06T04:07:22.082Z"
},
{
"block_position": 3,
"title": "People also search for",
"url": "/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-L5Wg8IL2sxPFxxcDEhVbocy-LJPZIvZySijw0ho2hfZ-KtV-sSEEJ9lw7JuEkXHDnRK5y4Dm8aqbiLwugbLbslwjG3hO_gpDTFZK2VoUGZPy2nrmOBCy0G3PoOfoiEtct2GSZlUz0uufG-xP8emtNzQKQpvjkAm5Zmi57iVZueiD62upz7-x2N3dAbwtm6FkInAPRw1yR91zuT7F3lEaPblTW3LaRwCDC0bvaRCh9x4N9zHgY1OOQa_rzts2jf5WpXcuw4Y%3D\u0026q=Burger+King\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4Qs9oBKAB6BAhzEAI",
"source": "",
"uploaded": "",
"uploaded_utc": "2024-05-06T04:07:22.082Z"
}
],
"related_questions": [
{
"question": "What\u0027s a number 7 at McDonald\u0027s?What\u0027s a number 7 at McDonald\u0027s?What\u0027s a number 7 at McDonald\u0027s?",
"answer": "",
"title": "",
"displayed_url": ""
},
{
"question": "Why is McDonald\u0027s changing their name?Why is McDonald\u0027s changing their name?Why is McDonald\u0027s changing their name?",
"answer": "",
"title": "",
"displayed_url": ""
},
{
"question": "What is the oldest still running Mcdonalds?What is the oldest still running Mcdonalds?What is the oldest still running Mcdonalds?",
"answer": "",
"title": "",
"displayed_url": ""
},
{
"question": "Why is McDonald\u0027s now WcDonald\u0027s?Why is McDonald\u0027s now WcDonald\u0027s?Why is McDonald\u0027s now WcDonald\u0027s?",
"answer": "",
"title": "",
"displayed_url": ""
}
],
"knowledge_graph": {
"title": "",
"type": "Fast-food restaurant company",
"image_urls": [
"https://serpstack-assets.apilayer.net/2418910010831954152.png"
],
"description": "McDonald\u0027s Corporation is an American multinational fast food chain, founded in 1940 as a restaurant operated by Richard and Maurice McDonald, in San Bernardino, California, United States.",
"source": {
"name": "Wikipedia",
"url": "https://en.wikipedia.org/wiki/McDonald\u0027s"
},
"people_also_search_for": [],
"known_attributes": [
{
"attribute": "kc:/business/business_operation:founder",
"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Ray+Kroc\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-LxARWRdbk5SkoY2sDn5Qq7yOmqYGei6qZ7sfJhsjZXBPgjMlLbS7824rpJOm69GzqVWMdoNIZiFX2T4A2td14sZOn4a1BexZLtZXHU7NZdF6VsWbGMVuiSYtXdev7uaUjEJKumiwlqTAATTebOriYTEBuSzC\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHgQAg",
"name": "Founder: ",
"value": "Ray Kroc"
},
{
"attribute": "kc:/organization/organization:ceo",
"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chris+Kempczinski\u0026si=ACC90nwLLwns5sISZcdzuISy7t-NHozt8Cbt6G3WNQfC9ekAgKFbjdEFCDgxLbt57EDZGosYDGiZuq1AcBhA6IhTOSZxfVSySuGQ3VDwmmTA7Z93n3K3596jAuZH9VVv5h8PyvKJSuGuSsQWviJTl3eKj2UL1ZIWuDgkjyVMnC47rN7j0G9PlHRCCLdQF7VDQ1gubTiC4onXqLRBTbwAj6a--PD6Jv_NoA%3D%3D\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHUQAg",
"name": "CEO: ",
"value": "Chris Kempczinski (Nov 1, 2019\u2013)"
},
{
"attribute": "kc:/business/employer:revenue",
"link": "",
"name": "Revenue: ",
"value": "25.49\u00a0billion USD (2023)"
},
{
"attribute": "kc:/organization/organization:founded",
"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Des+Plaines\u0026si=ACC90nyvvWro6QmnyY1IfSdgk5wwjB1r8BGd_IWRjXqmKPQqm_yqLtI_DBi5PXGOtg_Z3qrzzEP6mcih1nN7h5A7v6OefnEJiC7a8dBR-v9LxlRubfyR6vlMr3fZ3TmVKWwz9FRpvZb1eYNt-RM7KIDKQlwGEIgINvzhxjUrv6uxSmceduzxd8W7Pkz71XGwxF0F8OlSzHlx\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECG4QAg",
"name": "Founded: ",
"value": "April 15, 1955, Des Plaines, IL"
},
{
"attribute": "kc:/organization/organization:headquarters",
"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chicago\u0026si=ACC90nyvvWro6QmnyY1IfSdgk5wwjB1r8BGd_IWRjXqmKPQqm-46AEJ_kJbUIEvsvEEZqteiYJvXVXs2ScRNDvFFpjfeAaW3dxtpTGCgcsf5RMdi6IdzOdtjJMN3ZaFwqZOmdi7tC6r0Mh1O9bnP3HrVDB9hH02m7aA6f70dCAfTdpOFnGxDU6wVMAI5MxWBE3wTugtUDOK-\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHYQAg",
"name": "Headquarters: ",
"value": "Chicago, IL"
},
{
"attribute": "kc:/organization/organization:president",
"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Chris+Kempczinski\u0026si=ACC90nwLLwns5sISZcdzuISy7t-NHozt8Cbt6G3WNQfC9ekAgKFbjdEFCDgxLbt57EDZGosYDGiZuq1AcBhA6IhTOSZxfVSySuGQ3VDwmmTA7Z93n3K3596jAuZH9VVv5h8PyvKJSuGuSsQWviJTl3eKj2UL1ZIWuDgkjyVMnC47rN7j0G9PlHRCCLdQF7VDQ1gubTiC4onXqLRBTbwAj6a--PD6Jv_NoA%3D%3D\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHEQAg",
"name": "President: ",
"value": "Chris Kempczinski"
}
],
"website": "https://www.mcdonalds.com/us/en-us.html",
"profiles": [
{
"name": "Instagram",
"url": "https://www.instagram.com/McDonalds"
},
{
"name": "X (Twitter)",
"url": "https://twitter.com/McDonalds"
},
{
"name": "Facebook",
"url": "https://www.facebook.com/McDonaldsUS"
},
{
"name": "YouTube",
"url": "https://www.youtube.com/user/McDonaldsUS"
},
{
"name": "Pinterest",
"url": "https://www.pinterest.com/mcdonalds"
}
],
"founded": "April 15, 1955, Des Plaines, IL",
"headquarters": "Chicago, IL",
"founders": [
{
"name": "Ray Kroc",
"link": "http://www.google.com/search?safe=0\u0026sca_esv=c9c7fd42856085e2\u0026sca_upv=1\u0026gl=us\u0026hl=en\u0026q=Ray+Kroc\u0026si=ACC90nzx_D3_zUKRnpAjmO0UBLNxnt7EyN4YYdru6U3bxLI-LxARWRdbk5SkoY2sDn5Qq7yOmqYGei6qZ7sfJhsjZXBPgjMlLbS7824rpJOm69GzqVWMdoNIZiFX2T4A2td14sZOn4a1BexZLtZXHU7NZdF6VsWbGMVuiSYtXdev7uaUjEJKumiwlqTAATTebOriYTEBuSzC\u0026sa=X\u0026ved=2ahUKEwjF55alk_iFAxXlamwGHbqgAs4QmxMoAHoECHgQAg"
}
]
}
}

50
backend/apps/rag/utils.py
View File

@ -19,8 +19,24 @@ from langchain.retrievers import (
)
from typing import Optional
from config import SRC_LOG_LEVELS, CHROMA_CLIENT
from apps.rag.search.brave import search_brave
from apps.rag.search.google_pse import search_google_pse
from apps.rag.search.main import SearchResult
from apps.rag.search.searxng import search_searxng
from apps.rag.search.serper import search_serper
from apps.rag.search.serpstack import search_serpstack
from config import (
SRC_LOG_LEVELS,
CHROMA_CLIENT,
SEARXNG_QUERY_URL,
GOOGLE_PSE_API_KEY,
GOOGLE_PSE_ENGINE_ID,
BRAVE_SEARCH_API_KEY,
SERPSTACK_API_KEY,
SERPSTACK_HTTPS,
SERPER_API_KEY,
)
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
@ -515,3 +531,35 @@ class RerankCompressor(BaseDocumentCompressor):
)
final_results.append(doc)
return final_results
def search_web(query: str) -> list[SearchResult]:
"""Search the web using a search engine and return the results as a list of SearchResult objects.
Will look for a search engine API key in environment variables in the following order:
- SEARXNG_QUERY_URL
- GOOGLE_PSE_API_KEY + GOOGLE_PSE_ENGINE_ID
- BRAVE_SEARCH_API_KEY
- SERPSTACK_API_KEY
- SERPER_API_KEY
Args:
query (str): The query to search for
"""
try:
if SEARXNG_QUERY_URL:
return search_searxng(SEARXNG_QUERY_URL, query)
elif GOOGLE_PSE_API_KEY and GOOGLE_PSE_ENGINE_ID:
return search_google_pse(GOOGLE_PSE_API_KEY, GOOGLE_PSE_ENGINE_ID, query)
elif BRAVE_SEARCH_API_KEY:
return search_brave(BRAVE_SEARCH_API_KEY, query)
elif SERPSTACK_API_KEY:
return search_serpstack(
SERPSTACK_API_KEY, query, https_enabled=SERPSTACK_HTTPS
)
elif SERPER_API_KEY:
return search_serper(SERPER_API_KEY, query)
else:
raise Exception("No search engine API key found in environment variables")
except Exception as e:
log.error(f"Web search failed: {e}")
return []

8
backend/config.py
View File

@ -533,6 +533,14 @@ RAG_OPENAI_API_KEY = os.getenv("RAG_OPENAI_API_KEY", OPENAI_API_KEY)
ENABLE_LOCAL_WEB_FETCH = os.getenv("ENABLE_LOCAL_WEB_FETCH", "False").lower() == "true"
SEARXNG_QUERY_URL = os.getenv("SEARXNG_QUERY_URL", "")
GOOGLE_PSE_API_KEY = os.getenv("GOOGLE_PSE_API_KEY", "")
GOOGLE_PSE_ENGINE_ID = os.getenv("GOOGLE_PSE_ENGINE_ID", "")
BRAVE_SEARCH_API_KEY = os.getenv("BRAVE_SEARCH_API_KEY", "")
SERPSTACK_API_KEY = os.getenv("SERPSTACK_API_KEY", "")
SERPSTACK_HTTPS = os.getenv("SERPSTACK_HTTPS", "True").lower() == "true"
SERPER_API_KEY = os.getenv("SERPER_API_KEY", "")
####################################
# Transcribe
####################################