Moving code out of playwright branch

This commit is contained in:
Rory 2025-02-03 18:47:26 -06:00
parent 22746c7a3f
commit 1b581b714f
54 changed files with 6 additions and 84 deletions

View File

@ -1,5 +1,3 @@
import validators
from typing import Optional
from urllib.parse import urlparse
@ -12,8 +10,6 @@ def get_filtered_results(results, filter_list):
filtered_results = []
for result in results:
url = result.get("url") or result.get("link", "")
if not validators.url(url):
continue
domain = urlparse(url).netloc
if any(domain.endswith(filtered_domain) for filtered_domain in filter_list):
filtered_results.append(result)

View File

@ -48,16 +48,6 @@ def validate_url(url: Union[str, Sequence[str]]):
else:
return False
def safe_validate_urls(url: Sequence[str]) -> Sequence[str]:
valid_urls = []
for u in url:
try:
if validate_url(u):
valid_urls.append(u)
except ValueError:
continue
return valid_urls
def resolve_hostname(hostname):
# Get address information
addr_info = socket.getaddrinfo(hostname, None)
@ -253,11 +243,12 @@ def get_web_loader(
verify_ssl: bool = True,
requests_per_second: int = 2,
):
# Check if the URLs are valid
safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
# Check if the URL is valid
if not validate_url(urls):
raise ValueError(ERROR_MESSAGES.INVALID_URL)
web_loader_args = {
"urls": safe_urls,
"urls": urls,
"verify_ssl": verify_ssl,
"requests_per_second": requests_per_second,
"continue_on_failure": True
@ -270,6 +261,6 @@ def get_web_loader(
WebLoaderClass = RAG_WEB_LOADERS[RAG_WEB_LOADER.value]
web_loader = WebLoaderClass(**web_loader_args)
log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(safe_urls))
log.debug("Using RAG_WEB_LOADER %s for %s URLs", web_loader.__class__.__name__, len(urls))
return web_loader

View File

@ -1239,10 +1239,8 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
@router.post("/process/web/search")
async def process_web_search(
request: Request, form_data: SearchForm, extra_params: dict, user=Depends(get_verified_user)
request: Request, form_data: SearchForm, user=Depends(get_verified_user)
):
event_emitter = extra_params["__event_emitter__"]
try:
logging.info(
f"trying to web search with {request.app.state.config.RAG_WEB_SEARCH_ENGINE, form_data.query}"
@ -1260,18 +1258,6 @@ async def process_web_search(
log.debug(f"web_results: {web_results}")
await event_emitter(
{
"type": "status",
"data": {
"action": "web_search",
"description": "Loading {{count}} sites",
"urls": [result.link for result in web_results],
"done": False
},
}
)
try:
collection_name = form_data.collection_name
if collection_name == "" or collection_name is None:

View File

@ -443,7 +443,6 @@ async def chat_web_search_handler(
"query": searchQuery,
}
),
extra_params=extra_params,
user=user
)

View File

@ -585,7 +585,6 @@
: ''} text-gray-500 dark:text-gray-500 text-base line-clamp-1 text-wrap"
>
<!-- $i18n.t(`Searching "{{searchQuery}}"`) -->
<!-- $i18n.t(`Loading {{count}} sites`) -->
{#if status?.description.includes('{{searchQuery}}')}
{$i18n.t(status?.description, {
searchQuery: status?.query

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "يمكن أن تصدر بعض الأخطاء. لذلك يجب التحقق من المعلومات المهمة",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs могат да правят грешки. Проверете важните данни.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM ভুল করতে পারে। গুরুত্বপূর্ণ তথ্য যাচাই করে নিন।",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Escoltant...",
"Llama.cpp": "Llama.cpp",
"LLMs can make mistakes. Verify important information.": "Els models de llenguatge poden cometre errors. Verifica la informació important.",
"Loading {{count}} sites": "",
"Local": "Local",
"Local Models": "Models locals",
"Lost": "Perdut",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Ang mga LLM mahimong masayop. ",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Poslouchání...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM mohou dělat chyby. Ověřte si důležité informace.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Lokální modely",
"Lost": "Ztracený",

View File

@ -565,7 +565,6 @@
"Listening...": "Lytter...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM'er kan lave fejl. Bekræft vigtige oplysninger.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Lokale modeller",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Höre zu...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs können Fehler machen. Überprüfe wichtige Informationen.",
"Loading {{count}} sites": "",
"Local": "Lokal",
"Local Models": "Lokale Modelle",
"Lost": "Verloren",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs can make borks. Verify important info.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Ακούγεται...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Τα LLM μπορούν να κάνουν λάθη. Επαληθεύστε σημαντικές πληροφορίες.",
"Loading {{count}} sites": "",
"Local": "Τοπικό",
"Local Models": "Τοπικά Μοντέλα",
"Lost": "Χαμένος",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Escuchando...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Los LLM pueden cometer errores. Verifica la información importante.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Modelos locales",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Entzuten...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMek akatsak egin ditzakete. Egiaztatu informazio garrantzitsua.",
"Loading {{count}} sites": "",
"Local": "Lokala",
"Local Models": "Modelo lokalak",
"Lost": "Galduta",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "مدل\u200cهای زبانی بزرگ می\u200cتوانند اشتباه کنند. اطلاعات مهم را راستی\u200cآزمایی کنید.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Kuuntelee...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Kielimallit voivat tehdä virheitä. Tarkista tärkeät tiedot.",
"Loading {{count}} sites": "",
"Local": "Paikallinen",
"Local Models": "Paikalliset mallit",
"Lost": "Mennyt",

View File

@ -565,7 +565,6 @@
"Listening...": "En train d'écouter...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Les LLM peuvent faire des erreurs. Vérifiez les informations importantes.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Modèles locaux",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Écoute en cours...",
"Llama.cpp": "Llama.cpp",
"LLMs can make mistakes. Verify important information.": "Les LLM peuvent faire des erreurs. Vérifiez les informations importantes.",
"Loading {{count}} sites": "",
"Local": "Local",
"Local Models": "Modèles locaux",
"Lost": "Perdu",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "מודלים בשפה טבעית יכולים לטעות. אמת מידע חשוב.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "एलएलएम गलतियाँ कर सकते हैं। महत्वपूर्ण जानकारी सत्यापित करें.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Slušam...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM-ovi mogu pogriješiti. Provjerite važne informacije.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Lokalni modeli",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Hallgatás...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Az LLM-ek hibázhatnak. Ellenőrizze a fontos információkat.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Helyi modellek",
"Lost": "Elveszett",

View File

@ -565,7 +565,6 @@
"Listening...": "Mendengarkan",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM dapat membuat kesalahan. Verifikasi informasi penting.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Model Lokal",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Éisteacht...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Is féidir le LLManna botúin a dhéanamh. Fíoraigh faisnéis thábhachtach.",
"Loading {{count}} sites": "",
"Local": "Áitiúil",
"Local Models": "Múnlaí Áitiúla",
"Lost": "Cailleadh",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Gli LLM possono commettere errori. Verifica le informazioni importanti.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM は間違いを犯す可能性があります。重要な情報を検証してください。",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "ローカルモデル",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "შესაძლოა LLM-ებმა შეცდომები დაუშვან. გადაამოწმეთ მნიშვნელოვანი ინფორმაცია.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "듣는 중...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM에 오류가 있을 수 있습니다. 중요한 정보는 확인이 필요합니다.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "로컬 모델",
"Lost": "패배",

View File

@ -565,7 +565,6 @@
"Listening...": "Klausoma...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Dideli kalbos modeliai gali klysti. Patikrinkite atsakymų teisingumą.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Lokalūs modeliai",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Mendengar...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM boleh membuat kesilapan. Sahkan maklumat penting",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Model Tempatan",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Lytter ...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Språkmodeller kan gjøre feil. Kontroller viktige opplysninger.",
"Loading {{count}} sites": "",
"Local": "Lokal",
"Local Models": "Lokale modeller",
"Lost": "Tapt",

View File

@ -565,7 +565,6 @@
"Listening...": "Aan het luisteren...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs kunnen fouten maken. Verifieer belangrijke informatie.",
"Loading {{count}} sites": "",
"Local": "Lokaal",
"Local Models": "Lokale modellen",
"Lost": "Verloren",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs ਗਲਤੀਆਂ ਕਰ ਸਕਦੇ ਹਨ। ਮਹੱਤਵਪੂਰਨ ਜਾਣਕਾਰੀ ਦੀ ਪੁਸ਼ਟੀ ਕਰੋ।",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMy mogą popełniać błędy. Zweryfikuj ważne informacje.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Escutando...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs podem cometer erros. Verifique informações importantes.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Modelos Locais",
"Lost": "Perdeu",

View File

@ -565,7 +565,6 @@
"Listening...": "A escutar...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs podem cometer erros. Verifique informações importantes.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Modelos Locais",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Ascult...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM-urile pot face greșeli. Verificați informațiile importante.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Modele Locale",
"Lost": "Pierdut",

View File

@ -565,7 +565,6 @@
"Listening...": "Слушаю...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs могут допускать ошибки. Проверяйте важную информацию.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Локальные модели",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Počúvanie...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM môžu robiť chyby. Overte si dôležité informácie.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Lokálne modely",
"Lost": "Stratený",

View File

@ -565,7 +565,6 @@
"Listening...": "Слушам...",
"Llama.cpp": "Llama.cpp",
"LLMs can make mistakes. Verify important information.": "ВЈМ-ови (LLM-ови) могу правити грешке. Проверите важне податке.",
"Loading {{count}} sites": "",
"Local": "Локално",
"Local Models": "Локални модели",
"Lost": "Пораза",

View File

@ -565,7 +565,6 @@
"Listening...": "Lyssnar...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM:er kan göra misstag. Granska viktig information.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "Lokala modeller",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "กำลังฟัง...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLMs สามารถทำผิดพลาดได้ ตรวจสอบข้อมูลสำคัญ",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "โมเดลท้องถิ่น",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "Dinleniyor...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "LLM'ler hata yapabilir. Önemli bilgileri doğrulayın.",
"Loading {{count}} sites": "",
"Local": "Yerel",
"Local Models": "Yerel Modeller",
"Lost": "Kayıp",

View File

@ -565,7 +565,6 @@
"Listening...": "Слухаю...",
"Llama.cpp": "Llama.cpp",
"LLMs can make mistakes. Verify important information.": "LLMs можуть помилятися. Перевірте важливу інформацію.",
"Loading {{count}} sites": "",
"Local": "Локальний",
"Local Models": "Локальні моделі",
"Lost": "Втрачене",

View File

@ -565,7 +565,6 @@
"Listening...": "سن رہے ہیں...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "ایل ایل ایم غلطیاں کر سکتے ہیں اہم معلومات کی تصدیق کریں",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "مقامی ماڈلز",
"Lost": "گم شدہ",

View File

@ -565,7 +565,6 @@
"Listening...": "Đang nghe...",
"Llama.cpp": "",
"LLMs can make mistakes. Verify important information.": "Hệ thống có thể tạo ra nội dung không chính xác hoặc sai. Hãy kiểm chứng kỹ lưỡng thông tin trước khi tiếp nhận và sử dụng.",
"Loading {{count}} sites": "",
"Local": "",
"Local Models": "",
"Lost": "",

View File

@ -565,7 +565,6 @@
"Listening...": "正在倾听...",
"Llama.cpp": "Llama.cpp",
"LLMs can make mistakes. Verify important information.": "大语言模型可能会生成误导性错误信息,请对关键信息加以验证。",
"Loading {{count}} sites": "",
"Local": "本地",
"Local Models": "本地模型",
"Lost": "落败",

View File

@ -565,7 +565,6 @@
"Listening...": "正在聆聽...",
"Llama.cpp": "Llama.cpp",
"LLMs can make mistakes. Verify important information.": "大型語言模型可能會出錯。請驗證重要資訊。",
"Loading {{count}} sites": "",
"Local": "本機",
"Local Models": "本機模型",
"Lost": "已遺失",