Merge pull request #7025 from yeounhak/dev

fix: Enable RAG_WEB_SEARCH_CONCURRENT_REQUESTS
This commit is contained in:
Timothy Jaeryang Baek 2024-11-18 20:34:27 -08:00 committed by GitHub
commit 8bb3061101
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -1283,9 +1283,11 @@ def process_web_search(form_data: SearchForm, user=Depends(get_verified_user)):
urls = [result.link for result in web_results]
loader = get_web_loader(
urls, verify_ssl=app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION
urls,
verify_ssl=app.state.config.ENABLE_RAG_WEB_LOADER_SSL_VERIFICATION,
requests_per_second=app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS
)
docs = loader.load()
docs = loader.aload()
save_docs_to_vector_db(docs, collection_name, overwrite=True)