fix: web, youtube attachment issue

This commit is contained in:
Timothy J. Baek 2024-10-06 19:44:02 -07:00
parent 95616e92d7
commit ee22ba9676
2 changed files with 26 additions and 12 deletions

View File

@ -923,15 +923,22 @@ def process_youtube_video(form_data: ProcessUrlForm, user=Depends(get_verified_u
translation=app.state.YOUTUBE_LOADER_TRANSLATION, translation=app.state.YOUTUBE_LOADER_TRANSLATION,
) )
docs = loader.load() docs = loader.load()
text_content = " ".join([doc.page_content for doc in docs]) content = " ".join([doc.page_content for doc in docs])
log.debug(f"text_content: {text_content}") log.debug(f"text_content: {content}")
save_docs_to_vector_db(docs, collection_name, overwrite=True) save_docs_to_vector_db(docs, collection_name, overwrite=True)
return { return {
"status": True, "status": True,
"collection_name": collection_name, "collection_name": collection_name,
"filename": form_data.url, "filename": form_data.url,
"content": text_content, "file": {
"data": {
"content": content,
},
"meta": {
"name": form_data.url,
},
},
} }
except Exception as e: except Exception as e:
log.exception(e) log.exception(e)
@ -954,15 +961,22 @@ def process_web(form_data: ProcessUrlForm, user=Depends(get_verified_user)):
requests_per_second=app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS, requests_per_second=app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
) )
docs = loader.load() docs = loader.load()
text_content = " ".join([doc.page_content for doc in docs]) content = " ".join([doc.page_content for doc in docs])
log.debug(f"text_content: {text_content}") log.debug(f"text_content: {content}")
save_docs_to_vector_db(docs, collection_name, overwrite=True) save_docs_to_vector_db(docs, collection_name, overwrite=True)
return { return {
"status": True, "status": True,
"collection_name": collection_name, "collection_name": collection_name,
"filename": form_data.url, "filename": form_data.url,
"content": text_content, "file": {
"data": {
"content": content,
},
"meta": {
"name": form_data.url,
},
},
} }
except Exception as e: except Exception as e:
log.exception(e) log.exception(e)

View File

@ -34,7 +34,7 @@
type: 'doc', type: 'doc',
name: url, name: url,
collection_name: '', collection_name: '',
status: false, status: 'uploading',
url: url, url: url,
error: '' error: ''
}; };
@ -44,10 +44,10 @@
const res = await processWeb(localStorage.token, '', url); const res = await processWeb(localStorage.token, '', url);
if (res) { if (res) {
fileItem.status = 'processed'; fileItem.status = 'uploaded';
fileItem.collection_name = res.collection_name; fileItem.collection_name = res.collection_name;
fileItem.file = { fileItem.file = {
content: res.content, ...res.file,
...fileItem.file ...fileItem.file
}; };
@ -67,7 +67,7 @@
type: 'doc', type: 'doc',
name: url, name: url,
collection_name: '', collection_name: '',
status: false, status: 'uploading',
url: url, url: url,
error: '' error: ''
}; };
@ -77,10 +77,10 @@
const res = await processYoutubeVideo(localStorage.token, url); const res = await processYoutubeVideo(localStorage.token, url);
if (res) { if (res) {
fileItem.status = 'processed'; fileItem.status = 'uploaded';
fileItem.collection_name = res.collection_name; fileItem.collection_name = res.collection_name;
fileItem.file = { fileItem.file = {
content: res.content, ...res.file,
...fileItem.file ...fileItem.file
}; };
files = files; files = files;