Merge pull request #12571 from hurxxxx/feat/reindex-knowledge-files

feat: reindex knowledge files
This commit is contained in:
Tim Jaeryang Baek 2025-04-08 22:10:35 -07:00 committed by GitHub
commit b130b55f5b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
55 changed files with 236 additions and 5 deletions

View File

@ -159,6 +159,72 @@ async def create_new_knowledge(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.FILE_EXISTS,
)
############################
# ReindexKnowledgeFiles
############################
@router.post("/reindex", response_model=bool)
async def reindex_knowledge_files(
request: Request,
user=Depends(get_verified_user)
):
if user.role != "admin":
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=ERROR_MESSAGES.UNAUTHORIZED,
)
knowledge_bases = Knowledges.get_knowledge_bases()
log.info(f"Starting reindexing for {len(knowledge_bases)} knowledge bases")
for knowledge_base in knowledge_bases:
try:
files = Files.get_files_by_ids(knowledge_base.data.get("file_ids", []))
try:
if VECTOR_DB_CLIENT.has_collection(collection_name=knowledge_base.id):
VECTOR_DB_CLIENT.delete_collection(
collection_name=knowledge_base.id
)
except Exception as e:
log.error(f"Error deleting collection {knowledge_base.id}: {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Error deleting vector DB collection"
)
failed_files = []
for file in files:
try:
process_file(
request,
ProcessFileForm(file_id=file.id, collection_name=knowledge_base.id),
user=user,
)
except Exception as e:
log.error(f"Error processing file {file.filename} (ID: {file.id}): {str(e)}")
failed_files.append({"file_id": file.id, "error": str(e)})
continue
except Exception as e:
log.error(f"Error processing knowledge base {knowledge_base.id}: {str(e)}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Error processing knowledge base"
)
if failed_files:
log.warning(f"Failed to process {len(failed_files)} files in knowledge base {knowledge_base.id}")
for failed in failed_files:
log.warning(f"File ID: {failed['file_id']}, Error: {failed['error']}")
log.info("Reindexing completed successfully")
return True
############################
@ -676,3 +742,6 @@ def add_files_to_knowledge_batch(
return KnowledgeFilesResponse(
**knowledge.model_dump(), files=Files.get_files_by_ids(existing_file_ids)
)

View File

@ -345,3 +345,32 @@ export const deleteKnowledgeById = async (token: string, id: string) => {
return res;
};
export const reindexKnowledgeFiles = async (token: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/reindex`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.log(err);
return null;
});
if (error) {
throw error;
}
return res;
};

View File

@ -13,17 +13,16 @@
updateEmbeddingConfig,
getRerankingConfig,
updateRerankingConfig,
resetUploadDir,
getRAGConfig,
updateRAGConfig
} from '$lib/apis/retrieval';
import { knowledge, models } from '$lib/stores';
import { getKnowledgeBases } from '$lib/apis/knowledge';
import { uploadDir, deleteAllFiles, deleteFileById } from '$lib/apis/files';
import { reindexKnowledgeFiles} from '$lib/apis/knowledge';
import { deleteAllFiles } from '$lib/apis/files';
import ResetUploadDirConfirmDialog from '$lib/components/common/ConfirmDialog.svelte';
import ResetVectorDBConfirmDialog from '$lib/components/common/ConfirmDialog.svelte';
import ReindexKnowledgeFilesConfirmDialog from '$lib/components/common/ConfirmDialog.svelte';
import SensitiveInput from '$lib/components/common/SensitiveInput.svelte';
import Tooltip from '$lib/components/common/Tooltip.svelte';
import Switch from '$lib/components/common/Switch.svelte';
@ -31,12 +30,12 @@
const i18n = getContext('i18n');
let scanDirLoading = false;
let updateEmbeddingModelLoading = false;
let updateRerankingModelLoading = false;
let showResetConfirm = false;
let showResetUploadDirConfirm = false;
let showReindexConfirm = false;
let embeddingEngine = '';
let embeddingModel = '';
@ -333,6 +332,21 @@
}}
/>
<ReindexKnowledgeFilesConfirmDialog
bind:show={showReindexConfirm}
on:confirm={async () => {
const res = await reindexKnowledgeFiles(localStorage.token).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res) {
toast.success($i18n.t('Success'));
}
}}
/>
<form
class="flex flex-col h-full justify-between space-y-3 text-sm"
on:submit|preventDefault={() => {
@ -950,6 +964,21 @@
</button>
</div>
</div>
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
{$i18n.t('Reindex Knowledge Base Vectors')}
</div>
<div class="flex items-center relative">
<button
class="text-xs"
on:click={() => {
showReindexConfirm = true;
}}
>
{$i18n.t('Reindex')}
</button>
</div>
</div>
</div>
</div>
</div>

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "رفض عندما لا ينبغي أن يكون",
"Regenerate": "تجديد",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "ملاحظات الإصدار",
"Relevance": "",
"Remove": "إزالة",

View File

@ -878,6 +878,8 @@
"References from": "مراجع من",
"Refused when it shouldn't have": "رفض عندما لا ينبغي أن يكون",
"Regenerate": "تجديد",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "ملاحظات الإصدار",
"Relevance": "الصلة",
"Remove": "إزالة",

View File

@ -878,6 +878,8 @@
"References from": "Препратки от",
"Refused when it shouldn't have": "Отказано, когато не трябва да бъде",
"Regenerate": "Регенериране",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Бележки по изданието",
"Relevance": "Релевантност",
"Remove": "Изтриване",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "যদি উপযুক্ত নয়, তবে রেজিগেনেট করা হচ্ছে",
"Regenerate": "রেজিগেনেট করুন",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "রিলিজ নোটসমূহ",
"Relevance": "",
"Remove": "রিমুভ করুন",

View File

@ -878,6 +878,8 @@
"References from": "ནས་ལུང་འདྲེན།",
"Refused when it shouldn't have": "མི་དགོས་དུས་ཁས་མ་བླངས།",
"Regenerate": "བསྐྱར་བཟོ།",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "འགྲེམས་སྤེལ་མཆན་བུ།",
"Relevance": "འབྲེལ་ཡོད་རང་བཞིན།",
"Remove": "འདོར་བ།",

View File

@ -878,6 +878,8 @@
"References from": "Referències de",
"Refused when it shouldn't have": "Refusat quan no hauria d'haver estat",
"Regenerate": "Regenerar",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Notes de la versió",
"Relevance": "Rellevància",
"Remove": "Eliminar",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "",
"Regenerate": "",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Release Notes",
"Relevance": "",
"Remove": "",

View File

@ -878,6 +878,8 @@
"References from": "Reference z",
"Refused when it shouldn't have": "Odmítnuto, když nemělo být.",
"Regenerate": "Regenerovat",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Záznamy o vydání",
"Relevance": "Relevance",
"Remove": "Odebrat",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Afvist, når den ikke burde have været det",
"Regenerate": "Regenerer",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Udgivelsesnoter",
"Relevance": "",
"Remove": "Fjern",

View File

@ -878,6 +878,8 @@
"References from": "Referenzen aus",
"Refused when it shouldn't have": "Abgelehnt, obwohl es nicht hätte abgelehnt werden sollen",
"Regenerate": "Neu generieren",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Veröffentlichungshinweise",
"Relevance": "Relevanz",
"Remove": "Entfernen",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "",
"Regenerate": "",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Release Borks",
"Relevance": "",
"Remove": "",

View File

@ -878,6 +878,8 @@
"References from": "Αναφορές από",
"Refused when it shouldn't have": "Αρνήθηκε όταν δεν έπρεπε",
"Regenerate": "Αναγεννήστε",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Σημειώσεις Έκδοσης",
"Relevance": "Σχετικότητα",
"Remove": "Αφαίρεση",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "",
"Regenerate": "",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "",
"Relevance": "",
"Remove": "",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "",
"Regenerate": "",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "",
"Relevance": "",
"Remove": "",

View File

@ -878,6 +878,8 @@
"References from": "Referencias desde",
"Refused when it shouldn't have": "Rechazado cuando no debería haberlo hecho",
"Regenerate": "Regenerar",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Notas de la Versión",
"Relevance": "Relevancia",
"Remove": "Eliminar",

View File

@ -878,6 +878,8 @@
"References from": "Viited allikast",
"Refused when it shouldn't have": "Keeldus, kui ei oleks pidanud",
"Regenerate": "Regenereeri",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Väljalaskemärkmed",
"Relevance": "Asjakohasus",
"Remove": "Eemalda",

View File

@ -878,6 +878,8 @@
"References from": "Erreferentziak hemendik",
"Refused when it shouldn't have": "Ukatu duenean ukatu behar ez zuenean",
"Regenerate": "Bersortu",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Bertsio oharrak",
"Relevance": "Garrantzia",
"Remove": "Kendu",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "رد شده زمانی که باید نباشد",
"Regenerate": "ری\u200cسازی",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "یادداشت\u200cهای انتشار",
"Relevance": "ارتباط",
"Remove": "حذف",

View File

@ -878,6 +878,8 @@
"References from": "Viitteet lähteistä",
"Refused when it shouldn't have": "Kieltäytyi, vaikka ei olisi pitänyt",
"Regenerate": "Uudelleentuota",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Julkaisutiedot",
"Relevance": "Relevanssi",
"Remove": "Poista",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Refusé alors qu'il n'aurait pas dû l'être",
"Regenerate": "Regénérer",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Notes de publication",
"Relevance": "",
"Remove": "Retirer",

View File

@ -878,6 +878,8 @@
"References from": "Références de",
"Refused when it shouldn't have": "Refusé alors qu'il n'aurait pas dû l'être",
"Regenerate": "Regénérer",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Notes de mise à jour",
"Relevance": "Pertinence",
"Remove": "Retirer",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "נדחה כאשר לא היה צריך",
"Regenerate": "הפק מחדש",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "הערות שחרור",
"Relevance": "",
"Remove": "הסר",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "जब ऐसा नहीं होना चाहिए था तो मना कर दिया",
"Regenerate": "पुनः जेनरेट",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "रिलीज नोट्स",
"Relevance": "",
"Remove": "हटा दें",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Odbijen kada nije trebao biti",
"Regenerate": "Regeneriraj",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Bilješke o izdanju",
"Relevance": "",
"Remove": "Ukloni",

View File

@ -878,6 +878,8 @@
"References from": "Hivatkozások innen",
"Refused when it shouldn't have": "Elutasítva, amikor nem kellett volna",
"Regenerate": "Újragenerálás",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Kiadási jegyzetek",
"Relevance": "Relevancia",
"Remove": "Eltávolítás",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Menolak ketika seharusnya tidak",
"Regenerate": "Regenerasi",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Catatan Rilis",
"Relevance": "",
"Remove": "Hapus",

View File

@ -878,6 +878,8 @@
"References from": "Tagairtí ó",
"Refused when it shouldn't have": "Diúltaíodh nuair nár chóir dó",
"Regenerate": "Athghiniúint",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Nótaí Scaoilte",
"Relevance": "Ábharthacht",
"Remove": "Bain",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Rifiutato quando non avrebbe dovuto",
"Regenerate": "Rigenera",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Note di rilascio",
"Relevance": "",
"Remove": "Rimuovi",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "拒否すべきでないのに拒否した",
"Regenerate": "再生成",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "リリースノート",
"Relevance": "",
"Remove": "削除",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "უარა, როგორც უნდა იყოს",
"Regenerate": "თავიდან გენერაცია",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "გამოცემის შენიშვნები",
"Relevance": "შესაბამისობა",
"Remove": "წაშლა",

View File

@ -878,6 +878,8 @@
"References from": "출처",
"Refused when it shouldn't have": "허용되지 않았지만 허용되어야 합니다.",
"Regenerate": "재생성",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "릴리스 노트",
"Relevance": "관련도",
"Remove": "삭제",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Atmesta kai neturėtų būti atmesta",
"Regenerate": "Generuoti iš naujo",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Naujovės",
"Relevance": "",
"Remove": "Pašalinti",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Menolak dimana ia tidak sepatutnya",
"Regenerate": "Jana semula",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Nota Keluaran",
"Relevance": "",
"Remove": "Hapuskan",

View File

@ -878,6 +878,8 @@
"References from": "Henviser fra",
"Refused when it shouldn't have": "Avvist når det ikke burde ha blitt det",
"Regenerate": "Generer på nytt",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Utgivelsesnotater",
"Relevance": "Relevans",
"Remove": "Fjern",

View File

@ -878,6 +878,8 @@
"References from": "Referenties van",
"Refused when it shouldn't have": "Geweigerd terwijl het niet had moeten",
"Regenerate": "Regenereren",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Release-opmerkingen",
"Relevance": "Relevantie",
"Remove": "Verwijderen",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "ਜਦੋਂ ਇਹ ਨਹੀਂ ਹੋਣਾ ਚਾਹੀਦਾ ਸੀ ਤਾਂ ਇਨਕਾਰ ਕੀਤਾ",
"Regenerate": "ਮੁੜ ਬਣਾਓ",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "ਰਿਲੀਜ਼ ਨੋਟਸ",
"Relevance": "",
"Remove": "ਹਟਾਓ",

View File

@ -878,6 +878,8 @@
"References from": "Odniesienia do",
"Refused when it shouldn't have": "Odmówił, gdy nie powinien",
"Regenerate": "Wygeneruj ponownie",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Notatki do wydania",
"Relevance": "Trafność",
"Remove": "Usuń",

View File

@ -878,6 +878,8 @@
"References from": "Referências de",
"Refused when it shouldn't have": "Recusado quando não deveria",
"Regenerate": "Gerar novamente",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Notas de Lançamento",
"Relevance": "Relevância",
"Remove": "Remover",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Recusado quando não deveria",
"Regenerate": "Regenerar",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Notas de Lançamento",
"Relevance": "",
"Remove": "Remover",

View File

@ -878,6 +878,8 @@
"References from": "Referințe din",
"Refused when it shouldn't have": "Refuzat când nu ar fi trebuit",
"Regenerate": "Regenerare",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Note de Lansare",
"Relevance": "Relevanță",
"Remove": "Înlătură",

View File

@ -878,6 +878,8 @@
"References from": "Отсылки к",
"Refused when it shouldn't have": "Отказано в доступе, когда это не должно было произойти",
"Regenerate": "Перегенерировать",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Примечания к выпуску",
"Relevance": "Актуальность",
"Remove": "Удалить",

View File

@ -878,6 +878,8 @@
"References from": "Referencie z",
"Refused when it shouldn't have": "Odmietnuté, keď nemalo byť.",
"Regenerate": "Regenerovať",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Záznamy o vydaní",
"Relevance": "Relevancia",
"Remove": "Odstrániť",

View File

@ -878,6 +878,8 @@
"References from": "Референце од",
"Refused when it shouldn't have": "Одбијено када није требало",
"Regenerate": "Поново створи",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Напомене о издању",
"Relevance": "Примењивост",
"Remove": "Уклони",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "Avvisades när det inte borde ha gjort det",
"Regenerate": "Regenerera",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Versionsinformation",
"Relevance": "",
"Remove": "Ta bort",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "ปฏิเสธเมื่อไม่ควรทำ",
"Regenerate": "สร้างใหม่",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "บันทึกรุ่น",
"Relevance": "",
"Remove": "ลบ",

View File

@ -878,6 +878,8 @@
"References from": "",
"Refused when it shouldn't have": "",
"Regenerate": "",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "",
"Relevance": "",
"Remove": "",

View File

@ -878,6 +878,8 @@
"References from": "Referanslar arasından",
"Refused when it shouldn't have": "Reddedilmemesi gerekirken reddedildi",
"Regenerate": "Tekrar Oluştur",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Sürüm Notları",
"Relevance": "İlgili",
"Remove": "Kaldır",

View File

@ -878,6 +878,8 @@
"References from": "Посилання з",
"Refused when it shouldn't have": "Відмовив, коли не мав би",
"Regenerate": "Регенерувати",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Нотатки до випуску",
"Relevance": "Актуальність",
"Remove": "Видалити",

View File

@ -878,6 +878,8 @@
"References from": "سے حوالہ جات",
"Refused when it shouldn't have": "جب انکار نہیں ہونا چاہیے تھا، انکار کر دیا",
"Regenerate": "دوبارہ تخلیق کریں",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "ریلیز نوٹس",
"Relevance": "موزونیت",
"Remove": "ہٹا دیں",

View File

@ -878,6 +878,8 @@
"References from": "Tham khảo từ",
"Refused when it shouldn't have": "Từ chối trả lời mà nhẽ không nên làm vậy",
"Regenerate": "Tạo sinh lại câu trả lời",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "Mô tả những cập nhật mới",
"Relevance": "Mức độ liên quan",
"Remove": "Xóa",

View File

@ -878,6 +878,8 @@
"References from": "来自",
"Refused when it shouldn't have": "无理拒绝",
"Regenerate": "重新生成",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "更新日志",
"Relevance": "相关性",
"Remove": "移除",

View File

@ -878,6 +878,8 @@
"References from": "引用來源",
"Refused when it shouldn't have": "不應拒絕時拒絕了",
"Regenerate": "重新產生",
"Reindex": "",
"Reindex Knowledge Base Vectors": "",
"Release Notes": "釋出説明",
"Relevance": "相關性",
"Remove": "移除",