refac

2025-06-26 18:26:48 +00:00 · 2024-09-28 02:23:09 +02:00
parent 1b349016ff
commit af57a2c153
28 changed files with 1026 additions and 993 deletions
--- a/src/lib/apis/retrieval/index.ts
+++ b/src/lib/apis/retrieval/index.ts
@@ -170,284 +170,6 @@ export const updateQuerySettings = async (token: string, settings: QuerySettings
 	return res;
 };

-export const processFile = async (token: string, file_id: string) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/process/file`, {
-		method: 'POST',
-		headers: {
-			Accept: 'application/json',
-			'Content-Type': 'application/json',
-			authorization: `Bearer ${token}`
-		},
-		body: JSON.stringify({
-			file_id: file_id
-		})
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			console.log(err);
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
-export const uploadDocToVectorDB = async (token: string, collection_name: string, file: File) => {
-	const data = new FormData();
-	data.append('file', file);
-	data.append('collection_name', collection_name);
-
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/doc`, {
-		method: 'POST',
-		headers: {
-			Accept: 'application/json',
-			authorization: `Bearer ${token}`
-		},
-		body: data
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			console.log(err);
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
-export const uploadWebToVectorDB = async (token: string, collection_name: string, url: string) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/web`, {
-		method: 'POST',
-		headers: {
-			Accept: 'application/json',
-			'Content-Type': 'application/json',
-			authorization: `Bearer ${token}`
-		},
-		body: JSON.stringify({
-			url: url,
-			collection_name: collection_name
-		})
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			console.log(err);
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
-export const uploadYoutubeTranscriptionToVectorDB = async (token: string, url: string) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/youtube`, {
-		method: 'POST',
-		headers: {
-			Accept: 'application/json',
-			'Content-Type': 'application/json',
-			authorization: `Bearer ${token}`
-		},
-		body: JSON.stringify({
-			url: url
-		})
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			console.log(err);
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
-export const queryDoc = async (
-	token: string,
-	collection_name: string,
-	query: string,
-	k: number | null = null
-) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/query/doc`, {
-		method: 'POST',
-		headers: {
-			Accept: 'application/json',
-			'Content-Type': 'application/json',
-			authorization: `Bearer ${token}`
-		},
-		body: JSON.stringify({
-			collection_name: collection_name,
-			query: query,
-			k: k
-		})
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
-export const queryCollection = async (
-	token: string,
-	collection_names: string,
-	query: string,
-	k: number | null = null
-) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/query/collection`, {
-		method: 'POST',
-		headers: {
-			Accept: 'application/json',
-			'Content-Type': 'application/json',
-			authorization: `Bearer ${token}`
-		},
-		body: JSON.stringify({
-			collection_names: collection_names,
-			query: query,
-			k: k
-		})
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
-export const scanDocs = async (token: string) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/scan`, {
-		method: 'GET',
-		headers: {
-			Accept: 'application/json',
-			authorization: `Bearer ${token}`
-		}
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
-export const resetUploadDir = async (token: string) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/reset/uploads`, {
-		method: 'POST',
-		headers: {
-			Accept: 'application/json',
-			authorization: `Bearer ${token}`
-		}
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
-export const resetVectorDB = async (token: string) => {
-	let error = null;
-
-	const res = await fetch(`${RAG_API_BASE_URL}/reset/db`, {
-		method: 'POST',
-		headers: {
-			Accept: 'application/json',
-			authorization: `Bearer ${token}`
-		}
-	})
-		.then(async (res) => {
-			if (!res.ok) throw await res.json();
-			return res.json();
-		})
-		.catch((err) => {
-			error = err.detail;
-			return null;
-		});
-
-	if (error) {
-		throw error;
-	}
-
-	return res;
-};
-
 export const getEmbeddingConfig = async (token: string) => {
 	let error = null;

@@ -578,14 +300,140 @@ export const updateRerankingConfig = async (token: string, payload: RerankingMod
 	return res;
 };

-export const runWebSearch = async (
+export interface SearchDocument {
+	status: boolean;
+	collection_name: string;
+	filenames: string[];
+}
+
+export const processFile = async (token: string, file_id: string) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/process/file`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			file_id: file_id
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const processDocsDir = async (token: string) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/process/dir`, {
+		method: 'GET',
+		headers: {
+			Accept: 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const processYoutubeVideo = async (token: string, url: string) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/process/youtube`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			url: url
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const processWeb = async (token: string, collection_name: string, url: string) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/process/web`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			url: url,
+			collection_name: collection_name
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.log(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const processWebSearch = async (
 	token: string,
 	query: string,
 	collection_name?: string
 ): Promise<SearchDocument | null> => {
 	let error = null;

-	const res = await fetch(`${RAG_API_BASE_URL}/web/search`, {
+	const res = await fetch(`${RAG_API_BASE_URL}/process/web/search`, {
 		method: 'POST',
 		headers: {
 			'Content-Type': 'application/json',
@@ -613,8 +461,128 @@ export const runWebSearch = async (
 	return res;
 };

-export interface SearchDocument {
-	status: boolean;
-	collection_name: string;
-	filenames: string[];
-}
+export const queryDoc = async (
+	token: string,
+	collection_name: string,
+	query: string,
+	k: number | null = null
+) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/query/doc`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			collection_name: collection_name,
+			query: query,
+			k: k
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const queryCollection = async (
+	token: string,
+	collection_names: string,
+	query: string,
+	k: number | null = null
+) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/query/collection`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({
+			collection_names: collection_names,
+			query: query,
+			k: k
+		})
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const resetUploadDir = async (token: string) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/reset/uploads`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const resetVectorDB = async (token: string) => {
+	let error = null;
+
+	const res = await fetch(`${RAG_API_BASE_URL}/reset/db`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			authorization: `Bearer ${token}`
+		}
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.catch((err) => {
+			error = err.detail;
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
--- a/src/lib/components/admin/Settings/Documents.svelte
+++ b/src/lib/components/admin/Settings/Documents.svelte
@@ -7,7 +7,7 @@
 	import { deleteAllFiles, deleteFileById } from '$lib/apis/files';
 	import {
 		getQuerySettings,
-		scanDocs,
+		processDocsDir,
 		updateQuerySettings,
 		resetVectorDB,
 		getEmbeddingConfig,
@@ -63,7 +63,7 @@

 	const scanHandler = async () => {
 		scanDirLoading = true;
-		const res = await scanDocs(localStorage.token);
+		const res = await processDocsDir(localStorage.token);
 		scanDirLoading = false;

 		if (res) {
--- a/src/lib/components/chat/Chat.svelte
+++ b/src/lib/components/chat/Chat.svelte
@@ -52,7 +52,7 @@
 		updateChatById
 	} from '$lib/apis/chats';
 	import { generateOpenAIChatCompletion } from '$lib/apis/openai';
-	import { runWebSearch } from '$lib/apis/retrieval';
+	import { processWebSearch } from '$lib/apis/retrieval';
 	import { createOpenAITextStream } from '$lib/apis/streaming';
 	import { queryMemory } from '$lib/apis/memories';
 	import { getAndUpdateUserLocation, getUserSettings } from '$lib/apis/users';
@@ -1737,7 +1737,7 @@
 		});
 		history.messages[responseMessageId] = responseMessage;

-		const results = await runWebSearch(localStorage.token, searchQuery).catch((error) => {
+		const results = await processWebSearch(localStorage.token, searchQuery).catch((error) => {
 			console.log(error);
 			toast.error(error);

--- a/src/lib/components/chat/Controls/Controls.svelte
+++ b/src/lib/components/chat/Controls/Controls.svelte
@@ -46,6 +46,9 @@
 								chatFiles.splice(fileIdx, 1);
 								chatFiles = chatFiles;
 							}}
+							on:click={() => {
+								console.log(file);
+							}}
 						/>
 					{/each}
 				</div>
--- a/src/lib/components/chat/MessageInput/Commands.svelte
+++ b/src/lib/components/chat/MessageInput/Commands.svelte
@@ -9,7 +9,7 @@
 	import Models from './Commands/Models.svelte';

 	import { removeLastWordFromString } from '$lib/utils';
-	import { uploadWebToVectorDB, uploadYoutubeTranscriptionToVectorDB } from '$lib/apis/retrieval';
+	import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval';

 	export let prompt = '';
 	export let files = [];
@@ -41,7 +41,7 @@

 		try {
 			files = [...files, doc];
-			const res = await uploadWebToVectorDB(localStorage.token, '', url);
+			const res = await processWeb(localStorage.token, '', url);

 			if (res) {
 				doc.status = 'processed';
@@ -69,7 +69,7 @@

 		try {
 			files = [...files, doc];
-			const res = await uploadYoutubeTranscriptionToVectorDB(localStorage.token, url);
+			const res = await processYoutubeVideo(localStorage.token, url);

 			if (res) {
 				doc.status = 'processed';
--- a/src/lib/components/common/FileItem.svelte
+++ b/src/lib/components/common/FileItem.svelte
@@ -8,8 +8,6 @@
 	export let colorClassName = 'bg-white dark:bg-gray-800';
 	export let url: string | null = null;

-	export let clickHandler: Function | null = null;
-
 	export let dismissible = false;
 	export let status = 'processed';

@@ -17,7 +15,7 @@
 	export let type: string;
 	export let size: number;

-	function formatSize(size) {
+	const formatSize = (size) => {
 		if (size == null) return 'Unknown size';
 		if (typeof size !== 'number' || size < 0) return 'Invalid size';
 		if (size === 0) return '0 B';
@@ -29,7 +27,7 @@
 			unitIndex++;
 		}
 		return `${size.toFixed(1)} ${units[unitIndex]}`;
-	}
+	};
 </script>

 <div class="relative group">
@@ -37,17 +35,7 @@
 		class="h-14 {className} flex items-center space-x-3 {colorClassName} rounded-xl border border-gray-100 dark:border-gray-800 text-left"
 		type="button"
 		on:click={async () => {
-			if (clickHandler === null) {
-				if (url) {
-					if (type === 'file') {
-						window.open(`${url}/content`, '_blank').focus();
-					} else {
-						window.open(`${url}`, '_blank').focus();
-					}
-				}
-			} else {
-				clickHandler();
-			}
+			dispatch('click');
 		}}
 	>
 		<div class="p-4 py-[1.1rem] bg-red-400 text-white rounded-l-xl">