From 83986620ee6595fcd6a879490c71c5708db3d315 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Tue, 18 Jun 2024 14:15:08 -0700 Subject: [PATCH] refac --- backend/apps/rag/main.py | 3 ++ backend/apps/webui/models/files.py | 9 +++++ backend/apps/webui/routers/files.py | 18 +++++++++ src/lib/apis/files/index.ts | 1 - src/lib/apis/rag/index.ts | 1 + src/lib/components/chat/MessageInput.svelte | 5 +-- src/lib/utils/characters/index.ts | 38 +++++++++++++++---- .../workspace/models/create/+page.svelte | 2 + 8 files changed, 66 insertions(+), 11 deletions(-) diff --git a/backend/apps/rag/main.py b/backend/apps/rag/main.py index 3bd7303bd..00ae4e892 100644 --- a/backend/apps/rag/main.py +++ b/backend/apps/rag/main.py @@ -1136,6 +1136,7 @@ def store_doc( class ProcessDocForm(BaseModel): file_id: str + collection_name: Optional[str] = None @app.post("/process/doc") @@ -1148,6 +1149,8 @@ def process_doc( file_path = file.meta.get("path", f"{UPLOAD_DIR}/{file.filename}") f = open(file_path, "rb") + + collection_name = form_data.collection_name if collection_name == None: collection_name = calculate_sha256(f)[:63] f.close() diff --git a/backend/apps/webui/models/files.py b/backend/apps/webui/models/files.py index c34bd46d8..c62fa4019 100644 --- a/backend/apps/webui/models/files.py +++ b/backend/apps/webui/models/files.py @@ -99,5 +99,14 @@ class FilesTable: except: return False + def delete_all_files(self) -> bool: + try: + query = File.delete() + query.execute() # Remove the rows, return number of rows removed. + + return True + except: + return False + Files = FilesTable(DB) diff --git a/backend/apps/webui/routers/files.py b/backend/apps/webui/routers/files.py index a231f7bb1..471079e4c 100644 --- a/backend/apps/webui/routers/files.py +++ b/backend/apps/webui/routers/files.py @@ -53,6 +53,7 @@ def upload_file( # replace filename with uuid id = str(uuid.uuid4()) + filename = f"{id}_{filename}" file_path = f"{UPLOAD_DIR}/{filename}" contents = file.file.read() @@ -143,3 +144,20 @@ async def delete_file_by_id(id: str, user=Depends(get_verified_user)): status_code=status.HTTP_401_UNAUTHORIZED, detail=ERROR_MESSAGES.NOT_FOUND, ) + + +############################ +# Delete All Files +############################ + + +@router.delete("/all") +async def delete_all_files(user=Depends(get_admin_user)): + result = Files.delete_all_files() + if result: + return {"message": "All files deleted successfully"} + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.DEFAULT("Error deleting files"), + ) diff --git a/src/lib/apis/files/index.ts b/src/lib/apis/files/index.ts index 30222b5d5..d467d2889 100644 --- a/src/lib/apis/files/index.ts +++ b/src/lib/apis/files/index.ts @@ -9,7 +9,6 @@ export const uploadFile = async (token: string, file: File) => { method: 'POST', headers: { Accept: 'application/json', - 'Content-Type': 'application/json', authorization: `Bearer ${token}` }, body: data diff --git a/src/lib/apis/rag/index.ts b/src/lib/apis/rag/index.ts index 5639830c1..50f236e06 100644 --- a/src/lib/apis/rag/index.ts +++ b/src/lib/apis/rag/index.ts @@ -171,6 +171,7 @@ export const processDocToVectorDB = async (token: string, file_id: string) => { method: 'POST', headers: { Accept: 'application/json', + 'Content-Type': 'application/json', authorization: `Bearer ${token}` }, body: JSON.stringify({ diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte index f6bc595b2..ce619e31d 100644 --- a/src/lib/components/chat/MessageInput.svelte +++ b/src/lib/components/chat/MessageInput.svelte @@ -147,7 +147,6 @@ if (res) { fileItem.status = 'processed'; - fileItem.collection_name = res.collection_name; files = files; } } catch (e) { @@ -523,12 +522,12 @@ {/if} - {:else if file.type === 'doc'} + {:else if ['doc', 'file'].includes(file.type)}
- {#if file.upload_status} + {#if file.status === 'processed'} { const extractCharacter = (json) => { function getTrimmedValue(json, keys) { - return keys.map((key) => json[key]).find((value) => value && value.trim()); + return keys + .map((key) => { + const keyParts = key.split('.'); + let value = json; + for (const part of keyParts) { + if (value && value[part] != null) { + value = value[part]; + } else { + value = null; + break; + } + } + return value && value.trim(); + }) + .find((value) => value); } - const name = getTrimmedValue(json, ['char_name', 'name']); - const summary = getTrimmedValue(json, ['personality', 'title']); - const personality = getTrimmedValue(json, ['char_persona', 'description']); - const scenario = getTrimmedValue(json, ['world_scenario', 'scenario']); - const greeting = getTrimmedValue(json, ['char_greeting', 'greeting', 'first_mes']); - const examples = getTrimmedValue(json, ['example_dialogue', 'mes_example', 'definition']); + const name = getTrimmedValue(json, ['char_name', 'name', 'data.name']); + const summary = getTrimmedValue(json, ['personality', 'title', 'data.description']); + const personality = getTrimmedValue(json, ['char_persona', 'description', 'data.personality']); + const scenario = getTrimmedValue(json, ['world_scenario', 'scenario', 'data.scenario']); + const greeting = getTrimmedValue(json, [ + 'char_greeting', + 'greeting', + 'first_mes', + 'data.first_mes' + ]); + const examples = getTrimmedValue(json, [ + 'example_dialogue', + 'mes_example', + 'definition', + 'data.mes_example' + ]); return { name, summary, personality, scenario, greeting, examples }; }; diff --git a/src/routes/(app)/workspace/models/create/+page.svelte b/src/routes/(app)/workspace/models/create/+page.svelte index eb2ed761b..b870dec58 100644 --- a/src/routes/(app)/workspace/models/create/+page.svelte +++ b/src/routes/(app)/workspace/models/create/+page.svelte @@ -226,6 +226,8 @@ return null; }); + console.log(character); + if (character && character.character) { character = character.character; console.log(character);