From ae97a96379ed85d14e2b70cbd5ede233f1178534 Mon Sep 17 00:00:00 2001 From: Anuraag Jain Date: Sun, 18 Feb 2024 21:29:47 +0200 Subject: [PATCH 1/6] WIP feat: cancel model download --- .../components/chat/Settings/Models.svelte | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/src/lib/components/chat/Settings/Models.svelte b/src/lib/components/chat/Settings/Models.svelte index aafa333f3..cb557b55c 100644 --- a/src/lib/components/chat/Settings/Models.svelte +++ b/src/lib/components/chat/Settings/Models.svelte @@ -52,15 +52,17 @@ // Remove the downloaded model delete modelDownloadStatus[modelName]; - console.log(data); + modelDownloadStatus = {...modelDownloadStatus} + + console.log('Cleaned:',modelDownloadStatus); if (!data.success) { toast.error(data.error); } else { - toast.success(`Model '${modelName}' has been successfully downloaded.`); + toast.success(`Model '${sanitizedModelTag}' has been successfully downloaded.`); const notification = new Notification(WEBUI_NAME, { - body: `Model '${modelName}' has been successfully downloaded.`, + body: `Model '${sanitizedModelTag}' has been successfully downloaded.`, icon: '/favicon.png' }); @@ -266,6 +268,7 @@ downloadProgress = 100; } modelDownloadStatus[opts.modelName] = { + reader, pullProgress: downloadProgress, digest: data.digest }; @@ -286,6 +289,15 @@ opts.callback({ success: true, modelName: opts.modelName }); } }; + const deleteModelPull = async(model: string) => { + const {reader} = modelDownloadStatus[model]; + if(reader){ + await reader.cancel(); + toast.success(`${model} download has been canceled`); + delete modelDownloadStatus[model]; + } + + }
@@ -364,12 +376,36 @@
{model}
-
+
{modelDownloadStatus[model].pullProgress ?? 0}% +
+ + +
+
{modelDownloadStatus[model].digest}
From d291821bf39f939c7170be7506a1da8633e42a2c Mon Sep 17 00:00:00 2001 From: Anuraag Jain Date: Sat, 23 Mar 2024 20:56:18 +0200 Subject: [PATCH 2/6] refac: delete button UI --- .gitignore | 2 +- .../components/chat/Settings/Models.svelte | 24 ++++++++----------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 528e1f830..2ccac4d50 100644 --- a/.gitignore +++ b/.gitignore @@ -166,7 +166,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Logs logs diff --git a/src/lib/components/chat/Settings/Models.svelte b/src/lib/components/chat/Settings/Models.svelte index 973e3dd22..35386e1e9 100644 --- a/src/lib/components/chat/Settings/Models.svelte +++ b/src/lib/components/chat/Settings/Models.svelte @@ -473,8 +473,10 @@ const {reader} = modelDownloadStatus[model]; if(reader){ await reader.cancel(); - toast.success(`${model} download has been canceled`); delete modelDownloadStatus[model]; + await deleteModel(localStorage.token, model); + toast.success(`${model} download has been canceled`); + } } @@ -611,6 +613,7 @@
{model}
+
+
{modelDownloadStatus[model].digest}
From 7e6d49948657348240546b6f5b10bb8f4db47ac1 Mon Sep 17 00:00:00 2001 From: Anuraag Jain Date: Sat, 23 Mar 2024 21:00:01 +0200 Subject: [PATCH 3/6] refac: remove conosle.log --- src/lib/components/chat/Settings/Models.svelte | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/components/chat/Settings/Models.svelte b/src/lib/components/chat/Settings/Models.svelte index 35386e1e9..e2695fb59 100644 --- a/src/lib/components/chat/Settings/Models.svelte +++ b/src/lib/components/chat/Settings/Models.svelte @@ -161,8 +161,6 @@ modelDownloadStatus = {...modelDownloadStatus} - console.log('Cleaned:',modelDownloadStatus); - if (!data.success) { toast.error(data.error); } else { From 642ca5a94a0d9fd683c00cd80cb5272486cb936a Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 23 Mar 2024 12:46:06 -0700 Subject: [PATCH 4/6] refac: ui --- .../components/chat/Settings/Models.svelte | 61 +++++++++++-------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/src/lib/components/chat/Settings/Models.svelte b/src/lib/components/chat/Settings/Models.svelte index e2695fb59..2d0b1d7ad 100644 --- a/src/lib/components/chat/Settings/Models.svelte +++ b/src/lib/components/chat/Settings/Models.svelte @@ -159,7 +159,7 @@ // Remove the downloaded model delete modelDownloadStatus[modelName]; - modelDownloadStatus = {...modelDownloadStatus} + modelDownloadStatus = { ...modelDownloadStatus }; if (!data.success) { toast.error(data.error); @@ -467,17 +467,15 @@ ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => false); liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token); }); - const deleteModelPull = async(model: string) => { - const {reader} = modelDownloadStatus[model]; - if(reader){ + const deleteModelPull = async (model: string) => { + const { reader } = modelDownloadStatus[model]; + if (reader) { await reader.cancel(); delete modelDownloadStatus[model]; await deleteModel(localStorage.token, model); toast.success(`${model} download has been canceled`); - } - - } + };
@@ -611,25 +609,38 @@
{model}
-
-
- {modelDownloadStatus[model].pullProgress ?? 0}% +
+
+ {modelDownloadStatus[model].pullProgress ?? 0}% +
+
- -
{modelDownloadStatus[model].digest}
From 244f34c24e0a56433c2a0c996994d5271af41159 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 23 Mar 2024 13:12:23 -0700 Subject: [PATCH 5/6] refac: cancel download --- src/lib/apis/ollama/index.ts | 2 +- .../components/chat/Settings/Models.svelte | 124 ++++++++++++------ src/routes/(app)/+page.svelte | 6 +- src/routes/(app)/c/[id]/+page.svelte | 6 +- src/routes/(app)/playground/+page.svelte | 8 +- 5 files changed, 93 insertions(+), 53 deletions(-) diff --git a/src/lib/apis/ollama/index.ts b/src/lib/apis/ollama/index.ts index 2047fedef..ba04e0605 100644 --- a/src/lib/apis/ollama/index.ts +++ b/src/lib/apis/ollama/index.ts @@ -271,7 +271,7 @@ export const generateChatCompletion = async (token: string = '', body: object) = return [res, controller]; }; -export const cancelChatCompletion = async (token: string = '', requestId: string) => { +export const cancelOllamaRequest = async (token: string = '', requestId: string) => { let error = null; const res = await fetch(`${OLLAMA_API_BASE_URL}/cancel/${requestId}`, { diff --git a/src/lib/components/chat/Settings/Models.svelte b/src/lib/components/chat/Settings/Models.svelte index 2d0b1d7ad..d961b72b0 100644 --- a/src/lib/components/chat/Settings/Models.svelte +++ b/src/lib/components/chat/Settings/Models.svelte @@ -7,7 +7,8 @@ deleteModel, getOllamaUrls, getOllamaVersion, - pullModel + pullModel, + cancelOllamaRequest } from '$lib/apis/ollama'; import { WEBUI_API_BASE_URL, WEBUI_BASE_URL } from '$lib/constants'; import { WEBUI_NAME, models, user } from '$lib/stores'; @@ -364,12 +365,24 @@ for (const line of lines) { if (line !== '') { let data = JSON.parse(line); + console.log(data); if (data.error) { throw data.error; } if (data.detail) { throw data.detail; } + + if (data.id) { + modelDownloadStatus[opts.modelName] = { + ...modelDownloadStatus[opts.modelName], + requestId: data.id, + reader, + done: false + }; + console.log(data); + } + if (data.status) { if (data.digest) { let downloadProgress = 0; @@ -379,12 +392,17 @@ downloadProgress = 100; } modelDownloadStatus[opts.modelName] = { - reader, + ...modelDownloadStatus[opts.modelName], pullProgress: downloadProgress, digest: data.digest }; } else { toast.success(data.status); + + modelDownloadStatus[opts.modelName] = { + ...modelDownloadStatus[opts.modelName], + done: data.status === 'success' + }; } } } @@ -397,7 +415,14 @@ opts.callback({ success: false, error, modelName: opts.modelName }); } } - opts.callback({ success: true, modelName: opts.modelName }); + + console.log(modelDownloadStatus[opts.modelName]); + + if (modelDownloadStatus[opts.modelName].done) { + opts.callback({ success: true, modelName: opts.modelName }); + } else { + opts.callback({ success: false, error: 'Download canceled', modelName: opts.modelName }); + } } }; @@ -467,10 +492,13 @@ ollamaVersion = await getOllamaVersion(localStorage.token).catch((error) => false); liteLLMModelInfo = await getLiteLLMModelInfo(localStorage.token); }); - const deleteModelPull = async (model: string) => { - const { reader } = modelDownloadStatus[model]; + + const cancelModelPullHandler = async (model: string) => { + const { reader, requestId } = modelDownloadStatus[model]; if (reader) { await reader.cancel(); + + await cancelOllamaRequest(localStorage.token, requestId); delete modelDownloadStatus[model]; await deleteModel(localStorage.token, model); toast.success(`${model} download has been canceled`); @@ -606,46 +634,58 @@ {#if Object.keys(modelDownloadStatus).length > 0} {#each Object.keys(modelDownloadStatus) as model} -
-
{model}
-
-
-
- {modelDownloadStatus[model].pullProgress ?? 0}% + {#if 'pullProgress' in modelDownloadStatus[model]} +
+
{model}
+
+
+
+
+ {modelDownloadStatus[model].pullProgress ?? 0}% +
+
+ + + +
- -
-
- {modelDownloadStatus[model].digest} + {#if 'digest' in modelDownloadStatus[model]} +
+ {modelDownloadStatus[model].digest} +
+ {/if}
-
+ {/if} {/each} {/if}
diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte index 0fca312ae..417ddccda 100644 --- a/src/routes/(app)/+page.svelte +++ b/src/routes/(app)/+page.svelte @@ -19,7 +19,7 @@ } from '$lib/stores'; import { copyToClipboard, splitStream } from '$lib/utils'; - import { generateChatCompletion, cancelChatCompletion, generateTitle } from '$lib/apis/ollama'; + import { generateChatCompletion, cancelOllamaRequest, generateTitle } from '$lib/apis/ollama'; import { addTagById, createNewChat, @@ -104,7 +104,7 @@ const initNewChat = async () => { if (currentRequestId !== null) { - await cancelChatCompletion(localStorage.token, currentRequestId); + await cancelOllamaRequest(localStorage.token, currentRequestId); currentRequestId = null; } window.history.replaceState(history.state, '', `/`); @@ -372,7 +372,7 @@ if (stopResponseFlag) { controller.abort('User: Stop Response'); - await cancelChatCompletion(localStorage.token, currentRequestId); + await cancelOllamaRequest(localStorage.token, currentRequestId); } currentRequestId = null; diff --git a/src/routes/(app)/c/[id]/+page.svelte b/src/routes/(app)/c/[id]/+page.svelte index faa15b4b6..836fc90a4 100644 --- a/src/routes/(app)/c/[id]/+page.svelte +++ b/src/routes/(app)/c/[id]/+page.svelte @@ -19,7 +19,7 @@ } from '$lib/stores'; import { copyToClipboard, splitStream, convertMessagesToHistory } from '$lib/utils'; - import { generateChatCompletion, generateTitle, cancelChatCompletion } from '$lib/apis/ollama'; + import { generateChatCompletion, generateTitle, cancelOllamaRequest } from '$lib/apis/ollama'; import { addTagById, createNewChat, @@ -382,7 +382,7 @@ if (stopResponseFlag) { controller.abort('User: Stop Response'); - await cancelChatCompletion(localStorage.token, currentRequestId); + await cancelOllamaRequest(localStorage.token, currentRequestId); } currentRequestId = null; @@ -843,7 +843,7 @@ shareEnabled={messages.length > 0} initNewChat={async () => { if (currentRequestId !== null) { - await cancelChatCompletion(localStorage.token, currentRequestId); + await cancelOllamaRequest(localStorage.token, currentRequestId); currentRequestId = null; } diff --git a/src/routes/(app)/playground/+page.svelte b/src/routes/(app)/playground/+page.svelte index 737eff224..d8e9320dc 100644 --- a/src/routes/(app)/playground/+page.svelte +++ b/src/routes/(app)/playground/+page.svelte @@ -13,7 +13,7 @@ } from '$lib/constants'; import { WEBUI_NAME, config, user, models, settings } from '$lib/stores'; - import { cancelChatCompletion, generateChatCompletion } from '$lib/apis/ollama'; + import { cancelOllamaRequest, generateChatCompletion } from '$lib/apis/ollama'; import { generateOpenAIChatCompletion } from '$lib/apis/openai'; import { splitStream } from '$lib/utils'; @@ -52,7 +52,7 @@ // const cancelHandler = async () => { // if (currentRequestId) { - // const res = await cancelChatCompletion(localStorage.token, currentRequestId); + // const res = await cancelOllamaRequest(localStorage.token, currentRequestId); // currentRequestId = null; // loading = false; // } @@ -95,7 +95,7 @@ const { value, done } = await reader.read(); if (done || stopResponseFlag) { if (stopResponseFlag) { - await cancelChatCompletion(localStorage.token, currentRequestId); + await cancelOllamaRequest(localStorage.token, currentRequestId); } currentRequestId = null; @@ -181,7 +181,7 @@ const { value, done } = await reader.read(); if (done || stopResponseFlag) { if (stopResponseFlag) { - await cancelChatCompletion(localStorage.token, currentRequestId); + await cancelOllamaRequest(localStorage.token, currentRequestId); } currentRequestId = null; From e008738f30081e4697a1b3c3278e2e51d47539ad Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 23 Mar 2024 13:12:54 -0700 Subject: [PATCH 6/6] feat: cancel download from backend --- backend/apps/ollama/main.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/backend/apps/ollama/main.py b/backend/apps/ollama/main.py index 6f56f3cf6..2283774e0 100644 --- a/backend/apps/ollama/main.py +++ b/backend/apps/ollama/main.py @@ -234,11 +234,26 @@ async def pull_model( def get_request(): nonlocal url nonlocal r + + request_id = str(uuid.uuid4()) try: + REQUEST_POOL.append(request_id) def stream_content(): - for chunk in r.iter_content(chunk_size=8192): - yield chunk + try: + yield json.dumps({"id": request_id, "done": False}) + "\n" + + for chunk in r.iter_content(chunk_size=8192): + if request_id in REQUEST_POOL: + yield chunk + else: + print("User: canceled request") + break + finally: + if hasattr(r, "close"): + r.close() + if request_id in REQUEST_POOL: + REQUEST_POOL.remove(request_id) r = requests.request( method="POST", @@ -259,6 +274,7 @@ async def pull_model( try: return await run_in_threadpool(get_request) + except Exception as e: print(e) error_detail = "Open WebUI: Server Connection Error"