From 431b710e46b3a4fcd17a9d687ff9715a93e693e2 Mon Sep 17 00:00:00 2001 From: David Young Date: Fri, 29 Dec 2023 16:09:59 +0000 Subject: [PATCH 1/3] Settings were being referenced incorrectly and as a result, no settings were being sent to openai endpoint. Added num_predict slider in advanced UI. Also added seed, stop and max_tokens (uses num_predict) being sent to openai endpoint. --- .../components/chat/Settings/Advanced.svelte | 48 ++++++++++++++++++- src/lib/components/chat/SettingsModal.svelte | 7 ++- src/routes/(app)/+page.svelte | 11 +++-- src/routes/(app)/c/[id]/+page.svelte | 11 +++-- .../(app)/modelfiles/create/+page.svelte | 4 +- 5 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/lib/components/chat/Settings/Advanced.svelte b/src/lib/components/chat/Settings/Advanced.svelte index a188f107a..41ad69f33 100644 --- a/src/lib/components/chat/Settings/Advanced.svelte +++ b/src/lib/components/chat/Settings/Advanced.svelte @@ -12,7 +12,8 @@ top_k: '', top_p: '', tfs_z: '', - num_ctx: '' + num_ctx: '', + num_predict: '' }; @@ -507,4 +508,49 @@ {/if} +
+
+
Max Tokens
+ + +
+ + {#if options.num_predict !== ''} +
+
+ +
+
+ +
+
+ {/if} +
diff --git a/src/lib/components/chat/SettingsModal.svelte b/src/lib/components/chat/SettingsModal.svelte index 18aa6ecad..12405537b 100644 --- a/src/lib/components/chat/SettingsModal.svelte +++ b/src/lib/components/chat/SettingsModal.svelte @@ -52,7 +52,8 @@ top_p: '', stop: '', tfs_z: '', - num_ctx: '' + num_ctx: '', + num_predict: '' }; // Models @@ -644,6 +645,7 @@ options.top_k = settings.top_k ?? ''; options.top_p = settings.top_p ?? ''; options.num_ctx = settings.num_ctx ?? ''; + options.num_predict = settings.num_predict ?? ''; options = { ...options, ...settings.options }; options.stop = (settings?.options?.stop ?? []).join(','); @@ -1122,7 +1124,8 @@ top_k: options.top_k !== '' ? options.top_k : undefined, top_p: options.top_p !== '' ? options.top_p : undefined, tfs_z: options.tfs_z !== '' ? options.tfs_z : undefined, - num_ctx: options.num_ctx !== '' ? options.num_ctx : undefined + num_ctx: options.num_ctx !== '' ? options.num_ctx : undefined, + num_predict: options.num_predict !== '' ? options.num_predict : undefined } }); show = false; diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte index 6f272a62e..0655cd912 100644 --- a/src/routes/(app)/+page.svelte +++ b/src/routes/(app)/+page.svelte @@ -368,10 +368,13 @@ } : { content: message.content }) })), - temperature: $settings.temperature ?? undefined, - top_p: $settings.top_p ?? undefined, - num_ctx: $settings.num_ctx ?? undefined, - frequency_penalty: $settings.repeat_penalty ?? undefined + seed: $settings.options.seed ?? undefined, + stop: $settings.options.stop ?? undefined, + temperature: $settings.options.temperature ?? undefined, + top_p: $settings.options.top_p ?? undefined, + num_ctx: $settings.options.num_ctx ?? undefined, + frequency_penalty: $settings.options.repeat_penalty ?? undefined, + max_tokens: $settings.options.num_predict ?? undefined, }) } ).catch((err) => { diff --git a/src/routes/(app)/c/[id]/+page.svelte b/src/routes/(app)/c/[id]/+page.svelte index 9600f2982..8954a21fb 100644 --- a/src/routes/(app)/c/[id]/+page.svelte +++ b/src/routes/(app)/c/[id]/+page.svelte @@ -395,10 +395,13 @@ } : { content: message.content }) })), - temperature: $settings.temperature ?? undefined, - top_p: $settings.top_p ?? undefined, - num_ctx: $settings.num_ctx ?? undefined, - frequency_penalty: $settings.repeat_penalty ?? undefined + seed: $settings.options.seed ?? undefined, + stop: $settings.options.stop ?? undefined, + temperature: $settings.options.temperature ?? undefined, + top_p: $settings.options.top_p ?? undefined, + num_ctx: $settings.options.num_ctx ?? undefined, + frequency_penalty: $settings.options.repeat_penalty ?? undefined, + max_tokens: $settings.options.num_predict ?? undefined, }) } ).catch((err) => { diff --git a/src/routes/(app)/modelfiles/create/+page.svelte b/src/routes/(app)/modelfiles/create/+page.svelte index 506edb9f7..23a0cffc0 100644 --- a/src/routes/(app)/modelfiles/create/+page.svelte +++ b/src/routes/(app)/modelfiles/create/+page.svelte @@ -51,7 +51,8 @@ top_k: '', top_p: '', tfs_z: '', - num_ctx: '' + num_ctx: '', + num_predict: '' }; let modelfileCreator = null; @@ -73,6 +74,7 @@ ${options.top_k !== '' ? `PARAMETER top_k ${options.top_k}` : ''} ${options.top_p !== '' ? `PARAMETER top_p ${options.top_p}` : ''} ${options.tfs_z !== '' ? `PARAMETER tfs_z ${options.tfs_z}` : ''} ${options.num_ctx !== '' ? `PARAMETER num_ctx ${options.num_ctx}` : ''} +${options.num_predict !== '' ? `PARAMETER num_predict ${options.num_predict}` : ''} SYSTEM """${system}"""`.replace(/^\s*\n/gm, ''); } From 0d6440f5c0162684776e1d5b5975d08b513ed2fb Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 30 Dec 2023 20:16:05 -0800 Subject: [PATCH 2/3] fix: num_predict default value should match ollama default https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values --- src/lib/components/chat/Settings/Advanced.svelte | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/components/chat/Settings/Advanced.svelte b/src/lib/components/chat/Settings/Advanced.svelte index 41ad69f33..5c70dc56a 100644 --- a/src/lib/components/chat/Settings/Advanced.svelte +++ b/src/lib/components/chat/Settings/Advanced.svelte @@ -516,7 +516,7 @@ class="p-1 px-3 text-xs flex rounded transition" type="button" on:click={() => { - options.num_predict = options.num_predict === '' ? 256 : ''; + options.num_predict = options.num_predict === '' ? 128 : ''; }} > {#if options.num_predict === ''} @@ -533,7 +533,7 @@ From 14440014e5dc5359a59315ba4c86021337cea2c2 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Sat, 30 Dec 2023 20:18:06 -0800 Subject: [PATCH 3/3] fix: new params do not require legacy fallback --- src/lib/components/chat/SettingsModal.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/components/chat/SettingsModal.svelte b/src/lib/components/chat/SettingsModal.svelte index 84a98acbb..58719edc5 100644 --- a/src/lib/components/chat/SettingsModal.svelte +++ b/src/lib/components/chat/SettingsModal.svelte @@ -652,7 +652,6 @@ options.top_k = settings.top_k ?? ''; options.top_p = settings.top_p ?? ''; options.num_ctx = settings.num_ctx ?? ''; - options.num_predict = settings.num_predict ?? ''; options = { ...options, ...settings.options }; options.stop = (settings?.options?.stop ?? []).join(',');