From 431b710e46b3a4fcd17a9d687ff9715a93e693e2 Mon Sep 17 00:00:00 2001
From: David Young <dy@Maximus.localdomain>
Date: Fri, 29 Dec 2023 16:09:59 +0000
Subject: [PATCH] Settings were being referenced incorrectly and as a result,
 no settings were being sent to openai endpoint. Added num_predict slider in
 advanced UI. Also added seed, stop and max_tokens (uses num_predict) being
 sent to openai endpoint.

---
 .../components/chat/Settings/Advanced.svelte  | 48 ++++++++++++++++++-
 src/lib/components/chat/SettingsModal.svelte  |  7 ++-
 src/routes/(app)/+page.svelte                 | 11 +++--
 src/routes/(app)/c/[id]/+page.svelte          | 11 +++--
 .../(app)/modelfiles/create/+page.svelte      |  4 +-
 5 files changed, 69 insertions(+), 12 deletions(-)

diff --git a/src/lib/components/chat/Settings/Advanced.svelte b/src/lib/components/chat/Settings/Advanced.svelte
index a188f107a..41ad69f33 100644
--- a/src/lib/components/chat/Settings/Advanced.svelte
+++ b/src/lib/components/chat/Settings/Advanced.svelte
@@ -12,7 +12,8 @@
 		top_k: '',
 		top_p: '',
 		tfs_z: '',
-		num_ctx: ''
+		num_ctx: '',
+		num_predict: ''
 	};
 </script>
 
@@ -507,4 +508,49 @@
 			</div>
 		{/if}
 	</div>
+	<div class=" py-0.5 w-full justify-between">
+		<div class="flex w-full justify-between">
+			<div class=" self-center text-xs font-medium">Max Tokens</div>
+
+			<button
+				class="p-1 px-3 text-xs flex rounded transition"
+				type="button"
+				on:click={() => {
+					options.num_predict = options.num_predict === '' ? 256 : '';
+				}}
+			>
+				{#if options.num_predict === ''}
+					<span class="ml-2 self-center"> Default </span>
+				{:else}
+					<span class="ml-2 self-center"> Custom </span>
+				{/if}
+			</button>
+		</div>
+
+		{#if options.num_predict !== ''}
+			<div class="flex mt-0.5 space-x-2">
+				<div class=" flex-1">
+					<input
+						id="steps-range"
+						type="range"
+						min="1"
+						max="16000"
+						step="1"
+						bind:value={options.num_predict}
+						class="w-full h-2 rounded-lg appearance-none cursor-pointer dark:bg-gray-700"
+					/>
+				</div>
+				<div class="">
+					<input
+						bind:value={options.num_predict}
+						type="number"
+						class=" bg-transparent text-center w-14"
+						min="1"
+						max="16000"
+						step="1"
+					/>
+				</div>
+			</div>
+		{/if}
+	</div>
 </div>
diff --git a/src/lib/components/chat/SettingsModal.svelte b/src/lib/components/chat/SettingsModal.svelte
index 18aa6ecad..12405537b 100644
--- a/src/lib/components/chat/SettingsModal.svelte
+++ b/src/lib/components/chat/SettingsModal.svelte
@@ -52,7 +52,8 @@
 		top_p: '',
 		stop: '',
 		tfs_z: '',
-		num_ctx: ''
+		num_ctx: '',
+		num_predict: ''
 	};
 
 	// Models
@@ -644,6 +645,7 @@
 		options.top_k = settings.top_k ?? '';
 		options.top_p = settings.top_p ?? '';
 		options.num_ctx = settings.num_ctx ?? '';
+		options.num_predict = settings.num_predict ?? '';
 		options = { ...options, ...settings.options };
 		options.stop = (settings?.options?.stop ?? []).join(',');
 
@@ -1122,7 +1124,8 @@
 											top_k: options.top_k !== '' ? options.top_k : undefined,
 											top_p: options.top_p !== '' ? options.top_p : undefined,
 											tfs_z: options.tfs_z !== '' ? options.tfs_z : undefined,
-											num_ctx: options.num_ctx !== '' ? options.num_ctx : undefined
+											num_ctx: options.num_ctx !== '' ? options.num_ctx : undefined,
+											num_predict: options.num_predict !== '' ? options.num_predict : undefined
 										}
 									});
 									show = false;
diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte
index 6f272a62e..0655cd912 100644
--- a/src/routes/(app)/+page.svelte
+++ b/src/routes/(app)/+page.svelte
@@ -368,10 +368,13 @@
 										  }
 										: { content: message.content })
 								})),
-							temperature: $settings.temperature ?? undefined,
-							top_p: $settings.top_p ?? undefined,
-							num_ctx: $settings.num_ctx ?? undefined,
-							frequency_penalty: $settings.repeat_penalty ?? undefined
+							seed: $settings.options.seed ?? undefined,
+							stop: $settings.options.stop ?? undefined,
+							temperature: $settings.options.temperature ?? undefined,
+							top_p: $settings.options.top_p ?? undefined,
+							num_ctx: $settings.options.num_ctx ?? undefined,
+							frequency_penalty: $settings.options.repeat_penalty ?? undefined,
+							max_tokens: $settings.options.num_predict ?? undefined,
 						})
 					}
 				).catch((err) => {
diff --git a/src/routes/(app)/c/[id]/+page.svelte b/src/routes/(app)/c/[id]/+page.svelte
index 9600f2982..8954a21fb 100644
--- a/src/routes/(app)/c/[id]/+page.svelte
+++ b/src/routes/(app)/c/[id]/+page.svelte
@@ -395,10 +395,13 @@
 										  }
 										: { content: message.content })
 								})),
-							temperature: $settings.temperature ?? undefined,
-							top_p: $settings.top_p ?? undefined,
-							num_ctx: $settings.num_ctx ?? undefined,
-							frequency_penalty: $settings.repeat_penalty ?? undefined
+							seed: $settings.options.seed ?? undefined,
+							stop: $settings.options.stop ?? undefined,
+							temperature: $settings.options.temperature ?? undefined,
+							top_p: $settings.options.top_p ?? undefined,
+							num_ctx: $settings.options.num_ctx ?? undefined,
+							frequency_penalty: $settings.options.repeat_penalty ?? undefined,
+							max_tokens: $settings.options.num_predict ?? undefined,
 						})
 					}
 				).catch((err) => {
diff --git a/src/routes/(app)/modelfiles/create/+page.svelte b/src/routes/(app)/modelfiles/create/+page.svelte
index 506edb9f7..23a0cffc0 100644
--- a/src/routes/(app)/modelfiles/create/+page.svelte
+++ b/src/routes/(app)/modelfiles/create/+page.svelte
@@ -51,7 +51,8 @@
 		top_k: '',
 		top_p: '',
 		tfs_z: '',
-		num_ctx: ''
+		num_ctx: '',
+		num_predict: ''
 	};
 
 	let modelfileCreator = null;
@@ -73,6 +74,7 @@ ${options.top_k !== '' ? `PARAMETER top_k ${options.top_k}` : ''}
 ${options.top_p !== '' ? `PARAMETER top_p ${options.top_p}` : ''}
 ${options.tfs_z !== '' ? `PARAMETER tfs_z ${options.tfs_z}` : ''}
 ${options.num_ctx !== '' ? `PARAMETER num_ctx ${options.num_ctx}` : ''}
+${options.num_predict !== '' ? `PARAMETER num_predict ${options.num_predict}` : ''}
 SYSTEM """${system}"""`.replace(/^\s*\n/gm, '');
 	}