From 67df928c7ae953e4b725c548de08c0b61ce7d1e6 Mon Sep 17 00:00:00 2001
From: Jun Siang Cheah <git@jscheah.me>
Date: Sun, 21 Apr 2024 10:45:07 +0100
Subject: [PATCH] feat: make chunk splitting a configurable option

---
 src/lib/apis/streaming/index.ts               |  9 ++++--
 .../components/chat/Settings/Interface.svelte | 28 +++++++++++++++++++
 src/lib/i18n/locales/en-US/translation.json   |  1 +
 src/routes/(app)/+page.svelte                 |  2 +-
 src/routes/(app)/c/[id]/+page.svelte          |  8 +++---
 5 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/src/lib/apis/streaming/index.ts b/src/lib/apis/streaming/index.ts
index 4d1d2ecec..5b89a4668 100644
--- a/src/lib/apis/streaming/index.ts
+++ b/src/lib/apis/streaming/index.ts
@@ -6,9 +6,14 @@ type TextStreamUpdate = {
 // createOpenAITextStream takes a ReadableStreamDefaultReader from an SSE response,
 // and returns an async generator that emits delta updates with large deltas chunked into random sized chunks
 export async function createOpenAITextStream(
-	messageStream: ReadableStreamDefaultReader
+	messageStream: ReadableStreamDefaultReader,
+	splitLargeDeltas: boolean
 ): Promise<AsyncGenerator<TextStreamUpdate>> {
-	return streamLargeDeltasAsRandomChunks(openAIStreamToIterator(messageStream));
+	let iterator = openAIStreamToIterator(messageStream);
+	if (splitLargeDeltas) {
+		iterator = streamLargeDeltasAsRandomChunks(iterator);
+	}
+	return iterator;
 }
 
 async function* openAIStreamToIterator(
diff --git a/src/lib/components/chat/Settings/Interface.svelte b/src/lib/components/chat/Settings/Interface.svelte
index ad9e05e7f..37d7fa4ea 100644
--- a/src/lib/components/chat/Settings/Interface.svelte
+++ b/src/lib/components/chat/Settings/Interface.svelte
@@ -17,11 +17,17 @@
 	let titleAutoGenerateModelExternal = '';
 	let fullScreenMode = false;
 	let titleGenerationPrompt = '';
+	let splitLargeChunks = false;
 
 	// Interface
 	let promptSuggestions = [];
 	let showUsername = false;
 
+	const toggleSplitLargeChunks = async () => {
+		splitLargeChunks = !splitLargeChunks;
+		saveSettings({ splitLargeChunks: splitLargeChunks });
+	};
+
 	const toggleFullScreenMode = async () => {
 		fullScreenMode = !fullScreenMode;
 		saveSettings({ fullScreenMode: fullScreenMode });
@@ -197,6 +203,28 @@
 					</button>
 				</div>
 			</div>
+
+			<div>
+				<div class=" py-0.5 flex w-full justify-between">
+					<div class=" self-center text-xs font-medium">
+						{$i18n.t('Fluidly stream large external response chunks')}
+					</div>
+
+					<button
+						class="p-1 px-3 text-xs flex rounded transition"
+						on:click={() => {
+							toggleSplitLargeChunks();
+						}}
+						type="button"
+					>
+						{#if splitLargeChunks === true}
+							<span class="ml-2 self-center">{$i18n.t('On')}</span>
+						{:else}
+							<span class="ml-2 self-center">{$i18n.t('Off')}</span>
+						{/if}
+					</button>
+				</div>
+			</div>
 		</div>
 
 		<hr class=" dark:border-gray-700" />
diff --git a/src/lib/i18n/locales/en-US/translation.json b/src/lib/i18n/locales/en-US/translation.json
index be89b1b01..fdfe804ba 100644
--- a/src/lib/i18n/locales/en-US/translation.json
+++ b/src/lib/i18n/locales/en-US/translation.json
@@ -152,6 +152,7 @@
 	"File Mode": "",
 	"File not found.": "",
 	"Fingerprint spoofing detected: Unable to use initials as avatar. Defaulting to default profile image.": "",
+	"Fluidly stream large external response chunks": "",
 	"Focus chat input": "",
 	"Format your variables using square brackets like this:": "",
 	"From (Base Model)": "",
diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte
index bd8676985..9fc261773 100644
--- a/src/routes/(app)/+page.svelte
+++ b/src/routes/(app)/+page.svelte
@@ -600,7 +600,7 @@
 				.pipeThrough(splitStream('\n'))
 				.getReader();
 
-			const textStream = await createOpenAITextStream(reader);
+			const textStream = await createOpenAITextStream(reader, $settings.splitLargeChunks);
 			console.log(textStream);
 
 			for await (const update of textStream) {
diff --git a/src/routes/(app)/c/[id]/+page.svelte b/src/routes/(app)/c/[id]/+page.svelte
index 2f8ad7d0b..c230eb5c1 100644
--- a/src/routes/(app)/c/[id]/+page.svelte
+++ b/src/routes/(app)/c/[id]/+page.svelte
@@ -552,9 +552,9 @@
 				messages: [
 					$settings.system
 						? {
-							role: 'system',
-							content: $settings.system
-						}
+								role: 'system',
+								content: $settings.system
+						  }
 						: undefined,
 					...messages
 				]
@@ -612,7 +612,7 @@
 				.pipeThrough(splitStream('\n'))
 				.getReader();
 
-			const textStream = await createOpenAITextStream(reader);
+			const textStream = await createOpenAITextStream(reader, $settings.splitLargeChunks);
 			console.log(textStream);
 
 			for await (const update of textStream) {