From e70cff67c2b9faf9c24bca6a5fd1e8b2a672a790 Mon Sep 17 00:00:00 2001
From: "Timothy J. Baek" <timothyjrbeck@gmail.com>
Date: Sat, 25 Nov 2023 00:21:07 -0800
Subject: [PATCH] feat: multimodal support prep

---
 src/lib/components/chat/MessageInput.svelte | 133 +++++++++++++++-----
 src/lib/components/chat/Messages.svelte     |  11 ++
 src/lib/components/layout/Sidebar.svelte    |   4 +-
 src/routes/(app)/+page.svelte               |   7 +-
 4 files changed, 117 insertions(+), 38 deletions(-)

diff --git a/src/lib/components/chat/MessageInput.svelte b/src/lib/components/chat/MessageInput.svelte
index 69760ced3..8c2ba7f70 100644
--- a/src/lib/components/chat/MessageInput.svelte
+++ b/src/lib/components/chat/MessageInput.svelte
@@ -1,5 +1,6 @@
 <script lang="ts">
 	import { settings } from '$lib/stores';
+	import toast from 'svelte-french-toast';
 	import Suggestions from './MessageInput/Suggestions.svelte';
 
 	export let submitPrompt: Function;
@@ -8,6 +9,11 @@
 	export let suggestions = 'true';
 	export let autoScroll = true;
 
+	let filesInputElement;
+	let inputFiles;
+
+	export let files = [];
+
 	export let fileUploadEnabled = false;
 	export let speechRecognitionEnabled = true;
 	export let speechRecognitionListening = false;
@@ -111,43 +117,82 @@
 			{/if}
 
 			<div class="bg-gradient-to-t from-white dark:from-gray-800 from-40% pb-2">
+				<input
+					bind:this={filesInputElement}
+					bind:files={inputFiles}
+					type="file"
+					hidden
+					on:change={() => {
+						let reader = new FileReader();
+						reader.onload = (event) => {
+							files = [
+								...files,
+								{
+									type: 'image',
+									url: `${event.target.result}`
+								}
+							];
+							inputFiles = null;
+						};
+
+						if (
+							inputFiles &&
+							inputFiles.length > 0 &&
+							['image/gif', 'image/jpeg', 'image/png'].includes(inputFiles[0]['type'])
+						) {
+							reader.readAsDataURL(inputFiles[0]);
+						} else {
+							toast.error(`Unsupported File Type '${inputFiles[0]['type']}'.`);
+							inputFiles = null;
+						}
+					}}
+				/>
 				<form
-					class=" flex relative w-full"
+					class=" flex flex-col relative w-full rounded-xl border dark:border-gray-600"
 					on:submit|preventDefault={() => {
 						submitPrompt(prompt);
 					}}
 				>
-					<textarea
-						id="chat-textarea"
-						class="rounded-xl dark:bg-gray-800 dark:text-gray-100 outline-none border dark:border-gray-600 w-full py-3
-                        {fileUploadEnabled ? 'pl-12' : 'pl-5'} {speechRecognitionEnabled
-							? 'pr-20'
-							: 'pr-12'} resize-none"
-						placeholder={speechRecognitionListening ? 'Listening...' : 'Send a message'}
-						bind:value={prompt}
-						on:keypress={(e) => {
-							if (e.keyCode == 13 && !e.shiftKey) {
-								e.preventDefault();
-							}
-							if (prompt !== '' && e.keyCode == 13 && !e.shiftKey) {
-								submitPrompt(prompt);
-							}
-						}}
-						rows="1"
-						on:input={(e) => {
-							e.target.style.height = '';
-							e.target.style.height = Math.min(e.target.scrollHeight, 200) + 2 + 'px';
-						}}
-					/>
+					{#if files.length > 0}
+						<div class="ml-2 mt-2 mb-1 flex space-x-2">
+							{#each files as file, fileIdx}
+								<div class=" relative group">
+									<img src={file.url} alt="input" class=" h-16 w-16 rounded-xl bg-cover" />
 
-					{#if fileUploadEnabled}
-						<div class=" absolute left-0 bottom-0">
-							<div class="pl-2.5 pb-[9px]">
+									<div class=" absolute -top-1 -right-1">
+										<button
+											class=" bg-gray-400 text-white border border-white rounded-full group-hover:visible invisible transition"
+											type="button"
+											on:click={() => {
+												files.splice(fileIdx, 1);
+												files = files;
+											}}
+										>
+											<svg
+												xmlns="http://www.w3.org/2000/svg"
+												viewBox="0 0 20 20"
+												fill="currentColor"
+												class="w-4 h-4"
+											>
+												<path
+													d="M6.28 5.22a.75.75 0 00-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 101.06 1.06L10 11.06l3.72 3.72a.75.75 0 101.06-1.06L11.06 10l3.72-3.72a.75.75 0 00-1.06-1.06L10 8.94 6.28 5.22z"
+												/>
+											</svg>
+										</button>
+									</div>
+								</div>
+							{/each}
+						</div>
+					{/if}
+
+					<div class=" flex">
+						{#if fileUploadEnabled}
+							<div class=" self-end mb-2 ml-1.5">
 								<button
-									class="  text-gray-600 dark:text-gray-200 transition rounded-lg p-1.5"
+									class="  text-gray-600 dark:text-gray-200 transition rounded-lg p-1 ml-1"
 									type="button"
 									on:click={() => {
-										console.log('file');
+										filesInputElement.click();
 									}}
 								>
 									<svg
@@ -164,15 +209,35 @@
 									</svg>
 								</button>
 							</div>
-						</div>
-					{/if}
+						{/if}
 
-					<div class=" absolute right-0 bottom-0">
-						<div class="pr-2.5 pb-[9px]">
+						<textarea
+							id="chat-textarea"
+							class=" dark:bg-gray-800 dark:text-gray-100 outline-none w-full py-3 px-2 {fileUploadEnabled
+								? ''
+								: ' pl-4'} rounded-xl resize-none"
+							placeholder={speechRecognitionListening ? 'Listening...' : 'Send a message'}
+							bind:value={prompt}
+							on:keypress={(e) => {
+								if (e.keyCode == 13 && !e.shiftKey) {
+									e.preventDefault();
+								}
+								if (prompt !== '' && e.keyCode == 13 && !e.shiftKey) {
+									submitPrompt(prompt);
+								}
+							}}
+							rows="1"
+							on:input={(e) => {
+								e.target.style.height = '';
+								e.target.style.height = Math.min(e.target.scrollHeight, 200) + 'px';
+							}}
+						/>
+
+						<div class="self-end mb-2 flex space-x-0.5 mr-2">
 							{#if messages.length == 0 || messages.at(-1).done == true}
 								{#if speechRecognitionEnabled}
 									<button
-										class=" text-gray-600 dark:text-gray-300 transition rounded-lg p-1 mr-0.5"
+										class=" text-gray-600 dark:text-gray-300 transition rounded-lg p-1.5 mr-0.5 self-center"
 										type="button"
 										on:click={() => {
 											speechRecognitionHandler();
@@ -233,7 +298,7 @@
 								<button
 									class="{prompt !== ''
 										? 'bg-black text-white hover:bg-gray-900 dark:bg-white dark:text-black dark:hover:bg-gray-100 '
-										: 'text-white bg-gray-100 dark:text-gray-800 dark:bg-gray-600 disabled'} transition rounded-lg p-1"
+										: 'text-white bg-gray-100 dark:text-gray-800 dark:bg-gray-600 disabled'} transition rounded-lg p-1 mr-0.5 w-7 h-7 self-center"
 									type="submit"
 									disabled={prompt === ''}
 								>
diff --git a/src/lib/components/chat/Messages.svelte b/src/lib/components/chat/Messages.svelte
index 503537e65..63855f467 100644
--- a/src/lib/components/chat/Messages.svelte
+++ b/src/lib/components/chat/Messages.svelte
@@ -409,6 +409,17 @@
 										</div>
 									{:else}
 										<div class="w-full">
+											{#if message.files}
+												<div class="my-3">
+													{#each message.files as file}
+														<div>
+															{#if file.type === 'image'}
+																<img src={file.url} alt="input" class=" max-h-96" />
+															{/if}
+														</div>
+													{/each}
+												</div>
+											{/if}
 											{message.content}
 
 											<div class=" flex justify-start space-x-1">
diff --git a/src/lib/components/layout/Sidebar.svelte b/src/lib/components/layout/Sidebar.svelte
index de1ca16a8..283621cad 100644
--- a/src/lib/components/layout/Sidebar.svelte
+++ b/src/lib/components/layout/Sidebar.svelte
@@ -119,7 +119,7 @@
 			</button>
 		</div>
 
-		<div class="px-2.5 flex justify-center my-1">
+		<!-- <div class="px-2.5 flex justify-center my-1">
 			<button
 				class="flex-grow flex space-x-3 rounded-md px-3 py-2 hover:bg-gray-900 transition"
 				on:click={async () => {
@@ -147,7 +147,7 @@
 					<div class=" self-center font-medium text-sm">Presets</div>
 				</div>
 			</button>
-		</div>
+		</div> -->
 
 		<div class="px-2.5 mt-1 mb-2 flex justify-center space-x-2">
 			<div class="flex w-full">
diff --git a/src/routes/(app)/+page.svelte b/src/routes/(app)/+page.svelte
index 9cc3e55c8..fee723c00 100644
--- a/src/routes/(app)/+page.svelte
+++ b/src/routes/(app)/+page.svelte
@@ -21,6 +21,7 @@
 
 	let title = '';
 	let prompt = '';
+	let files = [];
 
 	let messages = [];
 	let history = {
@@ -358,7 +359,8 @@
 				parentId: messages.length !== 0 ? messages.at(-1).id : null,
 				childrenIds: [],
 				role: 'user',
-				content: userPrompt
+				content: userPrompt,
+				files: files.length > 0 ? files : undefined
 			};
 
 			if (messages.length !== 0) {
@@ -369,6 +371,7 @@
 			history.currentId = userMessageId;
 
 			prompt = '';
+			files = [];
 
 			if (messages.length == 0) {
 				await $db.createNewChat({
@@ -477,5 +480,5 @@
 		</div>
 	</div>
 
-	<MessageInput bind:prompt bind:autoScroll {messages} {submitPrompt} {stopResponse} />
+	<MessageInput bind:prompt bind:files bind:autoScroll {messages} {submitPrompt} {stopResponse} />
 </div>