feat: image support

add the ability to add images to your prompt
This commit is contained in:
Dustin Loring 2025-01-15 13:41:51 -05:00
parent 8b4c63f011
commit 63cba68b50
6 changed files with 139 additions and 33 deletions

View File

@ -7,6 +7,7 @@ import { Workbench } from '~/components/workbench/Workbench.client';
import { classNames } from '~/utils/classNames'; import { classNames } from '~/utils/classNames';
import { Messages } from './Messages.client'; import { Messages } from './Messages.client';
import { SendButton } from './SendButton.client'; import { SendButton } from './SendButton.client';
import FilePreview from './FilePreview';
import styles from './BaseChat.module.scss'; import styles from './BaseChat.module.scss';
@ -21,10 +22,14 @@ interface BaseChatProps {
enhancingPrompt?: boolean; enhancingPrompt?: boolean;
promptEnhanced?: boolean; promptEnhanced?: boolean;
input?: string; input?: string;
files?: File[];
imageDataList?: string[];
handleStop?: () => void; handleStop?: () => void;
sendMessage?: (event: React.UIEvent, messageInput?: string) => void; sendMessage?: (event: React.UIEvent, messageInput?: string) => void;
handleInputChange?: (event: React.ChangeEvent<HTMLTextAreaElement>) => void; handleInputChange?: (event: React.ChangeEvent<HTMLTextAreaElement>) => void;
enhancePrompt?: () => void; enhancePrompt?: () => void;
onFileUpload?: (files: FileList) => void;
onRemoveFile?: (index: number) => void;
} }
const EXAMPLE_PROMPTS = [ const EXAMPLE_PROMPTS = [
@ -54,6 +59,10 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
handleInputChange, handleInputChange,
enhancePrompt, enhancePrompt,
handleStop, handleStop,
files,
imageDataList,
onFileUpload,
onRemoveFile,
}, },
ref, ref,
) => { ) => {
@ -173,6 +182,20 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
</> </>
)} )}
</IconButton> </IconButton>
<input
type="file"
id="image-upload"
accept="image/*"
multiple
className="hidden"
onChange={(e) => onFileUpload?.(e.target.files!)}
/>
<IconButton
title="Upload images"
onClick={() => document.getElementById('image-upload')?.click()}
>
<div className="i-ph:image text-xl" />
</IconButton>
</div> </div>
{input.length > 3 ? ( {input.length > 3 ? (
<div className="text-xs text-bolt-elements-textTertiary"> <div className="text-xs text-bolt-elements-textTertiary">
@ -180,6 +203,15 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
</div> </div>
) : null} ) : null}
</div> </div>
{files && files.length > 0 && (
<div className="px-4 pb-4">
<FilePreview
files={files}
imageDataList={imageDataList || []}
onRemove={onRemoveFile || (() => {})}
/>
</div>
)}
</div> </div>
<div className="bg-bolt-elements-background-depth-1 pb-6">{/* Ghost Element */}</div> <div className="bg-bolt-elements-background-depth-1 pb-6">{/* Ghost Element */}</div>
</div> </div>

View File

@ -68,6 +68,8 @@ export const ChatImpl = memo(({ initialMessages, storeMessageHistory }: ChatProp
useShortcuts(); useShortcuts();
const textareaRef = useRef<HTMLTextAreaElement>(null); const textareaRef = useRef<HTMLTextAreaElement>(null);
const [files, setFiles] = useState<File[]>([]);
const [imageDataList, setImageDataList] = useState<string[]>([]);
const [chatStarted, setChatStarted] = useState(initialMessages.length > 0); const [chatStarted, setChatStarted] = useState(initialMessages.length > 0);
@ -146,6 +148,24 @@ export const ChatImpl = memo(({ initialMessages, storeMessageHistory }: ChatProp
setChatStarted(true); setChatStarted(true);
}; };
const handleFileUpload = (fileList: FileList) => {
const newFiles = Array.from(fileList);
setFiles((prevFiles) => [...prevFiles, ...newFiles]);
newFiles.forEach((file) => {
const reader = new FileReader();
reader.onloadend = () => {
setImageDataList((prev) => [...prev, reader.result as string]);
};
reader.readAsDataURL(file);
});
};
const handleRemoveFile = (index: number) => {
setFiles((prevFiles) => prevFiles.filter((_, i) => i !== index));
setImageDataList((prevList) => prevList.filter((_, i) => i !== index));
};
const sendMessage = async (_event: React.UIEvent, messageInput?: string) => { const sendMessage = async (_event: React.UIEvent, messageInput?: string) => {
const _input = messageInput || input; const _input = messageInput || input;
@ -153,13 +173,6 @@ export const ChatImpl = memo(({ initialMessages, storeMessageHistory }: ChatProp
return; return;
} }
/**
* @note (delm) Usually saving files shouldn't take long but it may take longer if there
* many unsaved files. In that case we need to block user input and show an indicator
* of some kind so the user is aware that something is happening. But I consider the
* happy case to be no unsaved files and I would expect users to save their changes
* before they send another message.
*/
await workbenchStore.saveAllFiles(); await workbenchStore.saveAllFiles();
const fileModifications = workbenchStore.getFileModifcations(); const fileModifications = workbenchStore.getFileModifcations();
@ -170,29 +183,28 @@ export const ChatImpl = memo(({ initialMessages, storeMessageHistory }: ChatProp
if (fileModifications !== undefined) { if (fileModifications !== undefined) {
const diff = fileModificationsToHTML(fileModifications); const diff = fileModificationsToHTML(fileModifications);
/**
* If we have file modifications we append a new user message manually since we have to prefix
* the user input with the file modifications and we don't want the new user input to appear
* in the prompt. Using `append` is almost the same as `handleSubmit` except that we have to
* manually reset the input and we'd have to manually pass in file attachments. However, those
* aren't relevant here.
*/
append({ role: 'user', content: `${diff}\n\n${_input}` }); append({ role: 'user', content: `${diff}\n\n${_input}` });
/**
* After sending a new message we reset all modifications since the model
* should now be aware of all the changes.
*/
workbenchStore.resetAllFileModifications(); workbenchStore.resetAllFileModifications();
} else { } else {
append({ role: 'user', content: _input }); append({
role: 'user',
content: [
{
type: 'text',
text: `${_input}`,
},
...imageDataList.map((imageData) => ({
type: 'image',
image: imageData,
})),
] as any,
});
} }
setInput(''); setInput('');
setFiles([]);
setImageDataList([]);
resetEnhancer(); resetEnhancer();
textareaRef.current?.blur(); textareaRef.current?.blur();
}; };
@ -229,6 +241,10 @@ export const ChatImpl = memo(({ initialMessages, storeMessageHistory }: ChatProp
scrollTextArea(); scrollTextArea();
}); });
}} }}
files={files}
imageDataList={imageDataList}
onFileUpload={handleFileUpload}
onRemoveFile={handleRemoveFile}
/> />
); );
}); });

View File

@ -0,0 +1,35 @@
import React from 'react';
interface FilePreviewProps {
files: File[];
imageDataList: string[];
onRemove: (index: number) => void;
}
const FilePreview: React.FC<FilePreviewProps> = ({ files, imageDataList, onRemove }) => {
if (!files || files.length === 0) {
return null;
}
return (
<div className="flex flex-row overflow-x-auto -mt-2">
{files.map((file, index) => (
<div key={file.name + file.size} className="mr-2 relative">
{imageDataList[index] && (
<div className="relative pt-4 pr-4">
<img src={imageDataList[index]} alt={file.name} className="max-h-20" />
<button
onClick={() => onRemove(index)}
className="absolute top-1 right-1 z-10 bg-black rounded-full w-5 h-5 shadow-md hover:bg-gray-900 transition-colors flex items-center justify-center"
>
<div className="i-ph:x w-3 h-3 text-gray-200" />
</button>
</div>
)}
</div>
))}
</div>
);
};
export default FilePreview;

View File

@ -1,14 +1,37 @@
import { modificationsRegex } from '~/utils/diff'; import { modificationsRegex } from '~/utils/diff';
import { Markdown } from './Markdown'; import { Markdown } from './Markdown';
interface MessageContent {
type: string;
text?: string;
image?: string;
}
interface UserMessageProps { interface UserMessageProps {
content: string; content: string | MessageContent[];
} }
export function UserMessage({ content }: UserMessageProps) { export function UserMessage({ content }: UserMessageProps) {
return ( return (
<div className="overflow-hidden pt-[4px]"> <div className="overflow-hidden pt-[4px]">
{Array.isArray(content) ? (
<div className="flex flex-col gap-4">
{content.map((item, index) => {
if (item.type === 'text') {
return <Markdown key={index} limitedMarkdown>{sanitizeUserMessage(item.text || '')}</Markdown>;
} else if (item.type === 'image') {
return (
<div key={index} className="max-w-[300px]">
<img src={item.image} alt="User uploaded" className="rounded-lg" />
</div>
);
}
return null;
})}
</div>
) : (
<Markdown limitedMarkdown>{sanitizeUserMessage(content)}</Markdown> <Markdown limitedMarkdown>{sanitizeUserMessage(content)}</Markdown>
)}
</div> </div>
); );
} }

View File

@ -45,8 +45,6 @@
"@nanostores/react": "^0.7.3", "@nanostores/react": "^0.7.3",
"@radix-ui/react-dialog": "^1.1.4", "@radix-ui/react-dialog": "^1.1.4",
"@radix-ui/react-dropdown-menu": "^2.1.4", "@radix-ui/react-dropdown-menu": "^2.1.4",
"@remix-run/cloudflare": "^2.15.2",
"@remix-run/cloudflare-pages": "^2.15.2",
"@remix-run/react": "^2.15.2", "@remix-run/react": "^2.15.2",
"@types/file-saver": "^2.0.7", "@types/file-saver": "^2.0.7",
"@uiw/codemirror-theme-vscode": "^4.23.7", "@uiw/codemirror-theme-vscode": "^4.23.7",
@ -84,6 +82,8 @@
"devDependencies": { "devDependencies": {
"@blitz/eslint-plugin": "0.1.3", "@blitz/eslint-plugin": "0.1.3",
"@cloudflare/workers-types": "^4.20250109.0", "@cloudflare/workers-types": "^4.20250109.0",
"@remix-run/cloudflare": "^2.15.2",
"@remix-run/cloudflare-pages": "^2.15.2",
"@remix-run/dev": "^2.15.2", "@remix-run/dev": "^2.15.2",
"@types/diff": "^5.2.3", "@types/diff": "^5.2.3",
"@types/react": "^18.3.18", "@types/react": "^18.3.18",

View File

@ -77,12 +77,6 @@ importers:
'@radix-ui/react-dropdown-menu': '@radix-ui/react-dropdown-menu':
specifier: ^2.1.4 specifier: ^2.1.4
version: 2.1.4(@types/react-dom@18.3.5(@types/react@18.3.18))(@types/react@18.3.18)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) version: 2.1.4(@types/react-dom@18.3.5(@types/react@18.3.18))(@types/react@18.3.18)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@remix-run/cloudflare':
specifier: ^2.15.2
version: 2.15.2(@cloudflare/workers-types@4.20250109.0)(typescript@5.7.3)
'@remix-run/cloudflare-pages':
specifier: ^2.15.2
version: 2.15.2(@cloudflare/workers-types@4.20250109.0)(typescript@5.7.3)
'@remix-run/react': '@remix-run/react':
specifier: ^2.15.2 specifier: ^2.15.2
version: 2.15.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.7.3) version: 2.15.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.7.3)
@ -189,6 +183,12 @@ importers:
'@cloudflare/workers-types': '@cloudflare/workers-types':
specifier: ^4.20250109.0 specifier: ^4.20250109.0
version: 4.20250109.0 version: 4.20250109.0
'@remix-run/cloudflare':
specifier: ^2.15.2
version: 2.15.2(@cloudflare/workers-types@4.20250109.0)(typescript@5.7.3)
'@remix-run/cloudflare-pages':
specifier: ^2.15.2
version: 2.15.2(@cloudflare/workers-types@4.20250109.0)(typescript@5.7.3)
'@remix-run/dev': '@remix-run/dev':
specifier: ^2.15.2 specifier: ^2.15.2
version: 2.15.2(@remix-run/react@2.15.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.7.3))(@types/node@22.10.6)(sass@1.77.6)(typescript@5.7.3)(vite@5.4.11(@types/node@22.10.6)(sass@1.77.6))(wrangler@3.102.0(@cloudflare/workers-types@4.20250109.0)) version: 2.15.2(@remix-run/react@2.15.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.7.3))(@types/node@22.10.6)(sass@1.77.6)(typescript@5.7.3)(vite@5.4.11(@types/node@22.10.6)(sass@1.77.6))(wrangler@3.102.0(@cloudflare/workers-types@4.20250109.0))