UniqAI/bolt.new-any-llm
1
0
Fork 0
mirror of https://github.com/coleam00/bolt.new-any-llm synced 2024-12-27 14:22:32 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #332 from atrokhym/main

Browse Source 
HIGH PRIORITY - Attach images to prompts
This commit is contained in:
Cole Medin 2024-12-04 10:53:37 -06:00 committed by GitHub
commit 8c641443b7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 276 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
app/components/chat/BaseChat.tsx
View File

@ -22,6 +22,8 @@ import { ExportChatButton } from '~/components/chat/chatExportAndImport/ExportCh
import { ImportButtons } from '~/components/chat/chatExportAndImport/ImportButtons';
import { ExamplePrompts } from '~/components/chat/ExamplePrompts';
import FilePreview from './FilePreview';
// @ts-ignore TODO: Introduce proper types
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList, apiKeys }) => {
@ -85,8 +87,11 @@ interface BaseChatProps {
enhancePrompt?: () => void;
importChat?: (description: string, messages: Message[]) => Promise<void>;
exportChat?: () => void;
uploadedFiles?: File[];
setUploadedFiles?: (files: File[]) => void;
imageDataList?: string[];
setImageDataList?: (dataList: string[]) => void;
}
export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
(
{
@ -96,20 +101,24 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
showChat = true,
chatStarted = false,
isStreaming = false,
enhancingPrompt = false,
promptEnhanced = false,
messages,
input = '',
model,
setModel,
provider,
setProvider,
sendMessage,
input = '',
enhancingPrompt,
handleInputChange,
promptEnhanced,
enhancePrompt,
sendMessage,
handleStop,
importChat,
exportChat,
uploadedFiles = [],
setUploadedFiles,
imageDataList = [],
setImageDataList,
messages,
},
ref,
) => {
@ -159,6 +168,58 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
}
};
const handleFileUpload = () => {
const input = document.createElement('input');
input.type = 'file';
input.accept = 'image/*';
input.onchange = async (e) => {
const file = (e.target as HTMLInputElement).files?.[0];
if (file) {
const reader = new FileReader();
reader.onload = (e) => {
const base64Image = e.target?.result as string;
setUploadedFiles?.([...uploadedFiles, file]);
setImageDataList?.([...imageDataList, base64Image]);
};
reader.readAsDataURL(file);
}
};
input.click();
};
const handlePaste = async (e: React.ClipboardEvent) => {
const items = e.clipboardData?.items;
if (!items) {
return;
}
for (const item of items) {
if (item.type.startsWith('image/')) {
e.preventDefault();
const file = item.getAsFile();
if (file) {
const reader = new FileReader();
reader.onload = (e) => {
const base64Image = e.target?.result as string;
setUploadedFiles?.([...uploadedFiles, file]);
setImageDataList?.([...imageDataList, base64Image]);
};
reader.readAsDataURL(file);
}
break;
}
}
};
const baseChat = (
<div
ref={ref}
@ -276,7 +337,14 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
)}
</div>
</div>
<FilePreview
files={uploadedFiles}
imageDataList={imageDataList}
onRemove={(index) => {
setUploadedFiles?.(uploadedFiles.filter((_, i) => i !== index));
setImageDataList?.(imageDataList.filter((_, i) => i !== index));
}}
/>
<div
className={classNames(
'relative shadow-xs border border-bolt-elements-borderColor backdrop-blur rounded-lg',
@ -284,9 +352,41 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
>
<textarea
ref={textareaRef}
className={
'w-full pl-4 pt-4 pr-16 focus:outline-none resize-none text-bolt-elements-textPrimary placeholder-bolt-elements-textTertiary bg-transparent text-sm'
className={classNames(
'w-full pl-4 pt-4 pr-16 focus:outline-none resize-none text-bolt-elements-textPrimary placeholder-bolt-elements-textTertiary bg-transparent text-sm',
'transition-all duration-200',
'hover:border-bolt-elements-focus',
)}
onDragEnter={(e) => {
e.preventDefault();
e.currentTarget.style.border = '2px solid #1488fc';
}}
onDragOver={(e) => {
e.preventDefault();
e.currentTarget.style.border = '2px solid #1488fc';
}}
onDragLeave={(e) => {
e.preventDefault();
e.currentTarget.style.border = '1px solid var(--bolt-elements-borderColor)';
}}
onDrop={(e) => {
e.preventDefault();
e.currentTarget.style.border = '1px solid var(--bolt-elements-borderColor)';
const files = Array.from(e.dataTransfer.files);
files.forEach((file) => {
if (file.type.startsWith('image/')) {
const reader = new FileReader();
reader.onload = (e) => {
const base64Image = e.target?.result as string;
setUploadedFiles?.([...uploadedFiles, file]);
setImageDataList?.([...imageDataList, base64Image]);
};
reader.readAsDataURL(file);
}
});
}}
onKeyDown={(event) => {
if (event.key === 'Enter') {
if (event.shiftKey) {
@ -302,6 +402,7 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
onChange={(event) => {
handleInputChange?.(event);
}}
onPaste={handlePaste}
style={{
minHeight: TEXTAREA_MIN_HEIGHT,
maxHeight: TEXTAREA_MAX_HEIGHT,
@ -312,7 +413,7 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
<ClientOnly>
{() => (
<SendButton
show={input.length > 0 || isStreaming}
show={input.length > 0 || isStreaming || uploadedFiles.length > 0}
isStreaming={isStreaming}
onClick={(event) => {
if (isStreaming) {
@ -320,21 +421,28 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
return;
}
if (input.length > 0 || uploadedFiles.length > 0) {
sendMessage?.(event);
}
}}
/>
)}
</ClientOnly>
<div className="flex justify-between items-center text-sm p-4 pt-2">
<div className="flex gap-1 items-center">
<IconButton title="Upload file" className="transition-all" onClick={() => handleFileUpload()}>
<div className="i-ph:paperclip text-xl"></div>
</IconButton>
<IconButton
title="Enhance prompt"
disabled={input.length === 0 || enhancingPrompt}
className={classNames('transition-all', {
'opacity-100!': enhancingPrompt,
'text-bolt-elements-item-contentAccent! pr-1.5 enabled:hover:bg-bolt-elements-item-backgroundAccent!':
promptEnhanced,
})}
className={classNames(
'transition-all',
enhancingPrompt ? 'opacity-100' : '',
promptEnhanced ? 'text-bolt-elements-item-contentAccent' : '',
promptEnhanced ? 'pr-1.5' : '',
promptEnhanced ? 'enabled:hover:bg-bolt-elements-item-backgroundAccent' : '',
)}
onClick={() => enhancePrompt?.()}
>
{enhancingPrompt ? (

43
app/components/chat/Chat.client.tsx
View File

@ -12,7 +12,6 @@ import { useMessageParser, usePromptEnhancer, useShortcuts, useSnapScroll } from
import { description, useChatHistory } from '~/lib/persistence';
import { chatStore } from '~/lib/stores/chat';
import { workbenchStore } from '~/lib/stores/workbench';
import { fileModificationsToHTML } from '~/utils/diff';
import { DEFAULT_MODEL, DEFAULT_PROVIDER, PROMPT_COOKIE_KEY, PROVIDER_LIST } from '~/utils/constants';
import { cubicEasingFn } from '~/utils/easings';
import { createScopedLogger, renderLogger } from '~/utils/logger';
@ -89,8 +88,10 @@ export const ChatImpl = memo(
useShortcuts();
const textareaRef = useRef<HTMLTextAreaElement>(null);
const [chatStarted, setChatStarted] = useState(initialMessages.length > 0);
const [uploadedFiles, setUploadedFiles] = useState<File[]>([]); // Move here
const [imageDataList, setImageDataList] = useState<string[]>([]); // Move here
const [model, setModel] = useState(() => {
const savedModel = Cookies.get('selectedModel');
return savedModel || DEFAULT_MODEL;
@ -206,8 +207,6 @@ export const ChatImpl = memo(
runAnimation();
if (fileModifications !== undefined) {
const diff = fileModificationsToHTML(fileModifications);
/**
* If we have file modifications we append a new user message manually since we have to prefix
* the user input with the file modifications and we don't want the new user input to appear
@ -215,7 +214,19 @@ export const ChatImpl = memo(
* manually reset the input and we'd have to manually pass in file attachments. However, those
* aren't relevant here.
*/
append({ role: 'user', content: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${diff}\n\n${_input}` });
append({
role: 'user',
content: [
{
type: 'text',
text: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${_input}`,
},
...imageDataList.map((imageData) => ({
type: 'image',
image: imageData,
})),
] as any, // Type assertion to bypass compiler check
});
/**
* After sending a new message we reset all modifications since the model
@ -223,12 +234,28 @@ export const ChatImpl = memo(
*/
workbenchStore.resetAllFileModifications();
} else {
append({ role: 'user', content: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${_input}` });
append({
role: 'user',
content: [
{
type: 'text',
text: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${_input}`,
},
...imageDataList.map((imageData) => ({
type: 'image',
image: imageData,
})),
] as any, // Type assertion to bypass compiler check
});
}
setInput('');
Cookies.remove(PROMPT_COOKIE_KEY);
// Add file cleanup here
setUploadedFiles([]);
setImageDataList([]);
resetEnhancer();
textareaRef.current?.blur();
@ -321,6 +348,10 @@ export const ChatImpl = memo(
apiKeys,
);
}}
uploadedFiles={uploadedFiles}
setUploadedFiles={setUploadedFiles}
imageDataList={imageDataList}
setImageDataList={setImageDataList}
/>
);
},

35
app/components/chat/FilePreview.tsx Normal file
View File

@ -0,0 +1,35 @@
import React from 'react';
interface FilePreviewProps {
files: File[];
imageDataList: string[];
onRemove: (index: number) => void;
}
const FilePreview: React.FC<FilePreviewProps> = ({ files, imageDataList, onRemove }) => {
if (!files || files.length === 0) {
return null;
}
return (
<div className="flex flex-row overflow-x-auto -mt-2">
{files.map((file, index) => (
<div key={file.name + file.size} className="mr-2 relative">
{imageDataList[index] && (
<div className="relative pt-4 pr-4">
<img src={imageDataList[index]} alt={file.name} className="max-h-20" />
<button
onClick={() => onRemove(index)}
className="absolute top-1 right-1 z-10 bg-black rounded-full w-5 h-5 shadow-md hover:bg-gray-900 transition-colors flex items-center justify-center"
>
<div className="i-ph:x w-3 h-3 text-gray-200" />
</button>
</div>
)}
</div>
))}
</div>
);
};
export default FilePreview;

5
app/components/chat/SendButton.client.tsx
View File

@ -4,11 +4,12 @@ interface SendButtonProps {
show: boolean;
isStreaming?: boolean;
onClick?: (event: React.MouseEvent<HTMLButtonElement, MouseEvent>) => void;
onImagesSelected?: (images: File[]) => void;
}
const customEasingFn = cubicBezier(0.4, 0, 0.2, 1);
export function SendButton({ show, isStreaming, onClick }: SendButtonProps) {
export const SendButton = ({ show, isStreaming, onClick }: SendButtonProps) => {
return (
<AnimatePresence>
{show ? (
@ -30,4 +31,4 @@ export function SendButton({ show, isStreaming, onClick }: SendButtonProps) {
) : null}
</AnimatePresence>
);
}
};

42
app/components/chat/UserMessage.tsx
View File

@ -2,26 +2,52 @@
* @ts-nocheck
* Preventing TS checks with files presented in the video for a better presentation.
*/
import { modificationsRegex } from '~/utils/diff';
import { MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants';
import { Markdown } from './Markdown';
interface UserMessageProps {
content: string;
content: string | Array<{ type: string; text?: string; image?: string }>;
}
export function UserMessage({ content }: UserMessageProps) {
if (Array.isArray(content)) {
const textItem = content.find((item) => item.type === 'text');
const textContent = sanitizeUserMessage(textItem?.text || '');
const images = content.filter((item) => item.type === 'image' && item.image);
return (
<div className="overflow-hidden pt-[4px]">
<Markdown limitedMarkdown>{sanitizeUserMessage(content)}</Markdown>
<div className="flex items-start gap-4">
<div className="flex-1">
<Markdown limitedMarkdown>{textContent}</Markdown>
</div>
{images.length > 0 && (
<div className="flex-shrink-0 w-[160px]">
{images.map((item, index) => (
<div key={index} className="relative">
<img
src={item.image}
alt={`Uploaded image ${index + 1}`}
className="w-full h-[160px] rounded-lg object-cover border border-bolt-elements-borderColor"
/>
</div>
))}
</div>
)}
</div>
</div>
);
}
const textContent = sanitizeUserMessage(content);
return (
<div className="overflow-hidden pt-[4px]">
<Markdown limitedMarkdown>{textContent}</Markdown>
</div>
);
}
function sanitizeUserMessage(content: string) {
return content
.replace(modificationsRegex, '')
.replace(MODEL_REGEX, 'Using: $1')
.replace(PROVIDER_REGEX, ' ($1)\n\n')
.trim();
return content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '');
}

4
app/components/sidebar/Menu.client.tsx
View File

@ -33,7 +33,7 @@ const menuVariants = {
type DialogContent = { type: 'delete'; item: ChatHistoryItem } | null;
export function Menu() {
export const Menu = () => {
const { duplicateCurrentChat, exportChat } = useChatHistory();
const menuRef = useRef<HTMLDivElement>(null);
const [list, setList] = useState<ChatHistoryItem[]>([]);
@ -206,4 +206,4 @@ export function Menu() {
</div>
</motion.div>
);
}
};

7
app/lib/.server/llm/model.ts
View File

@ -128,7 +128,12 @@ export function getXAIModel(apiKey: OptionalApiKey, model: string) {
}
export function getModel(provider: string, model: string, env: Env, apiKeys?: Record<string, string>) {
const apiKey = getAPIKey(env, provider, apiKeys);
/*
* let apiKey; // Declare first
* let baseURL;
*/
const apiKey = getAPIKey(env, provider, apiKeys); // Then assign
const baseURL = getBaseURL(env, provider);
switch (provider) {

35
app/lib/.server/llm/stream-text.ts
View File

@ -26,16 +26,37 @@ export type Messages = Message[];
export type StreamingOptions = Omit<Parameters<typeof _streamText>[0], 'model'>;
function extractPropertiesFromMessage(message: Message): { model: string; provider: string; content: string } {
// Extract model
const modelMatch = message.content.match(MODEL_REGEX);
const textContent = Array.isArray(message.content)
? message.content.find((item) => item.type === 'text')?.text || ''
: message.content;
const modelMatch = textContent.match(MODEL_REGEX);
const providerMatch = textContent.match(PROVIDER_REGEX);
/*
* Extract model
* const modelMatch = message.content.match(MODEL_REGEX);
*/
const model = modelMatch ? modelMatch[1] : DEFAULT_MODEL;
// Extract provider
const providerMatch = message.content.match(PROVIDER_REGEX);
/*
* Extract provider
* const providerMatch = message.content.match(PROVIDER_REGEX);
*/
const provider = providerMatch ? providerMatch[1] : DEFAULT_PROVIDER;
// Remove model and provider lines from content
const cleanedContent = message.content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '').trim();
const cleanedContent = Array.isArray(message.content)
? message.content.map((item) => {
if (item.type === 'text') {
return {
type: 'text',
text: item.text?.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, ''),
};
}
return item; // Preserve image_url and other types as is
})
: textContent.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '');
return { model, provider, content: cleanedContent };
}
@ -65,10 +86,10 @@ export function streamText(messages: Messages, env: Env, options?: StreamingOpti
const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS;
return _streamText({
...options,
model: getModel(currentProvider, currentModel, env, apiKeys),
system: getSystemPrompt(),
maxTokens: dynamicMaxTokens,
messages: convertToCoreMessages(processedMessages),
...options,
});
}

6
app/lib/stores/files.ts
View File

@ -212,9 +212,5 @@ function isBinaryFile(buffer: Uint8Array | undefined) {
* array buffer.
*/
function convertToBuffer(view: Uint8Array): Buffer {
const buffer = new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
Object.setPrototypeOf(buffer, Buffer.prototype);
return buffer as Buffer;
return Buffer.from(view.buffer, view.byteOffset, view.byteLength);
}

4
app/routes/api.chat.ts
View File

@ -32,8 +32,9 @@ function parseCookies(cookieHeader) {
}
async function chatAction({ context, request }: ActionFunctionArgs) {
const { messages } = await request.json<{
const { messages, model } = await request.json<{
messages: Messages;
model: string;
}>();
const cookieHeader = request.headers.get('Cookie');
@ -47,6 +48,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) {
const options: StreamingOptions = {
toolChoice: 'none',
apiKeys,
model,
onFinish: async ({ text: content, finishReason }) => {
if (finishReason !== 'length') {
return stream.close();

2
app/utils/logger.ts
View File

@ -11,7 +11,7 @@ interface Logger {
setLevel: (level: DebugLevel) => void;
}
let currentLevel: DebugLevel = (import.meta.env.VITE_LOG_LEVEL ?? import.meta.env.DEV) ? 'debug' : 'info';
let currentLevel: DebugLevel = import.meta.env.VITE_LOG_LEVEL ?? import.meta.env.DEV ? 'debug' : 'info';
const isWorker = 'HTMLRewriter' in globalThis;
const supportsColor = !isWorker;

3
vite.config.ts
View File

@ -19,8 +19,7 @@ export default defineConfig((config) => {
future: {
v3_fetcherPersist: true,
v3_relativeSplatPath: true,
v3_throwAbortReason: true,
v3_lazyRouteDiscovery: true,
v3_throwAbortReason: true
},
}),
UnoCSS(),