From c503fd244e93357ea1ddb94cf5aea6b938e5820a Mon Sep 17 00:00:00 2001 From: Brian Hackett Date: Tue, 18 Mar 2025 19:18:12 -0700 Subject: [PATCH] Updates for async chat format (#71) --- app/components/chat/AssistantMessage.tsx | 37 -- app/components/chat/BaseChat.tsx | 3 +- app/components/chat/Chat.client.tsx | 220 ++++-------- app/components/chat/ImportFolderButton.tsx | 2 +- app/components/chat/LoadProblemButton.tsx | 2 +- app/components/chat/MessageContents.tsx | 38 ++ app/components/chat/Messages.client.tsx | 14 +- app/components/chat/UserMessage.tsx | 47 --- .../chatExportAndImport/ImportButtons.tsx | 2 +- app/components/settings/data/DataTab.tsx | 2 +- app/components/sidebar/SaveReproduction.tsx | 13 - app/lib/common/prompts/prompts.ts | 103 ------ app/lib/hooks/pingTelemetry.ts | 12 - app/lib/persistence/db.ts | 2 +- app/lib/persistence/useChatHistory.ts | 11 +- app/lib/replay/Problems.ts | 4 +- app/lib/replay/SimulationPrompt.ts | 330 ++---------------- app/utils/folderImport.ts | 6 +- 18 files changed, 146 insertions(+), 702 deletions(-) delete mode 100644 app/components/chat/AssistantMessage.tsx create mode 100644 app/components/chat/MessageContents.tsx delete mode 100644 app/components/chat/UserMessage.tsx delete mode 100644 app/lib/common/prompts/prompts.ts diff --git a/app/components/chat/AssistantMessage.tsx b/app/components/chat/AssistantMessage.tsx deleted file mode 100644 index b549ce44..00000000 --- a/app/components/chat/AssistantMessage.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { memo } from 'react'; -import { Markdown } from './Markdown'; -import type { JSONValue } from 'ai'; - -interface AssistantMessageProps { - content: string; - annotations?: JSONValue[]; -} - -export function getAnnotationsTokensUsage(annotations: JSONValue[] | undefined) { - const filteredAnnotations = (annotations?.filter( - (annotation: JSONValue) => annotation && typeof annotation === 'object' && Object.keys(annotation).includes('type'), - ) || []) as { type: string; value: any }[]; - - const usage: { - completionTokens: number; - promptTokens: number; - totalTokens: number; - } = filteredAnnotations.find((annotation) => annotation.type === 'usage')?.value; - - return usage; -} - -export const AssistantMessage = memo(({ content, annotations }: AssistantMessageProps) => { - const usage = getAnnotationsTokensUsage(annotations); - - return ( -
- {usage && ( -
- Tokens: {usage.totalTokens} (prompt: {usage.promptTokens}, completion: {usage.completionTokens}) -
- )} - {content} -
- ); -}); diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index fbd43e78..544f09d4 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -9,7 +9,8 @@ import { IconButton } from '~/components/ui/IconButton'; import { Workbench } from '~/components/workbench/Workbench.client'; import { classNames } from '~/utils/classNames'; import { Messages } from './Messages.client'; -import { getPreviousRepositoryId, type Message } from '~/lib/persistence/useChatHistory'; +import { getPreviousRepositoryId } from '~/lib/persistence/useChatHistory'; +import type { Message } from '~/lib/persistence/message'; import { SendButton } from './SendButton.client'; import * as Tooltip from '@radix-ui/react-tooltip'; diff --git a/app/components/chat/Chat.client.tsx b/app/components/chat/Chat.client.tsx index b002d098..20adba4c 100644 --- a/app/components/chat/Chat.client.tsx +++ b/app/components/chat/Chat.client.tsx @@ -18,12 +18,11 @@ import { debounce } from '~/utils/debounce'; import { useSearchParams } from '@remix-run/react'; import { createSampler } from '~/utils/sampler'; import { - getSimulationRecording, - getSimulationEnhancedPrompt, simulationAddData, + simulationFinishData, simulationRepositoryUpdated, - shouldUseSimulation, - sendDeveloperChatMessage, + sendChatMessage, + type ChatReference, } from '~/lib/replay/SimulationPrompt'; import { getIFrameSimulationData } from '~/lib/replay/Recording'; import { getCurrentIFrame } from '~/components/workbench/Preview'; @@ -32,8 +31,9 @@ import { anthropicNumFreeUsesCookieName, anthropicApiKeyCookieName, maxFreeUses import { getNutLoginKey, submitFeedback } from '~/lib/replay/Problems'; import { ChatMessageTelemetry, pingTelemetry } from '~/lib/hooks/pingTelemetry'; import type { RejectChangeData } from './ApproveChange'; -import { generateRandomId } from '~/lib/replay/ReplayProtocolClient'; -import { getMessagesRepositoryId, getPreviousRepositoryId, type Message } from '~/lib/persistence/useChatHistory'; +import { assert, generateRandomId } from '~/lib/replay/ReplayProtocolClient'; +import { getMessagesRepositoryId, getPreviousRepositoryId } from '~/lib/persistence/useChatHistory'; +import type { Message } from '~/lib/persistence/message'; const toastAnimation = cssTransition({ enter: 'animated fadeInRight', @@ -64,15 +64,11 @@ async function flushSimulationData() { //console.log("HaveSimulationData", simulationData.length); // Add the simulation data to the chat. - await simulationAddData(simulationData); + simulationAddData(simulationData); } -let gLockSimulationData = false; - setInterval(async () => { - if (!gLockSimulationData) { - flushSimulationData(); - } + flushSimulationData(); }, 1000); export function Chat() { @@ -154,16 +150,6 @@ async function clearActiveChat() { gActiveChatMessageTelemetry = undefined; } -function buildMessageId(prefix: string, chatId: string) { - return `${prefix}-${chatId}`; -} - -const EnhancedPromptPrefix = 'enhanced-prompt'; - -export function isEnhancedPromptMessage(message: Message): boolean { - return message.id.startsWith(EnhancedPromptPrefix); -} - export const ChatImpl = memo( ({ description, initialMessages, storeMessageHistory, importChat, exportChat }: ChatProps) => { const textareaRef = useRef(null); @@ -261,50 +247,6 @@ export const ChatImpl = memo( setChatStarted(true); }; - const createRecording = async (chatId: string) => { - let recordingId, message; - - try { - recordingId = await getSimulationRecording(); - message = `[Recording of the bug](https://app.replay.io/recording/${recordingId})\n\n`; - } catch (e) { - console.error('Error creating recording', e); - message = 'Error creating recording.'; - } - - const recordingMessage: Message = { - id: buildMessageId('create-recording', chatId), - role: 'assistant', - content: message, - }; - - return { recordingId, recordingMessage }; - }; - - const getEnhancedPrompt = async (chatId: string, userMessage: string) => { - let enhancedPrompt, - message, - hadError = false; - - try { - const mouseData = getCurrentMouseData(); - enhancedPrompt = await getSimulationEnhancedPrompt(messages, userMessage, mouseData); - message = `Explanation of the bug:\n\n${enhancedPrompt}`; - } catch (e) { - console.error('Error enhancing prompt', e); - message = 'Error enhancing prompt.'; - hadError = true; - } - - const enhancedPromptMessage: Message = { - id: buildMessageId(EnhancedPromptPrefix, chatId), - role: 'assistant', - content: message, - }; - - return { enhancedPrompt, enhancedPromptMessage, hadError }; - }; - const sendMessage = async (messageInput?: string) => { const _input = messageInput || input; const numAbortsAtStart = gNumAborts; @@ -340,130 +282,81 @@ export const ChatImpl = memo( setActiveChatId(chatId); const userMessage: Message = { - id: buildMessageId('user', chatId), + id: `user-${chatId}`, role: 'user', - content: [ - { - type: 'text', - text: _input, - }, - ...imageDataList.map((imageData) => ({ - type: 'image', - image: imageData, - })), - ] as any, // Type assertion to bypass compiler check + type: 'text', + content: _input, }; let newMessages = [...messages, userMessage]; + + imageDataList.forEach((imageData, index) => { + const imageMessage: Message = { + id: `image-${chatId}-${index}`, + role: 'user', + type: 'image', + dataURL: imageData, + }; + newMessages.push(imageMessage); + }); + setMessages(newMessages); // Add file cleanup here setUploadedFiles([]); setImageDataList([]); - let simulation = false; - - try { - simulation = chatStarted && (await shouldUseSimulation(_input)); - } catch (e) { - console.error('Error checking simulation', e); - } - - if (numAbortsAtStart != gNumAborts) { - return; - } - - console.log('UseSimulation', simulation); - - let simulationStatus = 'NoSimulation'; - - if (simulation) { - gActiveChatMessageTelemetry.startSimulation(); - - gLockSimulationData = true; - - try { - await flushSimulationData(); - - const createRecordingPromise = createRecording(chatId); - const enhancedPromptPromise = getEnhancedPrompt(chatId, _input); - - const { recordingId, recordingMessage } = await createRecordingPromise; - - if (numAbortsAtStart != gNumAborts) { - return; - } - - console.log('RecordingMessage', recordingMessage); - newMessages = [...newMessages, recordingMessage]; - setMessages(newMessages); - - if (recordingId) { - const info = await enhancedPromptPromise; - - if (numAbortsAtStart != gNumAborts) { - return; - } - - console.log('EnhancedPromptMessage', info.enhancedPromptMessage); - newMessages = [...newMessages, info.enhancedPromptMessage]; - setMessages(newMessages); - - simulationStatus = info.hadError ? 'PromptError' : 'Success'; - } else { - simulationStatus = 'RecordingError'; - } - - gActiveChatMessageTelemetry.endSimulation(simulationStatus); - } finally { - gLockSimulationData = false; - } - } + await flushSimulationData(); + simulationFinishData(); chatStore.setKey('aborted', false); runAnimation(); - gActiveChatMessageTelemetry.sendPrompt(simulationStatus); - - const responseMessageId = buildMessageId('response', chatId); - let responseMessageContent = ''; - let responseRepositoryId: string | undefined; - let hasResponseMessage = false; - - const updateResponseMessage = () => { + const addResponseMessage = (msg: Message) => { if (gNumAborts != numAbortsAtStart) { return; } newMessages = [...newMessages]; - if (hasResponseMessage) { + const lastMessage = newMessages[newMessages.length - 1]; + + if (lastMessage.id == msg.id) { newMessages.pop(); + assert(lastMessage.type == 'text', 'Last message must be a text message'); + assert(msg.type == 'text', 'Message must be a text message'); + newMessages.push({ + ...msg, + content: lastMessage.content + msg.content, + }); + } else { + newMessages.push(msg); } - newMessages.push({ - id: responseMessageId, - role: 'assistant', - content: responseMessageContent, - repositoryId: responseRepositoryId, - }); setMessages(newMessages); - hasResponseMessage = true; }; - const addResponseContent = (content: string) => { - responseMessageContent += content; - updateResponseMessage(); - }; + const references: ChatReference[] = []; + + const mouseData = getCurrentMouseData(); + + if (mouseData) { + references.push({ + kind: 'element', + selector: mouseData.selector, + x: mouseData.x, + y: mouseData.y, + width: mouseData.width, + height: mouseData.height, + }); + } try { - const repositoryId = getMessagesRepositoryId(newMessages); - responseRepositoryId = await sendDeveloperChatMessage(newMessages, repositoryId, addResponseContent); - updateResponseMessage(); + await sendChatMessage(newMessages, references, addResponseMessage); } catch (e) { + toast.error('Error sending message'); console.error('Error sending message', e); - addResponseContent('Error sending message.'); } if (gNumAborts != numAbortsAtStart) { @@ -480,9 +373,14 @@ export const ChatImpl = memo( textareaRef.current?.blur(); - if (responseRepositoryId) { + const existingRepositoryId = getMessagesRepositoryId(messages); + const responseRepositoryId = getMessagesRepositoryId(newMessages); + + if (responseRepositoryId && existingRepositoryId != responseRepositoryId) { simulationRepositoryUpdated(responseRepositoryId); - setApproveChangesMessageId(responseMessageId); + + const lastMessage = newMessages[newMessages.length - 1]; + setApproveChangesMessageId(lastMessage.id); } }; diff --git a/app/components/chat/ImportFolderButton.tsx b/app/components/chat/ImportFolderButton.tsx index c4df33eb..e73783dc 100644 --- a/app/components/chat/ImportFolderButton.tsx +++ b/app/components/chat/ImportFolderButton.tsx @@ -1,10 +1,10 @@ import React, { useState } from 'react'; -import type { Message } from 'ai'; import { toast } from 'react-toastify'; import { MAX_FILES, isBinaryFile, shouldIncludeFile } from '~/utils/fileUtils'; import { createChatFromFolder, getFileRepositoryContents } from '~/utils/folderImport'; import { logStore } from '~/lib/stores/logs'; // Assuming logStore is imported from this location import { createRepositoryImported } from '~/lib/replay/Repository'; +import type { Message } from '~/lib/persistence/message'; interface ImportFolderButtonProps { className?: string; diff --git a/app/components/chat/LoadProblemButton.tsx b/app/components/chat/LoadProblemButton.tsx index 974425b0..8ee55c90 100644 --- a/app/components/chat/LoadProblemButton.tsx +++ b/app/components/chat/LoadProblemButton.tsx @@ -1,5 +1,4 @@ import React, { useState } from 'react'; -import type { Message } from 'ai'; import { toast } from 'react-toastify'; import { createChatFromFolder } from '~/utils/folderImport'; import { logStore } from '~/lib/stores/logs'; @@ -7,6 +6,7 @@ import { assert } from '~/lib/replay/ReplayProtocolClient'; import type { BoltProblem } from '~/lib/replay/Problems'; import { getProblem } from '~/lib/replay/Problems'; import { createRepositoryImported } from '~/lib/replay/Repository'; +import type { Message } from '~/lib/persistence/message'; interface LoadProblemButtonProps { className?: string; diff --git a/app/components/chat/MessageContents.tsx b/app/components/chat/MessageContents.tsx new file mode 100644 index 00000000..0b5e5b16 --- /dev/null +++ b/app/components/chat/MessageContents.tsx @@ -0,0 +1,38 @@ +/* + * @ts-nocheck + * Preventing TS checks with files presented in the video for a better presentation. + */ +import { MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants'; +import { Markdown } from './Markdown'; +import type { Message } from '~/lib/persistence/message'; + +interface MessageContentsProps { + message: Message; +} + +export function MessageContents({ message }: MessageContentsProps) { + switch (message.type) { + case 'text': + return ( +
+ {stripMetadata(message.content)} +
+ ); + case 'image': + return ( +
+
+ +
+
+ ); + } +} + +function stripMetadata(content: string) { + return content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, ''); +} diff --git a/app/components/chat/Messages.client.tsx b/app/components/chat/Messages.client.tsx index 96322907..504c6cde 100644 --- a/app/components/chat/Messages.client.tsx +++ b/app/components/chat/Messages.client.tsx @@ -1,9 +1,9 @@ import React, { Suspense } from 'react'; import { classNames } from '~/utils/classNames'; -import { AssistantMessage } from './AssistantMessage'; -import { UserMessage } from './UserMessage'; import WithTooltip from '~/components/ui/Tooltip'; -import { getPreviousRepositoryId, type Message } from '~/lib/persistence/useChatHistory'; +import { getPreviousRepositoryId } from '~/lib/persistence/useChatHistory'; +import type { Message } from '~/lib/persistence/message'; +import { MessageContents } from './MessageContents'; interface MessagesProps { id?: string; @@ -20,7 +20,7 @@ export const Messages = React.forwardRef((props:
{messages.length > 0 ? messages.map((message, index) => { - const { role, content, id: messageId, repositoryId } = message; + const { role, id: messageId, repositoryId } = message; const previousRepositoryId = getPreviousRepositoryId(messages, index); const isUserMessage = role === 'user'; const isFirst = index === 0; @@ -47,11 +47,7 @@ export const Messages = React.forwardRef((props:
)}
- {isUserMessage ? ( - - ) : ( - - )} +
{previousRepositoryId && repositoryId && onRewind && (
diff --git a/app/components/chat/UserMessage.tsx b/app/components/chat/UserMessage.tsx deleted file mode 100644 index 6fd6fc0d..00000000 --- a/app/components/chat/UserMessage.tsx +++ /dev/null @@ -1,47 +0,0 @@ -/* - * @ts-nocheck - * Preventing TS checks with files presented in the video for a better presentation. - */ -import { MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants'; -import { Markdown } from './Markdown'; - -interface UserMessageProps { - content: string | Array<{ type: string; text?: string; image?: string }>; -} - -export function UserMessage({ content }: UserMessageProps) { - if (Array.isArray(content)) { - const textItem = content.find((item) => item.type === 'text'); - const textContent = stripMetadata(textItem?.text || ''); - const images = content.filter((item) => item.type === 'image' && item.image); - - return ( -
-
- {textContent && {textContent}} - {images.map((item, index) => ( - {`Image - ))} -
-
- ); - } - - const textContent = stripMetadata(content); - - return ( -
- {textContent} -
- ); -} - -function stripMetadata(content: string) { - return content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, ''); -} diff --git a/app/components/chat/chatExportAndImport/ImportButtons.tsx b/app/components/chat/chatExportAndImport/ImportButtons.tsx index 5ad8bb56..5fa39f4a 100644 --- a/app/components/chat/chatExportAndImport/ImportButtons.tsx +++ b/app/components/chat/chatExportAndImport/ImportButtons.tsx @@ -1,6 +1,6 @@ -import type { Message } from 'ai'; import { toast } from 'react-toastify'; import { ImportFolderButton } from '~/components/chat/ImportFolderButton'; +import type { Message } from '~/lib/persistence/message'; type ChatData = { messages?: Message[]; // Standard Bolt format diff --git a/app/components/settings/data/DataTab.tsx b/app/components/settings/data/DataTab.tsx index acdcac23..88f7d527 100644 --- a/app/components/settings/data/DataTab.tsx +++ b/app/components/settings/data/DataTab.tsx @@ -5,7 +5,7 @@ import { toast } from 'react-toastify'; import { database, deleteById, getAll, setMessages } from '~/lib/persistence'; import { logStore } from '~/lib/stores/logs'; import { classNames } from '~/utils/classNames'; -import type { Message } from 'ai'; +import type { Message } from '~/lib/persistence/message'; // List of supported providers that can have API keys const API_KEY_PROVIDERS = [ diff --git a/app/components/sidebar/SaveReproduction.tsx b/app/components/sidebar/SaveReproduction.tsx index 8e310289..b7acaed5 100644 --- a/app/components/sidebar/SaveReproduction.tsx +++ b/app/components/sidebar/SaveReproduction.tsx @@ -7,7 +7,6 @@ import { getOrFetchLastLoadedProblem } from '~/components/chat/LoadProblemButton import { getLastUserSimulationData, getLastSimulationChatMessages, - getSimulationRecordingId, isSimulatingOrHasFinished, } from '~/lib/replay/SimulationPrompt'; @@ -54,18 +53,6 @@ export function SaveReproductionModal() { } try { - const loadId = toast.loading('Waiting for recording...'); - - try { - /* - * Wait for simulation to finish. - * const recordingId = - */ - await getSimulationRecordingId(); - } finally { - toast.dismiss(loadId); - } - toast.info('Submitting reproduction...'); console.log('SubmitReproduction'); diff --git a/app/lib/common/prompts/prompts.ts b/app/lib/common/prompts/prompts.ts deleted file mode 100644 index 66d87393..00000000 --- a/app/lib/common/prompts/prompts.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { stripIndents } from '~/utils/stripIndent'; - -export const developerSystemPrompt = ` -You are Nut, an expert AI assistant and exceptional senior software developer with vast knowledge across multiple programming languages, frameworks, and best practices. - -For all designs you produce, make them beautiful and modern. - - - Use 2 spaces for code indentation - - - - Before providing a solution, BRIEFLY outline your implementation steps. - This helps ensure systematic thinking and clear communication. Your planning should: - - List concrete steps you'll take - - Identify key components needed - - Note potential challenges - - Be concise (2-4 lines maximum) - - Example responses: - - User: "Create a todo list app with local storage" - Assistant: "Sure. I'll start by: - 1. Set up Vite + React - 2. Create TodoList and TodoItem components - 3. Implement localStorage for persistence - 4. Add CRUD operations - - Let's start now. - - [Rest of response...]" - - User: "Help debug why my API calls aren't working" - Assistant: "Great. My first steps will be: - 1. Check network requests - 2. Verify API endpoint format - 3. Examine error handling - - [Rest of response...]" - - - -IMPORTANT: Use valid markdown only for all your responses and DO NOT use HTML tags! - -ULTRA IMPORTANT: Think first and reply with all the files needed to set up the project and get it running. -It is SUPER IMPORTANT to respond with this first. Create every needed file. - - - Make a bouncing ball with real gravity using React - - - Certainly! I'll create a bouncing ball with real gravity using React. We'll use the react-spring library for physics-based animations. - - - { - "name": "bouncing-ball", - "private": true, - "version": "0.0.0", - "type": "module", - "scripts": { - "dev": "vite", - "build": "vite build", - "preview": "vite preview" - }, - "dependencies": { - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-spring": "^9.7.1" - }, - "devDependencies": { - "@types/react": "^18.0.28", - "@types/react-dom": "^18.0.11", - "@vitejs/plugin-react": "^3.1.0", - "vite": "^4.2.0" - } - } - - - - ... - - - - ... - - - - ... - - - - ... - - - You can now view the bouncing ball animation in the preview. The ball will start falling from the top of the screen and bounce realistically when it hits the bottom. - - -`; - -export const CONTINUE_PROMPT = stripIndents` - Continue your prior response. IMPORTANT: Immediately begin from where you left off without any interruptions. - Do not repeat any content, including artifact and action tags. -`; diff --git a/app/lib/hooks/pingTelemetry.ts b/app/lib/hooks/pingTelemetry.ts index a4519bc6..e2f4df1b 100644 --- a/app/lib/hooks/pingTelemetry.ts +++ b/app/lib/hooks/pingTelemetry.ts @@ -43,16 +43,4 @@ export class ChatMessageTelemetry { abort(reason: string) { this._ping('AbortMessage', { reason }); } - - startSimulation() { - this._ping('StartSimulation'); - } - - endSimulation(status: string) { - this._ping('EndSimulation', { status }); - } - - sendPrompt(simulationStatus: string) { - this._ping('SendPrompt', { simulationStatus }); - } } diff --git a/app/lib/persistence/db.ts b/app/lib/persistence/db.ts index 3091ae8f..2f6b49ed 100644 --- a/app/lib/persistence/db.ts +++ b/app/lib/persistence/db.ts @@ -1,6 +1,6 @@ -import type { Message } from 'ai'; import { createScopedLogger } from '~/utils/logger'; import type { ChatHistoryItem } from './useChatHistory'; +import type { Message } from './message'; const logger = createScopedLogger('ChatHistory'); diff --git a/app/lib/persistence/useChatHistory.ts b/app/lib/persistence/useChatHistory.ts index 3585c04b..e74e4f2f 100644 --- a/app/lib/persistence/useChatHistory.ts +++ b/app/lib/persistence/useChatHistory.ts @@ -1,21 +1,12 @@ import { useLoaderData, useNavigate, useSearchParams } from '@remix-run/react'; import { useState, useEffect } from 'react'; import { atom } from 'nanostores'; -import type { Message as BaseMessage } from 'ai'; import { toast } from 'react-toastify'; import { logStore } from '~/lib/stores/logs'; // Import logStore import { getMessages, getNextId, openDatabase, setMessages, duplicateChat, createChatFromMessages } from './db'; import { loadProblem } from '~/components/chat/LoadProblemButton'; import { createAsyncSuspenseValue } from '~/lib/asyncSuspenseValue'; - -/* - * Messages in a chat's history. The repository may update in response to changes in the messages. - * Each message which changes the repository state must have a repositoryId. - */ -export interface Message extends BaseMessage { - // Describes the state of the project after changes in this message were applied. - repositoryId?: string; -} +import type { Message } from './message'; export interface ChatState { description: string; diff --git a/app/lib/replay/Problems.ts b/app/lib/replay/Problems.ts index 0f8edff1..08f749f2 100644 --- a/app/lib/replay/Problems.ts +++ b/app/lib/replay/Problems.ts @@ -2,7 +2,7 @@ import { toast } from 'react-toastify'; import { sendCommandDedicatedClient } from './ReplayProtocolClient'; -import type { ProtocolMessage } from './SimulationPrompt'; +import type { Message } from '~/lib/persistence/message'; import Cookies from 'js-cookie'; import { shouldUseSupabase } from '~/lib/supabase/client'; import { @@ -22,7 +22,7 @@ export interface BoltProblemComment { export interface BoltProblemSolution { simulationData: any; - messages: ProtocolMessage[]; + messages: Message[]; evaluator?: string; } diff --git a/app/lib/replay/SimulationPrompt.ts b/app/lib/replay/SimulationPrompt.ts index eaf0bc3f..545b0a2d 100644 --- a/app/lib/replay/SimulationPrompt.ts +++ b/app/lib/replay/SimulationPrompt.ts @@ -3,14 +3,11 @@ * the AI developer prompt. */ -import type { Message } from 'ai'; import type { SimulationData, SimulationPacket } from './SimulationData'; import { simulationDataVersion } from './SimulationData'; import { assert, generateRandomId, ProtocolClient } from './ReplayProtocolClient'; -import type { MouseData } from './Recording'; -import { developerSystemPrompt } from '~/lib/common/prompts/prompts'; import { updateDevelopmentServer } from './DevelopmentServer'; -import { isEnhancedPromptMessage } from '~/components/chat/Chat.client'; +import type { Message } from '~/lib/persistence/message'; function createRepositoryIdPacket(repositoryId: string): SimulationPacket { return { @@ -20,31 +17,19 @@ function createRepositoryIdPacket(repositoryId: string): SimulationPacket { }; } -type ProtocolMessageRole = 'user' | 'assistant' | 'system'; - -type ProtocolMessageText = { - type: 'text'; - role: ProtocolMessageRole; - content: string; -}; - -type ProtocolMessageImage = { - type: 'image'; - role: ProtocolMessageRole; - dataURL: string; -}; - -export type ProtocolMessage = ProtocolMessageText | ProtocolMessageImage; - -type ChatResponsePartCallback = (response: string) => void; - -type ChatMessageMode = 'recording' | 'static' | 'developer'; - -interface ChatMessageOptions { - baseRepositoryId?: string; - onResponsePart?: ChatResponsePartCallback; +interface ChatReferenceElement { + kind: 'element'; + selector: string; + width: number; + height: number; + x: number; + y: number; } +export type ChatReference = ChatReferenceElement; + +type ChatResponsePartCallback = (message: Message) => void; + class ChatManager { // Empty if this chat has been destroyed. client: ProtocolClient | undefined; @@ -52,9 +37,6 @@ class ChatManager { // Resolves when the chat has started. chatIdPromise: Promise; - // Resolves when the recording has been created. - recordingIdPromise: Promise | undefined; - // Whether all simulation data has been sent. simulationFinished?: boolean; @@ -133,63 +115,37 @@ class ChatManager { assert(!this.simulationFinished, 'Simulation has been finished'); assert(this.repositoryId, 'Expected repository ID'); - this.recordingIdPromise = (async () => { - assert(this.client, 'Chat has been destroyed'); - - const chatId = await this.chatIdPromise; - const { recordingId } = (await this.client.sendCommand({ - method: 'Nut.finishSimulationData', - params: { chatId }, - })) as { recordingId: string | undefined }; - - assert(recordingId, 'Recording ID not set'); - - return recordingId; - })(); - const allData = [createRepositoryIdPacket(this.repositoryId), ...this.pageData]; this.simulationFinished = true; return allData; } - async sendChatMessage(mode: ChatMessageMode, messages: ProtocolMessage[], options?: ChatMessageOptions) { + async sendChatMessage(messages: Message[], references: ChatReference[], onResponsePart: ChatResponsePartCallback) { assert(this.client, 'Chat has been destroyed'); const responseId = `response-${generateRandomId()}`; - let response: string = ''; const removeResponseListener = this.client.listenForMessage( 'Nut.chatResponsePart', - ({ responseId: eventResponseId, message }: { responseId: string; message: ProtocolMessage }) => { + ({ responseId: eventResponseId, message }: { responseId: string; message: Message }) => { if (responseId == eventResponseId) { - if (message.type == 'text') { - response += message.content; - options?.onResponsePart?.(message.content); - } + console.log('ChatResponse', chatId, message); + onResponsePart(message); } }, ); const chatId = await this.chatIdPromise; - console.log( - 'ChatSendMessage', - new Date().toISOString(), - chatId, - JSON.stringify({ mode, messages, baseRepositoryId: options?.baseRepositoryId }), - ); + console.log('ChatSendMessage', new Date().toISOString(), chatId, JSON.stringify({ messages, references })); - const { repositoryId } = (await this.client.sendCommand({ + await this.client.sendCommand({ method: 'Nut.sendChatMessage', - params: { chatId, responseId, mode, messages, baseRepositoryId: options?.baseRepositoryId }, - })) as { repositoryId?: string }; - - console.log('ChatResponse', chatId, repositoryId, response); + params: { chatId, responseId, messages, references }, + }); removeResponseListener(); - - return { response, repositoryId }; } } @@ -232,264 +188,40 @@ export async function simulationReloaded() { startChat(repositoryId, []); } -export async function simulationAddData(data: SimulationData) { +export function simulationAddData(data: SimulationData) { assert(gChatManager, 'Expected to have an active chat'); gChatManager.addPageData(data); } +export function simulationFinishData() { + assert(gChatManager, 'Expected to have an active chat'); + gChatManager.finishSimulationData(); +} + let gLastUserSimulationData: SimulationData | undefined; export function getLastUserSimulationData(): SimulationData | undefined { return gLastUserSimulationData; } -export async function getSimulationRecording(): Promise { - assert(gChatManager, 'Expected to have an active chat'); - - const simulationData = gChatManager.finishSimulationData(); - - /* - * The repository contents are part of the problem and excluded from the simulation data - * reported for solutions. - */ - gLastUserSimulationData = simulationData.filter((packet) => packet.kind != 'repositoryId'); - - console.log('SimulationData', new Date().toISOString(), JSON.stringify(simulationData)); - - assert(gChatManager.recordingIdPromise, 'Expected recording promise'); - - return gChatManager.recordingIdPromise; -} - export function isSimulatingOrHasFinished(): boolean { return gChatManager?.isValid() ?? false; } -export async function getSimulationRecordingId(): Promise { - assert(gChatManager, 'Chat not started'); - assert(gChatManager.recordingIdPromise, 'Expected recording promise'); +let gLastSimulationChatMessages: Message[] | undefined; - return gChatManager.recordingIdPromise; -} - -let gLastSimulationChatMessages: ProtocolMessage[] | undefined; - -export function getLastSimulationChatMessages(): ProtocolMessage[] | undefined { +export function getLastSimulationChatMessages(): Message[] | undefined { return gLastSimulationChatMessages; } -const simulationSystemPrompt = ` -The following user message describes a bug or other problem on the page which needs to be fixed. -You must respond with a useful explanation that will help the user understand the source of the problem. -Do not describe the specific fix needed. -`; - -export async function getSimulationEnhancedPrompt( - chatMessages: Message[], - userMessage: string, - mouseData: MouseData | undefined, -): Promise { - assert(gChatManager, 'Chat not started'); - assert(gChatManager.simulationFinished, 'Simulation not finished'); - - let system = simulationSystemPrompt; - - if (mouseData) { - system += `The user pointed to an element on the page `; - } - - const messages: ProtocolMessage[] = [ - { - role: 'system', - type: 'text', - content: system, - }, - { - role: 'user', - type: 'text', - content: userMessage, - }, - ]; - - gLastSimulationChatMessages = messages; - - const { response } = await gChatManager.sendChatMessage('recording', messages); - - return response; -} - -export async function shouldUseSimulation(messageInput: string) { - if (!gChatManager) { - gChatManager = new ChatManager(); - } - - const systemPrompt = ` -You are a helpful assistant that determines whether a user's message that is asking an AI -to make a change to an application should first perform a detailed analysis of the application's -behavior to generate a better answer. - -This is most helpful when the user is asking the AI to fix a problem with the application. -When making straightforward improvements to the application a detailed analysis is not necessary. - -The text of the user's message will be wrapped in \`\` tags. You must describe your -reasoning and then respond with either \`true\` or \`false\`. - `; - - const userMessage = ` -Here is the user message you need to evaluate: ${messageInput} - `; - - const messages: ProtocolMessage[] = [ - { - role: 'system', - type: 'text', - content: systemPrompt, - }, - { - role: 'user', - type: 'text', - content: userMessage, - }, - ]; - - const { response } = await gChatManager.sendChatMessage('static', messages); - - console.log('UseSimulationResponse', response); - - const match = /(.*?)<\/analyze>/.exec(response); - - if (match) { - return match[1] === 'true'; - } - - return false; -} - -function getProtocolRole(message: Message): 'user' | 'assistant' | 'system' { - switch (message.role) { - case 'user': - return 'user'; - case 'assistant': - case 'data': - return 'assistant'; - case 'system': - return 'system'; - default: - throw new Error(`Unknown message role: ${message.role}`); - } -} - -function removeBoltArtifacts(text: string): string { - const openTag = ' isEnhancedPromptMessage(msg)); - - if (lastEnhancedPromptMessage == -1) { - return false; - } - - const lastUserMessage = messages.findLastIndex((msg) => msg.role == 'user'); - - if (lastUserMessage == -1) { - return false; - } - - return lastUserMessage < lastEnhancedPromptMessage; -} - -export async function sendDeveloperChatMessage( +export async function sendChatMessage( messages: Message[], - baseRepositoryId: string | undefined, + references: ChatReference[], onResponsePart: ChatResponsePartCallback, ) { if (!gChatManager) { gChatManager = new ChatManager(); } - let systemPrompt = developerSystemPrompt; - - if (messagesHaveEnhancedPrompt(messages)) { - // Add directions to the LLM when we have an enhanced prompt describing the bug to fix. - const systemEnhancedPrompt = ` -ULTRA IMPORTANT: You have been given a detailed description of a bug you need to fix. -Focus specifically on fixing this bug. Do not guess about other problems. - `; - systemPrompt += systemEnhancedPrompt; - } - - const protocolMessages = buildProtocolMessages(messages); - protocolMessages.unshift({ - role: 'system', - type: 'text', - content: systemPrompt, - }); - - const { repositoryId } = await gChatManager.sendChatMessage('developer', protocolMessages, { - baseRepositoryId, - onResponsePart, - }); - - return repositoryId; + await gChatManager.sendChatMessage(messages, references, onResponsePart); } diff --git a/app/utils/folderImport.ts b/app/utils/folderImport.ts index 4030c09c..7515b34f 100644 --- a/app/utils/folderImport.ts +++ b/app/utils/folderImport.ts @@ -1,4 +1,4 @@ -import type { Message } from '~/lib/persistence/useChatHistory'; +import type { Message } from '~/lib/persistence/message'; import { generateId } from './fileUtils'; import JSZip from 'jszip'; @@ -43,15 +43,15 @@ export function createChatFromFolder(folderName: string, repositoryId: string): role: 'user', id: generateId(), content: `Import the "${folderName}" folder`, - createdAt: new Date(), + type: 'text', }; const filesMessage: Message = { role: 'assistant', content: filesContent, id: generateId(), - createdAt: new Date(), repositoryId, + type: 'text', }; const messages = [userMessage, filesMessage];