Updates for async chat format (#71)

app/components/chat/AssistantMessage.tsx

@@ -1,37 +0,0 @@
-import { memo } from 'react';
-import { Markdown } from './Markdown';
-import type { JSONValue } from 'ai';
-
-interface AssistantMessageProps {
-  content: string;
-  annotations?: JSONValue[];
-}
-
-export function getAnnotationsTokensUsage(annotations: JSONValue[] | undefined) {
-  const filteredAnnotations = (annotations?.filter(
-    (annotation: JSONValue) => annotation && typeof annotation === 'object' && Object.keys(annotation).includes('type'),
-  ) || []) as { type: string; value: any }[];
-
-  const usage: {
-    completionTokens: number;
-    promptTokens: number;
-    totalTokens: number;
-  } = filteredAnnotations.find((annotation) => annotation.type === 'usage')?.value;
-
-  return usage;
-}
-
-export const AssistantMessage = memo(({ content, annotations }: AssistantMessageProps) => {
-  const usage = getAnnotationsTokensUsage(annotations);
-
-  return (
-    <div className="overflow-hidden w-full">
-      {usage && (
-        <div className="text-sm text-bolt-elements-textSecondary mb-2">
-          Tokens: {usage.totalTokens} (prompt: {usage.promptTokens}, completion: {usage.completionTokens})
-        </div>
-      )}
-      <Markdown html>{content}</Markdown>
-    </div>
-  );
-});

app/components/chat/BaseChat.tsx

@@ -9,7 +9,8 @@ import { IconButton } from '~/components/ui/IconButton';
 import { Workbench } from '~/components/workbench/Workbench.client';
 import { classNames } from '~/utils/classNames';
 import { Messages } from './Messages.client';
-import { getPreviousRepositoryId, type Message } from '~/lib/persistence/useChatHistory';
+import { getPreviousRepositoryId } from '~/lib/persistence/useChatHistory';
+import type { Message } from '~/lib/persistence/message';
 import { SendButton } from './SendButton.client';
 import * as Tooltip from '@radix-ui/react-tooltip';
 

app/components/chat/Chat.client.tsx

@@ -18,12 +18,11 @@ import { debounce } from '~/utils/debounce';
 import { useSearchParams } from '@remix-run/react';
 import { createSampler } from '~/utils/sampler';
 import {
-  getSimulationRecording,
-  getSimulationEnhancedPrompt,
   simulationAddData,
+  simulationFinishData,
   simulationRepositoryUpdated,
-  shouldUseSimulation,
+  sendChatMessage,
-  sendDeveloperChatMessage,
+  type ChatReference,
 } from '~/lib/replay/SimulationPrompt';
 import { getIFrameSimulationData } from '~/lib/replay/Recording';
 import { getCurrentIFrame } from '~/components/workbench/Preview';
@@ -32,8 +31,9 @@ import { anthropicNumFreeUsesCookieName, anthropicApiKeyCookieName, maxFreeUses
 import { getNutLoginKey, submitFeedback } from '~/lib/replay/Problems';
 import { ChatMessageTelemetry, pingTelemetry } from '~/lib/hooks/pingTelemetry';
 import type { RejectChangeData } from './ApproveChange';
-import { generateRandomId } from '~/lib/replay/ReplayProtocolClient';
+import { assert, generateRandomId } from '~/lib/replay/ReplayProtocolClient';
-import { getMessagesRepositoryId, getPreviousRepositoryId, type Message } from '~/lib/persistence/useChatHistory';
+import { getMessagesRepositoryId, getPreviousRepositoryId } from '~/lib/persistence/useChatHistory';
+import type { Message } from '~/lib/persistence/message';
 
 const toastAnimation = cssTransition({
   enter: 'animated fadeInRight',
@@ -64,15 +64,11 @@ async function flushSimulationData() {
   //console.log("HaveSimulationData", simulationData.length);
 
   // Add the simulation data to the chat.
-  await simulationAddData(simulationData);
+  simulationAddData(simulationData);
 }
 
-let gLockSimulationData = false;
-
 setInterval(async () => {
-  if (!gLockSimulationData) {
+  flushSimulationData();
-    flushSimulationData();
-  }
 }, 1000);
 
 export function Chat() {
@@ -154,16 +150,6 @@ async function clearActiveChat() {
   gActiveChatMessageTelemetry = undefined;
 }
 
-function buildMessageId(prefix: string, chatId: string) {
-  return `${prefix}-${chatId}`;
-}
-
-const EnhancedPromptPrefix = 'enhanced-prompt';
-
-export function isEnhancedPromptMessage(message: Message): boolean {
-  return message.id.startsWith(EnhancedPromptPrefix);
-}
-
 export const ChatImpl = memo(
   ({ description, initialMessages, storeMessageHistory, importChat, exportChat }: ChatProps) => {
     const textareaRef = useRef<HTMLTextAreaElement>(null);
@@ -261,50 +247,6 @@ export const ChatImpl = memo(
       setChatStarted(true);
     };
 
-    const createRecording = async (chatId: string) => {
-      let recordingId, message;
-
-      try {
-        recordingId = await getSimulationRecording();
-        message = `[Recording of the bug](https://app.replay.io/recording/${recordingId})\n\n`;
-      } catch (e) {
-        console.error('Error creating recording', e);
-        message = 'Error creating recording.';
-      }
-
-      const recordingMessage: Message = {
-        id: buildMessageId('create-recording', chatId),
-        role: 'assistant',
-        content: message,
-      };
-
-      return { recordingId, recordingMessage };
-    };
-
-    const getEnhancedPrompt = async (chatId: string, userMessage: string) => {
-      let enhancedPrompt,
-        message,
-        hadError = false;
-
-      try {
-        const mouseData = getCurrentMouseData();
-        enhancedPrompt = await getSimulationEnhancedPrompt(messages, userMessage, mouseData);
-        message = `Explanation of the bug:\n\n${enhancedPrompt}`;
-      } catch (e) {
-        console.error('Error enhancing prompt', e);
-        message = 'Error enhancing prompt.';
-        hadError = true;
-      }
-
-      const enhancedPromptMessage: Message = {
-        id: buildMessageId(EnhancedPromptPrefix, chatId),
-        role: 'assistant',
-        content: message,
-      };
-
-      return { enhancedPrompt, enhancedPromptMessage, hadError };
-    };
-
     const sendMessage = async (messageInput?: string) => {
       const _input = messageInput || input;
       const numAbortsAtStart = gNumAborts;
@@ -340,130 +282,81 @@ export const ChatImpl = memo(
       setActiveChatId(chatId);
 
       const userMessage: Message = {
-        id: buildMessageId('user', chatId),
+        id: `user-${chatId}`,
         role: 'user',
-        content: [
+        type: 'text',
-          {
+        content: _input,
-            type: 'text',
-            text: _input,
-          },
-          ...imageDataList.map((imageData) => ({
-            type: 'image',
-            image: imageData,
-          })),
-        ] as any, // Type assertion to bypass compiler check
       };
 
       let newMessages = [...messages, userMessage];
+
+      imageDataList.forEach((imageData, index) => {
+        const imageMessage: Message = {
+          id: `image-${chatId}-${index}`,
+          role: 'user',
+          type: 'image',
+          dataURL: imageData,
+        };
+        newMessages.push(imageMessage);
+      });
+
       setMessages(newMessages);
 
       // Add file cleanup here
       setUploadedFiles([]);
       setImageDataList([]);
 
-      let simulation = false;
+      await flushSimulationData();
-
+      simulationFinishData();
-      try {
-        simulation = chatStarted && (await shouldUseSimulation(_input));
-      } catch (e) {
-        console.error('Error checking simulation', e);
-      }
-
-      if (numAbortsAtStart != gNumAborts) {
-        return;
-      }
-
-      console.log('UseSimulation', simulation);
-
-      let simulationStatus = 'NoSimulation';
-
-      if (simulation) {
-        gActiveChatMessageTelemetry.startSimulation();
-
-        gLockSimulationData = true;
-
-        try {
-          await flushSimulationData();
-
-          const createRecordingPromise = createRecording(chatId);
-          const enhancedPromptPromise = getEnhancedPrompt(chatId, _input);
-
-          const { recordingId, recordingMessage } = await createRecordingPromise;
-
-          if (numAbortsAtStart != gNumAborts) {
-            return;
-          }
-
-          console.log('RecordingMessage', recordingMessage);
-          newMessages = [...newMessages, recordingMessage];
-          setMessages(newMessages);
-
-          if (recordingId) {
-            const info = await enhancedPromptPromise;
-
-            if (numAbortsAtStart != gNumAborts) {
-              return;
-            }
-
-            console.log('EnhancedPromptMessage', info.enhancedPromptMessage);
-            newMessages = [...newMessages, info.enhancedPromptMessage];
-            setMessages(newMessages);
-
-            simulationStatus = info.hadError ? 'PromptError' : 'Success';
-          } else {
-            simulationStatus = 'RecordingError';
-          }
-
-          gActiveChatMessageTelemetry.endSimulation(simulationStatus);
-        } finally {
-          gLockSimulationData = false;
-        }
-      }
 
       chatStore.setKey('aborted', false);
 
       runAnimation();
 
-      gActiveChatMessageTelemetry.sendPrompt(simulationStatus);
+      const addResponseMessage = (msg: Message) => {
-
-      const responseMessageId = buildMessageId('response', chatId);
-      let responseMessageContent = '';
-      let responseRepositoryId: string | undefined;
-      let hasResponseMessage = false;
-
-      const updateResponseMessage = () => {
         if (gNumAborts != numAbortsAtStart) {
           return;
         }
 
         newMessages = [...newMessages];
 
-        if (hasResponseMessage) {
+        const lastMessage = newMessages[newMessages.length - 1];
+
+        if (lastMessage.id == msg.id) {
           newMessages.pop();
+          assert(lastMessage.type == 'text', 'Last message must be a text message');
+          assert(msg.type == 'text', 'Message must be a text message');
+          newMessages.push({
+            ...msg,
+            content: lastMessage.content + msg.content,
+          });
+        } else {
+          newMessages.push(msg);
         }
 
-        newMessages.push({
-          id: responseMessageId,
-          role: 'assistant',
-          content: responseMessageContent,
-          repositoryId: responseRepositoryId,
-        });
         setMessages(newMessages);
-        hasResponseMessage = true;
       };
 
-      const addResponseContent = (content: string) => {
+      const references: ChatReference[] = [];
-        responseMessageContent += content;
+
-        updateResponseMessage();
+      const mouseData = getCurrentMouseData();
-      };
+
+      if (mouseData) {
+        references.push({
+          kind: 'element',
+          selector: mouseData.selector,
+          x: mouseData.x,
+          y: mouseData.y,
+          width: mouseData.width,
+          height: mouseData.height,
+        });
+      }
 
       try {
-        const repositoryId = getMessagesRepositoryId(newMessages);
+        await sendChatMessage(newMessages, references, addResponseMessage);
-        responseRepositoryId = await sendDeveloperChatMessage(newMessages, repositoryId, addResponseContent);
-        updateResponseMessage();
       } catch (e) {
+        toast.error('Error sending message');
         console.error('Error sending message', e);
-        addResponseContent('Error sending message.');
       }
 
       if (gNumAborts != numAbortsAtStart) {
@@ -480,9 +373,14 @@ export const ChatImpl = memo(
 
       textareaRef.current?.blur();
 
-      if (responseRepositoryId) {
+      const existingRepositoryId = getMessagesRepositoryId(messages);
+      const responseRepositoryId = getMessagesRepositoryId(newMessages);
+
+      if (responseRepositoryId && existingRepositoryId != responseRepositoryId) {
         simulationRepositoryUpdated(responseRepositoryId);
-        setApproveChangesMessageId(responseMessageId);
+
+        const lastMessage = newMessages[newMessages.length - 1];
+        setApproveChangesMessageId(lastMessage.id);
       }
     };
 

app/components/chat/ImportFolderButton.tsx

@@ -1,10 +1,10 @@
 import React, { useState } from 'react';
-import type { Message } from 'ai';
 import { toast } from 'react-toastify';
 import { MAX_FILES, isBinaryFile, shouldIncludeFile } from '~/utils/fileUtils';
 import { createChatFromFolder, getFileRepositoryContents } from '~/utils/folderImport';
 import { logStore } from '~/lib/stores/logs'; // Assuming logStore is imported from this location
 import { createRepositoryImported } from '~/lib/replay/Repository';
+import type { Message } from '~/lib/persistence/message';
 
 interface ImportFolderButtonProps {
   className?: string;

app/components/chat/LoadProblemButton.tsx

@@ -1,5 +1,4 @@
 import React, { useState } from 'react';
-import type { Message } from 'ai';
 import { toast } from 'react-toastify';
 import { createChatFromFolder } from '~/utils/folderImport';
 import { logStore } from '~/lib/stores/logs';
@@ -7,6 +6,7 @@ import { assert } from '~/lib/replay/ReplayProtocolClient';
 import type { BoltProblem } from '~/lib/replay/Problems';
 import { getProblem } from '~/lib/replay/Problems';
 import { createRepositoryImported } from '~/lib/replay/Repository';
+import type { Message } from '~/lib/persistence/message';
 
 interface LoadProblemButtonProps {
   className?: string;

app/components/chat/MessageContents.tsx

@@ -0,0 +1,38 @@
+/*
+ * @ts-nocheck
+ * Preventing TS checks with files presented in the video for a better presentation.
+ */
+import { MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants';
+import { Markdown } from './Markdown';
+import type { Message } from '~/lib/persistence/message';
+
+interface MessageContentsProps {
+  message: Message;
+}
+
+export function MessageContents({ message }: MessageContentsProps) {
+  switch (message.type) {
+    case 'text':
+      return (
+        <div className="overflow-hidden pt-[4px]">
+          <Markdown html>{stripMetadata(message.content)}</Markdown>
+        </div>
+      );
+    case 'image':
+      return (
+        <div className="overflow-hidden pt-[4px]">
+          <div className="flex flex-col gap-4">
+            <img
+              src={message.dataURL}
+              className="max-w-full h-auto rounded-lg"
+              style={{ maxHeight: '512px', objectFit: 'contain' }}
+            />
+          </div>
+        </div>
+      );
+  }
+}
+
+function stripMetadata(content: string) {
+  return content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '');
+}

app/components/chat/Messages.client.tsx

@@ -1,9 +1,9 @@
 import React, { Suspense } from 'react';
 import { classNames } from '~/utils/classNames';
-import { AssistantMessage } from './AssistantMessage';
-import { UserMessage } from './UserMessage';
 import WithTooltip from '~/components/ui/Tooltip';
-import { getPreviousRepositoryId, type Message } from '~/lib/persistence/useChatHistory';
+import { getPreviousRepositoryId } from '~/lib/persistence/useChatHistory';
+import type { Message } from '~/lib/persistence/message';
+import { MessageContents } from './MessageContents';
 
 interface MessagesProps {
   id?: string;
@@ -20,7 +20,7 @@ export const Messages = React.forwardRef<HTMLDivElement, MessagesProps>((props:
     <div id={id} ref={ref} className={props.className}>
       {messages.length > 0
         ? messages.map((message, index) => {
-            const { role, content, id: messageId, repositoryId } = message;
+            const { role, id: messageId, repositoryId } = message;
             const previousRepositoryId = getPreviousRepositoryId(messages, index);
             const isUserMessage = role === 'user';
             const isFirst = index === 0;
@@ -47,11 +47,7 @@ export const Messages = React.forwardRef<HTMLDivElement, MessagesProps>((props:
                     </div>
                   )}
                   <div className="grid grid-col-1 w-full">
-                    {isUserMessage ? (
+                    <MessageContents message={message} />
-                      <UserMessage content={content} />
-                    ) : (
-                      <AssistantMessage content={content} annotations={message.annotations} />
-                    )}
                   </div>
                   {previousRepositoryId && repositoryId && onRewind && (
                     <div className="flex gap-2 flex-col lg:flex-row">

app/components/chat/UserMessage.tsx

@@ -1,47 +0,0 @@
-/*
- * @ts-nocheck
- * Preventing TS checks with files presented in the video for a better presentation.
- */
-import { MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants';
-import { Markdown } from './Markdown';
-
-interface UserMessageProps {
-  content: string | Array<{ type: string; text?: string; image?: string }>;
-}
-
-export function UserMessage({ content }: UserMessageProps) {
-  if (Array.isArray(content)) {
-    const textItem = content.find((item) => item.type === 'text');
-    const textContent = stripMetadata(textItem?.text || '');
-    const images = content.filter((item) => item.type === 'image' && item.image);
-
-    return (
-      <div className="overflow-hidden pt-[4px]">
-        <div className="flex flex-col gap-4">
-          {textContent && <Markdown html>{textContent}</Markdown>}
-          {images.map((item, index) => (
-            <img
-              key={index}
-              src={item.image}
-              alt={`Image ${index + 1}`}
-              className="max-w-full h-auto rounded-lg"
-              style={{ maxHeight: '512px', objectFit: 'contain' }}
-            />
-          ))}
-        </div>
-      </div>
-    );
-  }
-
-  const textContent = stripMetadata(content);
-
-  return (
-    <div className="overflow-hidden pt-[4px]">
-      <Markdown html>{textContent}</Markdown>
-    </div>
-  );
-}
-
-function stripMetadata(content: string) {
-  return content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '');
-}

app/components/chat/chatExportAndImport/ImportButtons.tsx

@@ -1,6 +1,6 @@
-import type { Message } from 'ai';
 import { toast } from 'react-toastify';
 import { ImportFolderButton } from '~/components/chat/ImportFolderButton';
+import type { Message } from '~/lib/persistence/message';
 
 type ChatData = {
   messages?: Message[]; // Standard Bolt format

app/components/settings/data/DataTab.tsx

@@ -5,7 +5,7 @@ import { toast } from 'react-toastify';
 import { database, deleteById, getAll, setMessages } from '~/lib/persistence';
 import { logStore } from '~/lib/stores/logs';
 import { classNames } from '~/utils/classNames';
-import type { Message } from 'ai';
+import type { Message } from '~/lib/persistence/message';
 
 // List of supported providers that can have API keys
 const API_KEY_PROVIDERS = [

app/components/sidebar/SaveReproduction.tsx

@@ -7,7 +7,6 @@ import { getOrFetchLastLoadedProblem } from '~/components/chat/LoadProblemButton
 import {
   getLastUserSimulationData,
   getLastSimulationChatMessages,
-  getSimulationRecordingId,
   isSimulatingOrHasFinished,
 } from '~/lib/replay/SimulationPrompt';
 
@@ -54,18 +53,6 @@ export function SaveReproductionModal() {
     }
 
     try {
-      const loadId = toast.loading('Waiting for recording...');
-
-      try {
-        /*
-         * Wait for simulation to finish.
-         * const recordingId =
-         */
-        await getSimulationRecordingId();
-      } finally {
-        toast.dismiss(loadId);
-      }
-
       toast.info('Submitting reproduction...');
       console.log('SubmitReproduction');
 

app/lib/common/prompts/prompts.ts

@@ -1,103 +0,0 @@
-import { stripIndents } from '~/utils/stripIndent';
-
-export const developerSystemPrompt = `
-You are Nut, an expert AI assistant and exceptional senior software developer with vast knowledge across multiple programming languages, frameworks, and best practices.
-
-For all designs you produce, make them beautiful and modern.
-
-<code_formatting_info>
-  Use 2 spaces for code indentation
-</code_formatting_info>
-
-<chain_of_thought_instructions>
-  Before providing a solution, BRIEFLY outline your implementation steps.
-  This helps ensure systematic thinking and clear communication. Your planning should:
-  - List concrete steps you'll take
-  - Identify key components needed
-  - Note potential challenges
-  - Be concise (2-4 lines maximum)
-
-  Example responses:
-
-  User: "Create a todo list app with local storage"
-  Assistant: "Sure. I'll start by:
-  1. Set up Vite + React
-  2. Create TodoList and TodoItem components
-  3. Implement localStorage for persistence
-  4. Add CRUD operations
-  
-  Let's start now.
-
-  [Rest of response...]"
-
-  User: "Help debug why my API calls aren't working"
-  Assistant: "Great. My first steps will be:
-  1. Check network requests
-  2. Verify API endpoint format
-  3. Examine error handling
-  
-  [Rest of response...]"
-
-</chain_of_thought_instructions>
-
-IMPORTANT: Use valid markdown only for all your responses and DO NOT use HTML tags!
-
-ULTRA IMPORTANT: Think first and reply with all the files needed to set up the project and get it running.
-It is SUPER IMPORTANT to respond with this first. Create every needed file.
-
-<example>
-  <user_query>Make a bouncing ball with real gravity using React</user_query>
-
-  <assistant_response>
-    Certainly! I'll create a bouncing ball with real gravity using React. We'll use the react-spring library for physics-based animations.
-
-    <file path="package.json">
-      {
-        "name": "bouncing-ball",
-        "private": true,
-        "version": "0.0.0",
-        "type": "module",
-        "scripts": {
-          "dev": "vite",
-          "build": "vite build",
-          "preview": "vite preview"
-        },
-        "dependencies": {
-          "react": "^18.2.0",
-          "react-dom": "^18.2.0",
-          "react-spring": "^9.7.1"
-        },
-        "devDependencies": {
-          "@types/react": "^18.0.28",
-          "@types/react-dom": "^18.0.11",
-          "@vitejs/plugin-react": "^3.1.0",
-          "vite": "^4.2.0"
-        }
-      }
-    </file>
-
-    <file path="index.html">
-      ...
-    </file>
-
-    <file path="src/main.jsx">
-      ...
-    </file>
-
-    <file path="src/index.css">
-      ...
-    </file>
-
-    <file path="src/App.jsx">
-      ...
-    </file>
-
-    You can now view the bouncing ball animation in the preview. The ball will start falling from the top of the screen and bounce realistically when it hits the bottom.
-  </assistant_response>
-</example>
-`;
-
-export const CONTINUE_PROMPT = stripIndents`
-  Continue your prior response. IMPORTANT: Immediately begin from where you left off without any interruptions.
-  Do not repeat any content, including artifact and action tags.
-`;

app/lib/hooks/pingTelemetry.ts

@@ -43,16 +43,4 @@ export class ChatMessageTelemetry {
   abort(reason: string) {
     this._ping('AbortMessage', { reason });
   }
-
-  startSimulation() {
-    this._ping('StartSimulation');
-  }
-
-  endSimulation(status: string) {
-    this._ping('EndSimulation', { status });
-  }
-
-  sendPrompt(simulationStatus: string) {
-    this._ping('SendPrompt', { simulationStatus });
-  }
 }

app/lib/persistence/db.ts

@@ -1,6 +1,6 @@
-import type { Message } from 'ai';
 import { createScopedLogger } from '~/utils/logger';
 import type { ChatHistoryItem } from './useChatHistory';
+import type { Message } from './message';
 
 const logger = createScopedLogger('ChatHistory');
 

app/lib/persistence/useChatHistory.ts

@@ -1,21 +1,12 @@
 import { useLoaderData, useNavigate, useSearchParams } from '@remix-run/react';
 import { useState, useEffect } from 'react';
 import { atom } from 'nanostores';
-import type { Message as BaseMessage } from 'ai';
 import { toast } from 'react-toastify';
 import { logStore } from '~/lib/stores/logs'; // Import logStore
 import { getMessages, getNextId, openDatabase, setMessages, duplicateChat, createChatFromMessages } from './db';
 import { loadProblem } from '~/components/chat/LoadProblemButton';
 import { createAsyncSuspenseValue } from '~/lib/asyncSuspenseValue';
-
+import type { Message } from './message';
-/*
- * Messages in a chat's history. The repository may update in response to changes in the messages.
- * Each message which changes the repository state must have a repositoryId.
- */
-export interface Message extends BaseMessage {
-  // Describes the state of the project after changes in this message were applied.
-  repositoryId?: string;
-}
 
 export interface ChatState {
   description: string;

app/lib/replay/Problems.ts

@@ -2,7 +2,7 @@
 
 import { toast } from 'react-toastify';
 import { sendCommandDedicatedClient } from './ReplayProtocolClient';
-import type { ProtocolMessage } from './SimulationPrompt';
+import type { Message } from '~/lib/persistence/message';
 import Cookies from 'js-cookie';
 import { shouldUseSupabase } from '~/lib/supabase/client';
 import {
@@ -22,7 +22,7 @@ export interface BoltProblemComment {
 
 export interface BoltProblemSolution {
   simulationData: any;
-  messages: ProtocolMessage[];
+  messages: Message[];
   evaluator?: string;
 }
 

app/lib/replay/SimulationPrompt.ts

@@ -3,14 +3,11 @@
  * the AI developer prompt.
  */
 
-import type { Message } from 'ai';
 import type { SimulationData, SimulationPacket } from './SimulationData';
 import { simulationDataVersion } from './SimulationData';
 import { assert, generateRandomId, ProtocolClient } from './ReplayProtocolClient';
-import type { MouseData } from './Recording';
-import { developerSystemPrompt } from '~/lib/common/prompts/prompts';
 import { updateDevelopmentServer } from './DevelopmentServer';
-import { isEnhancedPromptMessage } from '~/components/chat/Chat.client';
+import type { Message } from '~/lib/persistence/message';
 
 function createRepositoryIdPacket(repositoryId: string): SimulationPacket {
   return {
@@ -20,31 +17,19 @@ function createRepositoryIdPacket(repositoryId: string): SimulationPacket {
   };
 }
 
-type ProtocolMessageRole = 'user' | 'assistant' | 'system';
+interface ChatReferenceElement {
-
+  kind: 'element';
-type ProtocolMessageText = {
+  selector: string;
-  type: 'text';
+  width: number;
-  role: ProtocolMessageRole;
+  height: number;
-  content: string;
+  x: number;
-};
+  y: number;
-
-type ProtocolMessageImage = {
-  type: 'image';
-  role: ProtocolMessageRole;
-  dataURL: string;
-};
-
-export type ProtocolMessage = ProtocolMessageText | ProtocolMessageImage;
-
-type ChatResponsePartCallback = (response: string) => void;
-
-type ChatMessageMode = 'recording' | 'static' | 'developer';
-
-interface ChatMessageOptions {
-  baseRepositoryId?: string;
-  onResponsePart?: ChatResponsePartCallback;
 }
 
+export type ChatReference = ChatReferenceElement;
+
+type ChatResponsePartCallback = (message: Message) => void;
+
 class ChatManager {
   // Empty if this chat has been destroyed.
   client: ProtocolClient | undefined;
@@ -52,9 +37,6 @@ class ChatManager {
   // Resolves when the chat has started.
   chatIdPromise: Promise<string>;
 
-  // Resolves when the recording has been created.
-  recordingIdPromise: Promise<string> | undefined;
-
   // Whether all simulation data has been sent.
   simulationFinished?: boolean;
 
@@ -133,63 +115,37 @@ class ChatManager {
     assert(!this.simulationFinished, 'Simulation has been finished');
     assert(this.repositoryId, 'Expected repository ID');
 
-    this.recordingIdPromise = (async () => {
-      assert(this.client, 'Chat has been destroyed');
-
-      const chatId = await this.chatIdPromise;
-      const { recordingId } = (await this.client.sendCommand({
-        method: 'Nut.finishSimulationData',
-        params: { chatId },
-      })) as { recordingId: string | undefined };
-
-      assert(recordingId, 'Recording ID not set');
-
-      return recordingId;
-    })();
-
     const allData = [createRepositoryIdPacket(this.repositoryId), ...this.pageData];
     this.simulationFinished = true;
 
     return allData;
   }
 
-  async sendChatMessage(mode: ChatMessageMode, messages: ProtocolMessage[], options?: ChatMessageOptions) {
+  async sendChatMessage(messages: Message[], references: ChatReference[], onResponsePart: ChatResponsePartCallback) {
     assert(this.client, 'Chat has been destroyed');
 
     const responseId = `response-${generateRandomId()}`;
 
-    let response: string = '';
     const removeResponseListener = this.client.listenForMessage(
       'Nut.chatResponsePart',
-      ({ responseId: eventResponseId, message }: { responseId: string; message: ProtocolMessage }) => {
+      ({ responseId: eventResponseId, message }: { responseId: string; message: Message }) => {
         if (responseId == eventResponseId) {
-          if (message.type == 'text') {
+          console.log('ChatResponse', chatId, message);
-            response += message.content;
+          onResponsePart(message);
-            options?.onResponsePart?.(message.content);
-          }
         }
       },
     );
 
     const chatId = await this.chatIdPromise;
 
-    console.log(
+    console.log('ChatSendMessage', new Date().toISOString(), chatId, JSON.stringify({ messages, references }));
-      'ChatSendMessage',
-      new Date().toISOString(),
-      chatId,
-      JSON.stringify({ mode, messages, baseRepositoryId: options?.baseRepositoryId }),
-    );
 
-    const { repositoryId } = (await this.client.sendCommand({
+    await this.client.sendCommand({
       method: 'Nut.sendChatMessage',
-      params: { chatId, responseId, mode, messages, baseRepositoryId: options?.baseRepositoryId },
+      params: { chatId, responseId, messages, references },
-    })) as { repositoryId?: string };
+    });
-
-    console.log('ChatResponse', chatId, repositoryId, response);
 
     removeResponseListener();
-
-    return { response, repositoryId };
   }
 }
 
@@ -232,264 +188,40 @@ export async function simulationReloaded() {
   startChat(repositoryId, []);
 }
 
-export async function simulationAddData(data: SimulationData) {
+export function simulationAddData(data: SimulationData) {
   assert(gChatManager, 'Expected to have an active chat');
   gChatManager.addPageData(data);
 }
 
+export function simulationFinishData() {
+  assert(gChatManager, 'Expected to have an active chat');
+  gChatManager.finishSimulationData();
+}
+
 let gLastUserSimulationData: SimulationData | undefined;
 
 export function getLastUserSimulationData(): SimulationData | undefined {
   return gLastUserSimulationData;
 }
 
-export async function getSimulationRecording(): Promise<string> {
-  assert(gChatManager, 'Expected to have an active chat');
-
-  const simulationData = gChatManager.finishSimulationData();
-
-  /*
-   * The repository contents are part of the problem and excluded from the simulation data
-   * reported for solutions.
-   */
-  gLastUserSimulationData = simulationData.filter((packet) => packet.kind != 'repositoryId');
-
-  console.log('SimulationData', new Date().toISOString(), JSON.stringify(simulationData));
-
-  assert(gChatManager.recordingIdPromise, 'Expected recording promise');
-
-  return gChatManager.recordingIdPromise;
-}
-
 export function isSimulatingOrHasFinished(): boolean {
   return gChatManager?.isValid() ?? false;
 }
 
-export async function getSimulationRecordingId(): Promise<string> {
+let gLastSimulationChatMessages: Message[] | undefined;
-  assert(gChatManager, 'Chat not started');
-  assert(gChatManager.recordingIdPromise, 'Expected recording promise');
 
-  return gChatManager.recordingIdPromise;
+export function getLastSimulationChatMessages(): Message[] | undefined {
-}
-
-let gLastSimulationChatMessages: ProtocolMessage[] | undefined;
-
-export function getLastSimulationChatMessages(): ProtocolMessage[] | undefined {
   return gLastSimulationChatMessages;
 }
 
-const simulationSystemPrompt = `
+export async function sendChatMessage(
-The following user message describes a bug or other problem on the page which needs to be fixed.
-You must respond with a useful explanation that will help the user understand the source of the problem.
-Do not describe the specific fix needed.
-`;
-
-export async function getSimulationEnhancedPrompt(
-  chatMessages: Message[],
-  userMessage: string,
-  mouseData: MouseData | undefined,
-): Promise<string> {
-  assert(gChatManager, 'Chat not started');
-  assert(gChatManager.simulationFinished, 'Simulation not finished');
-
-  let system = simulationSystemPrompt;
-
-  if (mouseData) {
-    system += `The user pointed to an element on the page <element selector=${JSON.stringify(mouseData.selector)} height=${mouseData.height} width=${mouseData.width} x=${mouseData.x} y=${mouseData.y} />`;
-  }
-
-  const messages: ProtocolMessage[] = [
-    {
-      role: 'system',
-      type: 'text',
-      content: system,
-    },
-    {
-      role: 'user',
-      type: 'text',
-      content: userMessage,
-    },
-  ];
-
-  gLastSimulationChatMessages = messages;
-
-  const { response } = await gChatManager.sendChatMessage('recording', messages);
-
-  return response;
-}
-
-export async function shouldUseSimulation(messageInput: string) {
-  if (!gChatManager) {
-    gChatManager = new ChatManager();
-  }
-
-  const systemPrompt = `
-You are a helpful assistant that determines whether a user's message that is asking an AI
-to make a change to an application should first perform a detailed analysis of the application's
-behavior to generate a better answer.
-
-This is most helpful when the user is asking the AI to fix a problem with the application.
-When making straightforward improvements to the application a detailed analysis is not necessary.
-
-The text of the user's message will be wrapped in \`<user_message>\` tags. You must describe your
-reasoning and then respond with either \`<analyze>true</analyze>\` or \`<analyze>false</analyze>\`.
-  `;
-
-  const userMessage = `
-Here is the user message you need to evaluate: <user_message>${messageInput}</user_message>
-  `;
-
-  const messages: ProtocolMessage[] = [
-    {
-      role: 'system',
-      type: 'text',
-      content: systemPrompt,
-    },
-    {
-      role: 'user',
-      type: 'text',
-      content: userMessage,
-    },
-  ];
-
-  const { response } = await gChatManager.sendChatMessage('static', messages);
-
-  console.log('UseSimulationResponse', response);
-
-  const match = /<analyze>(.*?)<\/analyze>/.exec(response);
-
-  if (match) {
-    return match[1] === 'true';
-  }
-
-  return false;
-}
-
-function getProtocolRole(message: Message): 'user' | 'assistant' | 'system' {
-  switch (message.role) {
-    case 'user':
-      return 'user';
-    case 'assistant':
-    case 'data':
-      return 'assistant';
-    case 'system':
-      return 'system';
-    default:
-      throw new Error(`Unknown message role: ${message.role}`);
-  }
-}
-
-function removeBoltArtifacts(text: string): string {
-  const openTag = '<boltArtifact';
-  const closeTag = '</boltArtifact>';
-
-  while (true) {
-    const openTagIndex = text.indexOf(openTag);
-
-    if (openTagIndex === -1) {
-      break;
-    }
-
-    const prefix = text.substring(0, openTagIndex);
-
-    const closeTagIndex = text.indexOf(closeTag, openTagIndex + openTag.length);
-
-    if (closeTagIndex === -1) {
-      text = prefix;
-    } else {
-      text = prefix + text.substring(closeTagIndex + closeTag.length);
-    }
-  }
-
-  return text;
-}
-
-function buildProtocolMessages(messages: Message[]): ProtocolMessage[] {
-  const rv: ProtocolMessage[] = [];
-
-  for (const msg of messages) {
-    const role = getProtocolRole(msg);
-
-    if (Array.isArray(msg.content)) {
-      for (const content of msg.content) {
-        switch (content.type) {
-          case 'text':
-            rv.push({
-              role,
-              type: 'text',
-              content: removeBoltArtifacts(content.text),
-            });
-            break;
-          case 'image':
-            rv.push({
-              role,
-              type: 'image',
-              dataURL: content.image,
-            });
-            break;
-          default:
-            console.error('Unknown message content', content);
-        }
-      }
-    } else if (typeof msg.content == 'string') {
-      rv.push({
-        role,
-        type: 'text',
-        content: msg.content,
-      });
-    }
-  }
-
-  return rv;
-}
-
-function messagesHaveEnhancedPrompt(messages: Message[]): boolean {
-  const lastEnhancedPromptMessage = messages.findLastIndex((msg) => isEnhancedPromptMessage(msg));
-
-  if (lastEnhancedPromptMessage == -1) {
-    return false;
-  }
-
-  const lastUserMessage = messages.findLastIndex((msg) => msg.role == 'user');
-
-  if (lastUserMessage == -1) {
-    return false;
-  }
-
-  return lastUserMessage < lastEnhancedPromptMessage;
-}
-
-export async function sendDeveloperChatMessage(
   messages: Message[],
-  baseRepositoryId: string | undefined,
+  references: ChatReference[],
   onResponsePart: ChatResponsePartCallback,
 ) {
   if (!gChatManager) {
     gChatManager = new ChatManager();
   }
 
-  let systemPrompt = developerSystemPrompt;
+  await gChatManager.sendChatMessage(messages, references, onResponsePart);
-
-  if (messagesHaveEnhancedPrompt(messages)) {
-    // Add directions to the LLM when we have an enhanced prompt describing the bug to fix.
-    const systemEnhancedPrompt = `
-ULTRA IMPORTANT: You have been given a detailed description of a bug you need to fix.
-Focus specifically on fixing this bug. Do not guess about other problems.
-    `;
-    systemPrompt += systemEnhancedPrompt;
-  }
-
-  const protocolMessages = buildProtocolMessages(messages);
-  protocolMessages.unshift({
-    role: 'system',
-    type: 'text',
-    content: systemPrompt,
-  });
-
-  const { repositoryId } = await gChatManager.sendChatMessage('developer', protocolMessages, {
-    baseRepositoryId,
-    onResponsePart,
-  });
-
-  return repositoryId;
 }

app/utils/folderImport.ts

@@ -1,4 +1,4 @@
-import type { Message } from '~/lib/persistence/useChatHistory';
+import type { Message } from '~/lib/persistence/message';
 import { generateId } from './fileUtils';
 import JSZip from 'jszip';
 
@@ -43,15 +43,15 @@ export function createChatFromFolder(folderName: string, repositoryId: string):
     role: 'user',
     id: generateId(),
     content: `Import the "${folderName}" folder`,
-    createdAt: new Date(),
+    type: 'text',
   };
 
   const filesMessage: Message = {
     role: 'assistant',
     content: filesContent,
     id: generateId(),
-    createdAt: new Date(),
     repositoryId,
+    type: 'text',
   };
 
   const messages = [userMessage, filesMessage];