From 1e7af9f5c8beb9a134d8bbf9a6cb79e83e7f5cad Mon Sep 17 00:00:00 2001
From: Brian Hackett <bhackett1024@gmail.com>
Date: Mon, 3 Mar 2025 08:54:08 -0800
Subject: [PATCH] Support feedback for individual prompts (#43)

---
 app/components/chat/ApproveChange.tsx     | 121 ++++++++++++++++++++++
 app/components/chat/BaseChat.tsx          |  54 ++++------
 app/components/chat/Chat.client.tsx       | 107 ++++++++++++++++++-
 app/components/chat/Messages.client.tsx   |  53 +++++++++-
 app/components/chat/SendButton.client.tsx |  21 ++--
 app/lib/.server/llm/chat-anthropic.ts     |  16 +--
 app/lib/hooks/pingTelemetry.ts            |  13 +++
 app/lib/hooks/useSimulation.ts            |  16 +++
 app/routes/api.ping-telemetry.ts          |  28 +++++
 app/routes/api.use-simulation.ts          |  55 ++++++++++
 10 files changed, 427 insertions(+), 57 deletions(-)
 create mode 100644 app/components/chat/ApproveChange.tsx
 create mode 100644 app/lib/hooks/pingTelemetry.ts
 create mode 100644 app/lib/hooks/useSimulation.ts
 create mode 100644 app/routes/api.ping-telemetry.ts
 create mode 100644 app/routes/api.use-simulation.ts

diff --git a/app/components/chat/ApproveChange.tsx b/app/components/chat/ApproveChange.tsx
new file mode 100644
index 00000000..a15ba08d
--- /dev/null
+++ b/app/components/chat/ApproveChange.tsx
@@ -0,0 +1,121 @@
+import React, { useRef, useState } from 'react';
+import { classNames } from '~/utils/classNames';
+import { TEXTAREA_MIN_HEIGHT } from './BaseChat';
+
+export interface RejectChangeData {
+  explanation: string;
+  shareProject: boolean;
+  retry: boolean;
+}
+
+interface ApproveChangeProps {
+  onApprove: () => void;
+  onReject: (data: RejectChangeData) => void;
+}
+
+const ApproveChange: React.FC<ApproveChangeProps> = ({ onApprove, onReject }) => {
+  const textareaRef = useRef<HTMLTextAreaElement>(null);
+  const [hasRejected, setHasRejected] = useState(false);
+  const [shareProject, setShareProject] = useState(false);
+
+  if (hasRejected) {
+    const performReject = (retry: boolean) => {
+      const explanation = textareaRef.current?.value ?? '';
+      onReject({
+        explanation,
+        shareProject,
+        retry,
+      });
+    };
+
+    return (
+      <>
+        <div
+          className={classNames(
+            'relative shadow-xs border border-bolt-elements-borderColor backdrop-blur rounded-lg bg-red-50 mt-3',
+          )}
+        >
+          <textarea
+            ref={textareaRef}
+            className={classNames(
+              'w-full pl-4 pt-4 pr-25 outline-none resize-none text-bolt-elements-textPrimary placeholder-bolt-elements-textTertiary bg-transparent text-sm',
+              'transition-all duration-200',
+              'hover:border-bolt-elements-focus'
+            )}
+            onKeyDown={(event) => {
+              if (event.key === 'Enter') {
+                if (event.shiftKey) {
+                  return;
+                }
+
+                event.preventDefault();
+                performReject(true);
+              }
+            }}
+            style={{
+              minHeight: TEXTAREA_MIN_HEIGHT,
+              maxHeight: 400,
+            }}
+            placeholder="What's wrong with the changes?"
+            translate="no"
+          />
+        </div>
+
+        <div className="flex items-center gap-2 w-full mb-2">
+          <input
+            type="checkbox"
+            id="share-project"
+            checked={shareProject}
+            onChange={(event) => setShareProject(event.target.checked)}
+            className="rounded border-red-300 text-red-500 focus:ring-red-500"
+          />
+          <label htmlFor="share-project" className="text-sm text-red-600">
+            Share with Nut team
+          </label>
+        </div>
+
+        <div className="flex items-center gap-1 w-full h-[30px] pt-2">
+          <button
+            onClick={() => performReject(false)}
+            className="flex-1 h-[30px] flex justify-center items-center bg-red-100 border border-red-500 text-red-500 hover:bg-red-200 hover:text-red-600 transition-colors rounded"
+            aria-label="Cancel changes"
+            title="Cancel changes"
+          >
+            <div className="i-ph:x-bold"></div>
+          </button>
+          <button
+            onClick={() => performReject(true)}
+            className="flex-1 h-[30px] flex justify-center items-center bg-green-100 border border-green-500 text-green-500 hover:bg-green-200 hover:text-green-600 transition-colors rounded"
+            aria-label="Try changes again"
+            title="Try changes again"
+          >
+            <div className="i-ph:repeat-bold"></div>
+          </button>
+        </div>
+      </>
+    );
+  }
+
+  return (
+    <div className="flex items-center gap-1 w-full h-[30px] pt-2">
+      <button
+        onClick={() => setHasRejected(true)}
+        className="flex-1 h-[30px] flex justify-center items-center bg-red-100 border border-red-500 text-red-500 hover:bg-red-200 hover:text-red-600 transition-colors rounded"
+        aria-label="Reject change"
+        title="Reject change"
+      >
+        <div className="i-ph:thumbs-down-bold"></div>
+      </button>
+      <button
+        onClick={onApprove}
+        className="flex-1 h-[30px] flex justify-center items-center bg-green-100 border border-green-500 text-green-500 hover:bg-green-200 hover:text-green-600 transition-colors rounded"
+        aria-label="Approve change"
+        title="Approve change"
+      >
+        <div className="i-ph:thumbs-up-bold"></div>
+      </button>
+    </div>
+  );
+};
+
+export default ApproveChange;
diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx
index eccd80ff..4f33ab00 100644
--- a/app/components/chat/BaseChat.tsx
+++ b/app/components/chat/BaseChat.tsx
@@ -26,8 +26,9 @@ import { ModelSelector } from '~/components/chat/ModelSelector';
 import { SpeechRecognitionButton } from '~/components/chat/SpeechRecognition';
 import { ScreenshotStateManager } from './ScreenshotStateManager';
 import { toast } from 'react-toastify';
+import type { RejectChangeData } from './ApproveChange';
 
-const TEXTAREA_MIN_HEIGHT = 76;
+export const TEXTAREA_MIN_HEIGHT = 76;
 
 interface BaseChatProps {
   textareaRef?: React.RefObject<HTMLTextAreaElement> | undefined;
@@ -42,7 +43,7 @@ interface BaseChatProps {
   promptEnhanced?: boolean;
   input?: string;
   handleStop?: () => void;
-  sendMessage?: (event: React.UIEvent, messageInput?: string, simulation?: boolean) => void;
+  sendMessage?: (messageInput?: string) => void;
   handleInputChange?: (event: React.ChangeEvent<HTMLTextAreaElement>) => void;
   enhancePrompt?: () => void;
   importChat?: (description: string, messages: Message[]) => Promise<void>;
@@ -52,6 +53,9 @@ interface BaseChatProps {
   imageDataList?: string[];
   setImageDataList?: (dataList: string[]) => void;
   onRewind?: (messageId: string, contents: string) => void;
+  approveChangesMessageId?: string;
+  onApproveChange?: (messageId: string) => void;
+  onRejectChange?: (lastMessageId: string, rewindMessageId: string, contents: string, data: RejectChangeData) => void;
 }
 
 export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
@@ -79,6 +83,9 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
       setImageDataList,
       messages,
       onRewind,
+      approveChangesMessageId,
+      onApproveChange,
+      onRejectChange,
     },
     ref,
   ) => {
@@ -139,9 +146,9 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
       }
     };
 
-    const handleSendMessage = (event: React.UIEvent, messageInput?: string, simulation?: boolean) => {
+    const handleSendMessage = (event: React.UIEvent, messageInput?: string) => {
       if (sendMessage) {
-        sendMessage(event, messageInput, simulation);
+        sendMessage(messageInput);
 
         if (recognition) {
           recognition.abort(); // Stop current recognition
@@ -244,6 +251,9 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                       messages={messages}
                       isStreaming={isStreaming}
                       onRewind={onRewind}
+                      approveChangesMessageId={approveChangesMessageId}
+                      onApproveChange={onApproveChange}
+                      onRejectChange={onRejectChange}
                     />
                   ) : null;
                 }}
@@ -377,33 +387,15 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                   />
                   <ClientOnly>
                     {() => (
-                      <>
-                        <SendButton
-                          show={input.length > 0 || isStreaming || uploadedFiles.length > 0}
-                          fixBug={false}
-                          isStreaming={isStreaming}
-                          onClick={(event) => {
-                            if (isStreaming) {
-                              handleStop?.();
-                              return;
-                            }
-
-                            if (input.length > 0 || uploadedFiles.length > 0) {
-                              handleSendMessage?.(event);
-                            }
-                          }}
-                        />
-                        <SendButton
-                          show={(input.length > 0 || uploadedFiles.length > 0) && chatStarted}
-                          fixBug={true}
-                          isStreaming={isStreaming}
-                          onClick={(event) => {
-                            if (input.length > 0 || uploadedFiles.length > 0) {
-                              handleSendMessage?.(event, undefined, true);
-                            }
-                          }}
-                        />
-                      </>
+                      <SendButton
+                        show={(input.length > 0 || uploadedFiles.length > 0) && chatStarted}
+                        isStreaming={isStreaming}
+                        onClick={(event) => {
+                          if (input.length > 0 || uploadedFiles.length > 0) {
+                            handleSendMessage?.(event);
+                          }
+                        }}
+                      />
                     )}
                   </ClientOnly>
                   <div className="flex justify-between items-center text-sm p-4 pt-2">
diff --git a/app/components/chat/Chat.client.tsx b/app/components/chat/Chat.client.tsx
index 76f15a33..1e1da7d4 100644
--- a/app/components/chat/Chat.client.tsx
+++ b/app/components/chat/Chat.client.tsx
@@ -29,7 +29,10 @@ import { getCurrentMouseData } from '../workbench/PointSelector';
 import { anthropicNumFreeUsesCookieName, anthropicApiKeyCookieName, MaxFreeUses } from '~/utils/freeUses';
 import type { FileMap } from '~/lib/stores/files';
 import { shouldIncludeFile } from '~/utils/fileUtils';
-import { getNutLoginKey } from '~/lib/replay/Problems';
+import { getNutLoginKey, submitFeedback } from '~/lib/replay/Problems';
+import { shouldUseSimulation } from '~/lib/hooks/useSimulation';
+import { pingTelemetry } from '~/lib/hooks/pingTelemetry';
+import type { RejectChangeData } from './ApproveChange';
 
 const toastAnimation = cssTransition({
   enter: 'animated fadeInRight',
@@ -185,6 +188,7 @@ export const ChatImpl = memo(
     const [simulationLoading, setSimulationLoading] = useState(false);
     const files = useStore(workbenchStore.files);
     const { promptId } = useSettings();
+    const [approveChangesMessageId, setApproveChangesMessageId] = useState<string | undefined>(undefined);
 
     const { showChat } = useStore(chatStore);
 
@@ -327,7 +331,7 @@ export const ChatImpl = memo(
       return { enhancedPrompt, enhancedPromptMessage };
     }
 
-    const sendMessage = async (_event: React.UIEvent, messageInput?: string, simulation?: boolean) => {
+    const sendMessage = async (messageInput?: string) => {
       const _input = messageInput || input;
       const numAbortsAtStart = gNumAborts;
 
@@ -363,6 +367,16 @@ export const ChatImpl = memo(
 
       let simulationEnhancedPrompt: string | undefined;
 
+      const simulation = await shouldUseSimulation(messages, _input);
+
+      if (numAbortsAtStart != gNumAborts) {
+        return;
+      }
+
+      console.log("UseSimulation", simulation);
+
+      let didEnhancePrompt = false;
+
       if (simulation) {
         gLockSimulationData = true;
         try {
@@ -388,6 +402,7 @@ export const ChatImpl = memo(
             }
 
             simulationEnhancedPrompt = info.enhancedPrompt;
+            didEnhancePrompt = true;
 
             console.log("EnhancedPromptMessage", info.enhancedPromptMessage);
             setMessages([...messages, info.enhancedPromptMessage]);
@@ -445,7 +460,15 @@ export const ChatImpl = memo(
         const { contentBase64 } = await workbenchStore.generateZipBase64();
         saveProjectContents(lastMessage.id, { content: contentBase64 });
         gLastProjectContents = contentBase64;
+        setApproveChangesMessageId(lastMessage.id);
       }
+
+      await pingTelemetry("Chat.SendMessage", {
+        numMessages: messages.length,
+        simulation,
+        didEnhancePrompt,
+        loginKey: getNutLoginKey(),
+      });
     };
 
     const onRewind = async (messageId: string, contents: string) => {
@@ -462,6 +485,83 @@ export const ChatImpl = memo(
         setParsedMessages(newParsedMessages);
         setMessages(messages.slice(0, messageIndex + 1));
       }
+
+      await pingTelemetry("Chat.Rewind", {
+        numMessages: messages.length,
+        rewindIndex: messageIndex,
+        loginKey: getNutLoginKey(),
+      });
+    };
+
+    const flashScreen = async () => {
+      const flash = document.createElement('div');
+      flash.style.position = 'fixed';
+      flash.style.top = '0';
+      flash.style.left = '0';
+      flash.style.width = '100%';
+      flash.style.height = '100%';
+      flash.style.backgroundColor = 'rgba(0, 255, 0, 0.3)';
+      flash.style.zIndex = '9999';
+      flash.style.pointerEvents = 'none';
+      document.body.appendChild(flash);
+
+      // Fade out and remove after 500ms
+      setTimeout(() => {
+        flash.style.transition = 'opacity 0.5s';
+        flash.style.opacity = '0';
+        setTimeout(() => {
+          document.body.removeChild(flash);
+        }, 500);
+      }, 200);
+    };
+
+    const onApproveChange = async (messageId: string) => {
+      console.log("ApproveChange", messageId);
+
+      setApproveChangesMessageId(undefined);
+
+      await flashScreen();
+
+      await pingTelemetry("Chat.ApproveChange", {
+        numMessages: messages.length,
+        loginKey: getNutLoginKey(),
+      });
+    };
+
+    const onRejectChange = async (messageId: string, rewindMessageId: string, projectContents: string, data: RejectChangeData) => {
+      console.log("RejectChange", messageId, data);
+
+      setApproveChangesMessageId(undefined);
+
+      const message = messages.find((message) => message.id === messageId);
+      const messageContents = message?.content ?? "";
+
+      await onRewind(rewindMessageId, projectContents);
+
+      let shareProjectSuccess = false;
+      if (data.shareProject) {
+        const feedbackData: any = {
+          explanation: data.explanation,
+          retry: data.retry,
+          chatMessages: messages,
+          repositoryContents: getLastProjectContents(),
+          loginKey: getNutLoginKey(),
+        };
+
+        shareProjectSuccess = await submitFeedback(feedbackData);
+      }
+
+      if (data.retry) {
+        sendMessage(messageContents);
+      }
+
+      await pingTelemetry("Chat.RejectChange", {
+        retry: data.retry,
+        shareProject: data.shareProject,
+        shareProjectSuccess,
+        numMessages: messages.length,
+        loginKey: getNutLoginKey(),
+      });
     };
 
     /**
@@ -535,6 +635,9 @@ export const ChatImpl = memo(
         imageDataList={imageDataList}
         setImageDataList={setImageDataList}
         onRewind={onRewind}
+        approveChangesMessageId={approveChangesMessageId}
+        onApproveChange={onApproveChange}
+        onRejectChange={onRejectChange}
       />
     );
   },
diff --git a/app/components/chat/Messages.client.tsx b/app/components/chat/Messages.client.tsx
index eec429c0..10eca85e 100644
--- a/app/components/chat/Messages.client.tsx
+++ b/app/components/chat/Messages.client.tsx
@@ -8,6 +8,7 @@ import { forkChat } from '~/lib/persistence/db';
 import { toast } from 'react-toastify';
 import WithTooltip from '~/components/ui/Tooltip';
 import { assert, sendCommandDedicatedClient } from '~/lib/replay/ReplayProtocolClient';
+import ApproveChange, { type RejectChangeData } from './ApproveChange';
 
 interface MessagesProps {
   id?: string;
@@ -15,6 +16,9 @@ interface MessagesProps {
   isStreaming?: boolean;
   messages?: Message[];
   onRewind?: (messageId: string, contents: string) => void;
+  approveChangesMessageId?: string;
+  onApproveChange?: (messageId: string) => void;
+  onRejectChange?: (lastMessageId: string, rewindMessageId: string, contents: string, data: RejectChangeData) => void;
 }
 
 interface ProjectContents {
@@ -32,14 +36,13 @@ function hasFileModifications(content: string) {
 }
 
 export const Messages = React.forwardRef<HTMLDivElement, MessagesProps>((props: MessagesProps, ref) => {
-  const { id, isStreaming = false, messages = [], onRewind } = props;
+  const { id, isStreaming = false, messages = [], onRewind, approveChangesMessageId, onApproveChange, onRejectChange } = props;
 
   const getLastMessageProjectContents = (index: number) => {
     // The message index is for the model response, and the project
     // contents will be associated with the last message present when
     // the user prompt was sent to the model. This could be either two
-    // or three messages back, depending on whether the "fix bug"
-    // button was clicked.
+    // or three messages back, depending on whether a bug explanation was added.
     const beforeUserMessage = messages[index - 2];
     const contents = gProjectContentsByMessageId.get(beforeUserMessage?.id);
     if (!contents) {
@@ -56,6 +59,28 @@ export const Messages = React.forwardRef<HTMLDivElement, MessagesProps>((props:
     return { messageId: beforeUserMessage.id, contents };
   };
 
+  const showApproveChange = (() => {
+    if (isStreaming) {
+      return false;
+    }
+
+    if (!messages.length) {
+      return false;
+    }
+
+    const lastMessageProjectContents = getLastMessageProjectContents(messages.length - 1);
+    if (!lastMessageProjectContents) {
+      return false;
+    }
+
+    if (lastMessageProjectContents.messageId != approveChangesMessageId) {
+      return false;
+    }
+
+    const lastMessage = messages[messages.length - 1];
+    return hasFileModifications(lastMessage.content);
+  })();
+
   return (
     <div id={id} ref={ref} className={props.className}>
       {messages.length > 0
@@ -123,6 +148,28 @@ export const Messages = React.forwardRef<HTMLDivElement, MessagesProps>((props:
       {isStreaming && (
         <div className="text-center w-full text-bolt-elements-textSecondary i-svg-spinners:3-dots-fade text-4xl mt-4"></div>
       )}
+      {showApproveChange && (
+        <ApproveChange
+          onApprove={() => {
+            if (onApproveChange) {
+              const lastMessage = messages[messages.length - 1];
+              assert(lastMessage);
+              onApproveChange(lastMessage.id);
+            }
+          }}
+          onReject={(data) => {
+            if (onRejectChange) {
+              const lastMessage = messages[messages.length - 1];
+              assert(lastMessage);
+
+              const info = getLastMessageProjectContents(messages.length - 1);
+              assert(info);
+
+              onRejectChange(lastMessage.id, info.messageId, info.contents.content, data);
+            }
+          }}
+        />
+      )}
     </div>
   );
 });
diff --git a/app/components/chat/SendButton.client.tsx b/app/components/chat/SendButton.client.tsx
index 0115278b..370c5947 100644
--- a/app/components/chat/SendButton.client.tsx
+++ b/app/components/chat/SendButton.client.tsx
@@ -2,7 +2,6 @@ import { AnimatePresence, cubicBezier, motion } from 'framer-motion';
 
 interface SendButtonProps {
   show: boolean;
-  fixBug: boolean;
   isStreaming?: boolean;
   disabled?: boolean;
   onClick?: (event: React.MouseEvent<HTMLButtonElement, MouseEvent>) => void;
@@ -11,17 +10,13 @@ interface SendButtonProps {
 
 const customEasingFn = cubicBezier(0.4, 0, 0.2, 1);
 
-export const SendButton = ({ show, fixBug, isStreaming, disabled, onClick }: SendButtonProps) => {
-  const className = fixBug
-    ? "absolute flex justify-center items-center top-[18px] right-[60px] p-1 bg-accent-500 hover:brightness-94 color-white rounded-md w-[34px] h-[34px] transition-theme disabled:opacity-50 disabled:cursor-not-allowed"
-    : "absolute flex justify-center items-center top-[18px] right-[22px] p-1 bg-accent-500 hover:brightness-94 color-white rounded-md w-[34px] h-[34px] transition-theme disabled:opacity-50 disabled:cursor-not-allowed";
+export const SendButton = ({ show, isStreaming, disabled, onClick }: SendButtonProps) => {
+  const className = "absolute flex justify-center items-center top-[18px] right-[22px] p-1 bg-accent-500 hover:brightness-94 color-white rounded-md w-[34px] h-[34px] transition-theme disabled:opacity-50 disabled:cursor-not-allowed";
 
   // Determine tooltip text based on button state
-  const tooltipText = fixBug
-    ? "Fix Bug"
-    : isStreaming
+  const tooltipText = isStreaming
     ? "Stop Generation"
-    : "Make Improvement";
+    : "Chat";
 
   return (
     <AnimatePresence>
@@ -43,11 +38,9 @@ export const SendButton = ({ show, fixBug, isStreaming, disabled, onClick }: Sen
           }}
         >
           <div className="text-lg">
-            {fixBug
-              ? <div className="i-ph:bug-fill"></div>
-              : !isStreaming
-                ? <div className="i-ph:hand-fill"></div>
-                : <div className="i-ph:stop-circle-bold"></div>}
+            {!isStreaming
+             ? <div className="i-ph:hand-fill"></div>
+             : <div className="i-ph:stop-circle-bold"></div>}
           </div>
         </motion.button>
       ) : null}
diff --git a/app/lib/.server/llm/chat-anthropic.ts b/app/lib/.server/llm/chat-anthropic.ts
index ecb39025..ef46ce97 100644
--- a/app/lib/.server/llm/chat-anthropic.ts
+++ b/app/lib/.server/llm/chat-anthropic.ts
@@ -45,6 +45,7 @@ export interface AnthropicApiKey {
   isUser: boolean;
   userLoginKey?: string;
 }
+
 export interface AnthropicCall {
   systemPrompt: string;
   messages: MessageParam[];
@@ -53,7 +54,7 @@ export interface AnthropicCall {
   promptTokens: number;
 }
 
-const callAnthropic = wrapWithSpan(
+export const callAnthropic = wrapWithSpan(
   {
     name: "llm-call",
     attrs: {
@@ -63,11 +64,12 @@ const callAnthropic = wrapWithSpan(
   },
 
   // eslint-disable-next-line prefer-arrow-callback
-  async function callAnthropic(apiKey: AnthropicApiKey, systemPrompt: string, messages: MessageParam[]): Promise<AnthropicCall> {
+  async function callAnthropic(apiKey: AnthropicApiKey, reason: string, systemPrompt: string, messages: MessageParam[]): Promise<AnthropicCall> {
     const span = getCurrentSpan();
     span?.setAttributes({
       "llm.chat.calls": 1, // so we can SUM(llm.chat.calls) without doing a COUNT + filter
       "llm.chat.num_messages": messages.length,
+      "llm.chat.reason": reason,
       "llm.chat.is_user_api_key": apiKey.isUser,
       "llm.chat.user_login_key": apiKey.userLoginKey,
     });
@@ -140,7 +142,7 @@ async function restorePartialFile(
   state: ChatState,
   existingContent: string,
   newContent: string,
-  apiKey: AnthropicApiKey,
+  cx: AnthropicApiKey,
   responseDescription: string
 ) {
   const systemPrompt = `
@@ -190,7 +192,7 @@ ${responseDescription}
     },
   ];
 
-  const restoreCall = await callAnthropic(apiKey, systemPrompt, messages);
+  const restoreCall = await callAnthropic(cx, "RestorePartialFile", systemPrompt, messages);
 
   const OpenTag = "<restoredContent>";
   const CloseTag = "</restoredContent>";
@@ -299,7 +301,7 @@ interface FileContents {
   content: string;
 }
 
-async function fixupResponseFiles(state: ChatState, files: FileMap, apiKey: AnthropicApiKey, responseText: string) {
+async function fixupResponseFiles(state: ChatState, files: FileMap, cx: AnthropicApiKey, responseText: string) {
   const fileContents: FileContents[] = [];
 
   const messageParser = new StreamingMessageParser({
@@ -328,7 +330,7 @@ async function fixupResponseFiles(state: ChatState, files: FileMap, apiKey: Anth
         state,
         existingContent,
         newContent,
-        apiKey,
+        cx,
         responseDescription
       );
       restoreCalls.push(restoreCall);
@@ -362,7 +364,7 @@ export async function chatAnthropic(chatController: ChatStreamController, files:
     });
   }
 
-  const mainCall = await callAnthropic(apiKey, systemPrompt, messageParams);
+  const mainCall = await callAnthropic(apiKey, "SendChatMessage", systemPrompt, messageParams);
 
   const state: ChatState = {
     infos: [],
diff --git a/app/lib/hooks/pingTelemetry.ts b/app/lib/hooks/pingTelemetry.ts
new file mode 100644
index 00000000..b7702ca9
--- /dev/null
+++ b/app/lib/hooks/pingTelemetry.ts
@@ -0,0 +1,13 @@
+
+// FIXME ping telemetry server directly instead of going through the server.
+export async function pingTelemetry(event: string, data: any) {
+  const requestBody: any = {
+    event,
+    data,
+  };
+
+  await fetch('/api/ping-telemetry', {
+    method: 'POST',
+    body: JSON.stringify(requestBody),
+  });
+}
diff --git a/app/lib/hooks/useSimulation.ts b/app/lib/hooks/useSimulation.ts
new file mode 100644
index 00000000..d53de91d
--- /dev/null
+++ b/app/lib/hooks/useSimulation.ts
@@ -0,0 +1,16 @@
+import type { Message } from 'ai';
+
+export async function shouldUseSimulation(messages: Message[], messageInput: string) {
+  const requestBody: any = {
+    messages,
+    messageInput,
+  };
+
+  const response = await fetch('/api/use-simulation', {
+    method: 'POST',
+    body: JSON.stringify(requestBody),
+  });
+
+  const result = await response.json() as any;
+  return "useSimulation" in result && !!result.useSimulation;
+}
diff --git a/app/routes/api.ping-telemetry.ts b/app/routes/api.ping-telemetry.ts
new file mode 100644
index 00000000..07fa515c
--- /dev/null
+++ b/app/routes/api.ping-telemetry.ts
@@ -0,0 +1,28 @@
+import { json, type ActionFunctionArgs } from '@remix-run/cloudflare';
+import { getCurrentSpan, wrapWithSpan } from '~/lib/.server/otel';
+
+export async function action(args: ActionFunctionArgs) {
+  return pingTelemetryAction(args);
+}
+
+const pingTelemetryAction = wrapWithSpan(
+  {
+    name: "ping-telemetry",
+  },
+  async function pingTelemetryAction({ context, request }: ActionFunctionArgs) {
+    const { event, data } = await request.json<{
+      event: string;
+      data: any;
+    }>();
+
+    console.log("PingTelemetry", event, data);
+
+    const span = getCurrentSpan();
+    span?.setAttributes({
+      "telemetry.event": event,
+      "telemetry.data": data,
+    });
+
+    return json({ success: true });
+  }
+);
diff --git a/app/routes/api.use-simulation.ts b/app/routes/api.use-simulation.ts
new file mode 100644
index 00000000..8723d396
--- /dev/null
+++ b/app/routes/api.use-simulation.ts
@@ -0,0 +1,55 @@
+import { json, type ActionFunctionArgs } from '@remix-run/cloudflare';
+
+import { callAnthropic, type AnthropicApiKey } from '~/lib/.server/llm/chat-anthropic';
+import type { MessageParam } from '@anthropic-ai/sdk/resources/messages/messages.mjs';
+
+export async function action(args: ActionFunctionArgs) {
+  return useSimulationAction(args);
+}
+
+async function useSimulationAction({ context, request }: ActionFunctionArgs) {
+  const { messages, messageInput } = await request.json<{
+    messages: Message[];
+    messageInput: string;
+  }>();
+
+  const apiKey = context.cloudflare.env.ANTHROPIC_API_KEY;
+  if (!apiKey) {
+    throw new Error("Anthropic API key is not set");
+  }
+
+  const anthropicApiKey: AnthropicApiKey = {
+    key: apiKey,
+    isUser: false,
+    userLoginKey: undefined,
+  };
+
+  const systemPrompt = `
+You are a helpful assistant that determines whether a user's message that is asking an AI
+to make a change to an application should first perform a detailed analysis of the application's
+behavior to generate a better answer.
+
+This is most helpful when the user is asking the AI to fix a problem with the application.
+When making straightforward improvements to the application a detailed analysis is not necessary.
+
+The text of the user's message will be wrapped in \`<user_message>\` tags. You must describe your
+reasoning and then respond with either \`<analyze>true</analyze>\` or \`<analyze>false</analyze>\`.
+  `;
+
+  const message: MessageParam = {
+    role: "user",
+    content: `Here is the user message you need to evaluate: <user_message>${messageInput}</user_message>`,
+  };
+
+  const { responseText } = await callAnthropic(anthropicApiKey, "UseSimulation", systemPrompt, [message]);
+
+  console.log("UseSimulationResponse", responseText);
+
+  const match = /<analyze>(.*?)<\/analyze>/.exec(responseText);
+  if (match) {
+    const useSimulation = match[1] === "true";
+    return json({ useSimulation });
+  } else {
+    return json({ useSimulation: false });
+  }
+}