Assorted telemetry related fixes (#46)

app/components/chat/BaseChat.tsx

@@ -328,9 +328,14 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
          <ClientOnly>
            {() => (
              <SendButton
-               show={(input.length > 0 || uploadedFiles.length > 0) && chatStarted}
+               show={(isStreaming || input.length > 0 || uploadedFiles.length > 0) && chatStarted}
                isStreaming={isStreaming}
                onClick={(event) => {
+                 if (isStreaming) {
+                   handleStop?.();
+                   return;
+                 }
+
                  if (input.length > 0 || uploadedFiles.length > 0) {
                    handleSendMessage?.(event);
                  }

app/components/chat/Chat.client.tsx

@@ -31,7 +31,7 @@ import type { FileMap } from '~/lib/stores/files';
 import { shouldIncludeFile } from '~/utils/fileUtils';
 import { getNutLoginKey, submitFeedback } from '~/lib/replay/Problems';
 import { shouldUseSimulation } from '~/lib/hooks/useSimulation';
-import { pingTelemetry } from '~/lib/hooks/pingTelemetry';
+import { ChatMessageTelemetry, pingTelemetry } from '~/lib/hooks/pingTelemetry';
 import type { RejectChangeData } from './ApproveChange';
 
 const toastAnimation = cssTransition({
@@ -176,6 +176,8 @@ function filterFiles(files: FileMap): FileMap {
   return rv;
 }
 
+let gActiveChatMessageTelemetry: ChatMessageTelemetry | undefined;
+
 export const ChatImpl = memo(
   ({ description, initialMessages, storeMessageHistory, importChat, exportChat }: ChatProps) => {
     useShortcuts();
@@ -211,6 +213,12 @@ export const ChatImpl = memo(
       initialInput: Cookies.get(PROMPT_COOKIE_KEY) || '',
     });
 
+    // Once we are no longer loading the message is complete.
+    if (gActiveChatMessageTelemetry && !isLoading && !simulationLoading) {
+      gActiveChatMessageTelemetry.finish();
+      gActiveChatMessageTelemetry = undefined;
+    }
+
     useEffect(() => {
       const prompt = searchParams.get('prompt');
 
@@ -262,6 +270,11 @@ export const ChatImpl = memo(
       chatStore.setKey('aborted', true);
       workbenchStore.abortAllActions();
       setSimulationLoading(false);
+
+      if (gActiveChatMessageTelemetry) {
+        gActiveChatMessageTelemetry.abort("StopButtonClicked");
+        gActiveChatMessageTelemetry = undefined;
+      }
     };
 
     useEffect(() => {
@@ -312,7 +325,7 @@ export const ChatImpl = memo(
     };
 
     const getEnhancedPrompt = async (userMessage: string) => {
-      let enhancedPrompt, message;
+      let enhancedPrompt, message, hadError = false;
       try {
         const mouseData = getCurrentMouseData();
         enhancedPrompt = await getSimulationEnhancedPrompt(messages, userMessage, mouseData);
@@ -320,6 +333,7 @@ export const ChatImpl = memo(
       } catch (e) {
         console.error("Error enhancing prompt", e);
         message = "Error enhancing prompt.";
+        hadError = true;
       }
 
       const enhancedPromptMessage: Message = {
@@ -328,7 +342,7 @@ export const ChatImpl = memo(
         content: message,
       };
 
-      return { enhancedPrompt, enhancedPromptMessage };
+      return { enhancedPrompt, enhancedPromptMessage, hadError };
     }
 
     const sendMessage = async (messageInput?: string) => {
@@ -339,6 +353,8 @@ export const ChatImpl = memo(
         return;
       }
 
+      gActiveChatMessageTelemetry = new ChatMessageTelemetry(messages.length);
+
       const loginKey = getNutLoginKey();
 
       const apiKeyCookie = Cookies.get(anthropicApiKeyCookieName);
@@ -348,6 +364,8 @@ export const ChatImpl = memo(
         const numFreeUses = +(Cookies.get(anthropicNumFreeUsesCookieName) || 0);
         if (numFreeUses >= MaxFreeUses) {
           toast.error('All free uses consumed. Please set a login key or Anthropic API key in the "User Info" settings.');
+          gActiveChatMessageTelemetry.abort("NoFreeUses");
+          gActiveChatMessageTelemetry = undefined;
           return;
         }
 
@@ -367,7 +385,7 @@ export const ChatImpl = memo(
 
       let simulationEnhancedPrompt: string | undefined;
 
-      const simulation = await shouldUseSimulation(messages, _input);
+      const simulation = chatStarted && await shouldUseSimulation(messages, _input);
 
       if (numAbortsAtStart != gNumAborts) {
         return;
@@ -375,9 +393,10 @@ export const ChatImpl = memo(
 
       console.log("UseSimulation", simulation);
 
-      let didEnhancePrompt = false;
-
+      let simulationStatus = "NoSimulation";
       if (simulation) {
+        gActiveChatMessageTelemetry.startSimulation();
+
         gLockSimulationData = true;
         try {
           await flushSimulationData();
@@ -402,11 +421,16 @@ export const ChatImpl = memo(
             }
 
             simulationEnhancedPrompt = info.enhancedPrompt;
-            didEnhancePrompt = true;
 
             console.log("EnhancedPromptMessage", info.enhancedPromptMessage);
             setMessages([...messages, info.enhancedPromptMessage]);
+
+            simulationStatus = info.hadError ? "PromptError" : "Success";
+          } else {
+            simulationStatus = "RecordingError";
           }
+
+          gActiveChatMessageTelemetry.endSimulation(simulationStatus);
         } finally {
           gLockSimulationData = false;
         }
@@ -463,12 +487,7 @@ export const ChatImpl = memo(
         setApproveChangesMessageId(lastMessage.id);
       }
 
-      await pingTelemetry("Chat.SendMessage", {
-        numMessages: messages.length,
-        simulation,
-        didEnhancePrompt,
-        loginKey: getNutLoginKey(),
-      });
+      gActiveChatMessageTelemetry.sendPrompt(simulationStatus);
     };
 
     const onRewind = async (messageId: string, contents: string) => {
@@ -486,7 +505,7 @@ export const ChatImpl = memo(
         setMessages(messages.slice(0, messageIndex + 1));
       }
 
-      await pingTelemetry("Chat.Rewind", {
+      await pingTelemetry("RewindChat", {
         numMessages: messages.length,
         rewindIndex: messageIndex,
         loginKey: getNutLoginKey(),
@@ -522,7 +541,7 @@ export const ChatImpl = memo(
 
       await flashScreen();
 
-      await pingTelemetry("Chat.ApproveChange", {
+      await pingTelemetry("ApproveChange", {
         numMessages: messages.length,
         loginKey: getNutLoginKey(),
       });
@@ -555,7 +574,7 @@ export const ChatImpl = memo(
         sendMessage(messageContents);
       }
 
-      await pingTelemetry("Chat.RejectChange", {
+      await pingTelemetry("RejectChange", {
         retry: data.retry,
         shareProject: data.shareProject,
         shareProjectSuccess,

app/components/header/Header.tsx

@@ -22,6 +22,14 @@ export function Header() {
           <img src="/logo-styled.svg" alt="logo" className="w-[40px] inline-block rotate-90" />
         </a>
         <Feedback />
+        <a
+          href="https://www.replay.io/discord"
+          target="_blank"
+          rel="noopener noreferrer"
+          className="text-bolt-elements-textPrimary hover:text-accent"
+        >
+          <div className="i-ph:discord-logo-fill text-xl" />
+        </a>
       </div>
       {chat.started && ( // Display ChatDescription and HeaderActionButtons only when the chat has started.
         <>

app/lib/hooks/pingTelemetry.ts

@@ -1,8 +1,12 @@
 
-// FIXME ping telemetry server directly instead of going through the server.
+// FIXME ping telemetry server directly instead of going through the backend.
+
+import { getNutLoginKey } from "../replay/Problems";
+
+// We do this to work around CORS insanity.
 export async function pingTelemetry(event: string, data: any) {
   const requestBody: any = {
-    event,
+    event: "NutChat." + event,
     data,
   };
 
@@ -11,3 +15,45 @@ export async function pingTelemetry(event: string, data: any) {
     body: JSON.stringify(requestBody),
   });
 }
+
+// Manage telemetry events for a single chat message.
+
+export class ChatMessageTelemetry {
+  id: string;
+  numMessages: number;
+
+  constructor(numMessages: number) {
+    this.id = Math.random().toString(36).substring(2, 15);
+    this.numMessages = numMessages;
+    this.ping("StartMessage");
+  }
+
+  private ping(event: string, data: any = {}) {
+    pingTelemetry(event, {
+      ...data,
+      loginKey: getNutLoginKey(),
+      messageId: this.id,
+      numMessages: this.numMessages,
+    });
+  }
+
+  finish() {
+    this.ping("FinishMessage");
+  }
+
+  abort(reason: string) {
+    this.ping("AbortMessage", { reason });
+  }
+
+  startSimulation() {
+    this.ping("StartSimulation");
+  }
+
+  endSimulation(status: string) {
+    this.ping("EndSimulation", { status });
+  }
+
+  sendPrompt(simulationStatus: string) {
+    this.ping("SendPrompt", { simulationStatus });
+  }
+}

app/routes/api.ping-telemetry.ts

@@ -1,28 +1,39 @@
 import { json, type ActionFunctionArgs } from '@remix-run/cloudflare';
-import { getCurrentSpan, wrapWithSpan } from '~/lib/.server/otel';
+
+async function pingTelemetry(event: string, data: any): Promise<boolean> {
+  console.log("PingTelemetry", event, data);
+
+  try {
+    const response = await fetch("https://telemetry.replay.io/", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ event, ...data }),
+    });
+
+    if (!response.ok) {
+      console.error(`Telemetry request returned unexpected status: ${response.status}`);
+      return false;
+    }
+    return true;
+  } catch (error) {
+    console.error("Telemetry request failed:", error);
+    return false;
+  }
+}
 
 export async function action(args: ActionFunctionArgs) {
   return pingTelemetryAction(args);
 }
 
-const pingTelemetryAction = wrapWithSpan(
-  {
-    name: "ping-telemetry",
-  },
-  async function pingTelemetryAction({ context, request }: ActionFunctionArgs) {
+async function pingTelemetryAction({ context, request }: ActionFunctionArgs) {
   const { event, data } = await request.json<{
     event: string;
     data: any;
   }>();
 
-    console.log("PingTelemetry", event, data);
+  const success = await pingTelemetry(event, data);
 
-    const span = getCurrentSpan();
-    span?.setAttributes({
-      "telemetry.event": event,
-      "telemetry.data": data,
-    });
-
-    return json({ success: true });
-  }
-);
+  return json({ success });
+}