diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 97b82868..67593002 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -328,9 +328,14 @@ export const BaseChat = React.forwardRef( {() => ( 0 || uploadedFiles.length > 0) && chatStarted} + show={(isStreaming || input.length > 0 || uploadedFiles.length > 0) && chatStarted} isStreaming={isStreaming} onClick={(event) => { + if (isStreaming) { + handleStop?.(); + return; + } + if (input.length > 0 || uploadedFiles.length > 0) { handleSendMessage?.(event); } diff --git a/app/components/chat/Chat.client.tsx b/app/components/chat/Chat.client.tsx index 64fda213..ed03e2e5 100644 --- a/app/components/chat/Chat.client.tsx +++ b/app/components/chat/Chat.client.tsx @@ -31,7 +31,7 @@ import type { FileMap } from '~/lib/stores/files'; import { shouldIncludeFile } from '~/utils/fileUtils'; import { getNutLoginKey, submitFeedback } from '~/lib/replay/Problems'; import { shouldUseSimulation } from '~/lib/hooks/useSimulation'; -import { pingTelemetry } from '~/lib/hooks/pingTelemetry'; +import { ChatMessageTelemetry, pingTelemetry } from '~/lib/hooks/pingTelemetry'; import type { RejectChangeData } from './ApproveChange'; const toastAnimation = cssTransition({ @@ -176,6 +176,8 @@ function filterFiles(files: FileMap): FileMap { return rv; } +let gActiveChatMessageTelemetry: ChatMessageTelemetry | undefined; + export const ChatImpl = memo( ({ description, initialMessages, storeMessageHistory, importChat, exportChat }: ChatProps) => { useShortcuts(); @@ -211,6 +213,12 @@ export const ChatImpl = memo( initialInput: Cookies.get(PROMPT_COOKIE_KEY) || '', }); + // Once we are no longer loading the message is complete. + if (gActiveChatMessageTelemetry && !isLoading && !simulationLoading) { + gActiveChatMessageTelemetry.finish(); + gActiveChatMessageTelemetry = undefined; + } + useEffect(() => { const prompt = searchParams.get('prompt'); @@ -262,6 +270,11 @@ export const ChatImpl = memo( chatStore.setKey('aborted', true); workbenchStore.abortAllActions(); setSimulationLoading(false); + + if (gActiveChatMessageTelemetry) { + gActiveChatMessageTelemetry.abort("StopButtonClicked"); + gActiveChatMessageTelemetry = undefined; + } }; useEffect(() => { @@ -312,7 +325,7 @@ export const ChatImpl = memo( }; const getEnhancedPrompt = async (userMessage: string) => { - let enhancedPrompt, message; + let enhancedPrompt, message, hadError = false; try { const mouseData = getCurrentMouseData(); enhancedPrompt = await getSimulationEnhancedPrompt(messages, userMessage, mouseData); @@ -320,6 +333,7 @@ export const ChatImpl = memo( } catch (e) { console.error("Error enhancing prompt", e); message = "Error enhancing prompt."; + hadError = true; } const enhancedPromptMessage: Message = { @@ -328,7 +342,7 @@ export const ChatImpl = memo( content: message, }; - return { enhancedPrompt, enhancedPromptMessage }; + return { enhancedPrompt, enhancedPromptMessage, hadError }; } const sendMessage = async (messageInput?: string) => { @@ -339,6 +353,8 @@ export const ChatImpl = memo( return; } + gActiveChatMessageTelemetry = new ChatMessageTelemetry(messages.length); + const loginKey = getNutLoginKey(); const apiKeyCookie = Cookies.get(anthropicApiKeyCookieName); @@ -348,6 +364,8 @@ export const ChatImpl = memo( const numFreeUses = +(Cookies.get(anthropicNumFreeUsesCookieName) || 0); if (numFreeUses >= MaxFreeUses) { toast.error('All free uses consumed. Please set a login key or Anthropic API key in the "User Info" settings.'); + gActiveChatMessageTelemetry.abort("NoFreeUses"); + gActiveChatMessageTelemetry = undefined; return; } @@ -367,7 +385,7 @@ export const ChatImpl = memo( let simulationEnhancedPrompt: string | undefined; - const simulation = await shouldUseSimulation(messages, _input); + const simulation = chatStarted && await shouldUseSimulation(messages, _input); if (numAbortsAtStart != gNumAborts) { return; @@ -375,9 +393,10 @@ export const ChatImpl = memo( console.log("UseSimulation", simulation); - let didEnhancePrompt = false; - + let simulationStatus = "NoSimulation"; if (simulation) { + gActiveChatMessageTelemetry.startSimulation(); + gLockSimulationData = true; try { await flushSimulationData(); @@ -402,11 +421,16 @@ export const ChatImpl = memo( } simulationEnhancedPrompt = info.enhancedPrompt; - didEnhancePrompt = true; console.log("EnhancedPromptMessage", info.enhancedPromptMessage); setMessages([...messages, info.enhancedPromptMessage]); + + simulationStatus = info.hadError ? "PromptError" : "Success"; + } else { + simulationStatus = "RecordingError"; } + + gActiveChatMessageTelemetry.endSimulation(simulationStatus); } finally { gLockSimulationData = false; } @@ -463,12 +487,7 @@ export const ChatImpl = memo( setApproveChangesMessageId(lastMessage.id); } - await pingTelemetry("Chat.SendMessage", { - numMessages: messages.length, - simulation, - didEnhancePrompt, - loginKey: getNutLoginKey(), - }); + gActiveChatMessageTelemetry.sendPrompt(simulationStatus); }; const onRewind = async (messageId: string, contents: string) => { @@ -486,7 +505,7 @@ export const ChatImpl = memo( setMessages(messages.slice(0, messageIndex + 1)); } - await pingTelemetry("Chat.Rewind", { + await pingTelemetry("RewindChat", { numMessages: messages.length, rewindIndex: messageIndex, loginKey: getNutLoginKey(), @@ -522,7 +541,7 @@ export const ChatImpl = memo( await flashScreen(); - await pingTelemetry("Chat.ApproveChange", { + await pingTelemetry("ApproveChange", { numMessages: messages.length, loginKey: getNutLoginKey(), }); @@ -555,7 +574,7 @@ export const ChatImpl = memo( sendMessage(messageContents); } - await pingTelemetry("Chat.RejectChange", { + await pingTelemetry("RejectChange", { retry: data.retry, shareProject: data.shareProject, shareProjectSuccess, diff --git a/app/components/header/Header.tsx b/app/components/header/Header.tsx index 70b317e7..23236745 100644 --- a/app/components/header/Header.tsx +++ b/app/components/header/Header.tsx @@ -22,6 +22,14 @@ export function Header() { logo + +
+
{chat.started && ( // Display ChatDescription and HeaderActionButtons only when the chat has started. <> diff --git a/app/lib/hooks/pingTelemetry.ts b/app/lib/hooks/pingTelemetry.ts index b7702ca9..fa584246 100644 --- a/app/lib/hooks/pingTelemetry.ts +++ b/app/lib/hooks/pingTelemetry.ts @@ -1,8 +1,12 @@ -// FIXME ping telemetry server directly instead of going through the server. +// FIXME ping telemetry server directly instead of going through the backend. + +import { getNutLoginKey } from "../replay/Problems"; + +// We do this to work around CORS insanity. export async function pingTelemetry(event: string, data: any) { const requestBody: any = { - event, + event: "NutChat." + event, data, }; @@ -11,3 +15,45 @@ export async function pingTelemetry(event: string, data: any) { body: JSON.stringify(requestBody), }); } + +// Manage telemetry events for a single chat message. + +export class ChatMessageTelemetry { + id: string; + numMessages: number; + + constructor(numMessages: number) { + this.id = Math.random().toString(36).substring(2, 15); + this.numMessages = numMessages; + this.ping("StartMessage"); + } + + private ping(event: string, data: any = {}) { + pingTelemetry(event, { + ...data, + loginKey: getNutLoginKey(), + messageId: this.id, + numMessages: this.numMessages, + }); + } + + finish() { + this.ping("FinishMessage"); + } + + abort(reason: string) { + this.ping("AbortMessage", { reason }); + } + + startSimulation() { + this.ping("StartSimulation"); + } + + endSimulation(status: string) { + this.ping("EndSimulation", { status }); + } + + sendPrompt(simulationStatus: string) { + this.ping("SendPrompt", { simulationStatus }); + } +} diff --git a/app/routes/api.ping-telemetry.ts b/app/routes/api.ping-telemetry.ts index 07fa515c..29a9a455 100644 --- a/app/routes/api.ping-telemetry.ts +++ b/app/routes/api.ping-telemetry.ts @@ -1,28 +1,39 @@ import { json, type ActionFunctionArgs } from '@remix-run/cloudflare'; -import { getCurrentSpan, wrapWithSpan } from '~/lib/.server/otel'; + +async function pingTelemetry(event: string, data: any): Promise { + console.log("PingTelemetry", event, data); + + try { + const response = await fetch("https://telemetry.replay.io/", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ event, ...data }), + }); + + if (!response.ok) { + console.error(`Telemetry request returned unexpected status: ${response.status}`); + return false; + } + return true; + } catch (error) { + console.error("Telemetry request failed:", error); + return false; + } +} export async function action(args: ActionFunctionArgs) { return pingTelemetryAction(args); } -const pingTelemetryAction = wrapWithSpan( - { - name: "ping-telemetry", - }, - async function pingTelemetryAction({ context, request }: ActionFunctionArgs) { - const { event, data } = await request.json<{ - event: string; - data: any; - }>(); +async function pingTelemetryAction({ context, request }: ActionFunctionArgs) { + const { event, data } = await request.json<{ + event: string; + data: any; + }>(); - console.log("PingTelemetry", event, data); + const success = await pingTelemetry(event, data); - const span = getCurrentSpan(); - span?.setAttributes({ - "telemetry.event": event, - "telemetry.data": data, - }); - - return json({ success: true }); - } -); + return json({ success }); +}