Fixes for cloudflare deployment (#3)

app/components/chat/BaseChat.tsx

@@ -9,7 +9,6 @@ import { Menu } from '~/components/sidebar/Menu.client';
 import { IconButton } from '~/components/ui/IconButton';
 import { Workbench } from '~/components/workbench/Workbench.client';
 import { classNames } from '~/utils/classNames';
-import { MODEL_LIST, PROVIDER_LIST, initializeModelList } from '~/utils/constants';
 import { Messages } from './Messages.client';
 import { SendButton } from './SendButton.client';
 import { APIKeyManager } from './APIKeyManager';
@@ -25,7 +24,6 @@ import GitCloneButton from './GitCloneButton';
 import FilePreview from './FilePreview';
 import { ModelSelector } from '~/components/chat/ModelSelector';
 import { SpeechRecognitionButton } from '~/components/chat/SpeechRecognition';
-import type { IProviderSetting, ProviderInfo } from '~/types/model';
 import { ScreenshotStateManager } from './ScreenshotStateManager';
 import { toast } from 'react-toastify';
 
@@ -43,11 +41,6 @@ interface BaseChatProps {
   enhancingPrompt?: boolean;
   promptEnhanced?: boolean;
   input?: string;
-  model?: string;
-  setModel?: (model: string) => void;
-  provider?: ProviderInfo;
-  setProvider?: (provider: ProviderInfo) => void;
-  providerList?: ProviderInfo[];
   handleStop?: () => void;
   sendMessage?: (event: React.UIEvent, messageInput?: string, simulation?: boolean) => void;
   handleInputChange?: (event: React.ChangeEvent<HTMLTextAreaElement>) => void;
@@ -69,11 +62,6 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
       showChat = true,
       chatStarted = false,
       isStreaming = false,
-      model,
-      setModel,
-      provider,
-      setProvider,
-      providerList,
       input = '',
       enhancingPrompt,
       handleInputChange,
@@ -93,22 +81,6 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
     ref,
   ) => {
     const TEXTAREA_MAX_HEIGHT = chatStarted ? 400 : 200;
-    const [apiKeys, setApiKeys] = useState<Record<string, string>>(() => {
-      const savedKeys = Cookies.get('apiKeys');
-
-      if (savedKeys) {
-        try {
-          return JSON.parse(savedKeys);
-        } catch (error) {
-          console.error('Failed to parse API keys from cookies:', error);
-          return {};
-        }
-      }
-
-      return {};
-    });
-    const [modelList, setModelList] = useState(MODEL_LIST);
-    const [isModelSettingsCollapsed, setIsModelSettingsCollapsed] = useState(false);
     const [isListening, setIsListening] = useState(false);
     const [recognition, setRecognition] = useState<SpeechRecognition | null>(null);
     const [transcript, setTranscript] = useState('');
@@ -120,50 +92,6 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
     useEffect(() => {
       // Load API keys from cookies on component mount
 
-      let parsedApiKeys: Record<string, string> | undefined = {};
-
-      try {
-        const storedApiKeys = Cookies.get('apiKeys');
-
-        if (storedApiKeys) {
-          const parsedKeys = JSON.parse(storedApiKeys);
-
-          if (typeof parsedKeys === 'object' && parsedKeys !== null) {
-            setApiKeys(parsedKeys);
-            parsedApiKeys = parsedKeys;
-          }
-        }
-      } catch (error) {
-        console.error('Error loading API keys from cookies:', error);
-
-        // Clear invalid cookie data
-        Cookies.remove('apiKeys');
-      }
-
-      let providerSettings: Record<string, IProviderSetting> | undefined = undefined;
-
-      try {
-        const savedProviderSettings = Cookies.get('providers');
-
-        if (savedProviderSettings) {
-          const parsedProviderSettings = JSON.parse(savedProviderSettings);
-
-          if (typeof parsedProviderSettings === 'object' && parsedProviderSettings !== null) {
-            providerSettings = parsedProviderSettings;
-          }
-        }
-      } catch (error) {
-        console.error('Error loading Provider Settings from cookies:', error);
-
-        // Clear invalid cookie data
-        Cookies.remove('providers');
-      }
-
-      initializeModelList({ apiKeys: parsedApiKeys, providerSettings }).then((modelList) => {
-        console.log('Model List: ', modelList);
-        setModelList(modelList);
-      });
-
       if (typeof window !== 'undefined' && ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window)) {
         const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
         const recognition = new SpeechRecognition();
@@ -351,31 +279,6 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                   <rect className={classNames(styles.PromptEffectLine)} pathLength="100" strokeLinecap="round"></rect>
                   <rect className={classNames(styles.PromptShine)} x="48" y="24" width="70" height="1"></rect>
                 </svg>
-                <div>
-                  <div className={isModelSettingsCollapsed ? 'hidden' : ''}>
-                    <ModelSelector
-                      key={provider?.name + ':' + modelList.length}
-                      model={model}
-                      setModel={setModel}
-                      modelList={modelList}
-                      provider={provider}
-                      setProvider={setProvider}
-                      providerList={providerList || (PROVIDER_LIST as ProviderInfo[])}
-                      apiKeys={apiKeys}
-                    />
-                    {(providerList || []).length > 0 && provider && (
-                      <APIKeyManager
-                        provider={provider}
-                        apiKey={apiKeys[provider.name] || ''}
-                        setApiKey={(key) => {
-                          const newApiKeys = { ...apiKeys, [provider.name]: key };
-                          setApiKeys(newApiKeys);
-                          Cookies.set('apiKeys', JSON.stringify(newApiKeys));
-                        }}
-                      />
-                    )}
-                  </div>
-                </div>
                 <FilePreview
                   files={uploadedFiles}
                   imageDataList={imageDataList}
@@ -476,7 +379,6 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                           show={input.length > 0 || isStreaming || uploadedFiles.length > 0}
                           simulation={false}
                           isStreaming={isStreaming}
-                          disabled={!providerList || providerList.length === 0}
                           onClick={(event) => {
                             if (isStreaming) {
                               handleStop?.();
@@ -492,7 +394,6 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                           show={(input.length > 0 || uploadedFiles.length > 0) && chatStarted}
                           simulation={true}
                           isStreaming={isStreaming}
-                          disabled={!providerList || providerList.length === 0}
                           onClick={(event) => {
                             if (input.length > 0 || uploadedFiles.length > 0) {
                               handleSendMessage?.(event, undefined, true);
@@ -530,20 +431,6 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                         disabled={isStreaming}
                       />
                       {chatStarted && <ClientOnly>{() => <ExportChatButton exportChat={exportChat} />}</ClientOnly>}
-                      <IconButton
-                        title="Model Settings"
-                        className={classNames('transition-all flex items-center gap-1', {
-                          'bg-bolt-elements-item-backgroundAccent text-bolt-elements-item-contentAccent':
-                            isModelSettingsCollapsed,
-                          'bg-bolt-elements-item-backgroundDefault text-bolt-elements-item-contentDefault':
-                            !isModelSettingsCollapsed,
-                        })}
-                        onClick={() => setIsModelSettingsCollapsed(!isModelSettingsCollapsed)}
-                        disabled={!providerList || providerList.length === 0}
-                      >
-                        <div className={`i-ph:caret-${isModelSettingsCollapsed ? 'right' : 'down'} text-lg`} />
-                        {isModelSettingsCollapsed ? <span className="text-xs">{model}</span> : <span />}
-                      </IconButton>
                     </div>
                     {input.length > 3 ? (
                       <div className="text-xs text-bolt-elements-textTertiary">

app/components/chat/Chat.client.tsx

@@ -12,18 +12,16 @@ import { useMessageParser, usePromptEnhancer, useShortcuts, useSnapScroll } from
 import { description, useChatHistory } from '~/lib/persistence';
 import { chatStore } from '~/lib/stores/chat';
 import { workbenchStore } from '~/lib/stores/workbench';
-import { DEFAULT_MODEL, DEFAULT_PROVIDER, PROMPT_COOKIE_KEY, PROVIDER_LIST } from '~/utils/constants';
+import { PROMPT_COOKIE_KEY } from '~/utils/constants';
 import { cubicEasingFn } from '~/utils/easings';
 import { createScopedLogger, renderLogger } from '~/utils/logger';
 import { BaseChat } from './BaseChat';
 import Cookies from 'js-cookie';
 import { debounce } from '~/utils/debounce';
 import { useSettings } from '~/lib/hooks/useSettings';
-import type { ProviderInfo } from '~/types/model';
 import { useSearchParams } from '@remix-run/react';
 import { createSampler } from '~/utils/sampler';
 import { saveProjectPrompt } from './Messages.client';
-import { uint8ArrayToBase64 } from '~/lib/replay/ReplayProtocolClient';
 import type { SimulationPromptClientData } from '~/lib/replay/SimulationPrompt';
 import { getIFrameSimulationData } from '~/lib/replay/Recording';
 import { getCurrentIFrame } from '../workbench/Preview';
@@ -120,27 +118,15 @@ export const ChatImpl = memo(
     const [imageDataList, setImageDataList] = useState<string[]>([]); // Move here
     const [searchParams, setSearchParams] = useSearchParams();
     const files = useStore(workbenchStore.files);
-    const { activeProviders, promptId } = useSettings();
+    const { promptId } = useSettings();
-
-    const [model, setModel] = useState(() => {
-      const savedModel = Cookies.get('selectedModel');
-      return savedModel || DEFAULT_MODEL;
-    });
-    const [provider, setProvider] = useState(() => {
-      const savedProvider = Cookies.get('selectedProvider');
-      return (PROVIDER_LIST.find((p) => p.name === savedProvider) || DEFAULT_PROVIDER) as ProviderInfo;
-    });
 
     const { showChat } = useStore(chatStore);
 
     const [animationScope, animate] = useAnimate();
 
-    const [apiKeys, setApiKeys] = useState<Record<string, string>>({});
-
     const { messages, isLoading, input, handleInputChange, setInput, stop, append } = useChat({
       api: '/api/chat',
       body: {
-        apiKeys,
         files,
         promptId,
       },
@@ -167,7 +153,6 @@ export const ChatImpl = memo(
     });
     useEffect(() => {
       const prompt = searchParams.get('prompt');
-      console.log(prompt, searchParams, model, provider);
 
       if (prompt) {
         setSearchParams({});
@@ -177,12 +162,12 @@ export const ChatImpl = memo(
           content: [
             {
               type: 'text',
-              text: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${prompt}`,
+              text: prompt,
             },
           ] as any, // Type assertion to bypass compiler check
         });
       }
-    }, [model, provider, searchParams]);
+    }, [searchParams]);
 
     const { enhancingPrompt, promptEnhanced, enhancePrompt, resetEnhancer } = usePromptEnhancer();
     const { parsedMessages, parseMessages } = useMessageParser();
@@ -294,7 +279,7 @@ export const ChatImpl = memo(
           content: [
             {
               type: 'text',
-              text: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${_input}`,
+              text: _input,
             },
             ...imageDataList.map((imageData) => ({
               type: 'image',
@@ -314,7 +299,7 @@ export const ChatImpl = memo(
           content: [
             {
               type: 'text',
-              text: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${_input}`,
+              text: _input,
             },
             ...imageDataList.map((imageData) => ({
               type: 'image',
@@ -363,24 +348,6 @@ export const ChatImpl = memo(
 
     const [messageRef, scrollRef] = useSnapScroll();
 
-    useEffect(() => {
-      const storedApiKeys = Cookies.get('apiKeys');
-
-      if (storedApiKeys) {
-        setApiKeys(JSON.parse(storedApiKeys));
-      }
-    }, []);
-
-    const handleModelChange = (newModel: string) => {
-      setModel(newModel);
-      Cookies.set('selectedModel', newModel, { expires: 30 });
-    };
-
-    const handleProviderChange = (newProvider: ProviderInfo) => {
-      setProvider(newProvider);
-      Cookies.set('selectedProvider', newProvider.name, { expires: 30 });
-    };
-
     return (
       <BaseChat
         ref={animationScope}
@@ -392,11 +359,6 @@ export const ChatImpl = memo(
         enhancingPrompt={enhancingPrompt}
         promptEnhanced={promptEnhanced}
         sendMessage={sendMessage}
-        model={model}
-        setModel={handleModelChange}
-        provider={provider}
-        setProvider={handleProviderChange}
-        providerList={activeProviders}
         messageRef={messageRef}
         scrollRef={scrollRef}
         handleInputChange={(e) => {
@@ -424,9 +386,6 @@ export const ChatImpl = memo(
               setInput(input);
               scrollTextArea();
             },
-            model,
-            provider,
-            apiKeys,
           );
         }}
         uploadedFiles={uploadedFiles}

app/lib/.server/llm/chat-anthropic.ts

@@ -0,0 +1,59 @@
+import type { CoreMessage } from "ai";
+import Anthropic from "@anthropic-ai/sdk";
+import { ChatStreamController } from "~/utils/chatStreamController";
+import type { ContentBlockParam, MessageParam } from "@anthropic-ai/sdk/resources/messages/messages.mjs";
+
+const MaxMessageTokens = 8192;
+
+function convertContentToAnthropic(content: any): ContentBlockParam[] {
+  if (typeof content === "string") {
+    return [{ type: "text", text: content }];
+  }
+  if (Array.isArray(content)) {
+    return content.flatMap(convertContentToAnthropic);
+  }
+  if (content.type === "text" && typeof content.text === "string") {
+    return [{ type: "text", text: content.text }];
+  }
+  console.log("AnthropicUnknownContent", JSON.stringify(content, null, 2));
+  return [];
+}
+
+export async function chatAnthropic(chatController: ChatStreamController, apiKey: string, systemPrompt: string, messages: CoreMessage[]) {
+  const anthropic = new Anthropic({ apiKey });
+
+  const messageParams: MessageParam[] = [];
+
+  messageParams.push({
+    role: "assistant",
+    content: systemPrompt,
+  });
+
+  for (const message of messages) {
+    const role = message.role == "user" ? "user" : "assistant";
+    const content = convertContentToAnthropic(message.content);
+    messageParams.push({
+      role,
+      content,
+    });
+  }
+
+  const response = await anthropic.messages.create({
+    model: "claude-3-5-sonnet-20241022",
+    messages: messageParams,
+    max_tokens: MaxMessageTokens,
+  });
+
+  for (const content of response.content) {
+    if (content.type === "text") {
+      chatController.writeText(content.text);
+    } else {
+      console.log("AnthropicUnknownResponse", JSON.stringify(content, null, 2));
+    }
+  }
+
+  const tokens = response.usage.input_tokens + response.usage.output_tokens;
+  console.log("AnthropicTokens", tokens);
+
+  chatController.writeUsage({ completionTokens: response.usage.output_tokens, promptTokens: response.usage.input_tokens });
+}

app/lib/.server/llm/stream-text.ts

@@ -142,16 +142,15 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid
   return { model, provider, content: cleanedContent };
 }
 
-export async function streamText(props: {
+export async function getStreamTextArguments(props: {
   messages: Messages;
   env: Env;
-  options?: StreamingOptions;
   apiKeys?: Record<string, string>;
   files?: FileMap;
   providerSettings?: Record<string, IProviderSetting>;
   promptId?: string;
 }) {
-  const { messages, env: serverEnv, options, apiKeys, files, providerSettings, promptId } = props;
+  const { messages, env: serverEnv, apiKeys, files, providerSettings, promptId } = props;
 
   // console.log({serverEnv});
 
@@ -202,9 +201,7 @@ export async function streamText(props: {
 
   const coreMessages = convertToCoreMessages(processedMessages as any);
 
-  console.log("QueryModel", JSON.stringify({ systemPrompt, coreMessages }));
+  return {
-
-  return _streamText({
     model: provider.getModelInstance({
       model: currentModel,
       serverEnv,
@@ -214,6 +211,18 @@ export async function streamText(props: {
     system: systemPrompt,
     maxTokens: dynamicMaxTokens,
     messages: coreMessages,
-    ...options,
+  };
-  });
+}
+
+export async function streamText(props: {
+  messages: Messages;
+  env: Env;
+  options?: StreamingOptions;
+  apiKeys?: Record<string, string>;
+  files?: FileMap;
+  providerSettings?: Record<string, IProviderSetting>;
+  promptId?: string;
+}) {
+  const args = await getStreamTextArguments(props);
+  return _streamText({ ...args, ...props.options });
 }

app/lib/hooks/usePromptEnhancer.ts

@@ -1,5 +1,4 @@
 import { useState } from 'react';
-import type { ProviderInfo } from '~/types/model';
 import { createScopedLogger } from '~/utils/logger';
 
 const logger = createScopedLogger('usePromptEnhancement');
@@ -15,24 +14,15 @@ export function usePromptEnhancer() {
 
   const enhancePrompt = async (
     input: string,
-    setInput: (value: string) => void,
+    setInput: (value: string) => void
-    model: string,
-    provider: ProviderInfo,
-    apiKeys?: Record<string, string>,
   ) => {
     setEnhancingPrompt(true);
     setPromptEnhanced(false);
 
     const requestBody: any = {
       message: input,
-      model,
-      provider,
     };
 
-    if (apiKeys) {
-      requestBody.apiKeys = apiKeys;
-    }
-
     const response = await fetch('/api/enhancer', {
       method: 'POST',
       body: JSON.stringify(requestBody),

app/lib/replay/Recording.ts

@@ -4,10 +4,11 @@ import { assert, stringToBase64, uint8ArrayToBase64 } from "./ReplayProtocolClie
 
 export interface SimulationResource {
   url: string;
-  requestBodyBase64: string;
+  requestBodyBase64?: string;
-  responseBodyBase64: string;
+  responseBodyBase64?: string;
-  responseStatus: number;
+  responseStatus?: number;
-  responseHeaders: Record<string, string>;
+  responseHeaders?: Record<string, string>;
+  error?: string;
 }
 
 enum SimulationInteractionKind {
@@ -122,7 +123,7 @@ export async function getMouseData(iframe: HTMLIFrameElement, position: { x: num
 
 // Add handlers to the current iframe's window.
 function addRecordingMessageHandler(messageHandlerId: string) {
-  const resources: Map<string, SimulationResource> = new Map();
+  const resources: SimulationResource[] = [];
   const interactions: SimulationInteraction[] = [];
   const indexedDBAccesses: IndexedDBAccess[] = [];
   const localStorageAccesses: LocalStorageAccess[] = [];
@@ -131,10 +132,7 @@ function addRecordingMessageHandler(messageHandlerId: string) {
 
   function addTextResource(path: string, text: string) {
     const url = (new URL(path, window.location.href)).href;
-    if (resources.has(url)) {
+    resources.push({
-      return;
-    }
-    resources.set(url, {
       url,
       requestBodyBase64: "",
       responseBodyBase64: stringToBase64(text),
@@ -147,7 +145,7 @@ function addRecordingMessageHandler(messageHandlerId: string) {
     return {
       locationHref: window.location.href,
       documentUrl: window.location.href,
-      resources: Array.from(resources.values()),
+      resources,
       interactions,
       indexedDBAccesses,
       localStorageAccesses,
@@ -475,10 +473,18 @@ function addRecordingMessageHandler(messageHandlerId: string) {
 
   const baseFetch = window.fetch;
   window.fetch = async (info, options) => {
-    const rv = await baseFetch(info, options);
     const url = info instanceof Request ? info.url : info.toString();
-    responseToURL.set(rv, url);
+    try {
-    return createProxy(rv);
+      const rv = await baseFetch(info, options);
+      responseToURL.set(rv, url);
+      return createProxy(rv);
+    } catch (error) {
+      resources.push({
+        url,
+        error: String(error),
+      });
+      throw error;
+    }
   };
 }
 

app/lib/replay/ReplayProtocolClient.ts

@@ -6,6 +6,16 @@ export function assert(condition: any, message: string = "Assertion failed!"): a
   }
 }
 
+export function defer<T>(): { promise: Promise<T>; resolve: (value: T) => void; reject: (reason?: any) => void } {
+  let resolve: (value: T) => void;
+  let reject: (reason?: any) => void;
+  const promise = new Promise<T>((_resolve, _reject) => {
+    resolve = _resolve;
+    reject = _reject;
+  });
+  return { promise, resolve: resolve!, reject: reject! };
+}
+
 export function uint8ArrayToBase64(data: Uint8Array) {
   let str = "";
   for (const byte of data) {

app/routes/api.chat.ts

@@ -1,36 +1,14 @@
 import { type ActionFunctionArgs } from '@remix-run/cloudflare';
-import { createDataStream } from 'ai';
-import { MAX_RESPONSE_SEGMENTS, MAX_TOKENS } from '~/lib/.server/llm/constants';
-import { CONTINUE_PROMPT } from '~/lib/common/prompts/prompts';
-import { streamText, type Messages, type StreamingOptions } from '~/lib/.server/llm/stream-text';
-import SwitchableStream from '~/lib/.server/llm/switchable-stream';
-import type { IProviderSetting } from '~/types/model';
 import { type SimulationChatMessage, type SimulationPromptClientData, performSimulationPrompt } from '~/lib/replay/SimulationPrompt';
 import { ChatStreamController } from '~/utils/chatStreamController';
 import { assert } from '~/lib/replay/ReplayProtocolClient';
+import { getStreamTextArguments, type Messages } from '~/lib/.server/llm/stream-text';
+import { chatAnthropic } from '~/lib/.server/llm/chat-anthropic';
 
 export async function action(args: ActionFunctionArgs) {
   return chatAction(args);
 }
 
-function parseCookies(cookieHeader: string): Record<string, string> {
-  const cookies: Record<string, string> = {};
-
-  const items = cookieHeader.split(';').map((cookie) => cookie.trim());
-
-  items.forEach((item) => {
-    const [name, ...rest] = item.split('=');
-
-    if (name && rest) {
-      const decodedName = decodeURIComponent(name.trim());
-      const decodedValue = decodeURIComponent(rest.join('=').trim());
-      cookies[decodedName] = decodedValue;
-    }
-  });
-
-  return cookies;
-}
-
 function extractMessageContent(baseContent: any): string {
   let content = baseContent;
 
@@ -68,138 +46,66 @@ async function chatAction({ context, request }: ActionFunctionArgs) {
     simulationClientData?: SimulationPromptClientData;
   }>();
 
+  let finished: (v?: any) => void;
+  context.cloudflare.ctx.waitUntil(new Promise((resolve) => finished = resolve));
+
   console.log("SimulationClientData", simulationClientData);
 
-  const cookieHeader = request.headers.get('Cookie');
-  const apiKeys = JSON.parse(parseCookies(cookieHeader || '').apiKeys || '{}');
-  const providerSettings: Record<string, IProviderSetting> = JSON.parse(
-    parseCookies(cookieHeader || '').providers || '{}',
-  );
-
-  const stream = new SwitchableStream();
-
-  const cumulativeUsage = {
-    completionTokens: 0,
-    promptTokens: 0,
-    totalTokens: 0,
-  };
-
   try {
-    if (simulationClientData) {
+    const { system, messages: coreMessages } = await getStreamTextArguments({
-      const chatHistory: SimulationChatMessage[] = [];
-      for (const { role, content } of messages) {
-        chatHistory.push({ role, content: extractMessageContent(content) });
-      }
-      const lastHistoryMessage = chatHistory.pop();
-      assert(lastHistoryMessage?.role == "user", "Last message in chat history must be a user message");
-      const userPrompt = lastHistoryMessage.content;
-
-      const anthropicApiKey = process.env.ANTHROPIC_API_KEY;
-      if (!anthropicApiKey) {
-        throw new Error("Anthropic API key is not set");
-      }
-
-      const { message, fileChanges } = await performSimulationPrompt(simulationClientData, userPrompt, chatHistory, anthropicApiKey);
-
-      const resultStream = new ReadableStream({
-        async start(controller) {
-          const chatController = new ChatStreamController(controller);
-
-          chatController.writeText(message + "\n");
-          chatController.writeFileChanges("Update Files", fileChanges);
-
-          /*
-          chatController.writeText("Hello World\n");
-          chatController.writeText("Hello World 2\n");
-          chatController.writeText("Hello\n World 3\n");
-          chatController.writeFileChanges("Rewrite Files", [{filePath: "src/services/llm.ts", contents: "FILE_CONTENTS_FIXME" }]);
-          chatController.writeAnnotation("usage", { completionTokens: 10, promptTokens: 20, totalTokens: 30 });
-          */
-
-          controller.close();
-          setTimeout(() => stream.close(), 1000);
-        },
-      });
-
-      stream.switchSource(resultStream);
-
-      return new Response(stream.readable, {
-        status: 200,
-        headers: {
-          contentType: 'text/plain; charset=utf-8',
-        },
-      });
-    }
-
-    const options: StreamingOptions = {
-      toolChoice: 'none',
-      onFinish: async ({ text: content, finishReason, usage }) => {
-        console.log("QueryModelFinished", usage, content);
-
-        if (usage) {
-          cumulativeUsage.completionTokens += usage.completionTokens || 0;
-          cumulativeUsage.promptTokens += usage.promptTokens || 0;
-          cumulativeUsage.totalTokens += usage.totalTokens || 0;
-        }
-
-        if (finishReason !== 'length') {
-          return stream
-            .switchSource(
-              createDataStream({
-                async execute(dataStream) {
-                  dataStream.writeMessageAnnotation({
-                    type: 'usage',
-                    value: {
-                      completionTokens: cumulativeUsage.completionTokens,
-                      promptTokens: cumulativeUsage.promptTokens,
-                      totalTokens: cumulativeUsage.totalTokens,
-                    },
-                  });
-                },
-                onError: (error: any) => `Custom error: ${error.message}`,
-              }),
-            )
-            .then(() => stream.close());
-        }
-
-        if (stream.switches >= MAX_RESPONSE_SEGMENTS) {
-          throw Error('Cannot continue message: Maximum segments reached');
-        }
-
-        const switchesLeft = MAX_RESPONSE_SEGMENTS - stream.switches;
-
-        console.log(`Reached max token limit (${MAX_TOKENS}): Continuing message (${switchesLeft} switches left)`);
-
-        messages.push({ role: 'assistant', content });
-        messages.push({ role: 'user', content: CONTINUE_PROMPT });
-
-        const result = await streamText({
-          messages,
-          env: context.cloudflare.env,
-          options,
-          apiKeys,
-          files,
-          providerSettings,
-          promptId,
-        });
-
-        return stream.switchSource(result.toDataStream());
-      },
-    };
-
-    const result = await streamText({
       messages,
       env: context.cloudflare.env,
-      options,
+      apiKeys: {},
-      apiKeys,
       files,
-      providerSettings,
+      providerSettings: undefined,
       promptId,
     });
 
-    stream.switchSource(result.toDataStream());
+    const anthropicApiKey = context.cloudflare.env.ANTHROPIC_API_KEY;
+    if (!anthropicApiKey) {
+      throw new Error("Anthropic API key is not set");
+    }
 
-    return new Response(stream.readable, {
+    const resultStream = new ReadableStream({
+      async start(controller) {
+        const chatController = new ChatStreamController(controller);
+
+        /*
+        chatController.writeText("Hello World\n");
+        chatController.writeText("Hello World 2\n");
+        chatController.writeText("Hello\n World 3\n");
+        chatController.writeFileChanges("Rewrite Files", [{filePath: "src/services/llm.ts", contents: "FILE_CONTENTS_FIXME" }]);
+        chatController.writeAnnotation("usage", { completionTokens: 10, promptTokens: 20, totalTokens: 30 });
+        */
+
+        try {
+          if (simulationClientData) {
+            const chatHistory: SimulationChatMessage[] = [];
+            for (const { role, content } of messages) {
+              chatHistory.push({ role, content: extractMessageContent(content) });
+            }
+            const lastHistoryMessage = chatHistory.pop();
+            assert(lastHistoryMessage?.role == "user", "Last message in chat history must be a user message");
+            const userPrompt = lastHistoryMessage.content;
+
+            const { message, fileChanges } = await performSimulationPrompt(simulationClientData, userPrompt, chatHistory, anthropicApiKey);
+
+            chatController.writeText(message + "\n");
+            chatController.writeFileChanges("Update Files", fileChanges);
+          } else {
+            await chatAnthropic(chatController, anthropicApiKey, system, coreMessages);
+          }
+        } catch (error: any) {
+          console.error(error);
+          chatController.writeText("Error: " + error.message);
+        }
+
+        controller.close();
+        setTimeout(finished, 1000);
+      },
+    });
+
+    return new Response(resultStream, {
       status: 200,
       headers: {
         contentType: 'text/plain; charset=utf-8',

app/utils/chatStreamController.ts

@@ -10,13 +10,16 @@ export interface ChatFileChange {
 
 export class ChatStreamController {
   private controller: ReadableStreamDefaultController;
+  private encoder: TextEncoder;
 
   constructor(controller: ReadableStreamDefaultController) {
     this.controller = controller;
+    this.encoder = new TextEncoder();
   }
 
   writeText(text: string) {
-    this.controller.enqueue(`0:${JSON.stringify(text)}\n`);
+    const data = this.encoder.encode(`0:${JSON.stringify(text)}\n`);
+    this.controller.enqueue(data);
   }
 
   writeFileChanges(title: string, fileChanges: ChatFileChange[]) {
@@ -29,6 +32,11 @@ export class ChatStreamController {
   }
 
   writeAnnotation(type: string, value: any) {
-    this.controller.enqueue(`8:[{"type":"${type}","value":${JSON.stringify(value)}}]\n`);
+    const data = this.encoder.encode(`8:[{"type":"${type}","value":${JSON.stringify(value)}}]\n`);
+    this.controller.enqueue(data);
+  }
+
+  writeUsage({ completionTokens, promptTokens }: { completionTokens: number, promptTokens: number }) {
+    this.writeAnnotation("usage", { completionTokens, promptTokens, totalTokens: completionTokens + promptTokens });
   }
 }

package.json

@@ -35,6 +35,7 @@
     "@ai-sdk/google": "^0.0.52",
     "@ai-sdk/mistral": "^0.0.43",
     "@ai-sdk/openai": "^0.0.66",
+    "@anthropic-ai/sdk": "^0.33.1",
     "@codemirror/autocomplete": "^6.18.3",
     "@codemirror/commands": "^6.7.1",
     "@codemirror/lang-cpp": "^6.0.2",

pnpm-lock.yaml

@@ -26,6 +26,9 @@ importers:
       '@ai-sdk/openai':
         specifier: ^0.0.66
         version: 0.0.66(zod@3.23.8)
+      '@anthropic-ai/sdk':
+        specifier: ^0.33.1
+        version: 0.33.1
       '@codemirror/autocomplete':
         specifier: ^6.18.3
         version: 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.4.1)(@codemirror/view@6.35.0)(@lezer/common@1.2.3)
@@ -435,6 +438,9 @@ packages:
   '@antfu/utils@0.7.10':
     resolution: {integrity: sha512-+562v9k4aI80m1+VuMHehNJWLOFjBnXn3tdOitzD0il5b7smkSBal4+a3oKiQTbrwMmN/TBUMDvbdoWDehgOww==}
 
+  '@anthropic-ai/sdk@0.33.1':
+    resolution: {integrity: sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==}
+
   '@babel/code-frame@7.26.2':
     resolution: {integrity: sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==}
     engines: {node: '>=6.9.0'}
@@ -2126,9 +2132,15 @@ packages:
   '@types/ms@0.7.34':
     resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==}
 
+  '@types/node-fetch@2.6.12':
+    resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==}
+
   '@types/node-forge@1.3.11':
     resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==}
 
+  '@types/node@18.19.70':
+    resolution: {integrity: sha512-RE+K0+KZoEpDUbGGctnGdkrLFwi1eYKTlIHNl2Um98mUkGsm1u2Ff6Ltd0e8DktTtC98uy7rSj+hO8t/QuLoVQ==}
+
   '@types/node@22.10.1':
     resolution: {integrity: sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==}
 
@@ -2396,6 +2408,10 @@ packages:
     engines: {node: '>=0.4.0'}
     hasBin: true
 
+  agentkeepalive@4.6.0:
+    resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==}
+    engines: {node: '>= 8.0.0'}
+
   aggregate-error@3.1.0:
     resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==}
     engines: {node: '>=8'}
@@ -2468,6 +2484,9 @@ packages:
   async-lock@1.4.1:
     resolution: {integrity: sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==}
 
+  asynckit@0.4.0:
+    resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
+
   available-typed-arrays@1.0.7:
     resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==}
     engines: {node: '>= 0.4'}
@@ -2692,6 +2711,10 @@ packages:
   colorjs.io@0.5.2:
     resolution: {integrity: sha512-twmVoizEW7ylZSN32OgKdXRmo1qg+wT5/6C3xu5b9QsWzSFAhHLn2xd8ro0diCsKfCj1RdaTP/nrcW+vAoQPIw==}
 
+  combined-stream@1.0.8:
+    resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
+    engines: {node: '>= 0.8'}
+
   comma-separated-tokens@2.0.3:
     resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==}
 
@@ -2869,6 +2892,10 @@ packages:
   defu@6.1.4:
     resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==}
 
+  delayed-stream@1.0.0:
+    resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
+    engines: {node: '>=0.4.0'}
+
   depd@2.0.0:
     resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
     engines: {node: '>= 0.8'}
@@ -3246,10 +3273,21 @@ packages:
     resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==}
     engines: {node: '>=14'}
 
+  form-data-encoder@1.7.2:
+    resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==}
+
+  form-data@4.0.1:
+    resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==}
+    engines: {node: '>= 6'}
+
   format@0.2.2:
     resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==}
     engines: {node: '>=0.4.x'}
 
+  formdata-node@4.4.1:
+    resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==}
+    engines: {node: '>= 12.20'}
+
   formdata-polyfill@4.0.10:
     resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
     engines: {node: '>=12.20.0'}
@@ -3463,6 +3501,9 @@ packages:
     resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==}
     engines: {node: '>=10.17.0'}
 
+  humanize-ms@1.2.1:
+    resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==}
+
   husky@9.1.7:
     resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==}
     engines: {node: '>=18'}
@@ -4232,6 +4273,15 @@ packages:
   node-fetch-native@1.6.4:
     resolution: {integrity: sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==}
 
+  node-fetch@2.7.0:
+    resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
+    engines: {node: 4.x || >=6.0.0}
+    peerDependencies:
+      encoding: ^0.1.0
+    peerDependenciesMeta:
+      encoding:
+        optional: true
+
   node-fetch@3.3.2:
     resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -5304,6 +5354,9 @@ packages:
     resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==}
     engines: {node: '>=6'}
 
+  tr46@0.0.3:
+    resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
+
   trim-lines@3.0.1:
     resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==}
 
@@ -5377,6 +5430,9 @@ packages:
   unconfig@0.5.5:
     resolution: {integrity: sha512-VQZ5PT9HDX+qag0XdgQi8tJepPhXiR/yVOkn707gJDKo31lGjRilPREiQJ9Z6zd/Ugpv6ZvO5VxVIcatldYcNQ==}
 
+  undici-types@5.26.5:
+    resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
+
   undici-types@6.20.0:
     resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==}
 
@@ -5667,6 +5723,16 @@ packages:
     resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
     engines: {node: '>= 8'}
 
+  web-streams-polyfill@4.0.0-beta.3:
+    resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==}
+    engines: {node: '>= 14'}
+
+  webidl-conversions@3.0.1:
+    resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
+
+  whatwg-url@5.0.0:
+    resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}
+
   which-typed-array@1.1.16:
     resolution: {integrity: sha512-g+N+GAWiRj66DngFwHvISJd+ITsyphZvD1vChfVg6cEdnzy53GzB3oy0fUNlvhz7H7+MiqhYr26qxQShCpKTTQ==}
     engines: {node: '>= 0.4'}
@@ -5904,6 +5970,18 @@ snapshots:
 
   '@antfu/utils@0.7.10': {}
 
+  '@anthropic-ai/sdk@0.33.1':
+    dependencies:
+      '@types/node': 18.19.70
+      '@types/node-fetch': 2.6.12
+      abort-controller: 3.0.0
+      agentkeepalive: 4.6.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.7.0
+    transitivePeerDependencies:
+      - encoding
+
   '@babel/code-frame@7.26.2':
     dependencies:
       '@babel/helper-validator-identifier': 7.25.9
@@ -7562,10 +7640,19 @@ snapshots:
 
   '@types/ms@0.7.34': {}
 
+  '@types/node-fetch@2.6.12':
+    dependencies:
+      '@types/node': 22.10.1
+      form-data: 4.0.1
+
   '@types/node-forge@1.3.11':
     dependencies:
       '@types/node': 22.10.1
 
+  '@types/node@18.19.70':
+    dependencies:
+      undici-types: 5.26.5
+
   '@types/node@22.10.1':
     dependencies:
       undici-types: 6.20.0
@@ -7975,6 +8062,10 @@ snapshots:
 
   acorn@8.14.0: {}
 
+  agentkeepalive@4.6.0:
+    dependencies:
+      humanize-ms: 1.2.1
+
   aggregate-error@3.1.0:
     dependencies:
       clean-stack: 2.2.0
@@ -8049,6 +8140,8 @@ snapshots:
 
   async-lock@1.4.1: {}
 
+  asynckit@0.4.0: {}
+
   available-typed-arrays@1.0.7:
     dependencies:
       possible-typed-array-names: 1.0.0
@@ -8307,6 +8400,10 @@ snapshots:
 
   colorjs.io@0.5.2: {}
 
+  combined-stream@1.0.8:
+    dependencies:
+      delayed-stream: 1.0.0
+
   comma-separated-tokens@2.0.3: {}
 
   common-tags@1.8.2: {}
@@ -8455,6 +8552,8 @@ snapshots:
 
   defu@6.1.4: {}
 
+  delayed-stream@1.0.0: {}
+
   depd@2.0.0: {}
 
   dequal@2.0.3: {}
@@ -8963,8 +9062,21 @@ snapshots:
       cross-spawn: 7.0.6
       signal-exit: 4.1.0
 
+  form-data-encoder@1.7.2: {}
+
+  form-data@4.0.1:
+    dependencies:
+      asynckit: 0.4.0
+      combined-stream: 1.0.8
+      mime-types: 2.1.35
+
   format@0.2.2: {}
 
+  formdata-node@4.4.1:
+    dependencies:
+      node-domexception: 1.0.0
+      web-streams-polyfill: 4.0.0-beta.3
+
   formdata-polyfill@4.0.10:
     dependencies:
       fetch-blob: 3.2.0
@@ -9249,6 +9361,10 @@ snapshots:
 
   human-signals@2.1.0: {}
 
+  humanize-ms@1.2.1:
+    dependencies:
+      ms: 2.1.3
+
   husky@9.1.7: {}
 
   iconv-lite@0.4.24:
@@ -10354,6 +10470,10 @@ snapshots:
 
   node-fetch-native@1.6.4: {}
 
+  node-fetch@2.7.0:
+    dependencies:
+      whatwg-url: 5.0.0
+
   node-fetch@3.3.2:
     dependencies:
       data-uri-to-buffer: 4.0.1
@@ -11497,6 +11617,8 @@ snapshots:
 
   totalist@3.0.1: {}
 
+  tr46@0.0.3: {}
+
   trim-lines@3.0.1: {}
 
   trough@2.2.0: {}
@@ -11562,6 +11684,8 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  undici-types@5.26.5: {}
+
   undici-types@6.20.0: {}
 
   undici@5.28.4:
@@ -11920,6 +12044,15 @@ snapshots:
 
   web-streams-polyfill@3.3.3: {}
 
+  web-streams-polyfill@4.0.0-beta.3: {}
+
+  webidl-conversions@3.0.1: {}
+
+  whatwg-url@5.0.0:
+    dependencies:
+      tr46: 0.0.3
+      webidl-conversions: 3.0.1
+
   which-typed-array@1.1.16:
     dependencies:
       available-typed-arrays: 1.0.7