UniqAI/bolt.diy
1
0
Fork 0
mirror of https://github.com/stackblitz-labs/bolt.diy synced 2025-06-26 18:26:38 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Improve logging for oversize messages

Browse Source
This commit is contained in:
Brian Hackett 2025-03-06 08:23:23 -08:00
parent 35ab0cbab1
commit 72fa945dc2
3 changed files with 45 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
app/lib/.server/llm/chat-anthropic.ts
View File

@ -40,10 +40,13 @@ function convertContentToAnthropic(content: any): ContentBlockParam[] {
return []; return [];
} }
export interface AnthropicApiKey { export interface ChatState {
key: string; apiKey: string;
isUser: boolean; isUser: boolean;
userLoginKey?: string; userLoginKey?: string;
// Info about how the chat was processed which will be conveyed back to the client.
infos: string[];
} }
export interface AnthropicCall { export interface AnthropicCall {
@ -112,9 +115,13 @@ function compressMessages(messages: MessageParam[]): MessageParam[] {
return compressed; return compressed;
} }
async function reduceMessageSize(messages: MessageParam[], systemPrompt: string, maximum: number): Promise<MessageParam[]> { async function reduceMessageSize(state: ChatState, messages: MessageParam[], systemPrompt: string, maximum: number): Promise<MessageParam[]> {
for (let i = 0; i < 5; i++) { for (let iteration = 0; iteration < 5; iteration++) {
const tokens = await countTokens(messages, systemPrompt); const tokens = await countTokens(messages, systemPrompt);
console.log(`AnthropicReduceMessageSize ${JSON.stringify({ iteration, tokens, maximum })}`);
state.infos.push(`AnthropicReduceMessageSize ${JSON.stringify({ iteration, tokens, maximum })}`);
if (tokens <= maximum) { if (tokens <= maximum) {
return messages; return messages;
} }
@ -125,8 +132,8 @@ async function reduceMessageSize(messages: MessageParam[], systemPrompt: string,
throw new Error("Message compression failed"); throw new Error("Message compression failed");
} }
async function callAnthropicRaw(apiKey: string, systemPrompt: string, messages: MessageParam[]): Promise<Anthropic.Messages.Message> { async function callAnthropicRaw(state: ChatState, systemPrompt: string, messages: MessageParam[]): Promise<Anthropic.Messages.Message> {
const anthropic = new Anthropic({ apiKey }); const anthropic = new Anthropic({ apiKey: state.apiKey });
try { try {
return await anthropic.messages.create({ return await anthropic.messages.create({
@ -137,14 +144,24 @@ async function callAnthropicRaw(apiKey: string, systemPrompt: string, messages:
}); });
} catch (e: any) { } catch (e: any) {
console.error("AnthropicError", e); console.error("AnthropicError", e);
console.log("AnthropicErrorMessages", JSON.stringify({ systemPrompt, messages }, null, 2)); state.infos.push(`AnthropicError: ${e}`);
try {
console.log(`AnthropicErrorData ${JSON.stringify(e.error)}`);
state.infos.push(`AnthropicErrorData ${JSON.stringify(e.error)}`);
} catch (e) {
console.log(`AnthropicErrorDataException ${e}`);
state.infos.push(`AnthropicErrorDataException ${e}`);
}
state.infos.push(`AnthropicErrorMessages ${JSON.stringify({ systemPrompt, messages })}`);
const info = maybeParseAnthropicErrorPromptTooLong(e); const info = maybeParseAnthropicErrorPromptTooLong(e);
if (info) { if (info) {
const { maximum } = info; const { maximum } = info;
const newMessages = await reduceMessageSize(messages, systemPrompt, maximum); const newMessages = await reduceMessageSize(state, messages, systemPrompt, maximum);
console.log("AnthropicCompressedMessages", JSON.stringify({ systemPrompt, newMessages }, null, 2)); state.infos.push(`AnthropicCompressedMessages ${JSON.stringify({ systemPrompt, newMessages })}`);
return await anthropic.messages.create({ return await anthropic.messages.create({
model: Model, model: Model,
@ -167,19 +184,19 @@ export const callAnthropic = wrapWithSpan(
}, },
// eslint-disable-next-line prefer-arrow-callback // eslint-disable-next-line prefer-arrow-callback
async function callAnthropic(apiKey: AnthropicApiKey, reason: string, systemPrompt: string, messages: MessageParam[]): Promise<AnthropicCall> { async function callAnthropic(state: ChatState, reason: string, systemPrompt: string, messages: MessageParam[]): Promise<AnthropicCall> {
const span = getCurrentSpan(); const span = getCurrentSpan();
span?.setAttributes({ span?.setAttributes({
"llm.chat.calls": 1, // so we can SUM(llm.chat.calls) without doing a COUNT + filter "llm.chat.calls": 1, // so we can SUM(llm.chat.calls) without doing a COUNT + filter
"llm.chat.num_messages": messages.length, "llm.chat.num_messages": messages.length,
"llm.chat.reason": reason, "llm.chat.reason": reason,
"llm.chat.is_user_api_key": apiKey.isUser, "llm.chat.is_user_api_key": state.isUser,
"llm.chat.user_login_key": apiKey.userLoginKey, "llm.chat.user_login_key": state.userLoginKey,
}); });
console.log("AnthropicMessageSend"); console.log("AnthropicMessageSend");
const response = await callAnthropicRaw(apiKey.key, systemPrompt, messages); const response = await callAnthropicRaw(state, systemPrompt, messages);
let responseText = ""; let responseText = "";
for (const content of response.content) { for (const content of response.content) {
@ -228,16 +245,10 @@ function shouldRestorePartialFile(existingContent: string, newContent: string):
return existingContent.length > newContent.length; return existingContent.length > newContent.length;
} }
interface ChatState {
// Info about how the chat was processed which will be conveyed back to the client.
infos: string[];
}
async function restorePartialFile( async function restorePartialFile(
state: ChatState, state: ChatState,
existingContent: string, existingContent: string,
newContent: string, newContent: string,
cx: AnthropicApiKey,
responseDescription: string responseDescription: string
) { ) {
const systemPrompt = ` const systemPrompt = `
@ -287,7 +298,7 @@ ${responseDescription}
}, },
]; ];
const restoreCall = await callAnthropic(cx, "RestorePartialFile", systemPrompt, messages); const restoreCall = await callAnthropic(state, "RestorePartialFile", systemPrompt, messages);
const OpenTag = "<restoredContent>"; const OpenTag = "<restoredContent>";
const CloseTag = "</restoredContent>"; const CloseTag = "</restoredContent>";
@ -396,7 +407,7 @@ interface FileContents {
content: string; content: string;
} }
async function fixupResponseFiles(state: ChatState, files: FileMap, cx: AnthropicApiKey, responseText: string) { async function fixupResponseFiles(state: ChatState, files: FileMap, responseText: string) {
const fileContents: FileContents[] = []; const fileContents: FileContents[] = [];
const messageParser = new StreamingMessageParser({ const messageParser = new StreamingMessageParser({
@ -425,7 +436,6 @@ async function fixupResponseFiles(state: ChatState, files: FileMap, cx: Anthropi
state, state,
existingContent, existingContent,
newContent, newContent,
cx,
responseDescription responseDescription
); );
restoreCalls.push(restoreCall); restoreCalls.push(restoreCall);
@ -447,7 +457,7 @@ export type ChatAnthropicInfo = {
infos: string[]; infos: string[];
} }
export async function chatAnthropic(chatController: ChatStreamController, files: FileMap, apiKey: AnthropicApiKey, systemPrompt: string, messages: CoreMessage[]) { export async function chatAnthropic(state: ChatState, chatController: ChatStreamController, files: FileMap, systemPrompt: string, messages: CoreMessage[]) {
const messageParams: MessageParam[] = []; const messageParams: MessageParam[] = [];
for (const message of messages) { for (const message of messages) {
@ -459,13 +469,9 @@ export async function chatAnthropic(chatController: ChatStreamController, files:
}); });
} }
const mainCall = await callAnthropic(apiKey, "SendChatMessage", systemPrompt, messageParams); const mainCall = await callAnthropic(state, "SendChatMessage", systemPrompt, messageParams);
const state: ChatState = { const { responseText, restoreCalls } = await fixupResponseFiles(state, files, mainCall.responseText);
infos: [],
};
const { responseText, restoreCalls } = await fixupResponseFiles(state, files, apiKey, mainCall.responseText);
chatController.writeText(responseText); chatController.writeText(responseText);

9
app/routes/api.chat.ts
View File

@ -2,7 +2,7 @@ import { type ActionFunctionArgs } from '@remix-run/cloudflare';
import { ChatStreamController } from '~/utils/chatStreamController'; import { ChatStreamController } from '~/utils/chatStreamController';
import { assert } from '~/lib/replay/ReplayProtocolClient'; import { assert } from '~/lib/replay/ReplayProtocolClient';
import { getStreamTextArguments, type FileMap, type Messages } from '~/lib/.server/llm/stream-text'; import { getStreamTextArguments, type FileMap, type Messages } from '~/lib/.server/llm/stream-text';
import { chatAnthropic, type AnthropicApiKey } from '~/lib/.server/llm/chat-anthropic'; import { chatAnthropic, type ChatState } from '~/lib/.server/llm/chat-anthropic';
import { ensureOpenTelemetryInitialized } from '~/lib/.server/otel-wrapper'; import { ensureOpenTelemetryInitialized } from '~/lib/.server/otel-wrapper';
export async function action(args: ActionFunctionArgs) { export async function action(args: ActionFunctionArgs) {
@ -47,10 +47,11 @@ async function chatAction({ context, request }: ActionFunctionArgs) {
throw new Error("Anthropic API key is not set"); throw new Error("Anthropic API key is not set");
} }
const anthropicApiKey: AnthropicApiKey = { const chatState: ChatState = {
key: apiKey, apiKey,
isUser: !!clientAnthropicApiKey, isUser: !!clientAnthropicApiKey,
userLoginKey: loginKey, userLoginKey: loginKey,
infos: [],
}; };
const resultStream = new ReadableStream({ const resultStream = new ReadableStream({
@ -67,7 +68,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) {
} }
try { try {
await chatAnthropic(chatController, files, anthropicApiKey, system, coreMessages); await chatAnthropic(chatState, chatController, files, system, coreMessages);
} catch (e) { } catch (e) {
console.error(e); console.error(e);
chatController.writeText(`Error chatting with Anthropic: ${e}`); chatController.writeText(`Error chatting with Anthropic: ${e}`);

10
app/routes/api.use-simulation.ts
View File

@ -1,6 +1,6 @@
import { json, type ActionFunctionArgs } from '@remix-run/cloudflare'; import { json, type ActionFunctionArgs } from '@remix-run/cloudflare';
import { callAnthropic, type AnthropicApiKey } from '~/lib/.server/llm/chat-anthropic'; import { callAnthropic, type ChatState } from '~/lib/.server/llm/chat-anthropic';
import type { MessageParam } from '@anthropic-ai/sdk/resources/messages/messages.mjs'; import type { MessageParam } from '@anthropic-ai/sdk/resources/messages/messages.mjs';
export async function action(args: ActionFunctionArgs) { export async function action(args: ActionFunctionArgs) {
@ -17,10 +17,10 @@ async function useSimulationAction({ context, request }: ActionFunctionArgs) {
throw new Error("Anthropic API key is not set"); throw new Error("Anthropic API key is not set");
} }
const anthropicApiKey: AnthropicApiKey = { const chatState: ChatState = {
key: apiKey, apiKey,
isUser: false, isUser: false,
userLoginKey: undefined, infos: [],
}; };
const systemPrompt = ` const systemPrompt = `
@ -40,7 +40,7 @@ reasoning and then respond with either \`<analyze>true</analyze>\` or \`<analyze
content: `Here is the user message you need to evaluate: <user_message>${messageInput}</user_message>`, content: `Here is the user message you need to evaluate: <user_message>${messageInput}</user_message>`,
}; };
const { responseText } = await callAnthropic(anthropicApiKey, "UseSimulation", systemPrompt, [message]); const { responseText } = await callAnthropic(chatState, "UseSimulation", systemPrompt, [message]);
console.log("UseSimulationResponse", responseText); console.log("UseSimulationResponse", responseText);