UniqAI/bolt.diy
1
0
Fork 0
mirror of https://github.com/stackblitz-labs/bolt.diy synced 2025-06-26 18:26:38 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Use Nut Chat API (#11)

Browse Source
This commit is contained in:
Brian Hackett
2025-01-29 10:45:18 -08:00
committed by GitHub
parent 45c7365f01
commit df8e2526ee
4 changed files with 297 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/components/chat/Chat.client.tsx
View File

@@ -23,7 +23,8 @@ import { useSearchParams } from '@remix-run/react';
import { createSampler } from '~/utils/sampler';
import { saveProjectContents } from './Messages.client';
import { getSimulationRecording, getSimulationEnhancedPrompt } from '~/lib/replay/SimulationPrompt';
import { getIFrameSimulationData, type SimulationData } from '~/lib/replay/Recording';
import { getIFrameSimulationData } from '~/lib/replay/Recording';
import type { SimulationData } from '~/lib/replay/SimulationData';
import { getCurrentIFrame } from '../workbench/Preview';
import { getCurrentMouseData } from '../workbench/PointSelector';
import { anthropicNumFreeUsesCookieName, anthropicApiKeyCookieName, MaxFreeUses } from '~/utils/freeUses';
@@ -280,11 +281,13 @@ export const ChatImpl = memo(
return { recordingId, recordingMessage };
};
const getEnhancedPrompt = async (recordingId: string, repositoryContents: string) => {
const getEnhancedPrompt = async (recordingId: string, userMessage: string) => {
let enhancedPrompt, message;
try {
const mouseData = getCurrentMouseData();
enhancedPrompt = await getSimulationEnhancedPrompt(recordingId, repositoryContents, mouseData);
console.log("MouseData", mouseData);
enhancedPrompt = await getSimulationEnhancedPrompt(recordingId, messages, userMessage);
message = `Explanation of the bug:\n\n${enhancedPrompt}`;
} catch (e) {
console.error("Error enhancing prompt", e);
@@ -346,7 +349,7 @@ export const ChatImpl = memo(
setInjectedMessages([...injectedMessages, { message: recordingMessage, previousId: messages[messages.length - 1].id }]);
if (recordingId) {
const info = await getEnhancedPrompt(recordingId, contentBase64);
const info = await getEnhancedPrompt(recordingId, _input);
if (numAbortsAtStart != gNumAborts) {
return;

125
app/lib/replay/Recording.ts
View File

@@ -1,76 +1,7 @@
// Manage state around recording Preview behavior for generating a Replay recording.
import { assert, stringToBase64, uint8ArrayToBase64 } from "./ReplayProtocolClient";
export interface SimulationResource {
url: string;
requestBodyBase64: string;
responseBodyBase64?: string;
responseStatus?: number;
responseHeaders?: Record<string, string>;
error?: string;
}
enum SimulationInteractionKind {
Click = "click",
DblClick = "dblclick",
KeyDown = "keydown",
}
export interface SimulationInteraction {
kind: SimulationInteractionKind;
// Elapsed time when the interaction occurred.
time: number;
// Selector of the element associated with the interaction.
selector: string;
// For mouse interactions, dimensions and position within the
// element where the event occurred.
width?: number;
height?: number;
x?: number;
y?: number;
// For keydown interactions, the key pressed.
key?: string;
}
interface IndexedDBAccess {
kind: "get" | "put" | "add";
key?: any;
item?: any;
storeName: string;
databaseName: string;
databaseVersion: number;
}
interface LocalStorageAccess {
kind: "get" | "set";
key: string;
value?: string;
}
export interface SimulationData {
// Contents of window.location.href.
locationHref: string;
// URL of the main document.
documentUrl: string;
// All resources accessed.
resources: SimulationResource[];
// All user interactions made.
interactions: SimulationInteraction[];
// All indexedDB accesses made.
indexedDBAccesses?: IndexedDBAccess[];
// All localStorage accesses made.
localStorageAccesses?: LocalStorageAccess[];
}
import type { IndexedDBAccess, LocalStorageAccess, NetworkResource, SimulationData, UserInteraction } from "./SimulationData";
// Our message event listener can trigger on messages from iframes we don't expect.
// This is a unique ID for the last time we injected the recording message handler
@@ -123,8 +54,8 @@ export async function getMouseData(iframe: HTMLIFrameElement, position: { x: num
// Add handlers to the current iframe's window.
function addRecordingMessageHandler(messageHandlerId: string) {
const resources: SimulationResource[] = [];
const interactions: SimulationInteraction[] = [];
const resources: NetworkResource[] = [];
const interactions: UserInteraction[] = [];
const indexedDBAccesses: IndexedDBAccess[] = [];
const localStorageAccesses: LocalStorageAccess[] = [];
@@ -147,14 +78,42 @@ function addRecordingMessageHandler(messageHandlerId: string) {
}
async function getSimulationData(): Promise<SimulationData> {
return {
locationHref: window.location.href,
documentUrl: window.location.href,
resources,
interactions,
indexedDBAccesses,
localStorageAccesses,
};
const data: SimulationData = [];
data.push({
kind: "locationHref",
href: window.location.href,
});
data.push({
kind: "documentURL",
url: window.location.href,
});
for (const resource of resources) {
data.push({
kind: "resource",
resource,
});
}
for (const interaction of interactions) {
data.push({
kind: "interaction",
interaction,
});
}
for (const indexedDBAccess of indexedDBAccesses) {
data.push({
kind: "indexedDB",
access: indexedDBAccess,
});
}
for (const localStorageAccess of localStorageAccesses) {
data.push({
kind: "localStorage",
access: localStorageAccess,
});
}
return data;
}
window.addEventListener("message", async (event) => {
@@ -249,7 +208,7 @@ function addRecordingMessageHandler(messageHandlerId: string) {
window.addEventListener("click", (event) => {
if (event.target) {
interactions.push({
kind: SimulationInteractionKind.Click,
kind: "click",
time: Date.now() - startTime,
...getMouseEventData(event)
});
@@ -259,7 +218,7 @@ function addRecordingMessageHandler(messageHandlerId: string) {
window.addEventListener("dblclick", (event) => {
if (event.target) {
interactions.push({
kind: SimulationInteractionKind.DblClick,
kind: "dblclick",
time: Date.now() - startTime,
...getMouseEventData(event)
});
@@ -269,7 +228,7 @@ function addRecordingMessageHandler(messageHandlerId: string) {
window.addEventListener("keydown", (event) => {
if (event.key) {
interactions.push({
kind: SimulationInteractionKind.KeyDown,
kind: "keydown",
time: Date.now() - startTime,
...getKeyboardEventData(event)
});

167
app/lib/replay/SimulationData.ts Normal file
View File

@@ -0,0 +1,167 @@
// Data structures for simulation.
export const SimulationDataVersion = "0.1";
// Simulation data specifying the server URL to connect to for static resources.
interface SimulationPacketServerURL {
kind: "serverURL";
url: string;
}
// Simulation data specifying the contents of the repository to set up a dev server
// for static resources.
interface SimulationPacketRepositoryContents {
kind: "repositoryContents";
contents: string; // base64 encoded zip of the repository.
}
// Simulation data specifying the contents of window.location.href.
interface SimulationPacketLocationHref {
kind: "locationHref";
href: string;
}
// Simulation data specifying the URL of the main document.
interface SimulationPacketDocumentURL {
kind: "documentURL";
url: string;
}
export interface NetworkResource {
url: string;
error?: string;
requestBodyBase64?: string;
responseBodyBase64?: string;
responseStatus?: number;
responseHeaders?: Record<string, string>;
}
interface SimulationPacketResource {
kind: "resource";
resource: NetworkResource;
}
export type UserInteractionKind = "click" | "dblclick" | "keydown";
export interface UserInteraction {
kind: UserInteractionKind;
// Elapsed time when the interaction occurred.
time: number;
// Selector of the element associated with the interaction.
selector: string;
// For mouse interactions, dimensions and position within the
// element where the event occurred.
width?: number;
height?: number;
x?: number;
y?: number;
// For keydown interactions, the key pressed.
key?: string;
}
interface SimulationPacketInteraction {
kind: "interaction";
interaction: UserInteraction;
}
export interface WebSocketCreate {
kind: "create";
socketId: number;
url: string;
}
interface WebSocketClose {
kind: "close";
socketId: number;
code: number;
reason: string;
}
export interface WebSocketSend {
kind: "send";
socketId: number;
binary: boolean;
text?: string;
encodedLength: number;
}
interface WebSocketConnected {
kind: "connected";
socketId: number;
subprotocol: string;
extensions: string;
}
export interface WebSocketNewMessage {
kind: "newMessage";
socketId: number;
binary: boolean;
text?: string;
encodedLength: number;
}
interface WebSocketOnError {
kind: "onError";
socketId: number;
}
interface WebSocketOnClose {
kind: "onClose";
socketId: number;
}
export type WebSocketEvent =
| WebSocketCreate
| WebSocketClose
| WebSocketSend
| WebSocketConnected
| WebSocketNewMessage
| WebSocketOnError
| WebSocketOnClose;
interface SimulationPacketWebSocket {
kind: "websocket";
event: WebSocketEvent;
}
export interface IndexedDBAccess {
kind: "get" | "put" | "add";
key?: any;
item?: any;
storeName: string;
databaseName: string;
databaseVersion: number;
}
interface SimulationPacketIndexedDB {
kind: "indexedDB";
access: IndexedDBAccess;
}
export interface LocalStorageAccess {
kind: "get" | "set";
key: string;
value?: string;
}
interface SimulationPacketLocalStorage {
kind: "localStorage";
access: LocalStorageAccess;
}
export type SimulationPacket =
| SimulationPacketServerURL
| SimulationPacketRepositoryContents
| SimulationPacketLocationHref
| SimulationPacketDocumentURL
| SimulationPacketResource
| SimulationPacketInteraction
| SimulationPacketWebSocket
| SimulationPacketIndexedDB
| SimulationPacketLocalStorage;
export type SimulationData = SimulationPacket[];

325
app/lib/replay/SimulationPrompt.ts
View File

@@ -1,263 +1,100 @@
// Core logic for using simulation data from a remote recording to enhance
// the AI developer prompt.
import { type SimulationData, type MouseData } from './Recording';
import { assert, ProtocolClient, sendCommandDedicatedClient } from './ReplayProtocolClient';
import JSZip from 'jszip';
interface RerecordGenerateParams {
rerecordData: SimulationData;
repositoryContents: string;
}
import type { Message } from 'ai';
import type { SimulationData } from './SimulationData';
import { SimulationDataVersion } from './SimulationData';
import { assert, ProtocolClient } from './ReplayProtocolClient';
export async function getSimulationRecording(
simulationData: SimulationData,
repositoryContents: string
): Promise<string> {
const params: RerecordGenerateParams = {
rerecordData: simulationData,
repositoryContents,
};
const rv = await sendCommandDedicatedClient({
method: "Recording.globalExperimentalCommand",
params: {
name: "rerecordGenerate",
params,
},
});
return (rv as { rval: { rerecordedRecordingId: string } }).rval.rerecordedRecordingId;
}
type ProtocolExecutionPoint = string;
export interface URLLocation {
sourceId: string;
line: number;
column: number;
url: string;
}
// A location within a recording and associated source contents.
export interface URLLocationWithSource extends URLLocation {
// Text from the application source indicating the location.
source: string;
}
interface ExecutionDataEntry {
// Value from the application source which is being described.
value?: string;
// Description of the contents of the value. If |value| is omitted
// this describes a control dependency for the location.
contents: string;
// Any associated execution point.
associatedPoint?: ProtocolExecutionPoint;
// Location in the recording of the associated execution point.
associatedLocation?: URLLocationWithSource;
// Any expression for the value at the associated point which flows to this one.
associatedValue?: string;
// Description of how data flows from the associated point to this one.
associatedDataflow?: string;
}
interface ExecutionDataPoint {
// Associated point.
point: ProtocolExecutionPoint;
// Location in the recording being described.
location: URLLocationWithSource;
// Entries describing the point.
entries: ExecutionDataEntry[];
}
// Initial point for analysis that is an uncaught exception thrown
// from application code called by React, causing the app to unmount.
interface RecordingFailureDataReactException {
kind: "ReactException";
errorText: string;
point: ProtocolExecutionPoint;
// Whether the exception was thrown by library code called at the point.
calleeFrame: boolean;
}
// Initial point for analysis that is an exception logged to the console.
interface RecordingFailureDataConsoleError {
kind: "ConsoleError";
errorText: string;
point: ProtocolExecutionPoint;
}
// Fallback failure data shows the React component tree at the end of the recording.
interface ReactComponentTree {
name: string;
children: ReactComponentTree[];
}
interface RecordingFailureDataComponentTree {
kind: "ComponentTree";
componentTree: ReactComponentTree;
}
export type RecordingFailureData =
| RecordingFailureDataReactException
| RecordingFailureDataConsoleError
| RecordingFailureDataComponentTree;
export interface ExecutionDataAnalysisResult {
// Points which were described.
points: ExecutionDataPoint[];
// If an expression was specified, the dataflow steps for that expression.
dataflow?: string[];
// The initial point which was analyzed. If no point was originally specified,
// another point will be picked based on any comments or other data in the recording.
point?: ProtocolExecutionPoint;
// Any comment text associated with the point.
commentText?: string;
// If the comment is on a React component, the name of the component.
reactComponentName?: string;
// If no point or comment was available, describes the failure associated with the
// initial point of the analysis.
failureData?: RecordingFailureData;
}
function trimFileName(url: string): string {
const lastSlash = url.lastIndexOf('/');
return url.slice(lastSlash + 1);
}
async function getSourceText(repositoryContents: string, fileName: string): Promise<string> {
const zip = new JSZip();
const binaryData = Buffer.from(repositoryContents, 'base64');
await zip.loadAsync(binaryData as any /* TS complains but JSZip works */);
for (const [path, file] of Object.entries(zip.files)) {
if (trimFileName(path) === fileName) {
return await file.async('string');
}
}
for (const path of Object.keys(zip.files)) {
console.log("RepositoryPath", path);
}
throw new Error(`File ${fileName} not found in repository`);
}
async function annotateSource(repositoryContents: string, fileName: string, source: string, annotation: string): Promise<string> {
const sourceText = await getSourceText(repositoryContents, fileName);
const sourceLines = sourceText.split('\n');
const lineIndex = sourceLines.findIndex(line => line.includes(source));
if (lineIndex === -1) {
throw new Error(`Source text ${source} not found in ${fileName}`);
}
let rv = "";
for (let i = lineIndex - 3; i < lineIndex + 3; i++) {
if (i < 0 || i >= sourceLines.length) {
continue;
}
if (i === lineIndex) {
const leadingSpaces = sourceLines[i].match(/^\s*/)![0];
rv += `${leadingSpaces}// ${annotation}\n`;
}
rv += `${sourceLines[i]}\n`;
}
return rv;
}
function describeComponentTree(componentTree: ReactComponentTree, indent: string): string {
let rv = "";
rv += `${indent}${componentTree.name}\n`;
for (const child of componentTree.children) {
rv += describeComponentTree(child, indent + " ");
}
return rv;
}
function codeBlock(text: string): string {
return "```\n" + text + (text.endsWith("\n") ? "" : "\n") + "```";
}
async function enhancePromptFromFailureData(
points: ExecutionDataPoint[],
failureData: RecordingFailureData,
repositoryContents: string
): Promise<string> {
const failurePoint = "point" in failureData ? points.find(p => p.point === failureData.point) : undefined;
let prompt = "";
let annotation;
switch (failureData.kind) {
case "ReactException":
prompt += "An exception was thrown which causes React to unmount the application.\n";
if (failureData.calleeFrame) {
annotation = `A function called from here is throwing the exception "${failureData.errorText}"`;
} else {
annotation = `This line is throwing the exception "${failureData.errorText}"`;
}
break;
case "ConsoleError":
prompt += "An exception was thrown and later logged to the console.\n";
annotation = `This line is throwing the exception "${failureData.errorText}"`;
break;
case "ComponentTree":
prompt += "The React component tree at the end of the recording:\n\n";
prompt += codeBlock(describeComponentTree(failureData.componentTree, ""));
break;
default:
throw new Error(`Unknown failure kind: ${(failureData as any).kind}`);
}
if (failurePoint) {
assert(annotation);
const pointText = failurePoint.location.source.trim();
const fileName = trimFileName(failurePoint.location.url);
const annotatedSource = await annotateSource(repositoryContents, fileName, pointText, annotation);
prompt += `Here is the affected code, in ${fileName}:\n\n`;
prompt += codeBlock(annotatedSource);
}
return prompt;
}
export async function getSimulationEnhancedPrompt(
recordingId: string,
repositoryContents: string,
mouseData: MouseData | undefined
): Promise<string> {
const client = new ProtocolClient();
await client.initialize();
try {
const createSessionRval = await client.sendCommand({ method: "Recording.createSession", params: { recordingId } });
const sessionId = (createSessionRval as { sessionId: string }).sessionId;
const { chatId } = await client.sendCommand({ method: "Nut.startChat", params: {} }) as { chatId: string };
const { rval } = await client.sendCommand({
method: "Session.experimentalCommand",
const repositoryContentsPacket = {
kind: "repositoryContents",
contents: repositoryContents,
};
const { recordingId } = await client.sendCommand({
method: "Nut.addSimulation",
params: {
name: "analyzeExecutionPoint",
params: { mouseData },
chatId,
version: SimulationDataVersion,
simulationData: [repositoryContentsPacket, ...simulationData],
completeData: true,
saveRecording: true,
},
sessionId,
}) as { rval: ExecutionDataAnalysisResult };
}) as { recordingId: string | undefined };
const { points, failureData } = rval;
assert(failureData, "No failure data");
if (!recordingId) {
throw new Error("Expected recording ID in result");
}
console.log("FailureData", JSON.stringify(failureData, null, 2));
const prompt = await enhancePromptFromFailureData(points, failureData, repositoryContents);
console.log("Enhanced prompt", prompt);
return prompt;
return recordingId;
} finally {
client.close();
}
}
type ProtocolMessage = {
role: "user" | "assistant" | "system";
type: "text";
content: string;
};
const SystemPrompt = `
The following user message describes a bug or other problem on the page which needs to be fixed.
You must respond with a useful explanation that will help the user understand the source of the problem.
Do not describe the specific fix needed.
`;
export async function getSimulationEnhancedPrompt(
recordingId: string,
chatMessages: Message[],
userMessage: string
): Promise<string> {
const client = new ProtocolClient();
await client.initialize();
try {
const { chatId } = await client.sendCommand({ method: "Nut.startChat", params: {} }) as { chatId: string };
await client.sendCommand({
method: "Nut.addRecording",
params: { chatId, recordingId },
});
const messages = [
{
role: "system",
type: "text",
content: SystemPrompt,
},
{
role: "user",
type: "text",
content: userMessage,
},
];
let response: string = "";
const removeListener = client.listenForMessage("Nut.chatResponsePart", ({ message }: { message: ProtocolMessage }) => {
console.log("ChatResponsePart", message);
response += message.content;
});
const responseId = "<response-id>";
await client.sendCommand({
method: "Nut.sendChatMessage",
params: { chatId, responseId, messages },
});
removeListener();
return response;
} finally {
client.close();
}