From ea5c6244a6f17a05447595119014414bcba9001d Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sat, 7 Dec 2024 15:58:13 +0530 Subject: [PATCH 1/6] feat(context optimization):improved context management and redused chat overhead --- app/components/chat/Chat.client.tsx | 3 +- app/lib/.server/llm/stream-text.ts | 92 ++++++++++++++++++++++++++++- app/lib/hooks/useMessageParser.ts | 4 +- app/lib/stores/workbench.ts | 10 +++- app/routes/api.chat.ts | 8 +-- 5 files changed, 107 insertions(+), 10 deletions(-) diff --git a/app/components/chat/Chat.client.tsx b/app/components/chat/Chat.client.tsx index c6a01ee..2818378 100644 --- a/app/components/chat/Chat.client.tsx +++ b/app/components/chat/Chat.client.tsx @@ -91,7 +91,7 @@ export const ChatImpl = memo( const [chatStarted, setChatStarted] = useState(initialMessages.length > 0); const [uploadedFiles, setUploadedFiles] = useState([]); // Move here const [imageDataList, setImageDataList] = useState([]); // Move here - + const files = useStore(workbenchStore.files); const [model, setModel] = useState(() => { const savedModel = Cookies.get('selectedModel'); return savedModel || DEFAULT_MODEL; @@ -111,6 +111,7 @@ export const ChatImpl = memo( api: '/api/chat', body: { apiKeys, + files, }, onError: (error) => { logger.error('Request failed\n\n', error); diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index f408ba2..589ba36 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -3,6 +3,7 @@ import { getModel } from '~/lib/.server/llm/model'; import { MAX_TOKENS } from './constants'; import { getSystemPrompt } from './prompts'; import { DEFAULT_MODEL, DEFAULT_PROVIDER, getModelList, MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants'; +import ignore from 'ignore'; interface ToolResult { toolCallId: string; @@ -22,6 +23,79 @@ export type Messages = Message[]; export type StreamingOptions = Omit[0], 'model'>; +export interface File { + type: 'file'; + content: string; + isBinary: boolean; +} + +export interface Folder { + type: 'folder'; +} + +type Dirent = File | Folder; + +export type FileMap = Record; + +function simplifyBoltActions(input: string): string { + // Using regex to match boltAction tags that have type="file" + const regex = /(]*type="file"[^>]*>)([\s\S]*?)(<\/boltAction>)/g; + + // Replace each matching occurrence + return input.replace(regex, (_0, openingTag, _2, closingTag) => { + return `${openingTag}\n ...\n ${closingTag}`; + }); +} + +// Common patterns to ignore, similar to .gitignore +const IGNORE_PATTERNS = [ + 'node_modules/**', + '.git/**', + 'dist/**', + 'build/**', + '.next/**', + 'coverage/**', + '.cache/**', + '.vscode/**', + '.idea/**', + '**/*.log', + '**/.DS_Store', + '**/npm-debug.log*', + '**/yarn-debug.log*', + '**/yarn-error.log*', + '**/*lock.json', + '**/*lock.yml', +]; +const ig = ignore().add(IGNORE_PATTERNS); + +function createFilesContext(files: FileMap) { + let filePaths = Object.keys(files); + filePaths = filePaths.filter((x) => { + const relPath = x.replace('/home/project/', ''); + return !ig.ignores(relPath); + }); + console.log(filePaths); + + const fileContexts = filePaths + .filter((x) => files[x] && files[x].type == 'file') + .map((path) => { + const dirent = files[path]; + + if (!dirent || dirent.type == 'folder') { + return ''; + } + + const codeWithLinesNumbers = dirent.content + .split('\n') + .map((v, i) => `${i + 1}|${v}`) + .join('\n'); + + return `\n${codeWithLinesNumbers}\n`; + }); + + return `Below are the code files present in the webcontainer:\ncode format:\n|\n ${fileContexts.join('\n\n')}\n\n`; +} + function extractPropertiesFromMessage(message: Message): { model: string; provider: string; content: string } { const textContent = Array.isArray(message.content) ? message.content.find((item) => item.type === 'text')?.text || '' @@ -63,6 +137,7 @@ export async function streamText( env: Env, options?: StreamingOptions, apiKeys?: Record, + files?: FileMap, ) { let currentModel = DEFAULT_MODEL; let currentProvider = DEFAULT_PROVIDER.name; @@ -77,6 +152,11 @@ export async function streamText( currentProvider = provider; + return { ...message, content }; + } else if (message.role == 'assistant') { + let content = message.content; + content = simplifyBoltActions(content); + return { ...message, content }; } @@ -87,9 +167,19 @@ export async function streamText( const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS; + let systemPrompt = getSystemPrompt(); + let codeContext = ''; + + if (files) { + codeContext = createFilesContext(files); + systemPrompt = `${systemPrompt}\n\n ${codeContext}`; + } + + console.log({ codeContext, processedMessages }); + return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys) as any, - system: getSystemPrompt(), + system: systemPrompt, maxTokens: dynamicMaxTokens, messages: convertToCoreMessages(processedMessages as any), ...options, diff --git a/app/lib/hooks/useMessageParser.ts b/app/lib/hooks/useMessageParser.ts index 97a063d..3510d00 100644 --- a/app/lib/hooks/useMessageParser.ts +++ b/app/lib/hooks/useMessageParser.ts @@ -23,14 +23,14 @@ const messageParser = new StreamingMessageParser({ logger.trace('onActionOpen', data.action); // we only add shell actions when when the close tag got parsed because only then we have the content - if (data.action.type !== 'shell') { + if (data.action.type === 'file') { workbenchStore.addAction(data); } }, onActionClose: (data) => { logger.trace('onActionClose', data.action); - if (data.action.type === 'shell') { + if (data.action.type !== 'file') { workbenchStore.addAction(data); } diff --git a/app/lib/stores/workbench.ts b/app/lib/stores/workbench.ts index de3a11e..90c3bd9 100644 --- a/app/lib/stores/workbench.ts +++ b/app/lib/stores/workbench.ts @@ -261,9 +261,9 @@ export class WorkbenchStore { this.artifacts.setKey(messageId, { ...artifact, ...state }); } addAction(data: ActionCallbackData) { - this._addAction(data); + // this._addAction(data); - // this.addToExecutionQueue(()=>this._addAction(data)) + this.addToExecutionQueue(() => this._addAction(data)); } async _addAction(data: ActionCallbackData) { const { messageId } = data; @@ -293,6 +293,12 @@ export class WorkbenchStore { unreachable('Artifact not found'); } + const action = artifact.runner.actions.get()[data.actionId]; + + if (action.executed) { + return; + } + if (data.action.type === 'file') { const wc = await webcontainer; const fullPath = nodePath.join(wc.workdir, data.action.filePath); diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index 0073274..e1b2372 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -29,9 +29,9 @@ function parseCookies(cookieHeader: string) { } async function chatAction({ context, request }: ActionFunctionArgs) { - const { messages } = await request.json<{ + const { messages, files } = await request.json<{ messages: Messages; - model: string; + files: any; }>(); const cookieHeader = request.headers.get('Cookie'); @@ -60,13 +60,13 @@ async function chatAction({ context, request }: ActionFunctionArgs) { messages.push({ role: 'assistant', content }); messages.push({ role: 'user', content: CONTINUE_PROMPT }); - const result = await streamText(messages, context.cloudflare.env, options, apiKeys); + const result = await streamText(messages, context.cloudflare.env, options, apiKeys, files); return stream.switchSource(result.toAIStream()); }, }; - const result = await streamText(messages, context.cloudflare.env, options, apiKeys); + const result = await streamText(messages, context.cloudflare.env, options, apiKeys, files); stream.switchSource(result.toAIStream()); From dfbbea110ef326364b5c8df56f1569c2a7481bda Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Tue, 10 Dec 2024 14:27:12 +0530 Subject: [PATCH 2/6] removed console logs --- app/lib/.server/llm/stream-text.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 589ba36..4aca822 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -74,7 +74,6 @@ function createFilesContext(files: FileMap) { const relPath = x.replace('/home/project/', ''); return !ig.ignores(relPath); }); - console.log(filePaths); const fileContexts = filePaths .filter((x) => files[x] && files[x].type == 'file') @@ -175,8 +174,6 @@ export async function streamText( systemPrompt = `${systemPrompt}\n\n ${codeContext}`; } - console.log({ codeContext, processedMessages }); - return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys) as any, system: systemPrompt, From 77425d61d9c24b1b2acc188aa0668a0295bf5bc1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 13 Dec 2024 20:38:54 +0000 Subject: [PATCH 3/6] chore: update commit hash to 8c4397a19f3eab2382082a39526d66385e9d2a49 --- app/commit.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/commit.json b/app/commit.json index 2a5e31e..5a4a539 100644 --- a/app/commit.json +++ b/app/commit.json @@ -1 +1 @@ -{ "commit": "d1fa70fc97dc7839ea8cd005feb03266f201cf4f" } +{ "commit": "8c4397a19f3eab2382082a39526d66385e9d2a49" } From 456c89429a8f8520d604a73bf37146efccbaec7f Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sat, 14 Dec 2024 02:25:36 +0530 Subject: [PATCH 4/6] fix: removed context optimization temporarily, to be moved to optional from menu --- app/commit.json | 2 +- app/lib/.server/llm/stream-text.ts | 7 ++++--- app/utils/constants.ts | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/commit.json b/app/commit.json index 2a5e31e..5a4a539 100644 --- a/app/commit.json +++ b/app/commit.json @@ -1 +1 @@ -{ "commit": "d1fa70fc97dc7839ea8cd005feb03266f201cf4f" } +{ "commit": "8c4397a19f3eab2382082a39526d66385e9d2a49" } diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index b178cc4..11ac99b 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -38,7 +38,7 @@ type Dirent = File | Folder; export type FileMap = Record; -function simplifyBoltActions(input: string): string { +export function simplifyBoltActions(input: string): string { // Using regex to match boltAction tags that have type="file" const regex = /(]*type="file"[^>]*>)([\s\S]*?)(<\/boltAction>)/g; @@ -156,8 +156,9 @@ export async function streamText(props: { return { ...message, content }; } else if (message.role == 'assistant') { - let content = message.content; - content = simplifyBoltActions(content); + const content = message.content; + + // content = simplifyBoltActions(content); return { ...message, content }; } diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 1e51619..3247fb2 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -462,7 +462,6 @@ async function getOpenRouterModels(): Promise { } async function getLMStudioModels(_apiKeys?: Record, settings?: IProviderSetting): Promise { - try { const baseUrl = settings?.baseUrl || import.meta.env.LMSTUDIO_API_BASE_URL || 'http://localhost:1234'; const response = await fetch(`${baseUrl}/v1/models`); From 9666b2ab67d25345542722ab9d870b36ad06252e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 13 Dec 2024 20:58:40 +0000 Subject: [PATCH 5/6] chore: update commit hash to 55094392cf4c5bc607aff796680ad50236a4cf20 --- app/commit.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/commit.json b/app/commit.json index 5a4a539..beebd2b 100644 --- a/app/commit.json +++ b/app/commit.json @@ -1 +1 @@ -{ "commit": "8c4397a19f3eab2382082a39526d66385e9d2a49" } +{ "commit": "55094392cf4c5bc607aff796680ad50236a4cf20" } From bbda3a1db4ceb583e7ca815fd1ee3c05f73f80a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 14 Dec 2024 02:09:03 +0000 Subject: [PATCH 6/6] chore: update commit hash to 9666b2ab67d25345542722ab9d870b36ad06252e --- app/commit.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/commit.json b/app/commit.json index beebd2b..ff64112 100644 --- a/app/commit.json +++ b/app/commit.json @@ -1 +1 @@ -{ "commit": "55094392cf4c5bc607aff796680ad50236a4cf20" } +{ "commit": "9666b2ab67d25345542722ab9d870b36ad06252e" }