From 3d2ab89cdc01b822a4fcfbd4ba4ec61ee66c3622 Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Mon, 25 Nov 2024 10:24:03 +0200 Subject: [PATCH 1/3] Proof of concept for folder import --- app/components/chat/BaseChat.tsx | 31 +++---- app/components/chat/ImportFolderButton.tsx | 100 +++++++++++++++++++++ 2 files changed, 113 insertions(+), 18 deletions(-) create mode 100644 app/components/chat/ImportFolderButton.tsx diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 1a33375..4da6493 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -20,6 +20,7 @@ import * as Tooltip from '@radix-ui/react-tooltip'; import styles from './BaseChat.module.scss'; import type { ProviderInfo } from '~/utils/types'; import { ExportChatButton } from '~/components/chat/ExportChatButton'; +import { ImportFolderButton } from '~/components/chat/ImportFolderButton'; const EXAMPLE_PROMPTS = [ { text: 'Build a todo app in React using Tailwind' }, @@ -184,31 +185,21 @@ export const BaseChat = React.forwardRef( reader.onload = async (e) => { try { - const content = e.target?.result as string; - const data = JSON.parse(content); - - if (!Array.isArray(data.messages)) { - toast.error('Invalid chat file format'); - } - - await importChat(data.description, data.messages); - toast.success('Chat imported successfully'); - } catch (error: unknown) { - if (error instanceof Error) { - toast.error('Failed to parse chat file: ' + error.message); - } else { - toast.error('Failed to parse chat file'); - } + const content = JSON.parse(e.target?.result as string); + await importChat(content.description || '', content.messages || []); + } catch (error) { + toast.error(`Invalid chat file format: ${error instanceof Error ? ': ' + error.message : ''}`); } }; - reader.onerror = () => toast.error('Failed to read chat file'); + + reader.onerror = () => { + toast.error('Something went wrong'); + }; reader.readAsText(file); } catch (error) { toast.error(error instanceof Error ? error.message : 'Failed to import chat'); } e.target.value = ''; // Reset file input - } else { - toast.error('Something went wrong'); } }} /> @@ -224,6 +215,10 @@ export const BaseChat = React.forwardRef(
Import Chat +
diff --git a/app/components/chat/ImportFolderButton.tsx b/app/components/chat/ImportFolderButton.tsx new file mode 100644 index 0000000..1444580 --- /dev/null +++ b/app/components/chat/ImportFolderButton.tsx @@ -0,0 +1,100 @@ +import React from 'react'; +import type { Message } from 'ai'; +import { toast } from 'react-toastify'; + +interface ImportFolderButtonProps { + className?: string; + importChat?: (description: string, messages: Message[]) => Promise; +} + +const IGNORED_FOLDERS = ['node_modules', '.git', 'dist', 'build', '.next', 'coverage', '.cache', '.vscode', '.idea']; + +const generateId = () => Math.random().toString(36).substring(2, 15); + +export const ImportFolderButton: React.FC = ({ className, importChat }) => { + const shouldIncludeFile = (path: string): boolean => { + return !IGNORED_FOLDERS.some((folder) => path.includes(`/${folder}/`)); + }; + + const createChatFromFolder = async (files: File[]) => { + const fileArtifacts = await Promise.all( + files.map(async (file) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + + reader.onload = () => { + const content = reader.result as string; + const relativePath = file.webkitRelativePath.split('/').slice(1).join('/'); + resolve( + ` +${content} +`, + ); + }; + reader.onerror = reject; + reader.readAsText(file); + }); + }), + ); + + const message: Message = { + role: 'assistant', + content: `I'll help you set up these files. + + +${fileArtifacts.join('\n\n')} +`, + id: generateId(), + createdAt: new Date(), + }; + + const userMessage: Message = { + role: 'user', + id: generateId(), + content: 'Import my files', + createdAt: new Date(), + }; + + const description = `Folder Import: ${files[0].webkitRelativePath.split('/')[0]}`; + + if (importChat) { + await importChat(description, [userMessage, message]); + } + }; + + return ( + <> + { + const allFiles = Array.from(e.target.files || []); + const filteredFiles = allFiles.filter((file) => shouldIncludeFile(file.webkitRelativePath)); + + try { + await createChatFromFolder(filteredFiles); + } catch (error) { + console.error('Failed to import folder:', error); + toast.error('Failed to import folder'); + } + + e.target.value = ''; // Reset file input + }} + {...({} as any)} // if removed webkitdirectory will throw errors as unknow attribute + /> + + + ); +}; From 050bf2028fc143bb76eb7fe2efb998018efac9c7 Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Mon, 25 Nov 2024 19:50:01 +0200 Subject: [PATCH 2/3] Added parsing if ignore file and added handling of binary files --- app/components/chat/ImportFolderButton.tsx | 74 ++++++++++++++++++++-- app/utils/logger.ts | 2 +- package.json | 1 + pnpm-lock.yaml | 9 +++ 4 files changed, 80 insertions(+), 6 deletions(-) diff --git a/app/components/chat/ImportFolderButton.tsx b/app/components/chat/ImportFolderButton.tsx index 1444580..5f822ee 100644 --- a/app/components/chat/ImportFolderButton.tsx +++ b/app/components/chat/ImportFolderButton.tsx @@ -1,22 +1,55 @@ import React from 'react'; import type { Message } from 'ai'; import { toast } from 'react-toastify'; +import ignore from 'ignore'; interface ImportFolderButtonProps { className?: string; importChat?: (description: string, messages: Message[]) => Promise; } -const IGNORED_FOLDERS = ['node_modules', '.git', 'dist', 'build', '.next', 'coverage', '.cache', '.vscode', '.idea']; +// Common patterns to ignore, similar to .gitignore +const IGNORE_PATTERNS = [ + 'node_modules/**', + '.git/**', + 'dist/**', + 'build/**', + '.next/**', + 'coverage/**', + '.cache/**', + '.vscode/**', + '.idea/**', + '**/*.log', + '**/.DS_Store', + '**/npm-debug.log*', + '**/yarn-debug.log*', + '**/yarn-error.log*', +]; +const ig = ignore().add(IGNORE_PATTERNS); const generateId = () => Math.random().toString(36).substring(2, 15); +const isBinaryFile = async (file: File): Promise => { + const chunkSize = 1024; // Read the first 1 KB of the file + const buffer = new Uint8Array(await file.slice(0, chunkSize).arrayBuffer()); + + for (let i = 0; i < buffer.length; i++) { + const byte = buffer[i]; + + if (byte === 0 || (byte < 32 && byte !== 9 && byte !== 10 && byte !== 13)) { + return true; // Found a binary character + } + } + + return false; +}; + export const ImportFolderButton: React.FC = ({ className, importChat }) => { const shouldIncludeFile = (path: string): boolean => { - return !IGNORED_FOLDERS.some((folder) => path.includes(`/${folder}/`)); + return !ig.ignores(path); }; - const createChatFromFolder = async (files: File[]) => { + const createChatFromFolder = async (files: File[], binaryFiles: string[]) => { const fileArtifacts = await Promise.all( files.map(async (file) => { return new Promise((resolve, reject) => { @@ -37,9 +70,14 @@ ${content} }), ); + const binaryFilesMessage = + binaryFiles.length > 0 + ? `\n\nSkipped ${binaryFiles.length} binary files:\n${binaryFiles.map((f) => `- ${f}`).join('\n')}` + : ''; + const message: Message = { role: 'assistant', - content: `I'll help you set up these files. + content: `I'll help you set up these files.${binaryFilesMessage} ${fileArtifacts.join('\n\n')} @@ -74,8 +112,34 @@ ${fileArtifacts.join('\n\n')} const allFiles = Array.from(e.target.files || []); const filteredFiles = allFiles.filter((file) => shouldIncludeFile(file.webkitRelativePath)); + if (filteredFiles.length === 0) { + toast.error('No files found in the selected folder'); + return; + } + try { - await createChatFromFolder(filteredFiles); + const fileChecks = await Promise.all( + filteredFiles.map(async (file) => ({ + file, + isBinary: await isBinaryFile(file), + })), + ); + + const textFiles = fileChecks.filter((f) => !f.isBinary).map((f) => f.file); + const binaryFilePaths = fileChecks + .filter((f) => f.isBinary) + .map((f) => f.file.webkitRelativePath.split('/').slice(1).join('/')); + + if (textFiles.length === 0) { + toast.error('No text files found in the selected folder'); + return; + } + + if (binaryFilePaths.length > 0) { + toast.info(`Skipping ${binaryFilePaths.length} binary files`); + } + + await createChatFromFolder(textFiles, binaryFilePaths); } catch (error) { console.error('Failed to import folder:', error); toast.error('Failed to import folder'); diff --git a/app/utils/logger.ts b/app/utils/logger.ts index 9b2c31c..1a5c932 100644 --- a/app/utils/logger.ts +++ b/app/utils/logger.ts @@ -11,7 +11,7 @@ interface Logger { setLevel: (level: DebugLevel) => void; } -let currentLevel: DebugLevel = (import.meta.env.VITE_LOG_LEVEL ?? import.meta.env.DEV) ? 'debug' : 'info'; +let currentLevel: DebugLevel = import.meta.env.VITE_LOG_LEVEL ?? import.meta.env.DEV ? 'debug' : 'info'; const isWorker = 'HTMLRewriter' in globalThis; const supportsColor = !isWorker; diff --git a/package.json b/package.json index 16c59f7..ad98403 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "diff": "^5.2.0", "file-saver": "^2.0.5", "framer-motion": "^11.2.12", + "ignore": "^6.0.2", "isbot": "^4.1.0", "istextorbinary": "^9.5.0", "jose": "^5.6.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 951d1a4..1c92ad2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -143,6 +143,9 @@ importers: framer-motion: specifier: ^11.2.12 version: 11.2.12(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + ignore: + specifier: ^6.0.2 + version: 6.0.2 isbot: specifier: ^4.1.0 version: 4.4.0 @@ -3399,6 +3402,10 @@ packages: resolution: {integrity: sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==} engines: {node: '>= 4'} + ignore@6.0.2: + resolution: {integrity: sha512-InwqeHHN2XpumIkMvpl/DCJVrAHgCsG5+cn1XlnLWGwtZBm8QJfSusItfrwx81CTp5agNZqpKU2J/ccC5nGT4A==} + engines: {node: '>= 4'} + immediate@3.0.6: resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} @@ -9290,6 +9297,8 @@ snapshots: ignore@5.3.1: {} + ignore@6.0.2: {} + immediate@3.0.6: {} immutable@4.3.7: {} From 15d490dda4503640f613f2dbf50927686a18fdc8 Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Mon, 25 Nov 2024 19:57:53 +0200 Subject: [PATCH 3/3] Merge with master fixes --- app/components/chat/BaseChat.tsx | 5 ++--- .../{ImportButton.tsx => ImportButtons.tsx} | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) rename app/components/chat/chatExportAndImport/{ImportButton.tsx => ImportButtons.tsx} (82%) diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 360d38d..0760078 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -19,9 +19,8 @@ import * as Tooltip from '@radix-ui/react-tooltip'; import styles from './BaseChat.module.scss'; import type { ProviderInfo } from '~/utils/types'; import { ExportChatButton } from '~/components/chat/chatExportAndImport/ExportChatButton'; -import { ImportButton } from '~/components/chat/chatExportAndImport/ImportButton'; +import { ImportButtons } from '~/components/chat/chatExportAndImport/ImportButtons'; import { ExamplePrompts } from '~/components/chat/ExamplePrompts'; -import { ImportFolderButton } from '~/components/chat/ImportFolderButton'; // eslint-disable-next-line @typescript-eslint/no-unused-vars const providerList = PROVIDER_LIST; @@ -311,7 +310,7 @@ export const BaseChat = React.forwardRef( - {!chatStarted && ImportButton(importChat)} + {!chatStarted && ImportButtons(importChat)} {!chatStarted && ExamplePrompts(sendMessage)} {() => } diff --git a/app/components/chat/chatExportAndImport/ImportButton.tsx b/app/components/chat/chatExportAndImport/ImportButtons.tsx similarity index 82% rename from app/components/chat/chatExportAndImport/ImportButton.tsx rename to app/components/chat/chatExportAndImport/ImportButtons.tsx index 91c699e..2b59574 100644 --- a/app/components/chat/chatExportAndImport/ImportButton.tsx +++ b/app/components/chat/chatExportAndImport/ImportButtons.tsx @@ -1,8 +1,9 @@ import type { Message } from 'ai'; import { toast } from 'react-toastify'; import React from 'react'; +import { ImportFolderButton } from '~/components/chat/ImportFolderButton'; -export function ImportButton(importChat: ((description: string, messages: Message[]) => Promise) | undefined) { +export function ImportButtons(importChat: ((description: string, messages: Message[]) => Promise) | undefined) { return (
Import Chat +