Merge pull request #582 from emcconnell/feat/image-select

Add Screenshot Capture and Management Features to Chat and Workbench Components

app/commit.json

@@ -1 +1 @@
-{ "commit": "eb53146937cbe49a84aaaaa59882df6db4a1e459" }
+{ "commit": "eb53146937cbe49a84aaaaa59882df6db4a1e459" }

app/components/chat/BaseChat.tsx

@@ -26,6 +26,7 @@ import FilePreview from './FilePreview';
 import { ModelSelector } from '~/components/chat/ModelSelector';
 import { SpeechRecognitionButton } from '~/components/chat/SpeechRecognition';
 import type { IProviderSetting, ProviderInfo } from '~/types/model';
+import { ScreenshotStateManager } from './ScreenshotStateManager';
 
 const TEXTAREA_MIN_HEIGHT = 76;
 
@@ -376,6 +377,16 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                     setImageDataList?.(imageDataList.filter((_, i) => i !== index));
                   }}
                 />
+                <ClientOnly>
+                  {() => (
+                    <ScreenshotStateManager
+                      setUploadedFiles={setUploadedFiles}
+                      setImageDataList={setImageDataList}
+                      uploadedFiles={uploadedFiles}
+                      imageDataList={imageDataList}
+                    />
+                  )}
+                </ClientOnly>
                 <div
                   className={classNames(
                     'relative shadow-xs border border-bolt-elements-borderColor backdrop-blur rounded-lg',
@@ -425,15 +436,17 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                         }
 
                         event.preventDefault();
-                        
+
                         if (isStreaming) {
                           handleStop?.();
                           return;
                         }
+
                         // ignore if using input method engine
                         if (event.nativeEvent.isComposing) {
-                          return
+                          return;
                         }
+
                         handleSendMessage?.(event);
                       }
                     }}

app/components/chat/ScreenshotStateManager.tsx

@@ -0,0 +1,33 @@
+import { useEffect } from 'react';
+
+interface ScreenshotStateManagerProps {
+  setUploadedFiles?: (files: File[]) => void;
+  setImageDataList?: (dataList: string[]) => void;
+  uploadedFiles: File[];
+  imageDataList: string[];
+}
+
+export const ScreenshotStateManager = ({
+  setUploadedFiles,
+  setImageDataList,
+  uploadedFiles,
+  imageDataList,
+}: ScreenshotStateManagerProps) => {
+  useEffect(() => {
+    if (setUploadedFiles && setImageDataList) {
+      (window as any).__BOLT_SET_UPLOADED_FILES__ = setUploadedFiles;
+      (window as any).__BOLT_SET_IMAGE_DATA_LIST__ = setImageDataList;
+      (window as any).__BOLT_UPLOADED_FILES__ = uploadedFiles;
+      (window as any).__BOLT_IMAGE_DATA_LIST__ = imageDataList;
+    }
+
+    return () => {
+      delete (window as any).__BOLT_SET_UPLOADED_FILES__;
+      delete (window as any).__BOLT_SET_IMAGE_DATA_LIST__;
+      delete (window as any).__BOLT_UPLOADED_FILES__;
+      delete (window as any).__BOLT_IMAGE_DATA_LIST__;
+    };
+  }, [setUploadedFiles, setImageDataList, uploadedFiles, imageDataList]);
+
+  return null;
+};

app/components/workbench/Preview.tsx

@@ -3,6 +3,7 @@ import { memo, useCallback, useEffect, useRef, useState } from 'react';
 import { IconButton } from '~/components/ui/IconButton';
 import { workbenchStore } from '~/lib/stores/workbench';
 import { PortDropdown } from './PortDropdown';
+import { ScreenshotSelector } from './ScreenshotSelector';
 
 type ResizeSide = 'left' | 'right' | null;
 
@@ -20,6 +21,7 @@ export const Preview = memo(() => {
 
   const [url, setUrl] = useState('');
   const [iframeUrl, setIframeUrl] = useState<string | undefined>();
+  const [isSelectionMode, setIsSelectionMode] = useState(false);
 
   // Toggle between responsive mode and device mode
   const [isDeviceModeOn, setIsDeviceModeOn] = useState(false);
@@ -218,12 +220,17 @@ export const Preview = memo(() => {
       )}
       <div className="bg-bolt-elements-background-depth-2 p-2 flex items-center gap-1.5">
         <IconButton icon="i-ph:arrow-clockwise" onClick={reloadPreview} />
-
+        <IconButton
+          icon="i-ph:selection"
+          onClick={() => setIsSelectionMode(!isSelectionMode)}
+          className={isSelectionMode ? 'bg-bolt-elements-background-depth-3' : ''}
+        />
         <div
           className="flex items-center gap-1 flex-grow bg-bolt-elements-preview-addressBar-background border border-bolt-elements-borderColor text-bolt-elements-preview-addressBar-text rounded-full px-3 py-1 text-sm hover:bg-bolt-elements-preview-addressBar-backgroundHover hover:focus-within:bg-bolt-elements-preview-addressBar-backgroundActive focus-within:bg-bolt-elements-preview-addressBar-backgroundActive
         focus-within-border-bolt-elements-borderColorActive focus-within:text-bolt-elements-preview-addressBar-textActive"
         >
           <input
+            title="URL"
             ref={inputRef}
             className="w-full bg-transparent outline-none"
             type="text"
@@ -281,7 +288,20 @@ export const Preview = memo(() => {
           }}
         >
           {activePreview ? (
-            <iframe ref={iframeRef} className="border-none w-full h-full bg-white" src={iframeUrl} allowFullScreen />
+            <>
+              <iframe
+                ref={iframeRef}
+                title="preview"
+                className="border-none w-full h-full bg-white"
+                src={iframeUrl}
+                allowFullScreen
+              />
+              <ScreenshotSelector
+                isSelectionMode={isSelectionMode}
+                setIsSelectionMode={setIsSelectionMode}
+                containerRef={iframeRef}
+              />
+            </>
           ) : (
             <div className="flex w-full h-full justify-center items-center bg-white">No preview available</div>
           )}

app/components/workbench/ScreenshotSelector.tsx

@@ -0,0 +1,293 @@
+import { memo, useCallback, useEffect, useRef, useState } from 'react';
+import { toast } from 'react-toastify';
+
+interface ScreenshotSelectorProps {
+  isSelectionMode: boolean;
+  setIsSelectionMode: (mode: boolean) => void;
+  containerRef: React.RefObject<HTMLElement>;
+}
+
+export const ScreenshotSelector = memo(
+  ({ isSelectionMode, setIsSelectionMode, containerRef }: ScreenshotSelectorProps) => {
+    const [isCapturing, setIsCapturing] = useState(false);
+    const [selectionStart, setSelectionStart] = useState<{ x: number; y: number } | null>(null);
+    const [selectionEnd, setSelectionEnd] = useState<{ x: number; y: number } | null>(null);
+    const mediaStreamRef = useRef<MediaStream | null>(null);
+    const videoRef = useRef<HTMLVideoElement | null>(null);
+
+    useEffect(() => {
+      // Cleanup function to stop all tracks when component unmounts
+      return () => {
+        if (videoRef.current) {
+          videoRef.current.pause();
+          videoRef.current.srcObject = null;
+          videoRef.current.remove();
+          videoRef.current = null;
+        }
+
+        if (mediaStreamRef.current) {
+          mediaStreamRef.current.getTracks().forEach((track) => track.stop());
+          mediaStreamRef.current = null;
+        }
+      };
+    }, []);
+
+    const initializeStream = async () => {
+      if (!mediaStreamRef.current) {
+        try {
+          const stream = await navigator.mediaDevices.getDisplayMedia({
+            audio: false,
+            video: {
+              displaySurface: 'window',
+              preferCurrentTab: true,
+              surfaceSwitching: 'include',
+              systemAudio: 'exclude',
+            },
+          } as MediaStreamConstraints);
+
+          // Add handler for when sharing stops
+          stream.addEventListener('inactive', () => {
+            if (videoRef.current) {
+              videoRef.current.pause();
+              videoRef.current.srcObject = null;
+              videoRef.current.remove();
+              videoRef.current = null;
+            }
+
+            if (mediaStreamRef.current) {
+              mediaStreamRef.current.getTracks().forEach((track) => track.stop());
+              mediaStreamRef.current = null;
+            }
+
+            setIsSelectionMode(false);
+            setSelectionStart(null);
+            setSelectionEnd(null);
+            setIsCapturing(false);
+          });
+
+          mediaStreamRef.current = stream;
+
+          // Initialize video element if needed
+          if (!videoRef.current) {
+            const video = document.createElement('video');
+            video.style.opacity = '0';
+            video.style.position = 'fixed';
+            video.style.pointerEvents = 'none';
+            video.style.zIndex = '-1';
+            document.body.appendChild(video);
+            videoRef.current = video;
+          }
+
+          // Set up video with the stream
+          videoRef.current.srcObject = stream;
+          await videoRef.current.play();
+        } catch (error) {
+          console.error('Failed to initialize stream:', error);
+          setIsSelectionMode(false);
+          toast.error('Failed to initialize screen capture');
+        }
+      }
+
+      return mediaStreamRef.current;
+    };
+
+    const handleCopySelection = useCallback(async () => {
+      if (!isSelectionMode || !selectionStart || !selectionEnd || !containerRef.current) {
+        return;
+      }
+
+      setIsCapturing(true);
+
+      try {
+        const stream = await initializeStream();
+
+        if (!stream || !videoRef.current) {
+          return;
+        }
+
+        // Wait for video to be ready
+        await new Promise((resolve) => setTimeout(resolve, 300));
+
+        // Create temporary canvas for full screenshot
+        const tempCanvas = document.createElement('canvas');
+        tempCanvas.width = videoRef.current.videoWidth;
+        tempCanvas.height = videoRef.current.videoHeight;
+
+        const tempCtx = tempCanvas.getContext('2d');
+
+        if (!tempCtx) {
+          throw new Error('Failed to get temporary canvas context');
+        }
+
+        // Draw the full video frame
+        tempCtx.drawImage(videoRef.current, 0, 0);
+
+        // Calculate scale factor between video and screen
+        const scaleX = videoRef.current.videoWidth / window.innerWidth;
+        const scaleY = videoRef.current.videoHeight / window.innerHeight;
+
+        // Get window scroll position
+        const scrollX = window.scrollX;
+        const scrollY = window.scrollY + 40;
+
+        // Get the container's position in the page
+        const containerRect = containerRef.current.getBoundingClientRect();
+
+        // Offset adjustments for more accurate clipping
+        const leftOffset = -9; // Adjust left position
+        const bottomOffset = -14; // Adjust bottom position
+
+        // Calculate the scaled coordinates with scroll offset and adjustments
+        const scaledX = Math.round(
+          (containerRect.left + Math.min(selectionStart.x, selectionEnd.x) + scrollX + leftOffset) * scaleX,
+        );
+        const scaledY = Math.round(
+          (containerRect.top + Math.min(selectionStart.y, selectionEnd.y) + scrollY + bottomOffset) * scaleY,
+        );
+        const scaledWidth = Math.round(Math.abs(selectionEnd.x - selectionStart.x) * scaleX);
+        const scaledHeight = Math.round(Math.abs(selectionEnd.y - selectionStart.y) * scaleY);
+
+        // Create final canvas for the cropped area
+        const canvas = document.createElement('canvas');
+        canvas.width = Math.round(Math.abs(selectionEnd.x - selectionStart.x));
+        canvas.height = Math.round(Math.abs(selectionEnd.y - selectionStart.y));
+
+        const ctx = canvas.getContext('2d');
+
+        if (!ctx) {
+          throw new Error('Failed to get canvas context');
+        }
+
+        // Draw the cropped area
+        ctx.drawImage(tempCanvas, scaledX, scaledY, scaledWidth, scaledHeight, 0, 0, canvas.width, canvas.height);
+
+        // Convert to blob
+        const blob = await new Promise<Blob>((resolve, reject) => {
+          canvas.toBlob((blob) => {
+            if (blob) {
+              resolve(blob);
+            } else {
+              reject(new Error('Failed to create blob'));
+            }
+          }, 'image/png');
+        });
+
+        // Create a FileReader to convert blob to base64
+        const reader = new FileReader();
+
+        reader.onload = (e) => {
+          const base64Image = e.target?.result as string;
+
+          // Find the textarea element
+          const textarea = document.querySelector('textarea');
+
+          if (textarea) {
+            // Get the setters from the BaseChat component
+            const setUploadedFiles = (window as any).__BOLT_SET_UPLOADED_FILES__;
+            const setImageDataList = (window as any).__BOLT_SET_IMAGE_DATA_LIST__;
+            const uploadedFiles = (window as any).__BOLT_UPLOADED_FILES__ || [];
+            const imageDataList = (window as any).__BOLT_IMAGE_DATA_LIST__ || [];
+
+            if (setUploadedFiles && setImageDataList) {
+              // Update the files and image data
+              const file = new File([blob], 'screenshot.png', { type: 'image/png' });
+              setUploadedFiles([...uploadedFiles, file]);
+              setImageDataList([...imageDataList, base64Image]);
+              toast.success('Screenshot captured and added to chat');
+            } else {
+              toast.error('Could not add screenshot to chat');
+            }
+          }
+        };
+        reader.readAsDataURL(blob);
+      } catch (error) {
+        console.error('Failed to capture screenshot:', error);
+        toast.error('Failed to capture screenshot');
+
+        if (mediaStreamRef.current) {
+          mediaStreamRef.current.getTracks().forEach((track) => track.stop());
+          mediaStreamRef.current = null;
+        }
+      } finally {
+        setIsCapturing(false);
+        setSelectionStart(null);
+        setSelectionEnd(null);
+        setIsSelectionMode(false); // Turn off selection mode after capture
+      }
+    }, [isSelectionMode, selectionStart, selectionEnd, containerRef, setIsSelectionMode]);
+
+    const handleSelectionStart = useCallback(
+      (e: React.MouseEvent) => {
+        e.preventDefault();
+        e.stopPropagation();
+
+        if (!isSelectionMode) {
+          return;
+        }
+
+        const rect = e.currentTarget.getBoundingClientRect();
+        const x = e.clientX - rect.left;
+        const y = e.clientY - rect.top;
+        setSelectionStart({ x, y });
+        setSelectionEnd({ x, y });
+      },
+      [isSelectionMode],
+    );
+
+    const handleSelectionMove = useCallback(
+      (e: React.MouseEvent) => {
+        e.preventDefault();
+        e.stopPropagation();
+
+        if (!isSelectionMode || !selectionStart) {
+          return;
+        }
+
+        const rect = e.currentTarget.getBoundingClientRect();
+        const x = e.clientX - rect.left;
+        const y = e.clientY - rect.top;
+        setSelectionEnd({ x, y });
+      },
+      [isSelectionMode, selectionStart],
+    );
+
+    if (!isSelectionMode) {
+      return null;
+    }
+
+    return (
+      <div
+        className="absolute inset-0 cursor-crosshair"
+        onMouseDown={handleSelectionStart}
+        onMouseMove={handleSelectionMove}
+        onMouseUp={handleCopySelection}
+        onMouseLeave={() => {
+          if (selectionStart) {
+            setSelectionStart(null);
+          }
+        }}
+        style={{
+          backgroundColor: isCapturing ? 'transparent' : 'rgba(0, 0, 0, 0.1)',
+          userSelect: 'none',
+          WebkitUserSelect: 'none',
+          pointerEvents: 'all',
+          opacity: isCapturing ? 0 : 1,
+          zIndex: 50,
+          transition: 'opacity 0.1s ease-in-out',
+        }}
+      >
+        {selectionStart && selectionEnd && !isCapturing && (
+          <div
+            className="absolute border-2 border-blue-500 bg-blue-200 bg-opacity-20"
+            style={{
+              left: Math.min(selectionStart.x, selectionEnd.x),
+              top: Math.min(selectionStart.y, selectionEnd.y),
+              width: Math.abs(selectionEnd.x - selectionStart.x),
+              height: Math.abs(selectionEnd.y - selectionStart.y),
+            }}
+          />
+        )}
+      </div>
+    );
+  },
+);

app/lib/stores/workbench.ts

@@ -297,7 +297,6 @@ export class WorkbenchStore {
 
     const action = artifact.runner.actions.get()[data.actionId];
 
-
     if (!action || action.executed) {
       return;
     }