Merge branch 'main' into perplexity-models

README.md

@@ -44,6 +44,7 @@ https://thinktank.ottomator.ai
 - ✅ Better prompt enhancing (@SujalXplores)
 - ✅ Attach images to prompts (@atrokhym)
 - ✅ Detect package.json and commands to auto install and run preview for folder and git import (@wonderwhy-er)
+- ✅ Selection tool to target changes visually (@emcconnell)
 - ⬜ **HIGH PRIORITY** - Prevent Bolt from rewriting files as often (file locking and diffs)
 - ⬜ **HIGH PRIORITY** - Better prompting for smaller LLMs (code window sometimes doesn't start)
 - ⬜ **HIGH PRIORITY** - Run agents in the backend as opposed to a single model call

app/commit.json

@@ -1 +1 @@
-{ "commit": "9666b2ab67d25345542722ab9d870b36ad06252e" }
+{ "commit": "ece0213500a94a6b29e29512c5040baf57884014" }

app/components/chat/BaseChat.tsx

@@ -26,6 +26,7 @@ import FilePreview from './FilePreview';
 import { ModelSelector } from '~/components/chat/ModelSelector';
 import { SpeechRecognitionButton } from '~/components/chat/SpeechRecognition';
 import type { IProviderSetting, ProviderInfo } from '~/types/model';
+import { ScreenshotStateManager } from './ScreenshotStateManager';
 
 const TEXTAREA_MIN_HEIGHT = 76;
 
@@ -376,6 +377,16 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                     setImageDataList?.(imageDataList.filter((_, i) => i !== index));
                   }}
                 />
+                <ClientOnly>
+                  {() => (
+                    <ScreenshotStateManager
+                      setUploadedFiles={setUploadedFiles}
+                      setImageDataList={setImageDataList}
+                      uploadedFiles={uploadedFiles}
+                      imageDataList={imageDataList}
+                    />
+                  )}
+                </ClientOnly>
                 <div
                   className={classNames(
                     'relative shadow-xs border border-bolt-elements-borderColor backdrop-blur rounded-lg',
@@ -431,6 +442,11 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
                           return;
                         }
 
+                        // ignore if using input method engine
+                        if (event.nativeEvent.isComposing) {
+                          return;
+                        }
+
                         handleSendMessage?.(event);
                       }
                     }}

app/components/chat/ScreenshotStateManager.tsx

@@ -0,0 +1,33 @@
+import { useEffect } from 'react';
+
+interface ScreenshotStateManagerProps {
+  setUploadedFiles?: (files: File[]) => void;
+  setImageDataList?: (dataList: string[]) => void;
+  uploadedFiles: File[];
+  imageDataList: string[];
+}
+
+export const ScreenshotStateManager = ({
+  setUploadedFiles,
+  setImageDataList,
+  uploadedFiles,
+  imageDataList,
+}: ScreenshotStateManagerProps) => {
+  useEffect(() => {
+    if (setUploadedFiles && setImageDataList) {
+      (window as any).__BOLT_SET_UPLOADED_FILES__ = setUploadedFiles;
+      (window as any).__BOLT_SET_IMAGE_DATA_LIST__ = setImageDataList;
+      (window as any).__BOLT_UPLOADED_FILES__ = uploadedFiles;
+      (window as any).__BOLT_IMAGE_DATA_LIST__ = imageDataList;
+    }
+
+    return () => {
+      delete (window as any).__BOLT_SET_UPLOADED_FILES__;
+      delete (window as any).__BOLT_SET_IMAGE_DATA_LIST__;
+      delete (window as any).__BOLT_UPLOADED_FILES__;
+      delete (window as any).__BOLT_IMAGE_DATA_LIST__;
+    };
+  }, [setUploadedFiles, setImageDataList, uploadedFiles, imageDataList]);
+
+  return null;
+};

app/components/settings/connections/ConnectionsTab.tsx

@@ -15,6 +15,7 @@ export default function ConnectionsTab() {
       hasToken: !!githubToken,
     });
     toast.success('GitHub credentials saved successfully!');
+    Cookies.set('git:github.com', JSON.stringify({ username: githubToken, password: 'x-oauth-basic' }));
   };
 
   return (

app/components/workbench/Preview.tsx

@@ -3,6 +3,7 @@ import { memo, useCallback, useEffect, useRef, useState } from 'react';
 import { IconButton } from '~/components/ui/IconButton';
 import { workbenchStore } from '~/lib/stores/workbench';
 import { PortDropdown } from './PortDropdown';
+import { ScreenshotSelector } from './ScreenshotSelector';
 
 type ResizeSide = 'left' | 'right' | null;
 
@@ -20,6 +21,7 @@ export const Preview = memo(() => {
 
   const [url, setUrl] = useState('');
   const [iframeUrl, setIframeUrl] = useState<string | undefined>();
+  const [isSelectionMode, setIsSelectionMode] = useState(false);
 
   // Toggle between responsive mode and device mode
   const [isDeviceModeOn, setIsDeviceModeOn] = useState(false);
@@ -218,12 +220,17 @@ export const Preview = memo(() => {
       )}
       <div className="bg-bolt-elements-background-depth-2 p-2 flex items-center gap-1.5">
         <IconButton icon="i-ph:arrow-clockwise" onClick={reloadPreview} />
-
+        <IconButton
+          icon="i-ph:selection"
+          onClick={() => setIsSelectionMode(!isSelectionMode)}
+          className={isSelectionMode ? 'bg-bolt-elements-background-depth-3' : ''}
+        />
         <div
           className="flex items-center gap-1 flex-grow bg-bolt-elements-preview-addressBar-background border border-bolt-elements-borderColor text-bolt-elements-preview-addressBar-text rounded-full px-3 py-1 text-sm hover:bg-bolt-elements-preview-addressBar-backgroundHover hover:focus-within:bg-bolt-elements-preview-addressBar-backgroundActive focus-within:bg-bolt-elements-preview-addressBar-backgroundActive
         focus-within-border-bolt-elements-borderColorActive focus-within:text-bolt-elements-preview-addressBar-textActive"
         >
           <input
+            title="URL"
             ref={inputRef}
             className="w-full bg-transparent outline-none"
             type="text"
@@ -281,7 +288,20 @@ export const Preview = memo(() => {
           }}
         >
           {activePreview ? (
-            <iframe ref={iframeRef} className="border-none w-full h-full bg-white" src={iframeUrl} allowFullScreen />
+            <>
+              <iframe
+                ref={iframeRef}
+                title="preview"
+                className="border-none w-full h-full bg-white"
+                src={iframeUrl}
+                allowFullScreen
+              />
+              <ScreenshotSelector
+                isSelectionMode={isSelectionMode}
+                setIsSelectionMode={setIsSelectionMode}
+                containerRef={iframeRef}
+              />
+            </>
           ) : (
             <div className="flex w-full h-full justify-center items-center bg-white">No preview available</div>
           )}

app/components/workbench/ScreenshotSelector.tsx

@@ -0,0 +1,293 @@
+import { memo, useCallback, useEffect, useRef, useState } from 'react';
+import { toast } from 'react-toastify';
+
+interface ScreenshotSelectorProps {
+  isSelectionMode: boolean;
+  setIsSelectionMode: (mode: boolean) => void;
+  containerRef: React.RefObject<HTMLElement>;
+}
+
+export const ScreenshotSelector = memo(
+  ({ isSelectionMode, setIsSelectionMode, containerRef }: ScreenshotSelectorProps) => {
+    const [isCapturing, setIsCapturing] = useState(false);
+    const [selectionStart, setSelectionStart] = useState<{ x: number; y: number } | null>(null);
+    const [selectionEnd, setSelectionEnd] = useState<{ x: number; y: number } | null>(null);
+    const mediaStreamRef = useRef<MediaStream | null>(null);
+    const videoRef = useRef<HTMLVideoElement | null>(null);
+
+    useEffect(() => {
+      // Cleanup function to stop all tracks when component unmounts
+      return () => {
+        if (videoRef.current) {
+          videoRef.current.pause();
+          videoRef.current.srcObject = null;
+          videoRef.current.remove();
+          videoRef.current = null;
+        }
+
+        if (mediaStreamRef.current) {
+          mediaStreamRef.current.getTracks().forEach((track) => track.stop());
+          mediaStreamRef.current = null;
+        }
+      };
+    }, []);
+
+    const initializeStream = async () => {
+      if (!mediaStreamRef.current) {
+        try {
+          const stream = await navigator.mediaDevices.getDisplayMedia({
+            audio: false,
+            video: {
+              displaySurface: 'window',
+              preferCurrentTab: true,
+              surfaceSwitching: 'include',
+              systemAudio: 'exclude',
+            },
+          } as MediaStreamConstraints);
+
+          // Add handler for when sharing stops
+          stream.addEventListener('inactive', () => {
+            if (videoRef.current) {
+              videoRef.current.pause();
+              videoRef.current.srcObject = null;
+              videoRef.current.remove();
+              videoRef.current = null;
+            }
+
+            if (mediaStreamRef.current) {
+              mediaStreamRef.current.getTracks().forEach((track) => track.stop());
+              mediaStreamRef.current = null;
+            }
+
+            setIsSelectionMode(false);
+            setSelectionStart(null);
+            setSelectionEnd(null);
+            setIsCapturing(false);
+          });
+
+          mediaStreamRef.current = stream;
+
+          // Initialize video element if needed
+          if (!videoRef.current) {
+            const video = document.createElement('video');
+            video.style.opacity = '0';
+            video.style.position = 'fixed';
+            video.style.pointerEvents = 'none';
+            video.style.zIndex = '-1';
+            document.body.appendChild(video);
+            videoRef.current = video;
+          }
+
+          // Set up video with the stream
+          videoRef.current.srcObject = stream;
+          await videoRef.current.play();
+        } catch (error) {
+          console.error('Failed to initialize stream:', error);
+          setIsSelectionMode(false);
+          toast.error('Failed to initialize screen capture');
+        }
+      }
+
+      return mediaStreamRef.current;
+    };
+
+    const handleCopySelection = useCallback(async () => {
+      if (!isSelectionMode || !selectionStart || !selectionEnd || !containerRef.current) {
+        return;
+      }
+
+      setIsCapturing(true);
+
+      try {
+        const stream = await initializeStream();
+
+        if (!stream || !videoRef.current) {
+          return;
+        }
+
+        // Wait for video to be ready
+        await new Promise((resolve) => setTimeout(resolve, 300));
+
+        // Create temporary canvas for full screenshot
+        const tempCanvas = document.createElement('canvas');
+        tempCanvas.width = videoRef.current.videoWidth;
+        tempCanvas.height = videoRef.current.videoHeight;
+
+        const tempCtx = tempCanvas.getContext('2d');
+
+        if (!tempCtx) {
+          throw new Error('Failed to get temporary canvas context');
+        }
+
+        // Draw the full video frame
+        tempCtx.drawImage(videoRef.current, 0, 0);
+
+        // Calculate scale factor between video and screen
+        const scaleX = videoRef.current.videoWidth / window.innerWidth;
+        const scaleY = videoRef.current.videoHeight / window.innerHeight;
+
+        // Get window scroll position
+        const scrollX = window.scrollX;
+        const scrollY = window.scrollY + 40;
+
+        // Get the container's position in the page
+        const containerRect = containerRef.current.getBoundingClientRect();
+
+        // Offset adjustments for more accurate clipping
+        const leftOffset = -9; // Adjust left position
+        const bottomOffset = -14; // Adjust bottom position
+
+        // Calculate the scaled coordinates with scroll offset and adjustments
+        const scaledX = Math.round(
+          (containerRect.left + Math.min(selectionStart.x, selectionEnd.x) + scrollX + leftOffset) * scaleX,
+        );
+        const scaledY = Math.round(
+          (containerRect.top + Math.min(selectionStart.y, selectionEnd.y) + scrollY + bottomOffset) * scaleY,
+        );
+        const scaledWidth = Math.round(Math.abs(selectionEnd.x - selectionStart.x) * scaleX);
+        const scaledHeight = Math.round(Math.abs(selectionEnd.y - selectionStart.y) * scaleY);
+
+        // Create final canvas for the cropped area
+        const canvas = document.createElement('canvas');
+        canvas.width = Math.round(Math.abs(selectionEnd.x - selectionStart.x));
+        canvas.height = Math.round(Math.abs(selectionEnd.y - selectionStart.y));
+
+        const ctx = canvas.getContext('2d');
+
+        if (!ctx) {
+          throw new Error('Failed to get canvas context');
+        }
+
+        // Draw the cropped area
+        ctx.drawImage(tempCanvas, scaledX, scaledY, scaledWidth, scaledHeight, 0, 0, canvas.width, canvas.height);
+
+        // Convert to blob
+        const blob = await new Promise<Blob>((resolve, reject) => {
+          canvas.toBlob((blob) => {
+            if (blob) {
+              resolve(blob);
+            } else {
+              reject(new Error('Failed to create blob'));
+            }
+          }, 'image/png');
+        });
+
+        // Create a FileReader to convert blob to base64
+        const reader = new FileReader();
+
+        reader.onload = (e) => {
+          const base64Image = e.target?.result as string;
+
+          // Find the textarea element
+          const textarea = document.querySelector('textarea');
+
+          if (textarea) {
+            // Get the setters from the BaseChat component
+            const setUploadedFiles = (window as any).__BOLT_SET_UPLOADED_FILES__;
+            const setImageDataList = (window as any).__BOLT_SET_IMAGE_DATA_LIST__;
+            const uploadedFiles = (window as any).__BOLT_UPLOADED_FILES__ || [];
+            const imageDataList = (window as any).__BOLT_IMAGE_DATA_LIST__ || [];
+
+            if (setUploadedFiles && setImageDataList) {
+              // Update the files and image data
+              const file = new File([blob], 'screenshot.png', { type: 'image/png' });
+              setUploadedFiles([...uploadedFiles, file]);
+              setImageDataList([...imageDataList, base64Image]);
+              toast.success('Screenshot captured and added to chat');
+            } else {
+              toast.error('Could not add screenshot to chat');
+            }
+          }
+        };
+        reader.readAsDataURL(blob);
+      } catch (error) {
+        console.error('Failed to capture screenshot:', error);
+        toast.error('Failed to capture screenshot');
+
+        if (mediaStreamRef.current) {
+          mediaStreamRef.current.getTracks().forEach((track) => track.stop());
+          mediaStreamRef.current = null;
+        }
+      } finally {
+        setIsCapturing(false);
+        setSelectionStart(null);
+        setSelectionEnd(null);
+        setIsSelectionMode(false); // Turn off selection mode after capture
+      }
+    }, [isSelectionMode, selectionStart, selectionEnd, containerRef, setIsSelectionMode]);
+
+    const handleSelectionStart = useCallback(
+      (e: React.MouseEvent) => {
+        e.preventDefault();
+        e.stopPropagation();
+
+        if (!isSelectionMode) {
+          return;
+        }
+
+        const rect = e.currentTarget.getBoundingClientRect();
+        const x = e.clientX - rect.left;
+        const y = e.clientY - rect.top;
+        setSelectionStart({ x, y });
+        setSelectionEnd({ x, y });
+      },
+      [isSelectionMode],
+    );
+
+    const handleSelectionMove = useCallback(
+      (e: React.MouseEvent) => {
+        e.preventDefault();
+        e.stopPropagation();
+
+        if (!isSelectionMode || !selectionStart) {
+          return;
+        }
+
+        const rect = e.currentTarget.getBoundingClientRect();
+        const x = e.clientX - rect.left;
+        const y = e.clientY - rect.top;
+        setSelectionEnd({ x, y });
+      },
+      [isSelectionMode, selectionStart],
+    );
+
+    if (!isSelectionMode) {
+      return null;
+    }
+
+    return (
+      <div
+        className="absolute inset-0 cursor-crosshair"
+        onMouseDown={handleSelectionStart}
+        onMouseMove={handleSelectionMove}
+        onMouseUp={handleCopySelection}
+        onMouseLeave={() => {
+          if (selectionStart) {
+            setSelectionStart(null);
+          }
+        }}
+        style={{
+          backgroundColor: isCapturing ? 'transparent' : 'rgba(0, 0, 0, 0.1)',
+          userSelect: 'none',
+          WebkitUserSelect: 'none',
+          pointerEvents: 'all',
+          opacity: isCapturing ? 0 : 1,
+          zIndex: 50,
+          transition: 'opacity 0.1s ease-in-out',
+        }}
+      >
+        {selectionStart && selectionEnd && !isCapturing && (
+          <div
+            className="absolute border-2 border-blue-500 bg-blue-200 bg-opacity-20"
+            style={{
+              left: Math.min(selectionStart.x, selectionEnd.x),
+              top: Math.min(selectionStart.y, selectionEnd.y),
+              width: Math.abs(selectionEnd.x - selectionStart.x),
+              height: Math.abs(selectionEnd.y - selectionStart.y),
+            }}
+          />
+        )}
+      </div>
+    );
+  },
+);

app/lib/stores/workbench.ts

@@ -16,6 +16,7 @@ import * as nodePath from 'node:path';
 import { extractRelativePath } from '~/utils/diff';
 import { description } from '~/lib/persistence';
 import Cookies from 'js-cookie';
+import { createSampler } from '~/utils/sampler';
 
 export interface ArtifactState {
   id: string;
@@ -280,7 +281,7 @@ export class WorkbenchStore {
 
   runAction(data: ActionCallbackData, isStreaming: boolean = false) {
     if (isStreaming) {
-      this._runAction(data, isStreaming);
+      this.actionStreamSampler(data, isStreaming);
     } else {
       this.addToExecutionQueue(() => this._runAction(data, isStreaming));
     }
@@ -296,7 +297,7 @@ export class WorkbenchStore {
 
     const action = artifact.runner.actions.get()[data.actionId];
 
-    if (action.executed) {
+    if (!action || action.executed) {
       return;
     }
 
@@ -329,6 +330,10 @@ export class WorkbenchStore {
     }
   }
 
+  actionStreamSampler = createSampler(async (data: ActionCallbackData, isStreaming: boolean = false) => {
+    return await this._runAction(data, isStreaming);
+  }, 100); // TODO: remove this magic number to have it configurable
+
   #getArtifact(id: string) {
     const artifacts = this.artifacts.get();
     return artifacts[id];

app/utils/constants.ts

@@ -139,11 +139,12 @@ const PROVIDER_LIST: ProviderInfo[] = [
   {
     name: 'Groq',
     staticModels: [
-      { name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
       { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
       { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+      { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },      
       { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
       { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+      { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
     ],
     getApiKeyLink: 'https://console.groq.com/keys',
   },

app/utils/sampler.ts

@@ -0,0 +1,49 @@
+/**
+ * Creates a function that samples calls at regular intervals and captures trailing calls.
+ * - Drops calls that occur between sampling intervals
+ * - Takes one call per sampling interval if available
+ * - Captures the last call if no call was made during the interval
+ *
+ * @param fn The function to sample
+ * @param sampleInterval How often to sample calls (in ms)
+ * @returns The sampled function
+ */
+export function createSampler<T extends (...args: any[]) => any>(fn: T, sampleInterval: number): T {
+  let lastArgs: Parameters<T> | null = null;
+  let lastTime = 0;
+  let timeout: NodeJS.Timeout | null = null;
+
+  // Create a function with the same type as the input function
+  const sampled = function (this: any, ...args: Parameters<T>) {
+    const now = Date.now();
+    lastArgs = args;
+
+    // If we're within the sample interval, just store the args
+    if (now - lastTime < sampleInterval) {
+      // Set up trailing call if not already set
+      if (!timeout) {
+        timeout = setTimeout(
+          () => {
+            timeout = null;
+            lastTime = Date.now();
+
+            if (lastArgs) {
+              fn.apply(this, lastArgs);
+              lastArgs = null;
+            }
+          },
+          sampleInterval - (now - lastTime),
+        );
+      }
+
+      return;
+    }
+
+    // If we're outside the interval, execute immediately
+    lastTime = now;
+    fn.apply(this, args);
+    lastArgs = null;
+  } as T;
+
+  return sampled;
+}