Support saving reproductions

.github/workflows/playwright.yaml

@@ -57,6 +57,7 @@ jobs:
           NUT_PASSWORD: ${{ secrets.NUT_PASSWORD }}
 
       - name: Upload test results
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: playwright-report
@@ -64,6 +65,7 @@ jobs:
           retention-days: 30
 
       - name: Deploy playwright report to Vercel
+        if: always()
         run: |
           cd playwright-report
           vercel link --project playwright-reports --yes --token=${{ secrets.VERCEL_TOKEN }}
@@ -111,6 +113,7 @@ jobs:
           NUT_PASSWORD: ${{ secrets.NUT_PASSWORD }}
 
       - name: Upload test results
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: playwright-report-supabase
@@ -118,6 +121,7 @@ jobs:
           retention-days: 30
 
       - name: Deploy Supabase playwright report to Vercel
+        if: always()
         run: |
           cd playwright-report
           vercel link --project playwright-reports --yes --token=${{ secrets.VERCEL_TOKEN }}

app/components/chat/LoadProblemButton.tsx

@@ -14,32 +14,11 @@ interface LoadProblemButtonProps {
 }
 
 export function setLastLoadedProblem(problem: BoltProblem) {
-  const problemSerialized = JSON.stringify(problem);
-
-  try {
   localStorage.setItem('loadedProblemId', problem.problemId);
-    localStorage.setItem('loadedProblem', problemSerialized);
-  } catch (error: any) {
-    // Remove loadedProblem, so we don't accidentally associate (e.g. reproduction) data with the wrong problem.
-    localStorage.removeItem('loadedProblem');
-    console.error(
-      `Failed to set last loaded problem (size=${(problemSerialized.length / 1024).toFixed(2)}kb):`,
-      error.stack || error,
-    );
-  }
 }
 
 export async function getOrFetchLastLoadedProblem(): Promise<BoltProblem | null> {
-  const problemJSON = localStorage.getItem('loadedProblem');
   let problem: BoltProblem | null = null;
-
-  if (problemJSON) {
-    problem = JSON.parse(problemJSON);
-  } else {
-    /*
-     * Problem might not have fit into localStorage.
-     * Try to re-load it from server.
-     */
   const problemId = localStorage.getItem('loadedProblemId');
 
   if (!problemId) {
@@ -52,9 +31,6 @@ export async function getOrFetchLastLoadedProblem(): Promise<BoltProblem | null>
     return null;
   }
 
-    setLastLoadedProblem(problem);
-  }
-
   return problem;
 }
 

app/components/settings/providers/APIKeysTab.tsx

@@ -11,6 +11,7 @@ export default function ConnectionsTab() {
   const [loginKey, setLoginKey] = useState(getNutLoginKey() || '');
   const numFreeUses = +(Cookies.get(anthropicNumFreeUsesCookieName) || 0);
 
+  const [loginKeyIsLoading, setLoginKeyIsLoading] = useState(false);
   const handleSaveAPIKey = async (key: string) => {
     if (key && !key.startsWith('sk-ant-')) {
       toast.error('Please provide a valid Anthropic API key');
@@ -37,10 +38,13 @@ export default function ConnectionsTab() {
     setLoginKey(key);
 
     try {
+      setLoginKeyIsLoading(true);
       await saveNutLoginKey(key);
       toast.success('Login key saved');
     } catch {
       toast.error('Failed to save login key');
+    } finally {
+      setLoginKeyIsLoading(false);
     }
   };
 
@@ -83,6 +87,8 @@ export default function ConnectionsTab() {
             type="text"
             placeholder="Enter your login key"
             value={loginKey}
+            data-testid="login-key-input"
+            data-isloading={loginKeyIsLoading}
             onChange={(e) => handleSaveLoginKey(e.target.value)}
             className="w-full bg-white dark:bg-bolt-elements-background-depth-4 relative px-2 py-1.5 rounded-md focus:outline-none placeholder-bolt-elements-textTertiary text-bolt-elements-textPrimary dark:text-bolt-elements-textPrimary border border-bolt-elements-borderColor disabled:opacity-50"
           />

app/components/sidebar/SaveProblem.tsx

@@ -2,11 +2,13 @@ import { toast } from 'react-toastify';
 import ReactModal from 'react-modal';
 import { useState, useEffect } from 'react';
 import { workbenchStore } from '~/lib/stores/workbench';
-import { getUsername, submitProblem, saveUsername, BoltProblemStatus } from '~/lib/replay/Problems';
+import { submitProblem, BoltProblemStatus } from '~/lib/replay/Problems';
 import type { BoltProblemInput } from '~/lib/replay/Problems';
 import { getRepositoryContents } from '~/lib/replay/Repository';
-import { shouldUseSupabase, getCurrentUser, isAuthenticated } from '~/lib/supabase/client';
+import { shouldUseSupabase, getCurrentUser } from '~/lib/supabase/client';
 import { authModalStore } from '~/lib/stores/authModal';
+import { authStatusStore } from '~/lib/stores/auth';
+import { useStore } from '@nanostores/react';
 
 ReactModal.setAppElement('#root');
 
@@ -20,27 +22,15 @@ export function SaveProblem() {
     username: '',
   });
   const [problemId, setProblemId] = useState<string | null>(null);
-  const [isLoggedIn, setIsLoggedIn] = useState<boolean | null>(null);
+  const isLoggedIn = useStore(authStatusStore.isLoggedIn);
+  const username = useStore(authStatusStore.username);
 
-  // Check authentication status and get username
+  // Update the username from the store when component mounts
   useEffect(() => {
-    async function checkAuthAndUsername() {
-      if (shouldUseSupabase()) {
-        const authenticated = await isAuthenticated();
-        setIsLoggedIn(authenticated);
-      } else {
-        setIsLoggedIn(true); // Always considered logged in when not using Supabase
-
-        const username = getUsername();
-
     if (username) {
       setFormData((prev) => ({ ...prev, username }));
     }
-      }
+  }, [username]);
-    }
-
-    checkAuthAndUsername();
-  }, []);
 
   const handleSaveProblem = (e: React.MouseEvent) => {
     e.preventDefault();
@@ -55,6 +45,11 @@ export function SaveProblem() {
       ...prev,
       [name]: value,
     }));
+
+    // Update username in the store if it's the username field
+    if (name === 'username') {
+      authStatusStore.updateUsername(value);
+    }
   };
 
   const handleSubmitProblem = async () => {
@@ -68,11 +63,6 @@ export function SaveProblem() {
       return;
     }
 
-    // Only save username to cookie if not using Supabase
-    if (!shouldUseSupabase()) {
-      saveUsername(formData.username);
-    }
-
     toast.info('Submitting problem...');
 
     const repositoryId = workbenchStore.repositoryId.get();
@@ -98,6 +88,7 @@ export function SaveProblem() {
 
     if (problemId) {
       setProblemId(problemId);
+      localStorage.setItem('loadedProblemId', problemId);
     }
   };
 

app/lib/replay/Problems.ts

@@ -16,7 +16,15 @@ import {
 import { getNutIsAdmin as getNutIsAdminFromSupabase } from '~/lib/supabase/client';
 import { updateIsAdmin, updateUsername } from '~/lib/stores/user';
 
+// Add global declaration for the problem property
+declare global {
+  interface Window {
+    __currentProblem__?: BoltProblem;
+  }
+}
+
 export interface BoltProblemComment {
+  id?: string;
   username?: string;
   content: string;
   timestamp: number;
@@ -99,10 +107,11 @@ export async function listAllProblems(): Promise<BoltProblemDescription[]> {
 }
 
 export async function getProblem(problemId: string): Promise<BoltProblem | null> {
-  if (shouldUseSupabase()) {
+  let problem: BoltProblem | null = null;
-    return supabaseGetProblem(problemId);
-  }
 
+  if (shouldUseSupabase()) {
+    problem = await supabaseGetProblem(problemId);
+  } else {
     try {
       if (!problemId) {
         toast.error('Invalid problem ID');
@@ -117,7 +126,7 @@ export async function getProblem(problemId: string): Promise<BoltProblem | null>
         },
       });
 
-    const problem = (rv as { rval: { problem: BoltProblem } }).rval.problem;
+      problem = (rv as { rval: { problem: BoltProblem } }).rval.problem;
 
       if (!problem) {
         toast.error('Problem not found');
@@ -129,8 +138,6 @@ export async function getProblem(problemId: string): Promise<BoltProblem | null>
         problem.repositoryContents = (problem as any).prompt.content;
         delete problem.prompt;
       }
-
-    return problem;
     } catch (error) {
       console.error('Error fetching problem', error);
 
@@ -141,8 +148,16 @@ export async function getProblem(problemId: string): Promise<BoltProblem | null>
         toast.error('Failed to fetch problem');
       }
     }
+  }
 
-  return null;
+  /*
+   * Only used for testing
+   */
+  if (problem) {
+    window.__currentProblem__ = problem;
+  }
+
+  return problem;
 }
 
 export async function submitProblem(problem: BoltProblemInput): Promise<string | null> {
@@ -177,11 +192,14 @@ export async function deleteProblem(problemId: string): Promise<void | undefined
   return undefined;
 }
 
+const nutLoginKeyCookieName = 'nutLoginKey';
+const nutIsAdminCookieName = 'nutIsAdmin';
+const nutUsernameCookieName = 'nutUsername';
+
 export async function updateProblem(problemId: string, problem: BoltProblemInput): Promise<BoltProblem | null> {
   if (shouldUseSupabase()) {
     await supabaseUpdateProblem(problemId, problem);
-  }
+  } else {
-
     try {
       if (!getNutIsAdmin()) {
         toast.error('Admin user required');
@@ -201,16 +219,13 @@ export async function updateProblem(problemId: string, problem: BoltProblemInput
       console.error('Error updating problem', error);
       toast.error('Failed to update problem');
     }
+  }
 
   const updatedProblem = await getProblem(problemId);
 
   return updatedProblem;
 }
 
-const nutLoginKeyCookieName = 'nutLoginKey';
-const nutIsAdminCookieName = 'nutIsAdmin';
-const nutUsernameCookieName = 'nutUsername';
-
 export function getNutLoginKey(): string | undefined {
   const cookieValue = Cookies.get(nutLoginKeyCookieName);
   return cookieValue?.length ? cookieValue : undefined;

app/lib/stores/auth.ts

@@ -2,11 +2,58 @@ import { atom } from 'nanostores';
 import { getSupabase } from '~/lib/supabase/client';
 import type { User, Session } from '@supabase/supabase-js';
 import { logStore } from './logs';
+import { useEffect, useState } from 'react';
+import { shouldUseSupabase, isAuthenticated } from '~/lib/supabase/client';
+import { getUsername, saveUsername } from '~/lib/replay/Problems';
 
 export const userStore = atom<User | null>(null);
 export const sessionStore = atom<Session | null>(null);
 export const isLoadingStore = atom<boolean>(true);
 
+// Auth status store for both Supabase and non-Supabase modes
+export const authStatusStore = {
+  isLoggedIn: atom<boolean | null>(null),
+  username: atom<string>(''),
+
+  // Initialize auth status store
+  async init() {
+    if (shouldUseSupabase()) {
+      // For Supabase, subscribe to the userStore
+      userStore.listen((user) => {
+        this.isLoggedIn.set(!!user);
+      });
+
+      // Check initial auth state
+      const authenticated = await isAuthenticated();
+      this.isLoggedIn.set(authenticated);
+    } else {
+      // For non-Supabase, always logged in
+      this.isLoggedIn.set(true);
+
+      // Get username from storage
+      const storedUsername = getUsername();
+
+      if (storedUsername) {
+        this.username.set(storedUsername);
+      }
+    }
+  },
+
+  // Update username (only meaningful in non-Supabase mode)
+  updateUsername(newUsername: string) {
+    this.username.set(newUsername);
+
+    if (!shouldUseSupabase()) {
+      saveUsername(newUsername);
+    }
+  },
+};
+
+// Initialize auth status store
+if (typeof window !== 'undefined') {
+  authStatusStore.init();
+}
+
 export async function initializeAuth() {
   try {
     isLoadingStore.set(true);
@@ -143,3 +190,25 @@ export async function signOut() {
     isLoadingStore.set(false);
   }
 }
+
+// Keep the hook for backwards compatibility, but implement it using the store
+export function useAuthStatus() {
+  const [isLoggedIn, setIsLoggedIn] = useState<boolean | null>(authStatusStore.isLoggedIn.get());
+  const [username, setUsername] = useState<string>(authStatusStore.username.get());
+
+  useEffect(() => {
+    const unsubscribeIsLoggedIn = authStatusStore.isLoggedIn.listen(setIsLoggedIn);
+    const unsubscribeUsername = authStatusStore.username.listen(setUsername);
+
+    return () => {
+      unsubscribeIsLoggedIn();
+      unsubscribeUsername();
+    };
+  }, []);
+
+  const updateUsername = (newUsername: string) => {
+    authStatusStore.updateUsername(newUsername);
+  };
+
+  return { isLoggedIn, username, updateUsername };
+}

app/lib/supabase/client.ts

@@ -18,8 +18,15 @@ export interface Database {
           repository_contents: Json;
           user_id: string | null;
           problem_comments: Database['public']['Tables']['problem_comments']['Row'][];
+          solution: Json;
+          repository_contents_path: string | null;
+          prompt_path: string | null;
+          solution_path: string | null;
         };
-        Insert: Omit<Database['public']['Tables']['problems']['Row'], 'created_at' | 'updated_at' | 'problem_comments'>;
+        Insert: Omit<
+          Database['public']['Tables']['problems']['Row'],
+          'created_at' | 'updated_at' | 'problem_comments' | 'solution'
+        >;
         Update: Partial<Database['public']['Tables']['problems']['Insert']>;
       };
       problem_comments: {

app/lib/supabase/problems.ts

@@ -5,6 +5,18 @@ import { getSupabase, type Database } from './client';
 import type { BoltProblem, BoltProblemDescription, BoltProblemInput, BoltProblemStatus } from '~/lib/replay/Problems';
 import { getUsername, getNutIsAdmin } from '~/lib/replay/Problems';
 
+async function downloadBlob(bucket: string, path: string) {
+  const supabase = getSupabase();
+  const { data, error } = await supabase.storage.from(bucket).download(path);
+
+  if (error) {
+    console.error('Error downloading blob:', error);
+    return null;
+  }
+
+  return data.text();
+}
+
 export async function supabaseListAllProblems(): Promise<BoltProblemDescription[]> {
   try {
     const { data, error } = await getSupabase()
@@ -72,61 +84,20 @@ export async function supabaseGetProblem(problemId: string): Promise<BoltProblem
     }
 
     // Fetch blob data from storage if paths are available
-    let repositoryContents = '';
+    let repositoryContents = data.repository_contents;
-    let prompt = undefined;
+    let solution = data.solution;
-    let solution = undefined;
+    const prompt = data.prompt;
 
     // Create a supabase instance for storage operations
     const supabase = getSupabase();
 
     // Fetch repository contents from storage if path is available
     if (data.repository_contents_path) {
-      try {
+      repositoryContents = (await downloadBlob('repository-contents', data.repository_contents_path)) || '';
-        const { data: blobData, error: blobError } = await supabase.storage
-          .from('repository-contents')
-          .download(data.repository_contents_path);
-
-        if (!blobError && blobData) {
-          repositoryContents = await blobData.text();
-        } else {
-          console.error('Error fetching repository contents:', blobError);
-        }
-      } catch (storageError) {
-        console.error('Error downloading repository contents:', storageError);
-      }
     }
 
-    // Fetch repository contents from storage if path is available
-    if (data.prompt_path) {
-      try {
-        const { data: blobData, error: blobError } = await supabase.storage.from('prompts').download(data.prompt_path);
-
-        if (!blobError && blobData) {
-          prompt = await blobData.text();
-        } else {
-          console.error('Error fetching repository contents:', blobError);
-        }
-      } catch (storageError) {
-        console.error('Error downloading repository contents:', storageError);
-      }
-    }
-
-    // Fetch solution from storage if path is available
     if (data.solution_path) {
-      try {
+      solution = JSON.parse((await downloadBlob('solutions', data.solution_path)) || '{}');
-        const { data: blobData, error: blobError } = await supabase.storage
-          .from('solutions')
-          .download(data.solution_path);
-
-        if (!blobError && blobData) {
-          const solutionText = await blobData.text();
-          solution = JSON.parse(solutionText);
-        } else {
-          console.error('Error fetching solution:', blobError);
-        }
-      } catch (storageError) {
-        console.error('Error downloading solution:', storageError);
-      }
     }
 
     // If the problem has a user_id, fetch the profile information
@@ -225,17 +196,14 @@ export async function supabaseUpdateProblem(problemId: string, problem: BoltProb
       return undefined;
     }
 
-    // Extract comments to add separately if needed
-    const comments = problem.comments || [];
-    delete (problem as any).comments;
-
     // Convert to Supabase format
     const updates: Database['public']['Tables']['problems']['Update'] = {
       title: problem.title,
       description: problem.description,
       status: problem.status,
       keywords: problem.keywords || [],
-      repository_contents: problem.repositoryContents,
+      repository_contents_path: problem.repositoryContents ? `problems/${problemId}.txt` : undefined,
+      solution_path: problem.solution ? `solutions/${problemId}.json` : undefined,
     };
 
     // Update the problem
@@ -245,27 +213,36 @@ export async function supabaseUpdateProblem(problemId: string, problem: BoltProb
       throw updateError;
     }
 
-    // Handle comments if they exist
+    if (updates.repository_contents_path) {
-    if (comments.length > 0) {
+      const { error: repositoryContentsError } = await getSupabase()
-      /**
+        .storage.from('repository-contents')
-       * Create a unique identifier for each comment based on content and timestamp.
+        .upload(updates.repository_contents_path, problem.repositoryContents);
-       * This allows us to use upsert with onConflict to avoid duplicates.
-       */
-      const commentInserts = comments.map((comment) => {
-        // Ensure timestamp is a valid number
-        const timestamp =
-          typeof comment.timestamp === 'number' && !isNaN(comment.timestamp) ? comment.timestamp : Date.now();
 
+      // @ts-ignore - ignore duplicate error
+      if (repositoryContentsError && repositoryContentsError.error !== 'Duplicate') {
+        throw repositoryContentsError;
+      }
+    }
+
+    if (updates.solution_path) {
+      const { error: solutionError } = await getSupabase()
+        .storage.from('solutions')
+        .upload(updates.solution_path, JSON.stringify(problem.solution), { upsert: true });
+
+      if (solutionError) {
+        throw solutionError;
+      }
+    }
+
+    // Handle comments if they exist
+    if (problem.comments && problem.comments.length > 0) {
+      const commentInserts = problem.comments
+        .filter((comment) => !comment.id)
+        .map((comment) => {
           return {
             problem_id: problemId,
             content: comment.content,
             username: comment.username || getUsername() || 'Anonymous',
-
-          /**
-           * Use timestamp as a unique identifier for the comment.
-           * This assumes that comments with the same timestamp are the same comment.
-           */
-          created_at: new Date(timestamp).toISOString(),
           };
         });
 
@@ -273,10 +250,7 @@ export async function supabaseUpdateProblem(problemId: string, problem: BoltProb
        * Use upsert with onConflict to avoid duplicates.
        * This will insert new comments and ignore existing ones based on created_at.
        */
-      const { error: commentsError } = await getSupabase().from('problem_comments').upsert(commentInserts, {
+      const { error: commentsError } = await getSupabase().from('problem_comments').insert(commentInserts);
-        onConflict: 'created_at',
-        ignoreDuplicates: true,
-      });
 
       if (commentsError) {
         throw commentsError;

playwright.config.ts

@@ -14,7 +14,7 @@ export default defineConfig({
   fullyParallel: true,
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
-  workers: process.env.CI ? 1 : undefined,
+  workers: 1,
   reporter: 'html',
   timeout: 60000, // Increase global timeout to 60 seconds
   use: {

tests/e2e/problem.spec.ts

@@ -24,43 +24,27 @@ test('Should be able to save a problem ', async ({ page }) => {
   await page.getByRole('link', { name: 'App goes blank getting' }).click();
   await page.getByRole('link', { name: 'Load Problem' }).click();
 
-  await expect(page.getByText('Import the "problem" folder')).toBeVisible({ timeout: 30000 });
-
   const useSupabase = await isSupabaseEnabled(page);
-
+  await expect(page.getByText('Import the "problem" folder')).toBeVisible({ timeout: 30000 });
-  if (useSupabase) {
-    await openSidebar(page);
-    await page.getByRole('button', { name: 'Save Problem' }).click();
-    await page.getByRole('button', { name: 'Log In' }).click();
-    await page.getByRole('textbox', { name: 'Email' }).click();
-
   await login(page);
 
   await expect(page.getByText('Import the "problem" folder')).toBeVisible({ timeout: 30000 });
 
-    await page.locator('[data-testid="sidebar-icon"]').click();
+  await openSidebar(page);
   await page.getByRole('button', { name: 'Save Problem' }).click();
 
   await page.locator('input[name="title"]').click();
   await page.locator('input[name="title"]').fill('[test] playwright');
   await page.locator('input[name="description"]').click();
   await page.locator('input[name="description"]').fill('...');
-    await page.getByRole('button', { name: 'Submit' }).click();
-    await page.getByRole('button', { name: 'Close' }).click();
-  } else {
-    await page.locator('[data-testid="sidebar-icon"]').click();
-    await page.getByRole('button', { name: 'Save Problem' }).click();
 
-    await page.locator('input[name="title"]').click();
+  if (!useSupabase) {
-    await page.locator('input[name="title"]').fill('[test] playwright');
-    await page.locator('input[name="description"]').click();
-    await page.locator('input[name="description"]').fill('...');
     await page.locator('input[name="username"]').click();
     await page.locator('input[name="username"]').fill('playwright');
+  }
 
   await page.getByRole('button', { name: 'Submit' }).click();
   await page.getByRole('button', { name: 'Close' }).click();
-  }
 });
 
 test('Should be able to update a problem', async ({ page }) => {
@@ -70,11 +54,7 @@ test('Should be able to update a problem', async ({ page }) => {
   await page.getByRole('link', { name: '[test] playwright' }).first().click();
   expect(await page.getByRole('textbox', { name: 'Set the title of the problem' })).not.toBeVisible();
 
-  if (await isSupabaseEnabled(page)) {
   await login(page);
-  } else {
-    await setLoginKey(page);
-  }
 
   const currentTime = new Date();
   const hours = currentTime.getHours().toString().padStart(2, '0');
@@ -117,11 +97,7 @@ test('Should be able to add a comment to a problem', async ({ page }) => {
   await page.getByRole('combobox').selectOption('all');
   await page.getByRole('link', { name: '[test] playwright' }).first().click();
 
-  if (await isSupabaseEnabled(page)) {
   await login(page);
-  } else {
-    await setLoginKey(page);
-  }
 
   // Add a comment to the problem
   const comment = `test comment ${Date.now().toString()}`;
@@ -138,12 +114,50 @@ test('Should be able to add a comment to a problem', async ({ page }) => {
 test('Confirm that admins see the "Save Reproduction" button', async ({ page }) => {
   await page.goto('/problems?showAll=true');
 
-  if (await isSupabaseEnabled(page)) {
   await login(page);
-  } else {
-    await setLoginKey(page);
-  }
-
   await openSidebar(page);
   await expect(page.getByRole('link', { name: 'Save Reproduction' })).toBeVisible();
 });
+
+test('Should be able to save a reproduction', async ({ page }) => {
+  await page.goto('/problems?showAll=true');
+  await page.getByRole('combobox').selectOption('all');
+  await page.getByRole('link', { name: '[test] tic tac toe' }).first().click();
+
+  const shouldUseSupabase = await isSupabaseEnabled(page);
+  await login(page);
+
+  await page.getByRole('link', { name: 'Load Problem' }).click();
+
+  // TODO: Find a way to interact with the tic tac toe board
+  // find the cell in the tic tac toe board inside the iframe
+  // const frameLocator = page.frameLocator('iframe[title="preview"]').first();
+  // await frameLocator.getByTestId('cell-0-0').click();
+
+  const message = `test message ${Date.now().toString()}`;
+
+  await page.getByRole('textbox', { name: 'How can we help you?' }).click();
+  await page.getByRole('textbox', { name: 'How can we help you?' }).fill(message);
+  await page.getByRole('button', { name: 'Chat', exact: true }).click();
+
+  await openSidebar(page);
+
+  await page.getByRole('link', { name: 'Save Reproduction' }).click();
+  await page.getByRole('button', { name: 'Submit' }).click();
+  await expect(page.getByText('Reproduction saved')).toBeVisible();
+
+  /*
+   * Check to see if __currentProblem__ is set and has the correct solution message
+   */
+  const currentProblem = await page.evaluate(() => {
+    // @ts-ignore - accessing window.__currentProblem__ which is defined at runtime
+    return window.__currentProblem__;
+  });
+
+  // Only supabase is working for now
+  if (shouldUseSupabase) {
+    // Check if the message is a text message before accessing content
+    const message3 = currentProblem?.solution?.messages[2];
+    expect(message3 && message3.type === 'text' ? (message3 as any).content : null).toBe(message);
+  }
+});

tests/e2e/setup/test-utils.ts

@@ -57,8 +57,15 @@ export async function getElementText(page: Page, selector: string): Promise<stri
 export async function openSidebar(page: Page): Promise<void> {
   await page.locator('[data-testid="sidebar-icon"]').click();
 }
-
 export async function login(page: Page): Promise<void> {
+  if (await isSupabaseEnabled(page)) {
+    await loginToSupabase(page);
+  } else {
+    await setLoginKey(page);
+  }
+}
+
+export async function loginToSupabase(page: Page): Promise<void> {
   await page.getByRole('button', { name: 'Sign In' }).click();
   await page.getByRole('textbox', { name: 'Email' }).click();
   await page.getByRole('textbox', { name: 'Email' }).fill(process.env.SUPABASE_TEST_USER_EMAIL || '');
@@ -77,6 +84,9 @@ export async function setLoginKey(page: Page): Promise<void> {
   await page.getByRole('textbox', { name: 'Enter your login key' }).click();
   await page.getByRole('textbox', { name: 'Enter your login key' }).fill(process.env.NUT_LOGIN_KEY || '');
 
+  // wait for loading to finish data-isloading="true"
+  await page.waitForSelector('input[data-isloading="false"]', { state: 'attached' });
+
   await page.getByRole('textbox', { name: 'Enter your username' }).click();
   await page.getByRole('textbox', { name: 'Enter your username' }).fill(process.env.NUT_USERNAME || '');