From 501488cec095a9f473054598c89a30167ec96a16 Mon Sep 17 00:00:00 2001 From: Jason Laster Date: Fri, 21 Mar 2025 09:05:13 -0700 Subject: [PATCH] Add migration scripts for the db --- migrate-problems/README.md | 86 ++++++++ migrate-problems/analyze-schema.ts | 307 ++++++++++++++++++++++++++++ migrate-problems/bun.lockb | Bin 0 -> 7849 bytes migrate-problems/export-problems.ts | 296 +++++++++++++++++++++++++++ migrate-problems/insert-problems.ts | 221 ++++++++++++++++++++ migrate-problems/package.json | 18 ++ migrate-problems/tsconfig.json | 12 ++ 7 files changed, 940 insertions(+) create mode 100644 migrate-problems/README.md create mode 100644 migrate-problems/analyze-schema.ts create mode 100755 migrate-problems/bun.lockb create mode 100644 migrate-problems/export-problems.ts create mode 100644 migrate-problems/insert-problems.ts create mode 100644 migrate-problems/package.json create mode 100644 migrate-problems/tsconfig.json diff --git a/migrate-problems/README.md b/migrate-problems/README.md new file mode 100644 index 00000000..029a2872 --- /dev/null +++ b/migrate-problems/README.md @@ -0,0 +1,86 @@ +# Migrate Problems + +This folder contains scripts to migrate problem data from Replay's WebSocket API to a Supabase database. + +## Overview + +These scripts handle the migration process for problems from Replay to Supabase: + +1. `export-problems.ts` - Fetches problems from Replay's API and saves them to JSON files +2. `insert-problems.ts` - Imports problems from JSON files into the Supabase database +3. `analyze-schema.ts` - Analyzes and compares local problem data with the Supabase database schema + +## Setup + +1. Create a `.env.local` file in this directory with your Supabase credentials: + +``` +SUPABASE_URL=https://your-project-url.supabase.co +SUPABASE_ANON_KEY=your-anon-key +SUPABASE_SERVICE_ROLE_KEY=your-service-role-key +``` + +2. Install dependencies: + +```bash +npm install +# or +bun install +``` + +## Usage + +### Export Problems from Replay + +Fetches all problems from Replay's WebSocket API and saves them as JSON files in the `../data` directory: + +```bash +bun export-problems.ts +``` + +To include test problems: + +```bash +bun export-problems.ts --include-test +``` + +### Analyze Schema + +Analyzes the structure of the exported problem files and compares with the Supabase database schema: + +```bash +bun analyze-schema.ts +``` + +### Import Problems to Supabase + +Imports problems from the `data` directory into Supabase: + +```bash +bun insert-problems.ts +``` + +**Warning**: This script deletes existing problems in the database that don't have "tic tac toe" in their title before importing. + +## Problem Data Structure + +Problems have the following structure: + +```typescript +interface BoltProblem { + version: number; + problemId: string; + timestamp: number; + title: string; + description: string; + status?: string; + keywords?: string[]; + username?: string; + user_id?: string; + repositoryContents: string; + comments?: BoltProblemComment[]; + solution?: BoltProblemSolution; +} +``` + +When imported to Supabase, large fields (repository contents, solutions, prompts) are stored in separate storage buckets. \ No newline at end of file diff --git a/migrate-problems/analyze-schema.ts b/migrate-problems/analyze-schema.ts new file mode 100644 index 00000000..5d66b5e0 --- /dev/null +++ b/migrate-problems/analyze-schema.ts @@ -0,0 +1,307 @@ +import { readdir, readFile } from 'fs/promises'; +import { join } from 'path'; +import { createClient } from '@supabase/supabase-js'; +import * as dotenv from 'dotenv'; + +// Load environment variables from .env file +dotenv.config(); + +// Path to the problem files directory +const problemsDir = join(process.cwd(), '..', 'data'); + +// Types for problems +interface BoltProblemComment { + username?: string; + content: string; + timestamp: number; +} + +interface BoltProblemSolution { + simulationData: any; + messages: any[]; + evaluator?: string; +} + +interface BoltProblem { + version: number; + problemId: string; + timestamp: number; + title: string; + description: string; + status?: string; + keywords?: string[]; + username?: string; + user_id?: string; + repositoryContents: string; + comments?: BoltProblemComment[]; + solution?: BoltProblemSolution; + [key: string]: any; // Allow for additional properties +} + +// Database connection details +const SUPABASE_URL = process.env.SUPABASE_URL || ''; +const SUPABASE_KEY = process.env.SUPABASE_ANON_KEY || process.env.SUPABASE_SERVICE_ROLE_KEY || ''; + +// Log connection details for debugging +console.log('Supabase URL detected:', SUPABASE_URL ? 'Yes' : 'No'); +console.log('Supabase key detected:', SUPABASE_KEY ? 'Yes' : 'No'); + +type ValueType = 'null' | 'string' | 'number' | 'boolean' | 'object' | string; + +function getValueType(value: any): ValueType { + if (value === null) { + return 'null'; + } + + if (Array.isArray(value)) { + const itemTypes = new Set(value.map((item) => getValueType(item))); + return `array<${Array.from(itemTypes).join(' | ')}>`; + } + + if (typeof value === 'object') { + return 'object'; + } + + return typeof value; +} + +async function getProblemFiles(): Promise { + try { + const files = await readdir(problemsDir); + return files.filter((file) => file.startsWith('problem-') && file.endsWith('.json') && !file.includes('summaries')); + } catch (error) { + console.error('Error reading problem files directory:', error); + return []; + } +} + +async function analyzeLocalSchema(problemFiles: string[]): Promise>> { + try { + // Track all fields and their types + const fieldTypes = new Map>(); + const fieldNullability = new Map(); + const fieldExamples = new Map(); + let hasShownComments = false; + + // Process each problem file + for (const file of problemFiles) { + const filePath = join(problemsDir, file); + const problemData = await readFile(filePath, 'utf8'); + const problem: BoltProblem = JSON.parse(problemData); + + // Special handling for comments + if (problem.comments && problem.comments.length > 0 && !hasShownComments) { + console.log('\nComments Analysis:'); + console.log('Example comment structure:'); + console.log(JSON.stringify(problem.comments[0], null, 2)); + + console.log('\nComment fields:'); + + if (problem.comments[0]) { + Object.entries(problem.comments[0]).forEach(([key, value]) => { + console.log(`${key}: ${typeof value} = ${JSON.stringify(value)}`); + }); + } + + hasShownComments = true; + } + + // Special handling for timestamp + if (problem.timestamp) { + const date = new Date(problem.timestamp); + + if (!fieldExamples.has('timestamp')) { + console.log('\nTimestamp Analysis:'); + console.log('Raw value:', problem.timestamp); + console.log('As date:', date.toISOString()); + console.log('Type:', typeof problem.timestamp); + } + } + + // Collect all keys and their types + for (const [key, value] of Object.entries(problem)) { + const valueType = getValueType(value); + + // Track types + if (!fieldTypes.has(key)) { + fieldTypes.set(key, new Set()); + } + + fieldTypes.get(key)?.add(valueType); + + // Track nullability + if (!fieldNullability.has(key)) { + fieldNullability.set(key, true); // assume nullable until proven otherwise + } + + if (value !== null && value !== undefined) { + fieldNullability.set(key, false); + } + + // Track example values (non-null) + if (value !== null && value !== undefined && !fieldExamples.has(key)) { + fieldExamples.set(key, JSON.stringify(value).slice(0, 50) + (JSON.stringify(value).length > 50 ? '...' : '')); + } + } + } + + // Generate schema report + console.log('\nLocal Problem Files Schema Analysis:\n'); + console.log('Field Name | Types | Nullable | Example Value'); + console.log('-'.repeat(80)); + + for (const [field, types] of fieldTypes) { + const typeStr = Array.from(types).join(' | '); + const nullable = fieldNullability.get(field) ? 'YES' : 'NO'; + const example = fieldExamples.get(field) || 'N/A'; + + console.log(`${field.padEnd(20)} | ${typeStr.padEnd(20)} | ${nullable.padEnd(8)} | ${example}`); + } + + return fieldTypes; + } catch (error) { + console.error('Error analyzing local schema:', error); + return new Map(); + } +} + +export async function analyzeDatabaseSchema(): Promise { + try { + const supabaseUrl = process.env.SUPABASE_URL; + const supabaseKey = process.env.SUPABASE_ANON_KEY; + + if (!supabaseUrl || !supabaseKey) { + console.log('Supabase URL or key not found in environment variables'); + return; + } + + console.log('\nFetching database schema for problems table...\n'); + + const supabase = createClient(supabaseUrl, supabaseKey); + + // Use direct SQL query to get the schema information + const { data: columnsData, error: columnsError } = await supabase + .from('pg_catalog.information_schema.columns') + .select('column_name, data_type, is_nullable, column_default') + .eq('table_name', 'problems') + .eq('table_schema', 'public'); + + if (columnsError) { + console.log(`Error in database schema fetch: ${columnsError.message}`); + return; + } + + if (!columnsData || columnsData.length === 0) { + console.log('No schema information found for the problems table'); + return; + } + + console.log('\nDatabase Problems Table Schema:\n'); + console.log('Column Name | Data Type | Nullable | Default Value'); + console.log('--------------------------------------------------------------------------------'); + + columnsData.forEach((column: any) => { + console.log( + `${column.column_name.padEnd(20)} | ${column.data_type.padEnd(20)} | ${column.is_nullable === 'YES' ? 'YES' : 'NO'.padEnd(8)} | ${column.column_default || ''}`, + ); + }); + + // Get a sample row to show example values + const { data: sampleData, error: sampleError } = await supabase.from('problems').select('*').limit(1); + + if (!sampleError && sampleData && sampleData.length > 0) { + console.log('\nExample values from first row:'); + const sampleRow = sampleData[0]; + + Object.entries(sampleRow).forEach(([key, value]) => { + const displayValue = + typeof value === 'object' + ? JSON.stringify(value).substring(0, 50) + (JSON.stringify(value).length > 50 ? '...' : '') + : String(value).substring(0, 50) + (String(value).length > 50 ? '...' : ''); + + console.log(`${key.padEnd(20)}: ${displayValue}`); + }); + } + + // Check if problem_comments table exists + console.log('\nChecking for problem_comments table...'); + + const { data: commentsTableData, error: commentsTableError } = await supabase + .from('pg_catalog.information_schema.tables') + .select('table_name') + .eq('table_name', 'problem_comments') + .eq('table_schema', 'public'); + + if (commentsTableError) { + console.log(`Error checking for problem_comments table: ${commentsTableError.message}`); + return; + } + + if (commentsTableData && commentsTableData.length > 0) { + console.log('problem_comments table found, fetching schema...'); + + const { data: commentColumnsData, error: commentColumnsError } = await supabase + .from('pg_catalog.information_schema.columns') + .select('column_name, data_type, is_nullable, column_default') + .eq('table_name', 'problem_comments') + .eq('table_schema', 'public'); + + if (commentColumnsError) { + console.log(`Error in problem_comments schema fetch: ${commentColumnsError.message}`); + return; + } + + console.log('\nDatabase Problem Comments Table Schema:\n'); + console.log('Column Name | Data Type | Nullable | Default Value'); + console.log('--------------------------------------------------------------------------------'); + + commentColumnsData.forEach((column: any) => { + console.log( + `${column.column_name.padEnd(20)} | ${column.data_type.padEnd(20)} | ${column.is_nullable === 'YES' ? 'YES' : 'NO'.padEnd(8)} | ${column.column_default || ''}`, + ); + }); + + // Check for foreign key relationship + const { data: fkData, error: fkError } = await supabase + .from('pg_catalog.information_schema.key_column_usage') + .select('column_name, constraint_name') + .eq('table_name', 'problem_comments') + .eq('table_schema', 'public'); + + if (!fkError && fkData && fkData.length > 0) { + console.log('\nForeign key relationships:'); + fkData.forEach((fk: any) => { + console.log(`${fk.column_name} -> ${fk.constraint_name}`); + }); + } + } else { + console.log('problem_comments table not found in the database'); + } + } catch (error) { + console.log('Error in database analysis:', error); + } +} + +async function main(): Promise { + try { + // First analyze local problem files + console.log(`Looking for problem files in: ${problemsDir}`); + + const problemFiles = await getProblemFiles(); + console.log(`Found ${problemFiles.length} problem files to analyze.`); + console.log(); + + // Analyze local schema, but we're not using it for comparison + await analyzeLocalSchema(problemFiles); + + // Then analyze database + await analyzeDatabaseSchema(); + + console.log("\nNote: Schema comparison skipped as we're directly logging database schema information."); + } catch (error) { + console.error('Error in main function:', error); + } +} + +// Run the analysis +main(); diff --git a/migrate-problems/bun.lockb b/migrate-problems/bun.lockb new file mode 100755 index 0000000000000000000000000000000000000000..eff59f76f77590b24f99d834dc3bf8782a488ef0 GIT binary patch literal 7849 zcmeHMX+RU#7EWSBL<+LlA_`*bQfxC2b`fzw6ql#sQmG3XAPK>cU?u^<0uhCWOBE}k z(280WWvK{-ih|VQQk9Adiuj&Oh1LZuwJy|x$~!0I!uV8(@A;!YJaBX7&fM?2=bo8! z=H}#sfp#idVHY5g*$HLgHUTP`0W_Xe9=KQ}kqUVtxgu1cK;D+@$aE%@*g}s&@^$wnjD4c9Og3MontrO0Po0 z2U1?uq)F3i$9*=bS6TYb@-30uLMkO2?B;$xq;TZ$o^p>9^}cDvt|b!7y_QD7(zB(7 z(<|f3x0I!(2V0a}-8NYL@kjB(fMs{ZXA(^t&<+2W;9W#lKMC;I$N0gfix7MT0MLHykJAOH9YgT_!0F`q#d|=`cSS|U%>g|6 zAMZb&d!0DIvDgqt@&;apXme5d8lQsQ4?RsYX3=Mvh0(@*%8!wZZJ8t)P!6{P~)bT$_y?p1upuhUrSA^$% z={k$DS$sQVg#Wl(^U8A%_-uR_BnDcbSp|;a%E6f{>Fx0n;$KSlYQ)(T~g7KT)5}(vQ^!VcWcTj zd6&!JWyS@!ddS7dcuTs^mNhj6OP=}1*hWbzqd)DR$}P%`8{x^GHTlVvMWq-6W@vb$Cy=A(59@jw(@gBOVpc9>;6fl)l6gXl6XWXb8C`@ zULVDpkX855v5fnx&-e0)4LucW^mR+2+10-3`WHUD`rZvo`-5AKncVlUUGreQ|G}fj z6Yd2cD<81w!I$@^%#CF5lAWN2*HyJPzkX9g=FID(v*%YITW_sr)-(9%vgUoILwc4s zXB@m;>C}>#_E-$k#T*;LeVbNy`r&omppC@k> z496<^i@(p0nE$E%z0-=g3z9QYCsTyU-A_&Xg~7{wPjN%9*O(WU`dW3CzV^{;;Y z*A?HTXI|eWSI*ns;J-k9eO|-j$0>9Ftjc!@Jn1xM`P04husn)BedYVap}SJ^nD-6# zw(Y`A^yW-YGq2=bOy8do7a!;U@1RX$yY27uYdx-v3f;CkuFu2^H`f_(OBWeE?RM{u zt7+3z@0^;Ubw>3!eb+@y-*}T@FXm$+TvsYd92i&}YIk(j2CLt$HaW}cIWaeWduD$+ zr1(&X*@mh+i_Ip@v2MAMv}NC{1$INuEbKOMdBf<~&EGuTIlFu{?!8IgVZQIVeC522 zYi-N=?0&c+Dks4?bpRD)lkoG2(WY*P>lR= zq*COUY7bXC#_VM9GWT&@zr&gJrL6hq_kW#UTh_9>reDQ(wJL#OiOIKH&)1BvUZKkV zarYBb6QAkvTNfERwwxN`TTM}x5urJ@`Pog2>$Wp^@p}Og?vC@tTTB1tE}G^W={}j$ zH2CC<{OD%SuT<5%e7~~(&&0_i7KQiE_4{GN*^j2~D;PC>^JVM1U#yU`zuwkro1@4( z&fq2QGvww5r#6>;6Y+G<7 zabaaf5uF*HnU*l^rdw{%i zh<;NuyuDp-%9C$--g-bM4(1SFm9G%9va5J5E1n zxVOW7819*HKHQ7oz5%~8;rke;!S^S=>wL6x73nx9+KKNnoD=6p8??)4MLy)ky)e#$ z^P^qZ$9*i?gtj3DZ9?1d8rp{Y4?HG~*nC@S*AeX(Jp+$<4lWB+Zs1*)cE9jDyo)x# zylPTM)8|M9N+GReu}I~QRNC~}{?3#=W#3VQBsD({+uwn58cy+9q)JJuf>@Po6H><{ zbwj|oQG7?B6T$#knL3!>*V=5HEiq^NJHb4B7E2CSz%S}^&gQc|z4EY!Iooq2n&7}9 z)k#tfc!BBiC){)vLsWo!ge2PCDrfIL9l4>Pl z_aV2UiC0|F=$Fg`-f^M$PApQRB{fm-f!3p>T1%><9Ci$NnsS8a4Lk_Xkebv_ z0Rv9MmA>4g=mb*ZB{fxzubnBrJ&RO&x_t6R+CCGsmyZN!(f#tq;^bd zwP3Xi<$%V$n(cqN^5S}8+`wvB&#P_gjfD-6>014K;-Q71;X(!Fy;Lbw$OO`vN`*uw z_O#)5gos;5h-i3}bf88bX))ieNKP`x9|rK8@wJ^EjaEwzDQQ}$0L}$fHp^#-COVT4+ZOcp-GKQW-|O+u13EVhOEOL{hS_&=8uEE5vpYv>gvVco28+ zPbtMw0V;3ADrGRNspEu5B@0wa#!n+(?B`AXQ32h0mR& zm2!nZjOTP)Li=!BL3;-l0P(N0BRXsbmW~dvK&o&?nuuV5GD6H#DWsj4)iDTYIyyW} zUeT}!10+FGUZ7kC4=0VT>x}J%!N~f;NgigPu9k`^4bZ4~np~!{YZx~P2pBzZ4klkD z8m$Qfc$&2bVZm~tOtQ3-wLL!&h_w(NA3W~Tm2qS`^an-|WFIGn9e4w~o*PF2f#z;I zz*-z}Ij|U&EJzY4X$$X8SeV0rhS}w{<=w7ULE_Dh86D^fMzDZC@ZF~~6*DA(irK}O zhCf(gC#&iRS#7ifiS`JC0p4dw(3Wo7Qfwz=9mx?`Iyw*zZe1>^J;MUD{TRIA05=;f L&}8Mm;m3aflPEa< literal 0 HcmV?d00001 diff --git a/migrate-problems/export-problems.ts b/migrate-problems/export-problems.ts new file mode 100644 index 00000000..a58c53b3 --- /dev/null +++ b/migrate-problems/export-problems.ts @@ -0,0 +1,296 @@ +// Script to fetch all problems and save them to a JSON file +import { mkdir, writeFile } from 'fs/promises'; +import { join } from 'path'; +import { WebSocket } from 'ws'; + +// Augment ImportMeta type for Bun +declare global { + interface ImportMeta { + dir: string; + } +} + +// Types from Problems.ts +interface BoltProblemComment { + username?: string; + content: string; + timestamp: number; +} + +interface BoltProblemSolution { + simulationData: any; + messages: any[]; + evaluator?: string; +} + +enum BoltProblemStatus { + Pending = 'Pending', + Unsolved = 'Unsolved', + Solved = 'Solved', +} + +interface BoltProblemDescription { + version: number; + problemId: string; + timestamp: number; + title: string; + description: string; + status?: BoltProblemStatus; + keywords?: string[]; +} + +interface BoltProblem extends BoltProblemDescription { + username?: string; + user_id?: string; + repositoryContents: string; + comments?: BoltProblemComment[]; + solution?: BoltProblemSolution; +} + +// URL of the Replay WebSocket server +const replayWsServer = 'wss://dispatch.replay.io'; + +// Helper functions from ReplayProtocolClient.ts +function assert(condition: any, message: string = 'Assertion failed!'): asserts condition { + if (!condition) { + throw new Error(message); + } +} + +interface Deferred { + promise: Promise; + resolve: (value: T) => void; + reject: (reason?: any) => void; +} + +function createDeferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (reason?: any) => void; + const promise = new Promise((_resolve, _reject) => { + resolve = _resolve; + reject = _reject; + }); + + return { promise, resolve, reject }; +} + +type EventListener = (params: any) => void; + +// ProtocolClient adapted for Bun/Node.js +class ProtocolClient { + openDeferred = createDeferred(); + eventListeners = new Map>(); + nextMessageId = 1; + pendingCommands = new Map }>(); + socket: WebSocket; + + constructor() { + console.log(`Creating WebSocket for ${replayWsServer}`); + this.socket = new WebSocket(replayWsServer); + this.socket.on('close', () => this.onSocketClose()); + this.socket.on('error', (error) => this.onSocketError(error)); + this.socket.on('open', () => this.onSocketOpen()); + this.socket.on('message', (data) => this.onSocketMessage(data)); + this.listenForMessage('Recording.sessionError', (error) => { + console.log(`Session error ${error}`); + }); + } + + initialize() { + return this.openDeferred.promise; + } + + close() { + this.socket.close(); + for (const info of this.pendingCommands.values()) { + info.deferred.reject(new Error('Client destroyed')); + } + this.pendingCommands.clear(); + } + + listenForMessage(method: string, callback: EventListener) { + let listeners = this.eventListeners.get(method); + if (listeners == null) { + listeners = new Set([callback]); + this.eventListeners.set(method, listeners); + } else { + listeners.add(callback); + } + return () => { + listeners!.delete(callback); + }; + } + + sendCommand(args: { method: string; params: any; sessionId?: string }) { + const id = this.nextMessageId++; + const { method, params, sessionId } = args; + console.log('Sending command', { id, method, params, sessionId }); + const command = { + id, + method, + params, + sessionId, + }; + this.socket.send(JSON.stringify(command)); + const deferred = createDeferred(); + this.pendingCommands.set(id, { method, deferred }); + return deferred.promise; + } + + onSocketClose() { + console.log('Socket closed'); + } + + onSocketError(error: any) { + console.log(`Socket error ${error}`); + } + + onSocketMessage(data: any) { + const { error, id, method, params, result } = JSON.parse(String(data)); + if (id) { + const info = this.pendingCommands.get(id); + assert(info, `Received message with unknown id: ${id}`); + this.pendingCommands.delete(id); + if (result) { + info.deferred.resolve(result); + } else if (error) { + console.error('ProtocolError', info.method, id, error); + info.deferred.reject(new Error(`Protocol error ${error.code}: ${error.message}`)); + } else { + info.deferred.reject(new Error('Channel error')); + } + } else if (this.eventListeners.has(method)) { + const callbacks = this.eventListeners.get(method); + if (callbacks) { + callbacks.forEach((callback) => callback(params)); + } + } else { + console.log('Received message without a handler', { method, params }); + } + } + + onSocketOpen() { + console.log('Socket opened'); + this.openDeferred.resolve(); + } +} + +// Send a single command with a one-use protocol client +async function sendCommandDedicatedClient(args: { method: string; params: any }) { + const client = new ProtocolClient(); + await client.initialize(); + try { + const rval = await client.sendCommand(args); + client.close(); + return rval; + } finally { + client.close(); + } +} + +// Function to list all problems (adapted from Problems.ts) +async function listAllProblems(includeTestProblems = false): Promise { + try { + const rv = await sendCommandDedicatedClient({ + method: 'Recording.globalExperimentalCommand', + params: { + name: 'listBoltProblems', + }, + }); + console.log('ListProblemsRval', rv); + let problems = rv.rval.problems.reverse(); + // Filter out test problems if not explicitly included + if (!includeTestProblems) { + problems = problems.filter((problem: BoltProblemDescription) => !problem.title.includes('[test]')); + } + return problems; + } catch (error) { + console.error('Error fetching problems', error); + return []; + } +} + +// Function to get a specific problem by ID (adapted from Problems.ts) +async function getProblem(problemId: string): Promise { + try { + if (!problemId) { + console.error('Invalid problem ID'); + return null; + } + + const rv = await sendCommandDedicatedClient({ + method: 'Recording.globalExperimentalCommand', + params: { + name: 'fetchBoltProblem', + params: { problemId }, + }, + }); + + const problem = (rv as { rval: { problem: BoltProblem } }).rval.problem; + + if (!problem) { + console.error('Problem not found'); + return null; + } + + // Handle legacy format + if ('prompt' in problem) { + // Convert from old format + (problem as any).repositoryContents = (problem as any).prompt.content; + delete (problem as any).prompt; + } + + return problem; + } catch (error) { + console.error('Error fetching problem', error); + return null; + } +} + +// Main function +async function main() { + try { + // Check if --include-test flag is provided + const includeTestProblems = process.argv.includes('--include-test'); + console.log(`Fetching problems${includeTestProblems ? ' (including test problems)' : ''}...`); + + // Get problem summaries + const problemDescriptions = await listAllProblems(includeTestProblems); + + // Create data directory if it doesn't exist + const dataDir = join(import.meta.dir, '../data'); + await mkdir(dataDir, { recursive: true }); + + // Save problem summaries + const summaryFile = join(dataDir, 'problem-summaries.json'); + await writeFile(summaryFile, JSON.stringify(problemDescriptions, null, 2)); + console.log(`Successfully saved ${problemDescriptions.length} problem summaries to ${summaryFile}`); + + // Fetch full problem details + console.log('Fetching full problem details...'); + let counter = 0; + let successCount = 0; + + for (const summary of problemDescriptions) { + counter++; + const { problemId, title } = summary; + console.log(`Fetching problem ${counter}/${problemDescriptions.length}: ${title} (${problemId})`); + + const fullProblem = await getProblem(problemId); + if (fullProblem) { + // Save each problem to its own file + const problemFile = join(dataDir, `problem-${problemId}.json`); + await writeFile(problemFile, JSON.stringify(fullProblem, null, 2)); + successCount++; + } + } + + console.log(`Successfully saved ${successCount} problems to individual files in ${dataDir}`); + } catch (error) { + console.error('Failed to save problems:', error); + process.exit(1); + } +} + +// Run the script +main(); diff --git a/migrate-problems/insert-problems.ts b/migrate-problems/insert-problems.ts new file mode 100644 index 00000000..21c04c14 --- /dev/null +++ b/migrate-problems/insert-problems.ts @@ -0,0 +1,221 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { createClient } from '@supabase/supabase-js'; +import * as dotenv from 'dotenv'; + +// Load environment variables from .env.local +dotenv.config({ path: '.env.local' }); + +// Define types based on the existing schema in analyze-schema.ts +interface BoltProblemSolution { + simulationData?: any; + messages?: any[]; + evaluator?: string; + [key: string]: any; +} + +interface BoltProblem { + version: number; + problemId: string; + timestamp: number; + title: string; + description: string; + status?: string; + keywords?: string[]; + username?: string; + user_id?: string; + repositoryContents?: string; + solution?: BoltProblemSolution; + prompt?: any; + [key: string]: any; +} + +// Initialize Supabase client with service role key +const supabaseUrl = process.env.SUPABASE_URL; +const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY; + +if (!supabaseUrl || !supabaseKey) { + console.error('Missing required environment variables: SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY'); + process.exit(1); +} + +const supabase = createClient(supabaseUrl, supabaseKey, { + auth: { + autoRefreshToken: true, + persistSession: true, + }, +}); + +async function deleteExistingProblems(): Promise { + try { + console.log('Deleting all problems that don\'t have "tic tac toe" in their title...'); + + // First, query to get all problem IDs that don't match the criteria + const { data: problemsToDelete, error: fetchError } = await supabase + .from('problems') + .select('id, title') + .not('title', 'ilike', '%tic tac toe%'); + + if (fetchError) { + console.error('Error fetching problems to delete:', fetchError.message); + throw fetchError; + } + + if (!problemsToDelete || problemsToDelete.length === 0) { + console.log('No problems to delete.'); + return; + } + + console.log(`Found ${problemsToDelete.length} problems to delete.`); + + // Delete the problems + const { error: deleteError } = await supabase.from('problems').delete().not('title', 'ilike', '%tic tac toe%'); + + if (deleteError) { + console.error('Error deleting problems:', deleteError.message); + throw deleteError; + } + + console.log(`Successfully deleted ${problemsToDelete.length} problems.`); + } catch (error) { + console.error('Error in deleteNonTicTacToeProblems:', error instanceof Error ? error.message : String(error)); + throw error; + } +} + +async function importProblems(): Promise { + try { + // First delete all problems that don't have "tic tac toe" in their title + await deleteExistingProblems(); + + // Get all problem files + const dataDir = path.join(process.cwd(), 'data'); + const files = await fs.readdir(dataDir); + const problemFiles = files.filter((file) => file.startsWith('problem-') && file.endsWith('.json')); + + console.log(`Found ${problemFiles.length} problem files to import`); + console.log('Starting import...\n'); + + let successCount = 0; + let errorCount = 0; + + for (const file of problemFiles) { + try { + await processProblemFile(file, dataDir); + successCount++; + } catch (error) { + errorCount++; + } + } + + console.log('\nImport Summary:'); + console.log(`Total files: ${problemFiles.length}`); + console.log(`Successfully imported: ${successCount}`); + console.log(`Failed to import: ${errorCount}`); + } catch (error) { + console.error('Import failed:', error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} + +async function uploadBlob(bucket: string, path: string, contents: string) { + const { error } = await supabase.storage.from(bucket).upload(path, contents); + + if (error && error.error !== 'Duplicate') { + console.error(` ❌ Error uploading ${path}:`, error.message, error); + throw error; + } else { + console.log(` ✅ Successfully uploaded ${path} to ${bucket}`); + } +} + +async function processProblemFile(file: string, dataDir: string) { + try { + if (file.includes('problem-summaries')) { + return; + } + + console.log(`Processing ${file}`); + + const filePath = path.join(dataDir, file); + const content = await fs.readFile(filePath, 'utf8'); + const problem: BoltProblem = JSON.parse(content); + + // Convert Unix timestamp to ISO string + const createdAt = new Date(problem.timestamp).toISOString(); + + // Extract keywords from the problem data if they exist + const keywords = Array.isArray(problem.keywords) ? problem.keywords : []; + + // Extract solution and prompt, defaulting to empty objects if not present + const solution = problem.solution || {}; + const prompt = problem.prompt || {}; + + // Validate repository_contents + let repositoryContents = ''; + + try { + repositoryContents = problem.repositoryContents || ''; + + if (repositoryContents && typeof repositoryContents !== 'string') { + console.warn(`Warning: Invalid repository_contents in ${file}, converting to string`); + repositoryContents = JSON.stringify(repositoryContents); + } + } catch (err) { + console.warn( + `Warning: Error processing repository_contents in ${file}:`, + err instanceof Error ? err.message : String(err), + ); + repositoryContents = ''; + } + + const repositoryContentsPath = `problem/${problem.problemId}.txt`; + await uploadBlob('repository-contents', repositoryContentsPath, repositoryContents); + + const solutionPath = `problem/${problem.problemId}.json`; + await uploadBlob('solutions', solutionPath, JSON.stringify(solution)); + + const promptPath = `problem/${problem.problemId}.json`; + await uploadBlob('prompts', promptPath, JSON.stringify(prompt)); + + // Insert into database + const { error } = await supabase + .from('problems') + .upsert({ + user_id: `97cde220-c22b-4eb5-849d-6946fb07ebc4`, + id: problem.problemId, + created_at: createdAt, + updated_at: createdAt, + title: problem.title, + description: problem.description, + status: problem.status || 'pending', + keywords, + repository_contents_path: repositoryContentsPath, + solution_path: solutionPath, + prompt_path: promptPath, + version: problem.version, + repository_contents: null, + solution: '', + prompt: '', + }) + .select() + .single(); + + if (error) { + console.error(` ❌ Error updating problem:`, error.message); + throw error; + } else { + console.log(` ✅ Successfully updated problem`); + return; + } + } catch (error) { + console.error(` ❌ Error processing ${file}: ${(error as any).message}`); + + if (error instanceof Error && error.stack) { + console.error(error.stack); + } + + throw error; + } +} +importProblems(); diff --git a/migrate-problems/package.json b/migrate-problems/package.json new file mode 100644 index 00000000..1fa32b87 --- /dev/null +++ b/migrate-problems/package.json @@ -0,0 +1,18 @@ +{ + "name": "migrate-problems", + "version": "1.0.0", + "description": "Script to migrate problems from Replay to Supabase", + "private": true, + "type": "module", + "main": "save-problems.ts", + "scripts": {}, + "dependencies": { + "@supabase/supabase-js": "^2.49.1", + "dotenv": "^16.4.7", + "ws": "^8.13.0" + }, + "devDependencies": { + "typescript": "^5.0.0", + "bun-types": "latest" + } +} \ No newline at end of file diff --git a/migrate-problems/tsconfig.json b/migrate-problems/tsconfig.json new file mode 100644 index 00000000..664ed797 --- /dev/null +++ b/migrate-problems/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "node", + "types": ["bun-types"], + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "noImplicitAny": false + } +} \ No newline at end of file