APAW/agent-evolution/scripts/watch-db.cjs

#!/usr/bin/env node
/**
 * Watch for changes in agent files and update database incrementally
 * 
 * Features:
 * 1. Poll git for changes in .kilo/agents/*.md and kilo-meta.json
 * 2. On change, only process changed files (incremental update)
 * 3. Determine change type (model_change vs prompt_change)
 * 4. Update database with new versions and metadata
 * 5. Export updated data to JSON
 * 6. Clean shutdown on SIGINT/SIGTERM
 * 7. Configurable polling interval via WATCH_INTERVAL_MS
 * 8. Logging to .kilo/logs/watch-db.log
 */

const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');

// Try to load sqlite3, but make it optional
let sqlite3;
try {
  sqlite3 = require('sqlite3').verbose();
} catch (e) {
  console.warn('sqlite3 not available, database functionality will be limited');
  sqlite3 = null;
}

// Configuration
const DB_PATH = path.join(__dirname, '../data/agent-evolutions.db');
const LOG_FILE = path.join(__dirname, '../../.kilo/logs/watch-db.log');
const WATCH_INTERVAL_MS = parseInt(process.env.WATCH_INTERVAL_MS || '60000'); // Default 60 seconds

// Ensure log directory exists
const logDir = path.dirname(LOG_FILE);
if (!fs.existsSync(logDir)) {
  fs.mkdirSync(logDir, { recursive: true });
}

// Logging function
function log(level, message) {
  const timestamp = new Date().toISOString();
  const logMessage = `[${timestamp}] ${level.toUpperCase()}: ${message}\n`;
  console.log(logMessage.trim());
  fs.appendFileSync(LOG_FILE, logMessage);
}

// Get current HEAD commit for tracked files
function getCurrentCommit() {
try {
const output = execSync(
'git log -1 --format="%H" -- .kilo/agents/*.md kilo-meta.json',
{ cwd: path.join(__dirname, '../..'), encoding: 'utf-8' }
).trim();
return output || null;
} catch (error) {
log('error', `Failed to get current commit: ${error.message}`);
return null;
}
}

// Get last synced commit from database
function getLastSyncedCommit(db) {
  // If sqlite3 is not available, return null to trigger first run
  if (!sqlite3) {
    return Promise.resolve(null);
  }
  
  return new Promise((resolve, reject) => {
    db.get("SELECT value FROM meta WHERE key = 'last_synced_commit'", (err, row) => {
      if (err) {
        reject(err);
      } else {
        resolve(row ? row.value : null);
      }
    });
  });
}

// Update last synced commit in database
function updateLastSyncedCommit(db, commit) {
  // If sqlite3 is not available, just resolve
  if (!sqlite3) {
    return Promise.resolve();
  }
  
  return new Promise((resolve, reject) => {
    db.run(
      "INSERT OR REPLACE INTO meta (key, value) VALUES ('last_synced_commit', ?)",
      [commit],
      (err) => {
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      }
    );
  });
}

// Get changed files between commits
function getChangedFiles(sinceCommit) {
try {
const command = sinceCommit 
? `git log --name-only --oneline ${sinceCommit}..HEAD -- .kilo/agents/*.md kilo-meta.json`
: 'git log -1 --name-only --oneline HEAD -- .kilo/agents/*.md kilo-meta.json';

const output = execSync(command, {
cwd: path.join(__dirname, '../..'),
encoding: 'utf-8'
});

// Parse output to get file paths
const lines = output.trim().split('\n');
const files = [];

for (const line of lines) {
if (line && (line.endsWith('.md') || line.endsWith('kilo-meta.json'))) {
files.push(line.trim());
}
}

return [...new Set(files)]; // Remove duplicates
} catch (error) {
log('error', `Failed to get changed files: ${error.message}`);
return [];
}
}

// Parse YAML frontmatter from file
function parseYamlFrontmatter(content) {
if (!content.startsWith('---')) return null;
const end = content.indexOf('---', 4);
if (end === -1) return null;

const frontmatter = content.slice(4, end).trim();
const lines = frontmatter.split('\n');
const result = {};

for (const line of lines) {
const match = line.match(/^([a-zA-Z_]+):\s*(.*)$/);
if (match) {
const key = match[1];
let value = match[2].trim();

// Remove quotes if present
if (value.startsWith('"') && value.endsWith('"')) {
value = value.slice(1, -1);
} else if (value.startsWith("'") && value.endsWith("'")) {
value = value.slice(1, -1);
}

result[key] = value;
}
}

return result;
}

// Get previous version of an agent from database
function getPreviousAgentVersion(db, agentName) {
  // If sqlite3 is not available, return null
  if (!sqlite3) {
    return Promise.resolve(null);
  }
  
  return new Promise((resolve, reject) => {
    db.get(
      "SELECT model FROM agent_versions WHERE agent_name = ? ORDER BY date DESC LIMIT 1",
      [agentName],
      (err, row) => {
        if (err) {
          reject(err);
        } else {
          resolve(row ? row.model : null);
        }
      }
    );
  });
}

// Insert new agent version into database
function insertAgentVersion(db, agentData) {
  // If sqlite3 is not available, just resolve
  if (!sqlite3) {
    return Promise.resolve();
  }
  
  return new Promise((resolve, reject) => {
    const {
      agent_name,
      model,
      date,
      commit,
      reason,
      change_type
    } = agentData;
    
    db.run(
      "INSERT INTO agent_versions (agent_name, model, date, commit, reason, change_type) VALUES (?, ?, ?, ?, ?, ?)",
      [agent_name, model, date, commit, reason, change_type],
      (err) => {
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      }
    );
  });
}

// Process changed agent file
async function processAgentFile(db, filePath, commitInfo) {
try {
const agentName = path.basename(filePath, '.md');
const fullPath = path.join(__dirname, '../../', filePath);

// Read file content
const content = fs.readFileSync(fullPath, 'utf-8');
const frontmatter = parseYamlFrontmatter(content);

if (!frontmatter) {
log('warn', `Could not parse frontmatter for ${filePath}`);
return;
}

// Get previous version
const previousModel = await getPreviousAgentVersion(db, agentName);

// Determine change type
let changeType = 'prompt_change';
if (previousModel && frontmatter.model !== previousModel) {
changeType = 'model_change';
}

// Insert new version
await insertAgentVersion(db, {
agent_name: agentName,
model: frontmatter.model || 'unknown',
date: commitInfo.date,
commit: commitInfo.hash,
reason: commitInfo.message,
change_type: changeType
});

log('info', `Processed ${agentName}: ${changeType}`);
} catch (error) {
log('error', `Failed to process ${filePath}: ${error.message}`);
}
}

// Process kilo-meta.json changes
async function processKiloMetaChanges(db, commitInfo) {
try {
const metaPath = path.join(__dirname, '../../kilo-meta.json');
const content = fs.readFileSync(metaPath, 'utf-8');
const metaData = JSON.parse(content);

// For each agent in meta, check if model changed
for (const [agentName, agentData] of Object.entries(metaData.agents)) {
// Get previous version
const previousModel = await getPreviousAgentVersion(db, agentName);

// Check if model changed
if (previousModel && agentData.model !== previousModel) {
// Insert model change record
await insertAgentVersion(db, {
agent_name: agentName,
model: agentData.model,
date: commitInfo.date,
commit: commitInfo.hash,
reason: commitInfo.message,
change_type: 'model_change'
});

log('info', `Processed ${agentName} model change in kilo-meta.json`);
}
}
} catch (error) {
log('error', `Failed to process kilo-meta.json changes: ${error.message}`);
}
}

// Export DB to JSON
function exportDbToJson() {
return new Promise((resolve, reject) => {
const exportScript = path.join(__dirname, 'export-db-to-json.cjs');

try {
// Check if export script exists
if (!fs.existsSync(exportScript)) {
log('error', `Export script not found: ${exportScript}`);
resolve();
return;
}

// Try to import and run the export function
const exportModule = require(exportScript);
if (typeof exportModule === 'function') {
log('info', 'Exporting database to JSON...');
exportModule();
log('info', 'Database exported to JSON successfully');
resolve();
} else {
// If that fails, try running as a child process
const { exec } = require('child_process');
exec(`node "${exportScript}"`, (error, stdout, stderr) => {
if (error) {
log('error', `Export script failed: ${error.message}`);
log('error', `stderr: ${stderr}`);
resolve();
} else {
log('info', 'Database exported to JSON successfully');
log('info', `stdout: ${stdout}`);
resolve();
}
});
}
} catch (error) {
log('error', `Failed to export database to JSON: ${error.message}`);
resolve(); // Don't fail the whole process if export fails
}
});
}

// Initialize database
function initializeDatabase() {
  // If sqlite3 is not available, return a mock database object
  if (!sqlite3) {
    return Promise.resolve({
      get: (sql, callback) => callback(null, null),
      run: (sql, params, callback) => callback && callback(null),
      close: (callback) => callback && callback(null)
    });
  }
  
  return new Promise((resolve, reject) => {
    const db = new sqlite3.Database(DB_PATH, (err) => {
      if (err) {
        reject(err);
        return;
      }
      
      // Create tables if they don't exist
      db.serialize(() => {
        db.run(`CREATE TABLE IF NOT EXISTS meta (
          key TEXT PRIMARY KEY, 
          value TEXT
        )`);
        
        db.run(`CREATE TABLE IF NOT EXISTS agents (
          name TEXT PRIMARY KEY, 
          current_model TEXT, 
          description TEXT, 
          mode TEXT, 
          color TEXT, 
          file_path TEXT, 
          last_updated TEXT
        )`);
        
        db.run(`CREATE TABLE IF NOT EXISTS agent_versions (
          id INTEGER PRIMARY KEY AUTOINCREMENT, 
          agent_name TEXT, 
          model TEXT, 
          date TEXT, 
          commit TEXT, 
          reason TEXT, 
          change_type TEXT, 
          score INTEGER
        )`);
        
        db.run(`CREATE TABLE IF NOT EXISTS recommendations (
          id INTEGER PRIMARY KEY AUTOINCREMENT, 
          agent_name TEXT, 
          recommended_model TEXT, 
          impact TEXT, 
          source TEXT, 
          generated_at TEXT, 
          applied INTEGER DEFAULT 0
        )`);
        
        resolve(db);
      });
    });
  });
}

// Main sync function
async function syncDatabase(db) {
try {
// Get last synced commit
const lastSyncedCommit = await getLastSyncedCommit(db);
log('info', `Last synced commit: ${lastSyncedCommit || 'none (first run)'}`);

// Get current commit
const currentCommit = getCurrentCommit();
if (!currentCommit) {
log('error', 'Could not determine current commit');
return;
}

log('info', `Current commit: ${currentCommit}`);

// If this is the first run, bootstrap with current commit
if (!lastSyncedCommit) {
log('info', 'First run, bootstrapping with current commit');
await updateLastSyncedCommit(db, currentCommit);
return;
}

// Check if there are changes
if (currentCommit === lastSyncedCommit) {
log('info', 'No changes detected');
return;
}

log('info', 'Changes detected, processing...');

// Get changed files
const changedFiles = getChangedFiles(lastSyncedCommit);
log('info', `Changed files: ${changedFiles.join(', ')}`);

// Get commit info
const commitInfo = {
hash: currentCommit,
date: new Date().toISOString(),
message: 'Incremental update'
};

// Process each changed file
for (const file of changedFiles) {
if (file.endsWith('.md') && file.startsWith('.kilo/agents/')) {
await processAgentFile(db, file, commitInfo);
} else if (file === 'kilo-meta.json') {
await processKiloMetaChanges(db, commitInfo);
}
}

// Update last synced commit
await updateLastSyncedCommit(db, currentCommit);

// Export to JSON
await exportDbToJson();

log('info', 'Database sync completed successfully');
} catch (error) {
log('error', `Database sync failed: ${error.message}`);
}
}

// Main function
async function main() {
let db;

try {
log('info', 'Starting agent evolution DB watcher');

// Initialize database
db = await initializeDatabase();
log('info', `Database initialized at ${DB_PATH}`);

// Initial sync
await syncDatabase(db);

// Set up polling
log('info', `Starting poll loop with interval ${WATCH_INTERVAL_MS}ms`);

const intervalId = setInterval(async () => {
await syncDatabase(db);
}, WATCH_INTERVAL_MS);

// Handle graceful shutdown
const shutdown = async (signal) => {
log('info', `Received ${signal}, shutting down gracefully...`);

// Clear interval
clearInterval(intervalId);

// Close database
if (db) {
db.close((err) => {
if (err) {
log('error', `Error closing database: ${err.message}`);
} else {
log('info', 'Database closed successfully');
}
process.exit(0);
});
} else {
process.exit(0);
}
};

process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));

} catch (error) {
log('error', `Failed to start watcher: ${error.message}`);
if (db) {
db.close();
}
process.exit(1);
}
}

// Run main function if script is executed directly
if (require.main === module) {
main();
}

module.exports = { 
syncDatabase, 
initializeDatabase, 
parseYamlFrontmatter,
getChangedFiles,
processAgentFile,
processKiloMetaChanges
};