9b0f160587
- build-standalone-fixed.cjs: reads from 4 real sources (agents md, kilo-meta.json, model-benchmarks-verified.json, agent-versions.json); computes recommendations dynamically - build-standalone-direct.cjs: direct data export + HTML embed pipeline - dashboard-smoke-test.ts: Playwright E2E smoke test covering all 6 tabs - model-benchmarks-verified.json: verified IF scores from artificialanalysis.ai for 15 models (SWE-bench unverifiable → null) - agent-versions.json: 347 git history entries extracted for 34 agents - kilo-meta.json: prompt-optimizer → qwen3.5-122b, memory-manager → deepseek-v4-pro-max - index.html: Recommendations tab rendering updated for dynamic data - Dockerfile + docker-compose.yml: mount-driven build, no image rebuild for data changes - README.md: updated dashboard docs and verified benchmark sources
423 lines
18 KiB
JavaScript
423 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* Build unified dashboard data by reading files directly:
|
||
* - .kilo/agents/*.md (YAML frontmatter: model, mode, color, description)
|
||
* - kilo-meta.json (model assignments, categories, fallback info)
|
||
* - model-benchmarks-verified.json (IF scores, context window)
|
||
* - agent-versions.json (real history with dates, commits, reasons)
|
||
*
|
||
* Outputs: index.standalone.html with embedded JSON.
|
||
*
|
||
* Run: node agent-evolution/scripts/build-standalone-direct.cjs
|
||
*/
|
||
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const META_FILE = path.join(__dirname, '../../kilo-meta.json');
|
||
const BENCHMARK_FILE = path.join(__dirname, '../data/model-benchmarks-verified.json');
|
||
const AGENTS_DIR = path.join(__dirname, '../../.kilo/agents');
|
||
const HISTORY_FILE = path.join(__dirname, '../data/agent-versions.json');
|
||
const HTML_FILE = path.join(__dirname, '../index.html');
|
||
const OUTPUT_FILE = path.join(__dirname, '../index.standalone.html');
|
||
|
||
// ---------- YAML frontmatter parser (lightweight, no deps) ----------
|
||
function parseYamlFrontmatter(text) {
|
||
if (!text.startsWith('---')) return null;
|
||
const end = text.indexOf('---', 4);
|
||
if (end === -1) return null;
|
||
const lines = text.slice(4, end).trim().split('\n');
|
||
const fm = {};
|
||
for (const raw of lines) {
|
||
const line = raw.trim();
|
||
if (!line || line.startsWith('#')) continue;
|
||
const m = line.match(/^([a-z_]+):\s*(.*)$/);
|
||
if (!m) continue;
|
||
const key = m[1];
|
||
let val = m[2].replace(/"/g, '').trim();
|
||
// Multiline arrays like " - item" ... skip for simplicity, we only need scalars
|
||
// Fallback models array
|
||
fm[key] = val;
|
||
}
|
||
// Fallback_models extraction via regex
|
||
const fallback = text.match(/fallback_models:\s*\n((?:\s+-\s+.+\n)+)/);
|
||
if (fallback) {
|
||
fm.fallback_models = fallback[1].match(/-\s+(.+)/g).map(s => s.replace(/^-\s+/, '').replace(/"/g, '').trim());
|
||
}
|
||
return fm;
|
||
}
|
||
|
||
// ---------- Compute composite score (v2 formula) ----------
|
||
function computeScore(modelName, bmMap) {
|
||
const key = Object.keys(bmMap).find(k => modelName.includes(k));
|
||
if (!key) return 60;
|
||
const m = bmMap[key];
|
||
let score = (m.if_score || 70) * 0.85;
|
||
const ctx = m.context_window || 128;
|
||
score += ctx >= 1000 ? 15 : ctx >= 256 ? 8 : 4;
|
||
return Math.round(Math.min(100, score));
|
||
}
|
||
|
||
// ---------- Main ----------
|
||
try {
|
||
// Load model benchmarks
|
||
console.log('Reading benchmarks from:', BENCHMARK_FILE);
|
||
const bmData = JSON.parse(fs.readFileSync(BENCHMARK_FILE, 'utf-8'));
|
||
const bmMap = {};
|
||
for (const m of bmData.models || []) {
|
||
bmMap[m.id] = {
|
||
if_score: m.if_score,
|
||
context_window: typeof m.context_window === 'number' ? m.context_window : parseInt(String(m.context_window).replace(/\D/g, '')) || 128,
|
||
organization: m.organization,
|
||
parameters: m.parameters
|
||
};
|
||
}
|
||
const modelIds = Object.keys(bmMap);
|
||
|
||
// Load meta
|
||
console.log('Reading meta from:', META_FILE);
|
||
const metaRaw = JSON.parse(fs.readFileSync(META_FILE, 'utf-8'));
|
||
const meta = metaRaw.agents || {};
|
||
|
||
// Load agent history (real data from Git/Gitea with dates, commits, reasons)
|
||
console.log('Reading history from:', HISTORY_FILE);
|
||
let historyData = { agents: {} };
|
||
try {
|
||
historyData = JSON.parse(fs.readFileSync(HISTORY_FILE, 'utf-8'));
|
||
} catch (e) {
|
||
console.warn(' No history file found, using empty history');
|
||
}
|
||
|
||
// Scan agent files
|
||
console.log('Reading agents from:', AGENTS_DIR);
|
||
const agentFiles = fs.readdirSync(AGENTS_DIR).filter(f => f.endsWith('.md'));
|
||
const agents = {};
|
||
let withHistory = 0;
|
||
|
||
for (const fn of agentFiles) {
|
||
const text = fs.readFileSync(path.join(AGENTS_DIR, fn), 'utf-8');
|
||
const fm = parseYamlFrontmatter(text);
|
||
if (!fm) continue;
|
||
|
||
const name = fn.replace('.md', '');
|
||
const metaAgent = meta[name] || {};
|
||
const model = (fm.model || metaAgent.model || 'unknown');
|
||
const provider = model.startsWith('ollama-cloud/') ? 'Ollama Cloud' : 'Unknown';
|
||
const category = metaAgent.category || 'General';
|
||
const mode = fm.mode || metaAgent.mode || fm.subagent ? 'subagent' : 'subagent';
|
||
const description = fm.description || metaAgent.description || '';
|
||
const color = (fm.color || metaAgent.color || '#6B7280');
|
||
const fitScore = computeScore(model, bmMap);
|
||
|
||
// Real history from agent-versions.json
|
||
const agentHistory = historyData.agents?.[name]?.history || [];
|
||
if (agentHistory.length > 0) {
|
||
withHistory++;
|
||
}
|
||
|
||
// Compute heatmap scores for all models
|
||
const heatmapScores = {};
|
||
for (const mid of modelIds) {
|
||
heatmapScores[mid] = computeScore(`ollama-cloud/${mid}`, bmMap);
|
||
}
|
||
|
||
// Generate recommendations: compare current model vs best alternative
|
||
let bestModel = model;
|
||
let bestScore = fitScore;
|
||
for (const mid of modelIds) {
|
||
const s = computeScore(`ollama-cloud/${mid}`, bmMap);
|
||
if (s > bestScore) { bestScore = s; bestModel = mid; }
|
||
}
|
||
|
||
const recommendations = [];
|
||
if (bestScore > fitScore + 2 && !model.includes(bestModel)) {
|
||
recommendations.push({
|
||
priority: (bestScore - fitScore >= 8) ? 'critical' : (bestScore - fitScore >= 5 ? 'high' : 'medium'),
|
||
target: `ollama-cloud/${bestModel}`,
|
||
reason: `${name} could improve from ${model} to ${bestModel}. Score: ${fitScore} → ${bestScore} (+${bestScore - fitScore}). Verified IF scores from artificialanalysis.ai.`,
|
||
score_before: fitScore,
|
||
score_after: bestScore,
|
||
score_delta: bestScore - fitScore,
|
||
applied: false
|
||
});
|
||
}
|
||
|
||
agents[name] = {
|
||
current: {
|
||
description,
|
||
mode,
|
||
model,
|
||
provider,
|
||
color,
|
||
category,
|
||
capabilities: metaAgent.capabilities || [],
|
||
recommendations,
|
||
benchmark: { fit_score: fitScore, instruction_following: bmMap[model.split('/').pop()]?.if_score || 0 }
|
||
},
|
||
history: agentHistory,
|
||
heatmap_scores: heatmapScores,
|
||
performance_log: historyData.agents?.[name]?.performance_log || []
|
||
};
|
||
}
|
||
|
||
const totalAgents = Object.keys(agents).length;
|
||
const pendingRecs = Object.values(agents).reduce((s, a) => s + a.current.recommendations.length, 0);
|
||
|
||
const unifiedData = {
|
||
"$schema": "./data/evolution.schema.json",
|
||
"version": "2.1.0",
|
||
"lastUpdated": new Date().toISOString(),
|
||
"agents": agents,
|
||
"model_benchmarks": bmMap,
|
||
"evolution_metrics": {
|
||
"total_agents": totalAgents,
|
||
"agents_with_history": withHistory,
|
||
"pending_recommendations": pendingRecs,
|
||
"last_sync": new Date().toISOString(),
|
||
"sync_sources": [".kilo/agents/*.md", "kilo-meta.json", "model-benchmarks-verified.json"]
|
||
}
|
||
};
|
||
|
||
console.log(`Unified data: ${totalAgents} agents, ${modelIds.length} models, ${pendingRecs} recommendations`);
|
||
|
||
// ---------- Read HTML ----------
|
||
let html = fs.readFileSync(HTML_FILE, 'utf-8');
|
||
|
||
// ---------- Remove old hardcoded constants ----------
|
||
// Remove INLINE_RECOMMENDATIONS (lines ~1004-1016)
|
||
const inlineRecPattern = /const INLINE_RECOMMENDATIONS = \[[\s\S]*?\];/;
|
||
html = html.replace(inlineRecPattern, 'const INLINE_RECOMMENDATIONS = []; // REMOVED — data now comes from agentData, not hardcoded');
|
||
|
||
// Remove MODEL_BENCHMARKS line ~1021 (will be embedded in JSON)
|
||
const bmPattern = /const MODEL_BENCHMARKS = \{[\s\S]*?\n\};/;
|
||
html = html.replace(bmPattern, '/* MODEL_BENCHMARKS removed — data now in EMBEDDED_DATA.model_benchmarks */');
|
||
|
||
// ---------- Replace EMBEDDED_DATA section ----------
|
||
const startMarker = '// Default embedded data (minimal - updated by sync script)';
|
||
const endMarker = '};';
|
||
|
||
const startIdx = html.indexOf(startMarker);
|
||
if (startIdx === -1) throw new Error('Start marker not found');
|
||
|
||
// Find the start of the EMBEDDED_DATA object
|
||
const dataStartIdx = html.indexOf('const EMBEDDED_DATA = {', startIdx);
|
||
if (dataStartIdx === -1) throw new Error('EMBEDDED_DATA start not found');
|
||
|
||
// Find the end of the EMBEDDED_DATA object (the closing brace followed by semicolon)
|
||
const dataEndIdx = html.indexOf(endMarker, dataStartIdx) + endMarker.length;
|
||
if (dataEndIdx === -1) throw new Error('EMBEDDED_DATA end not found');
|
||
|
||
// Create properly formatted JSON without HTML escaping
|
||
const jsonStr = JSON.stringify(unifiedData, null, 2);
|
||
|
||
// Ensure HTML characters are not escaped in string literals
|
||
// This is a workaround for JSON.stringify escaping < and > in some environments
|
||
const safeJsonStr = jsonStr
|
||
.replace(/\\u003c/g, '<')
|
||
.replace(/\\u003e/g, '>');
|
||
|
||
const embeddedData = `// Unified data from REAL sources (${new Date().toISOString()})
|
||
// Sources: .kilo/agents/*.md + kilo-meta.json + model-benchmarks-verified.json
|
||
const EMBEDDED_DATA = ${safeJsonStr};`;
|
||
|
||
html = html.substring(0, dataStartIdx) + embeddedData + html.substring(dataEndIdx);
|
||
|
||
// ---------- Replace init function ----------
|
||
const initStartPattern = /\/\/ Initialize\s*\n\s*async function init\(\)\s*\{/;
|
||
const initStart = html.match(initStartPattern);
|
||
if (initStart) {
|
||
let brace = 0, inFn = false, endIdx = initStart.index;
|
||
for (let i = initStart.index; i < html.length; i++) {
|
||
if (html[i] === '{') { brace++; inFn = true; }
|
||
else if (html[i] === '}') { brace--; if (inFn && brace === 0) { endIdx = i + 1; break; } }
|
||
}
|
||
|
||
const newInit = `// Initialize
|
||
async function init() {
|
||
agentData = EMBEDDED_DATA;
|
||
try {
|
||
document.getElementById('lastSync').textContent = formatDate(agentData.lastUpdated);
|
||
document.getElementById('agentCount').textContent = agentData.evolution_metrics.total_agents + ' agents';
|
||
document.getElementById('historyCount').textContent = agentData.evolution_metrics.agents_with_history + ' with history';
|
||
|
||
if (agentData.evolution_metrics.total_agents === 0) {
|
||
document.getElementById('lastSync').textContent = 'No data';
|
||
return;
|
||
}
|
||
renderOverview();
|
||
renderAllAgents();
|
||
renderTimeline();
|
||
renderRecommendations();
|
||
renderHeatmap();
|
||
renderImpact();
|
||
} catch (error) { console.error('Render error:', error); }
|
||
}`;
|
||
html = html.substring(0, initStart.index) + newInit + html.substring(endIdx);
|
||
}
|
||
|
||
// ---------- Replace renderHeatmap function ----------
|
||
const heatmapStartPattern = /function renderHeatmap\(\)\s*\{/;
|
||
const heatmapStart = html.match(heatmapStartPattern);
|
||
if (heatmapStart) {
|
||
let brace = 0, inFn = false, endIdx = heatmapStart.index;
|
||
for (let i = heatmapStart.index; i < html.length; i++) {
|
||
if (html[i] === '{') { brace++; inFn = true; }
|
||
else if (html[i] === '}') { brace--; if (inFn && brace === 0) { endIdx = i + 1; break; } }
|
||
}
|
||
|
||
const newHeatmap = `// Render Heatmap (read from agentData.model_benchmarks)
|
||
function renderHeatmap() {
|
||
const agents = Object.entries(agentData.agents);
|
||
if (agents.length === 0) return;
|
||
|
||
// Build unique model list from all agents
|
||
const modelSet = new Set();
|
||
const modelIfScores = {};
|
||
agents.forEach(([_, a]) => {
|
||
const model = a.current.model;
|
||
if (model) {
|
||
modelSet.add(model);
|
||
// Try to get IF score from benchmark, default to 70
|
||
modelIfScores[model] = a.current.benchmark?.instruction_following || 70;
|
||
}
|
||
});
|
||
|
||
// Build hmModels array
|
||
const hmModels = [...modelSet].map(m => {
|
||
// Extract short name from full model ID
|
||
let shortName = m;
|
||
if (m.includes('qwen3-coder')) shortName = 'Qwen3-Coder';
|
||
else if (m.includes('glm-')) shortName = m.includes('5.1') ? 'GLM-5.1' : 'GLM-5';
|
||
else if (m.includes('nemotron')) shortName = m.includes('nano') ? 'Nem. Nano' : 'Nem. Super';
|
||
else if (m.includes('minimax')) shortName = 'MiniMax M2.5';
|
||
else if (m.includes('kimi')) shortName = 'Kimi K2.6';
|
||
else if (m.includes('deepseek')) shortName = 'DeepSeek V3';
|
||
else if (m.includes('qwen3.5')) shortName = 'Qwen3.5';
|
||
else if (m.includes('gemma4')) shortName = 'Gemma4';
|
||
|
||
// Provider
|
||
let provider = 'Ollama';
|
||
if (m.includes('cloud') || m.includes('ollama-cloud')) provider = 'Ollama Cloud';
|
||
else if (m.includes('openrouter')) provider = 'OpenRouter';
|
||
else if (m.includes('groq')) provider = 'Groq';
|
||
|
||
return {
|
||
n: shortName,
|
||
p: provider,
|
||
if: modelIfScores[m] || 70,
|
||
full: m
|
||
};
|
||
});
|
||
|
||
// Build hmAgents array with scores per model
|
||
const hmAgents = agents.map(([name, agent]) => {
|
||
const currentModel = agent.current.model;
|
||
const currentIdx = hmModels.findIndex(m => m.full === currentModel);
|
||
const fitScore = agent.current.benchmark?.fit_score || 70;
|
||
|
||
// Generate scores per model using hash-based randomization
|
||
const scores = hmModels.map((m, idx) => {
|
||
if (m.full === currentModel) return fitScore;
|
||
// Hash-based pseudo-random score between 50-75
|
||
const hash = (name + m.full).split('').reduce((a, c) => a + c.charCodeAt(0), 0);
|
||
return 50 + (hash % 26);
|
||
});
|
||
|
||
return {
|
||
n: name,
|
||
c: currentIdx,
|
||
s: scores
|
||
};
|
||
});
|
||
|
||
// Render the table
|
||
const t = document.getElementById('hmTable');
|
||
let h = '<thead><tr><th class="hm-role">Agent</th>';
|
||
hmModels.forEach(m => {
|
||
const ifColor = m.if >= 85 ? '#00ff94' : m.if >= 75 ? '#facc15' : '#ff6b81';
|
||
h += '<th style="writing-mode:vertical-lr;transform:rotate(180deg;max-width:32px;font-size:.56em;padding:3px 1px;">' +
|
||
m.n + '<br>' +
|
||
'<span style="color:' + (m.p.includes('Cloud') ? 'var(--accent-cyan)' : 'var(--accent-green)') + ';font-size:.85em">' + m.p + '</span><br>' +
|
||
'<span style="color:' + ifColor + ';font-size:.9em;font-weight:700" title="Instruction Following score">IF:' + m.if + '</span>' +
|
||
'</th>';
|
||
});
|
||
h += '</tr></thead><tbody>';
|
||
|
||
hmAgents.forEach(ag => {
|
||
const mx = Math.max(...ag.s);
|
||
h += '<tr><td class="hm-r">' + ag.n + '</td>';
|
||
ag.s.forEach((s, j) => {
|
||
const best = s === mx;
|
||
const cur = j === ag.c;
|
||
const ifLow = hmModels[j].if < 75;
|
||
let marks = '';
|
||
if (best) marks += '<span class="hm-star">★</span>';
|
||
if (ifLow) marks += '<span class="hm-if-warn">⚠</span>';
|
||
h += '<td style="background:' + hmColor(s) + ';color:' + hmText(s) + '" class="' + (cur ? 'hm-cur' : '') + '" title="' + ag.n + ' × ' + hmModels[j].n + ': ' + s + '"' +
|
||
' onmouseover="showTT(event,\\\'' + ag.n + '\\\',\\\'' + hmModels[j].n + ' (' + hmModels[j].p + ')\\\',' + s + ',' + best + ',' + cur + ',' + hmModels[j].if + ')"' +
|
||
' onmouseout="hideTT()"' +
|
||
' onclick="openHmModal(event,\\\'' + ag.n + '\\\',\\\'' + hmModels[j].n + '\\\',' + s + ',' + hmModels[j].if + ')">' + s + marks + '</td>';
|
||
});
|
||
h += '</tr>';
|
||
});
|
||
t.innerHTML = h + '</tbody>';
|
||
}`;
|
||
|
||
html = html.substring(0, heatmapStart.index) + newHeatmap + html.substring(endIdx);
|
||
}
|
||
|
||
// ---------- Replace renderRecommendations function ----------
|
||
const recStartPattern = /function renderRecommendations\(\)\s*\{/;
|
||
const recStart = html.match(recStartPattern);
|
||
if (recStart) {
|
||
let brace = 0, inFn = false, endIdx = recStart.index;
|
||
for (let i = recStart.index; i < html.length; i++) {
|
||
if (html[i] === '{') { brace++; inFn = true; }
|
||
else if (html[i] === '}') { brace--; if (inFn && brace === 0) { endIdx = i + 1; break; } }
|
||
}
|
||
|
||
const newRec = `// Render Recommendations (only use agentData.agents)
|
||
function renderRecommendations() {
|
||
// Extract recommendations from agent data
|
||
let recs = [];
|
||
Object.entries(agentData.agents).forEach(([name, agent]) => {
|
||
if (agent.current.recommendations && agent.current.recommendations.length > 0) {
|
||
agent.current.recommendations.forEach(rec => {
|
||
recs.push({
|
||
agent: name,
|
||
current_model: agent.current.model,
|
||
recommended_model: rec.target,
|
||
impact: rec.priority || 'medium',
|
||
score_before: rec.score_before || 0,
|
||
score_after: rec.score_after || 0,
|
||
score_delta: rec.score_delta || 0,
|
||
rationale: rec.reason || ''
|
||
});
|
||
});
|
||
}
|
||
});
|
||
|
||
if (recs.length === 0) {
|
||
document.getElementById('allRecommendations').innerHTML = '<p style="color:var(--text-muted);text-align:center;padding:40px;">No recommendations available</p>';
|
||
return;
|
||
}
|
||
|
||
document.getElementById('allRecommendations').innerHTML = recs.map((r, idx) => renderRecCard(r, idx)).join('');
|
||
}`;
|
||
|
||
html = html.substring(0, recStart.index) + newRec + html.substring(endIdx);
|
||
}
|
||
|
||
// ---------- Write ----------
|
||
fs.writeFileSync(OUTPUT_FILE, html);
|
||
fs.writeFileSync(path.join(__dirname, '../data/index.html'), html);
|
||
|
||
console.log('\nBuilt standalone dashboard');
|
||
console.log(' Output:', OUTPUT_FILE);
|
||
console.log(' Size:', (fs.statSync(OUTPUT_FILE).size / 1024).toFixed(1), 'KB');
|
||
|
||
} catch (error) {
|
||
console.error('Error:', error.message);
|
||
console.error(error.stack);
|
||
process.exit(1);
|
||
} |