feat(scripts): add real-fit evaluation engine and supporting test scripts

- real-fit-engine.py: refactored to support --from-report, improved Ollama v1/chat/completions compatibility, agent name normalization
- run-focused-eval.py: run evaluations for specific agent/model pairs from CLI
- test_ollama_minimal.py/test_real_api.py: Ollama API connectivity tests
- real-fit-architecture.md: architecture overview document
- tests/scripts/: E2E landing test, analytics capture, evolution heatmap verification
- Remove real-fit-recalc.py (superseded by --from-report flag)
This commit is contained in:
Deploy Bot
2026-05-28 11:57:46 +01:00
parent a0e7bd99fb
commit 4071551476
10 changed files with 1219 additions and 312 deletions

View File

@@ -0,0 +1,89 @@
#!/usr/bin/env node
/**
* Quick capture + element check for Analytics Hierarchy Section
*/
const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const TARGET_URL = process.env.TARGET_URL || 'http://localhost:3002';
const OUTPUT_DIR = process.env.OUTPUT_DIR || '/app/tests/visual/current';
(async () => {
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
}
const browser = await chromium.launch({
headless: true,
args: ['--disable-setuid-sandbox', '--no-sandbox'],
});
const page = await browser.newPage({
viewport: { width: 1280, height: 900 },
});
console.log(`Navigating to: ${TARGET_URL}`);
await page.goto(TARGET_URL, { waitUntil: 'networkidle', timeout: 60000 });
await page.waitForTimeout(3000);
// Scroll to "Аналитическая иерархия"
const heading = page.locator('text=Аналитическая иерархия').first();
if (await heading.isVisible().catch(() => false)) {
console.log('Scrolling to Аналитическая иерархия section...');
await heading.scrollIntoViewIfNeeded();
await page.evaluate(() => window.scrollBy(0, -60));
await page.waitForTimeout(1500);
} else {
console.log('Heading not found, fallback scroll');
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight / 3));
await page.waitForTimeout(1500);
}
// Scroll further down to reveal cards 3 and 4 (heatmap, commands table)
await page.evaluate(() => window.scrollBy(0, 900));
await page.waitForTimeout(1000);
const screenshotPath = path.join(OUTPUT_DIR, 'analytics_section.png');
await page.screenshot({ path: screenshotPath, fullPage: false });
console.log(`Screenshot saved to: ${screenshotPath}`);
// Check for each card's evidence (use Russian text as it appears in the page)
const checks = [
{ label: 'Model tree with collapsible categories', text: 'Модели → Категории → Агенты' },
{ label: 'Category bars', text: 'Дистрибуция по категориям' },
{ label: 'Fit-score heatmap', text: 'Fit-score распределение' },
{ label: 'Commands table', text: 'Команды' },
];
const results = { visible: {}, issues: [] };
for (const c of checks) {
const found = await page.locator(`text=${c.text}`).first().isVisible({ timeout: 3000 }).catch(() => false);
if (found) {
const textContent = await page.locator(`text=${c.text}`).first().textContent({ timeout: 3000 }).catch(() => '');
results.visible[c.label] = textContent;
} else {
results.issues.push(`${c.label} (searching text "${c.text}") — NOT FOUND`);
}
}
const reportPath = path.join(OUTPUT_DIR, 'analytics_section_report.json');
fs.writeFileSync(reportPath, JSON.stringify(results, null, 2));
console.log(`Report saved to: ${reportPath}`);
// Also write summary to stdout
console.log('\n=== Scan Results ===');
if (Object.keys(results.visible).length === 4) {
console.log('All 4 analytics cards are visible.');
} else {
console.log(`Visible: ${Object.keys(results.visible).join(', ')}`);
console.log(`Missing: ${results.issues.join(', ')}`);
}
await browser.close();
})().catch((err) => {
console.error('Fatal error:', err);
process.exit(1);
});

View File

@@ -0,0 +1,381 @@
#!/usr/bin/env node
/**
* E2E Test Suite for APAW Landing Page
* Tests: page load, console errors, API state, analytics, heatmap modal,
* close interactions, visual regression.
*
* Usage: node e2e-landing-test.js
* Environment: TARGET_URL (default http://host.docker.internal:3002)
*/
const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const pixelmatch = require('pixelmatch');
const { PNG } = require('pngjs');
const { launchBrowser, newContext, navigateTo } = require('./lib/browser-launcher');
const TARGET_URL = process.env.TARGET_URL || 'http://host.docker.internal:3002';
const REPORTS_DIR = process.env.REPORTS_DIR || path.join(__dirname, '..', 'reports');
const BASELINE_DIR = process.env.BASELINE_DIR || path.join(__dirname, '..', 'visual', 'baseline');
const CURRENT_DIR = process.env.CURRENT_DIR || path.join(__dirname, '..', 'visual', 'current');
const VIEWPORT = { width: 1280, height: 900 };
async function main() {
console.log('═══════════════════════════════════════════════════');
console.log(' APAW Landing E2E Tests');
console.log('═══════════════════════════════════════════════════\n');
console.log(`Target: ${TARGET_URL}\n`);
for (const dir of [REPORTS_DIR, CURRENT_DIR]) {
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
}
const browser = await launchBrowser();
const context = await newContext(browser, { viewport: VIEWPORT });
const page = await context.newPage();
const consoleErrors = [];
const consoleWarnings = [];
const networkErrors = [];
let networkRequests = [];
const results = [];
page.on('console', msg => {
if (msg.type() === 'error') consoleErrors.push(msg.text());
else if (msg.type() === 'warning') consoleWarnings.push(msg.text());
});
page.on('requestfailed', request => {
networkErrors.push({ url: request.url(), failure: request.failure()?.errorText || 'Unknown' });
});
page.on('response', response => {
if (response.status() >= 400) {
networkErrors.push({ url: response.url(), status: response.status() });
}
});
// ============================================================
// Test 1: Page loads without console errors
// ============================================================
{
console.log('┌─────────────────────────────────────────────────┐');
console.log('│ Test 1: Page loads without console errors │');
console.log('└─────────────────────────────────────────────────┘');
try {
const response = await navigateTo(page, `${TARGET_URL}`, { waitUntil: 'commit', timeout: 30000, delay: 3000 });
const status = response?.status() || 0;
// Wait for analytics section to be present in DOM
await page.waitForSelector('#analytics', { timeout: 10000 }).catch(() => {});
const title = await page.title();
const pageLoaded = status === 200 && title.includes('APAW');
result(results, '1_page_load', pageLoaded && consoleErrors.length === 0, `HTTP ${status}, title: "${title}", console errors: ${consoleErrors.length}`);
} catch (e) {
result(results, '1_page_load', false, e.message);
}
}
// ============================================================
// Test 2: /api/state loads successfully
// ============================================================
{
console.log('\n┌─────────────────────────────────────────────────┐');
console.log('│ Test 2: /api/state loads successfully │');
console.log('└─────────────────────────────────────────────────┘');
try {
const apiResponse = await page.evaluate(async (url) => {
const res = await fetch(`${url}/api/state`);
const data = await res.json().catch(() => null);
return { status: res.status, ok: res.ok, hasAgents: !!(data && Array.isArray(data.agents) && data.agents.length > 0) };
}, TARGET_URL);
result(results, '2_api_state', apiResponse.ok && apiResponse.hasAgents,
`status=${apiResponse.status}, hasAgents=${apiResponse.hasAgents}`);
} catch (e) {
result(results, '2_api_state', false, e.message);
}
}
// ============================================================
// Test 3: #analytics section is visible with heatmap rendered
// ============================================================
{
console.log('\n┌─────────────────────────────────────────────────┐');
console.log('│ Test 3: #analytics visible + heatmap rendered │');
console.log('└─────────────────────────────────────────────────┘');
try {
const analytics = await page.locator('#analytics').first();
const isVisible = await analytics.isVisible().catch(() => false);
// Scroll to analytics
await page.evaluate(() => {
const el = document.getElementById('analytics');
if (el) el.scrollIntoView({ block: 'start' });
});
await page.waitForTimeout(800);
const heatmap = await page.locator('#fit-heatmap').first();
const heatmapVisible = await heatmap.isVisible().catch(() => false);
const cellCount = await heatmap.locator('.heatmap__cell').count().catch(() => 0);
result(results, '3_analytics_heatmap', isVisible && heatmapVisible && cellCount > 0,
`analytics visible=${isVisible}, heatmap visible=${heatmapVisible}, cells=${cellCount}`);
} catch (e) {
result(results, '3_analytics_heatmap', false, e.message);
}
}
// ============================================================
// Test 4: Clicking a heatmap cell opens #fit-modal
// ============================================================
{
console.log('\n┌─────────────────────────────────────────────────┐');
console.log('│ Test 4: Clicking heatmap cell opens #fit-modal │');
console.log('└─────────────────────────────────────────────────┘');
try {
const cell = await page.locator('#fit-heatmap .heatmap__cell').first();
await cell.scrollIntoViewIfNeeded();
await page.waitForTimeout(500);
await cell.click();
await page.waitForTimeout(500);
const modal = await page.locator('#fit-modal').first();
const modalVisible = await modal.isVisible().catch(() => false);
const isOpen = await modal.evaluate(el => el.classList.contains('is-open')).catch(() => false);
result(results, '4_click_opens_modal', modalVisible && isOpen,
`modal visible=${modalVisible}, class is-open=${isOpen}`);
} catch (e) {
result(results, '4_click_opens_modal', false, e.message);
}
}
// ============================================================
// Test 5: Modal displays agent name, model, fit score,
// breakdown dimensions, and explanation
// ============================================================
{
console.log('\n┌─────────────────────────────────────────────────┐');
console.log('│ Test 5: Modal content (name, model, score, etc) │');
console.log('└─────────────────────────────────────────────────┘');
try {
const modal = await page.locator('#fit-modal').first();
const agentName = await modal.locator('#modal-agent-name').textContent().catch(() => '');
const modelText = await modal.locator('#modal-model').textContent().catch(() => '');
const scoreText = await modal.locator('#modal-score').textContent().catch(() => '');
const explanation = await modal.locator('#modal-explanation').textContent().catch(() => '');
const dims = await modal.locator('#modal-breakdown .modal__dimension').count().catch(() => 0);
const nameOk = agentName.trim().length > 0 && agentName !== 'Agent';
const modelOk = modelText.trim().length > 0;
const scoreOk = !isNaN(parseInt(scoreText, 10)) && parseInt(scoreText, 10) > 0;
const dimsOk = dims >= 4;
const explOk = explanation.trim().length > 0;
result(results, '5_modal_content',
nameOk && modelOk && scoreOk && dimsOk && explOk,
`name="${agentName.trim()}", model="${modelText.trim()}", score="${scoreText.trim()}", dimensions=${dims}, explanation=${explOk ? 'present' : 'missing'}`);
} catch (e) {
result(results, '5_modal_content', false, e.message);
}
}
// ============================================================
// Test 6: Modal can be closed via close button and Escape key
// ============================================================
{
const modal = await page.locator('#fit-modal').first();
// 6a: close button
{
console.log('\n┌─────────────────────────────────────────────────┐');
console.log('│ Test 6a: Close via close button │');
console.log('└─────────────────────────────────────────────────┘');
try {
await modal.locator('.modal__close').click();
await page.waitForTimeout(600);
// If CSS transition leaves it briefly visible, wait a tick
const isOpen = await modal.evaluate(el => el.classList.contains('is-open')).catch(() => true);
const visible = await modal.isVisible().catch(() => true);
result(results, '6_close_button', !isOpen && !visible, `is-open=${isOpen}, visible=${visible}`);
} catch (e) {
result(results, '6_close_button', false, e.message);
}
}
// 6b: Escape key
{
console.log('\n┌─────────────────────────────────────────────────┐');
console.log('│ Test 6b: Close via Escape key │');
console.log('└─────────────────────────────────────────────────┘');
try {
// If modal is still open (bug), force close via JS
const stillOpen = await modal.evaluate(el => el.classList.contains('is-open')).catch(() => false);
if (stillOpen) await page.evaluate(() => { if (typeof closeFitModal === 'function') closeFitModal(); });
await page.waitForTimeout(400);
const cell = await page.locator('#fit-heatmap .heatmap__cell').first();
await cell.evaluate(el => el.scrollIntoView({ block: 'center' }));
await cell.click({ force: true });
await page.waitForTimeout(500);
await page.keyboard.press('Escape');
await page.waitForTimeout(500);
const isOpen = await modal.evaluate(el => el.classList.contains('is-open')).catch(() => true);
const visible = await modal.isVisible().catch(() => true);
result(results, '6_escape_key', !isOpen && !visible, `is-open=${isOpen}, visible=${visible}`);
} catch (e) {
result(results, '6_escape_key', false, e.message);
}
}
}
// ============================================================
// Screenshot of opened modal
// ============================================================
{
console.log('\n┌─────────────────────────────────────────────────┐');
console.log('│ Capturing modal screenshot │');
console.log('└─────────────────────────────────────────────────┘');
try {
const cell = await page.locator('#fit-heatmap .heatmap__cell').first();
await cell.click();
await page.waitForTimeout(600);
const modal = await page.locator('#fit-modal').first();
const modalBox = await modal.boundingBox().catch(() => null);
const screenshotPath = path.join(CURRENT_DIR, 'modal_opened.png');
if (modalBox) {
await page.screenshot({ path: screenshotPath, clip: modalBox });
} else {
await page.screenshot({ path: screenshotPath });
}
console.log(` ✅ Screenshot saved: ${screenshotPath}`);
result(results, 'screenshot_modal', true, screenshotPath);
} catch (e) {
console.log(` ❌ Screenshot failed: ${e.message}`);
result(results, 'screenshot_modal', false, e.message);
}
}
// ============================================================
// Test 7: No visual regressions from baseline
// ============================================================
{
console.log('\n┌─────────────────────────────────────────────────┐');
console.log('│ Test 7: Visual regression (baseline vs current) │');
console.log('└─────────────────────────────────────────────────┘');
const baselinePath = path.join(BASELINE_DIR, 'homepage_desktop.png');
const currentPath = path.join(CURRENT_DIR, 'homepage_desktop.png');
// Capture current homepage for comparison
try {
await navigateTo(page, `${TARGET_URL}`, { waitUntil: 'commit', delay: 3000 });
await page.screenshot({ path: currentPath, fullPage: true });
} catch (e) {
console.log(` ⚠️ Could not capture current screenshot: ${e.message}`);
}
if (!fs.existsSync(baselinePath)) {
console.log(` ⚠️ Baseline not found at ${baselinePath}`);
result(results, '7_visual_regression', null, 'SKIP: baseline missing');
} else if (!fs.existsSync(currentPath)) {
result(results, '7_visual_regression', false, 'Current screenshot capture failed');
} else {
try {
const baseline = PNG.sync.read(fs.readFileSync(baselinePath));
const current = PNG.sync.read(fs.readFileSync(currentPath));
if (baseline.width !== current.width || baseline.height !== current.height) {
result(results, '7_visual_regression', false, `Size mismatch: ${baseline.width}x${baseline.height} vs ${current.width}x${current.height}`);
} else {
const diff = new PNG({ width: baseline.width, height: baseline.height });
const numDiff = pixelmatch(baseline.data, current.data, diff.data, baseline.width, baseline.height, { threshold: 0.1 });
const diffPercent = (numDiff / (baseline.width * baseline.height)) * 100;
const passed = diffPercent <= 5.0; // 5% tolerance
result(results, '7_visual_regression', passed, `diff pixels=${numDiff} (${diffPercent.toFixed(2)}%)`);
if (!passed) {
const diffPath = path.join(CURRENT_DIR, 'homepage_desktop_diff.png');
fs.writeFileSync(diffPath, PNG.sync.write(diff));
console.log(` 📸 Diff saved: ${diffPath}`);
}
}
} catch (e) {
result(results, '7_visual_regression', false, e.message);
}
}
}
await context.close();
await browser.close();
// ============================================================
// Summary
// ============================================================
console.log('\n═══════════════════════════════════════════════════');
console.log(' Results Summary');
console.log('═══════════════════════════════════════════════════\n');
for (const r of results) {
const icon = r.pass === true ? '✅' : r.pass === false ? '❌' : '⏭️';
console.log(`${icon} ${r.name}`);
console.log(` ${r.detail}`);
}
console.log(`\n📊 Console errors: ${consoleErrors.length}`);
console.log(`📊 Console warnings: ${consoleWarnings.length}`);
console.log(`📊 Network errors: ${networkErrors.length}`);
const failures = results.filter(r => r.pass === false);
const passed = results.filter(r => r.pass === true);
const skipped = results.filter(r => r.pass === null);
console.log(`\n✅ Passed: ${passed.length}`);
console.log(`❌ Failed: ${failures.length}`);
console.log(`⏭️ Skipped: ${skipped.length}`);
const reportPath = path.join(REPORTS_DIR, 'e2e-landing-report.json');
fs.writeFileSync(reportPath, JSON.stringify({
timestamp: new Date().toISOString(),
targetUrl: TARGET_URL,
results,
summary: {
passed: passed.length,
failed: failures.length,
skipped: skipped.length,
consoleErrors: consoleErrors.length,
consoleWarnings: consoleWarnings.length,
networkErrors: networkErrors.length,
},
}, null, 2));
console.log(`\n📄 Report: ${reportPath}`);
process.exit(failures.length > 0 ? 1 : 0);
}
function result(list, name, pass, detail) {
list.push({ name, pass, detail });
const icon = pass === true ? '✅' : pass === false ? '❌' : '⏭️';
console.log(` ${icon} ${name}: ${detail}`);
}
main().catch(err => {
console.error('Fatal:', err);
process.exit(1);
});

View File

@@ -0,0 +1,79 @@
const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const TARGET = process.env.TARGET_URL || 'http://host.docker.internal:3003';
const OUT_DIR = process.env.OUT_DIR || path.join(__dirname, '..', 'reports');
(async () => {
if (!fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });
const browser = await chromium.launch({ headless: true });
const context = await browser.newContext({ viewport: { width: 1600, height: 1200 } });
const page = await context.newPage();
// Capture console & network errors
const consoleErrors = [];
const networkErrors = [];
page.on('console', msg => { if (msg.type() === 'error') consoleErrors.push(msg.text()); });
page.on('requestfailed', req => networkErrors.push({ url: req.url(), error: req.failure()?.errorText }));
page.on('response', res => { if (res.status() >= 400) networkErrors.push({ url: res.url(), status: res.status() }); });
console.log('[HEATMAP] Navigating to', TARGET);
await page.goto(TARGET, { waitUntil: 'domcontentloaded', timeout: 30000 });
await page.waitForTimeout(1500); // wait for fetch/dashboard-data
const tabBtn = page.locator('button.tab-btn', { hasText: /Heatmap/ }).first();
if (await tabBtn.count()) {
await tabBtn.click();
console.log('[HEATMAP] Clicked Heatmap tab');
} else {
console.log('[HEATMAP] No Heatmap tab found, tabs may already be active');
}
await page.waitForTimeout(2000); // let table build from JS
// Get table dimensions
const rows = await page.locator('#hmTable tbody tr').count().catch(() => 0);
const colCount = await page.locator('#hmTable thead th').count().catch(() => 0);
console.log(`[HEATMAP] Table: ${rows} rows, ${colCount} columns`);
// Screenshot full page of heatmap tab
const screenshotPath = path.join(OUT_DIR, 'heatmap.png');
await page.screenshot({ path: screenshotPath, fullPage: true });
console.log('[HEATMAP] Screenshot saved to', screenshotPath);
// Also screenshot just the table if possible
const tableScreenshotPath = path.join(OUT_DIR, 'heatmap-table.png');
const tableEl = page.locator('#hmTable').first();
if (await tableEl.count() && rows > 0) {
await tableEl.screenshot({ path: tableScreenshotPath });
console.log('[HEATMAP] Table screenshot saved to', tableScreenshotPath);
}
// Read cell data
const cellTexts = await page.locator('#hmTable tbody td').allTextContents().catch(() => []);
console.log('[HEATMAP] First 30 cell texts:', cellTexts.slice(0, 30).map(t => t.trim()));
// Dump innerHTML
const innerHTML = await page.locator('#hmTable').innerHTML().catch(() => null);
// Report
const report = {
target: TARGET,
table: { rows, colCount },
cellSamples: cellTexts.slice(0, 30).map(t => t.trim()),
consoleErrors,
networkErrors,
screenshots: [screenshotPath, tableScreenshotPath].filter(f => fs.existsSync(f)),
innerHTML: innerHTML ? innerHTML.slice(0, 2000) : null,
ok: rows > 0 && colCount > 0,
};
const reportPath = path.join(OUT_DIR, 'heatmap-report.json');
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
console.log('[HEATMAP] Report saved to', reportPath);
await browser.close();
process.exit(report.ok ? 0 : 1);
})();