4071551476
- real-fit-engine.py: refactored to support --from-report, improved Ollama v1/chat/completions compatibility, agent name normalization - run-focused-eval.py: run evaluations for specific agent/model pairs from CLI - test_ollama_minimal.py/test_real_api.py: Ollama API connectivity tests - real-fit-architecture.md: architecture overview document - tests/scripts/: E2E landing test, analytics capture, evolution heatmap verification - Remove real-fit-recalc.py (superseded by --from-report flag)
382 lines
18 KiB
JavaScript
382 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* E2E Test Suite for APAW Landing Page
|
|
* Tests: page load, console errors, API state, analytics, heatmap modal,
|
|
* close interactions, visual regression.
|
|
*
|
|
* Usage: node e2e-landing-test.js
|
|
* Environment: TARGET_URL (default http://host.docker.internal:3002)
|
|
*/
|
|
|
|
const { chromium } = require('playwright');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const pixelmatch = require('pixelmatch');
|
|
const { PNG } = require('pngjs');
|
|
const { launchBrowser, newContext, navigateTo } = require('./lib/browser-launcher');
|
|
|
|
const TARGET_URL = process.env.TARGET_URL || 'http://host.docker.internal:3002';
|
|
const REPORTS_DIR = process.env.REPORTS_DIR || path.join(__dirname, '..', 'reports');
|
|
const BASELINE_DIR = process.env.BASELINE_DIR || path.join(__dirname, '..', 'visual', 'baseline');
|
|
const CURRENT_DIR = process.env.CURRENT_DIR || path.join(__dirname, '..', 'visual', 'current');
|
|
|
|
const VIEWPORT = { width: 1280, height: 900 };
|
|
|
|
async function main() {
|
|
console.log('═══════════════════════════════════════════════════');
|
|
console.log(' APAW Landing E2E Tests');
|
|
console.log('═══════════════════════════════════════════════════\n');
|
|
console.log(`Target: ${TARGET_URL}\n`);
|
|
|
|
for (const dir of [REPORTS_DIR, CURRENT_DIR]) {
|
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
}
|
|
|
|
const browser = await launchBrowser();
|
|
const context = await newContext(browser, { viewport: VIEWPORT });
|
|
const page = await context.newPage();
|
|
|
|
const consoleErrors = [];
|
|
const consoleWarnings = [];
|
|
const networkErrors = [];
|
|
let networkRequests = [];
|
|
|
|
const results = [];
|
|
|
|
page.on('console', msg => {
|
|
if (msg.type() === 'error') consoleErrors.push(msg.text());
|
|
else if (msg.type() === 'warning') consoleWarnings.push(msg.text());
|
|
});
|
|
|
|
page.on('requestfailed', request => {
|
|
networkErrors.push({ url: request.url(), failure: request.failure()?.errorText || 'Unknown' });
|
|
});
|
|
|
|
page.on('response', response => {
|
|
if (response.status() >= 400) {
|
|
networkErrors.push({ url: response.url(), status: response.status() });
|
|
}
|
|
});
|
|
|
|
// ============================================================
|
|
// Test 1: Page loads without console errors
|
|
// ============================================================
|
|
{
|
|
console.log('┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Test 1: Page loads without console errors │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
try {
|
|
const response = await navigateTo(page, `${TARGET_URL}`, { waitUntil: 'commit', timeout: 30000, delay: 3000 });
|
|
const status = response?.status() || 0;
|
|
|
|
// Wait for analytics section to be present in DOM
|
|
await page.waitForSelector('#analytics', { timeout: 10000 }).catch(() => {});
|
|
|
|
const title = await page.title();
|
|
const pageLoaded = status === 200 && title.includes('APAW');
|
|
|
|
result(results, '1_page_load', pageLoaded && consoleErrors.length === 0, `HTTP ${status}, title: "${title}", console errors: ${consoleErrors.length}`);
|
|
} catch (e) {
|
|
result(results, '1_page_load', false, e.message);
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Test 2: /api/state loads successfully
|
|
// ============================================================
|
|
{
|
|
console.log('\n┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Test 2: /api/state loads successfully │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
try {
|
|
const apiResponse = await page.evaluate(async (url) => {
|
|
const res = await fetch(`${url}/api/state`);
|
|
const data = await res.json().catch(() => null);
|
|
return { status: res.status, ok: res.ok, hasAgents: !!(data && Array.isArray(data.agents) && data.agents.length > 0) };
|
|
}, TARGET_URL);
|
|
|
|
result(results, '2_api_state', apiResponse.ok && apiResponse.hasAgents,
|
|
`status=${apiResponse.status}, hasAgents=${apiResponse.hasAgents}`);
|
|
} catch (e) {
|
|
result(results, '2_api_state', false, e.message);
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Test 3: #analytics section is visible with heatmap rendered
|
|
// ============================================================
|
|
{
|
|
console.log('\n┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Test 3: #analytics visible + heatmap rendered │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
try {
|
|
const analytics = await page.locator('#analytics').first();
|
|
const isVisible = await analytics.isVisible().catch(() => false);
|
|
|
|
// Scroll to analytics
|
|
await page.evaluate(() => {
|
|
const el = document.getElementById('analytics');
|
|
if (el) el.scrollIntoView({ block: 'start' });
|
|
});
|
|
await page.waitForTimeout(800);
|
|
|
|
const heatmap = await page.locator('#fit-heatmap').first();
|
|
const heatmapVisible = await heatmap.isVisible().catch(() => false);
|
|
const cellCount = await heatmap.locator('.heatmap__cell').count().catch(() => 0);
|
|
|
|
result(results, '3_analytics_heatmap', isVisible && heatmapVisible && cellCount > 0,
|
|
`analytics visible=${isVisible}, heatmap visible=${heatmapVisible}, cells=${cellCount}`);
|
|
} catch (e) {
|
|
result(results, '3_analytics_heatmap', false, e.message);
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Test 4: Clicking a heatmap cell opens #fit-modal
|
|
// ============================================================
|
|
{
|
|
console.log('\n┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Test 4: Clicking heatmap cell opens #fit-modal │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
try {
|
|
const cell = await page.locator('#fit-heatmap .heatmap__cell').first();
|
|
await cell.scrollIntoViewIfNeeded();
|
|
await page.waitForTimeout(500);
|
|
await cell.click();
|
|
await page.waitForTimeout(500);
|
|
|
|
const modal = await page.locator('#fit-modal').first();
|
|
const modalVisible = await modal.isVisible().catch(() => false);
|
|
const isOpen = await modal.evaluate(el => el.classList.contains('is-open')).catch(() => false);
|
|
|
|
result(results, '4_click_opens_modal', modalVisible && isOpen,
|
|
`modal visible=${modalVisible}, class is-open=${isOpen}`);
|
|
} catch (e) {
|
|
result(results, '4_click_opens_modal', false, e.message);
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Test 5: Modal displays agent name, model, fit score,
|
|
// breakdown dimensions, and explanation
|
|
// ============================================================
|
|
{
|
|
console.log('\n┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Test 5: Modal content (name, model, score, etc) │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
try {
|
|
const modal = await page.locator('#fit-modal').first();
|
|
const agentName = await modal.locator('#modal-agent-name').textContent().catch(() => '');
|
|
const modelText = await modal.locator('#modal-model').textContent().catch(() => '');
|
|
const scoreText = await modal.locator('#modal-score').textContent().catch(() => '');
|
|
const explanation = await modal.locator('#modal-explanation').textContent().catch(() => '');
|
|
const dims = await modal.locator('#modal-breakdown .modal__dimension').count().catch(() => 0);
|
|
|
|
const nameOk = agentName.trim().length > 0 && agentName !== 'Agent';
|
|
const modelOk = modelText.trim().length > 0;
|
|
const scoreOk = !isNaN(parseInt(scoreText, 10)) && parseInt(scoreText, 10) > 0;
|
|
const dimsOk = dims >= 4;
|
|
const explOk = explanation.trim().length > 0;
|
|
|
|
result(results, '5_modal_content',
|
|
nameOk && modelOk && scoreOk && dimsOk && explOk,
|
|
`name="${agentName.trim()}", model="${modelText.trim()}", score="${scoreText.trim()}", dimensions=${dims}, explanation=${explOk ? 'present' : 'missing'}`);
|
|
} catch (e) {
|
|
result(results, '5_modal_content', false, e.message);
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Test 6: Modal can be closed via close button and Escape key
|
|
// ============================================================
|
|
{
|
|
const modal = await page.locator('#fit-modal').first();
|
|
|
|
// 6a: close button
|
|
{
|
|
console.log('\n┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Test 6a: Close via close button │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
try {
|
|
await modal.locator('.modal__close').click();
|
|
await page.waitForTimeout(600);
|
|
// If CSS transition leaves it briefly visible, wait a tick
|
|
const isOpen = await modal.evaluate(el => el.classList.contains('is-open')).catch(() => true);
|
|
const visible = await modal.isVisible().catch(() => true);
|
|
result(results, '6_close_button', !isOpen && !visible, `is-open=${isOpen}, visible=${visible}`);
|
|
} catch (e) {
|
|
result(results, '6_close_button', false, e.message);
|
|
}
|
|
}
|
|
|
|
// 6b: Escape key
|
|
{
|
|
console.log('\n┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Test 6b: Close via Escape key │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
try {
|
|
// If modal is still open (bug), force close via JS
|
|
const stillOpen = await modal.evaluate(el => el.classList.contains('is-open')).catch(() => false);
|
|
if (stillOpen) await page.evaluate(() => { if (typeof closeFitModal === 'function') closeFitModal(); });
|
|
await page.waitForTimeout(400);
|
|
|
|
const cell = await page.locator('#fit-heatmap .heatmap__cell').first();
|
|
await cell.evaluate(el => el.scrollIntoView({ block: 'center' }));
|
|
await cell.click({ force: true });
|
|
await page.waitForTimeout(500);
|
|
await page.keyboard.press('Escape');
|
|
await page.waitForTimeout(500);
|
|
const isOpen = await modal.evaluate(el => el.classList.contains('is-open')).catch(() => true);
|
|
const visible = await modal.isVisible().catch(() => true);
|
|
result(results, '6_escape_key', !isOpen && !visible, `is-open=${isOpen}, visible=${visible}`);
|
|
} catch (e) {
|
|
result(results, '6_escape_key', false, e.message);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Screenshot of opened modal
|
|
// ============================================================
|
|
{
|
|
console.log('\n┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Capturing modal screenshot │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
try {
|
|
const cell = await page.locator('#fit-heatmap .heatmap__cell').first();
|
|
await cell.click();
|
|
await page.waitForTimeout(600);
|
|
const modal = await page.locator('#fit-modal').first();
|
|
const modalBox = await modal.boundingBox().catch(() => null);
|
|
|
|
const screenshotPath = path.join(CURRENT_DIR, 'modal_opened.png');
|
|
if (modalBox) {
|
|
await page.screenshot({ path: screenshotPath, clip: modalBox });
|
|
} else {
|
|
await page.screenshot({ path: screenshotPath });
|
|
}
|
|
console.log(` ✅ Screenshot saved: ${screenshotPath}`);
|
|
result(results, 'screenshot_modal', true, screenshotPath);
|
|
} catch (e) {
|
|
console.log(` ❌ Screenshot failed: ${e.message}`);
|
|
result(results, 'screenshot_modal', false, e.message);
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Test 7: No visual regressions from baseline
|
|
// ============================================================
|
|
{
|
|
console.log('\n┌─────────────────────────────────────────────────┐');
|
|
console.log('│ Test 7: Visual regression (baseline vs current) │');
|
|
console.log('└─────────────────────────────────────────────────┘');
|
|
|
|
const baselinePath = path.join(BASELINE_DIR, 'homepage_desktop.png');
|
|
const currentPath = path.join(CURRENT_DIR, 'homepage_desktop.png');
|
|
|
|
// Capture current homepage for comparison
|
|
try {
|
|
await navigateTo(page, `${TARGET_URL}`, { waitUntil: 'commit', delay: 3000 });
|
|
await page.screenshot({ path: currentPath, fullPage: true });
|
|
} catch (e) {
|
|
console.log(` ⚠️ Could not capture current screenshot: ${e.message}`);
|
|
}
|
|
|
|
if (!fs.existsSync(baselinePath)) {
|
|
console.log(` ⚠️ Baseline not found at ${baselinePath}`);
|
|
result(results, '7_visual_regression', null, 'SKIP: baseline missing');
|
|
} else if (!fs.existsSync(currentPath)) {
|
|
result(results, '7_visual_regression', false, 'Current screenshot capture failed');
|
|
} else {
|
|
try {
|
|
const baseline = PNG.sync.read(fs.readFileSync(baselinePath));
|
|
const current = PNG.sync.read(fs.readFileSync(currentPath));
|
|
|
|
if (baseline.width !== current.width || baseline.height !== current.height) {
|
|
result(results, '7_visual_regression', false, `Size mismatch: ${baseline.width}x${baseline.height} vs ${current.width}x${current.height}`);
|
|
} else {
|
|
const diff = new PNG({ width: baseline.width, height: baseline.height });
|
|
const numDiff = pixelmatch(baseline.data, current.data, diff.data, baseline.width, baseline.height, { threshold: 0.1 });
|
|
const diffPercent = (numDiff / (baseline.width * baseline.height)) * 100;
|
|
|
|
const passed = diffPercent <= 5.0; // 5% tolerance
|
|
result(results, '7_visual_regression', passed, `diff pixels=${numDiff} (${diffPercent.toFixed(2)}%)`);
|
|
|
|
if (!passed) {
|
|
const diffPath = path.join(CURRENT_DIR, 'homepage_desktop_diff.png');
|
|
fs.writeFileSync(diffPath, PNG.sync.write(diff));
|
|
console.log(` 📸 Diff saved: ${diffPath}`);
|
|
}
|
|
}
|
|
} catch (e) {
|
|
result(results, '7_visual_regression', false, e.message);
|
|
}
|
|
}
|
|
}
|
|
|
|
await context.close();
|
|
await browser.close();
|
|
|
|
// ============================================================
|
|
// Summary
|
|
// ============================================================
|
|
console.log('\n═══════════════════════════════════════════════════');
|
|
console.log(' Results Summary');
|
|
console.log('═══════════════════════════════════════════════════\n');
|
|
|
|
for (const r of results) {
|
|
const icon = r.pass === true ? '✅' : r.pass === false ? '❌' : '⏭️';
|
|
console.log(`${icon} ${r.name}`);
|
|
console.log(` ${r.detail}`);
|
|
}
|
|
|
|
console.log(`\n📊 Console errors: ${consoleErrors.length}`);
|
|
console.log(`📊 Console warnings: ${consoleWarnings.length}`);
|
|
console.log(`📊 Network errors: ${networkErrors.length}`);
|
|
|
|
const failures = results.filter(r => r.pass === false);
|
|
const passed = results.filter(r => r.pass === true);
|
|
const skipped = results.filter(r => r.pass === null);
|
|
|
|
console.log(`\n✅ Passed: ${passed.length}`);
|
|
console.log(`❌ Failed: ${failures.length}`);
|
|
console.log(`⏭️ Skipped: ${skipped.length}`);
|
|
|
|
const reportPath = path.join(REPORTS_DIR, 'e2e-landing-report.json');
|
|
fs.writeFileSync(reportPath, JSON.stringify({
|
|
timestamp: new Date().toISOString(),
|
|
targetUrl: TARGET_URL,
|
|
results,
|
|
summary: {
|
|
passed: passed.length,
|
|
failed: failures.length,
|
|
skipped: skipped.length,
|
|
consoleErrors: consoleErrors.length,
|
|
consoleWarnings: consoleWarnings.length,
|
|
networkErrors: networkErrors.length,
|
|
},
|
|
}, null, 2));
|
|
console.log(`\n📄 Report: ${reportPath}`);
|
|
|
|
process.exit(failures.length > 0 ? 1 : 0);
|
|
}
|
|
|
|
function result(list, name, pass, detail) {
|
|
list.push({ name, pass, detail });
|
|
const icon = pass === true ? '✅' : pass === false ? '❌' : '⏭️';
|
|
console.log(` ${icon} ${name}: ${detail}`);
|
|
}
|
|
|
|
main().catch(err => {
|
|
console.error('Fatal:', err);
|
|
process.exit(1);
|
|
});
|