feat: add AI-powered visual regression testing infrastructure
- Add docker-compose.web-testing.yml with vlmkit service - Add 5 test scripts: capture, compare, pipeline, console-monitor, link-checker - Add vrt.config.json with 3 viewports + maskSelectors - Add package.json with @mizchi/vlmkit and Node >=24 - Add tests/README.md with VRT documentation - Update .gitignore for test artifacts Based on APAW issue #144 / milestone #106
This commit is contained in:
NW
249
tests/scripts/visual-test-pipeline.js
Normal file
249
tests/scripts/visual-test-pipeline.js
Normal file
@@ -0,0 +1,249 @@
|
||||
const { chromium } = require('playwright');
|
||||
const pixelmatch = require('pixelmatch');
|
||||
const { PNG } = require('pngjs');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const VIEWPORTS = {
|
||||
mobile: { width: 375, height: 667 },
|
||||
tablet: { width: 768, height: 1024 },
|
||||
desktop: { width: 1280, height: 720 },
|
||||
};
|
||||
|
||||
const TARGET_URL = process.env.TARGET_URL;
|
||||
const PAGES = (process.env.PAGES || '/').split(',').map(p => p.trim());
|
||||
const BASELINE_DIR = process.env.BASELINE_DIR || './screenshots/baseline';
|
||||
const CURRENT_DIR = process.env.CURRENT_DIR || './screenshots/current';
|
||||
const DIFF_DIR = process.env.DIFF_DIR || './screenshots/diffs';
|
||||
const REPORTS_DIR = process.env.REPORTS_DIR || './reports';
|
||||
const THRESHOLD = parseFloat(process.env.PIXELMATCH_THRESHOLD) || 0.05;
|
||||
|
||||
function slugify(pagePath) {
|
||||
return pagePath.replace(/^\//, '').replace(/[^a-zA-Z0-9_-]/g, '_') || 'home';
|
||||
}
|
||||
|
||||
function readPNG(filepath) {
|
||||
const data = fs.readFileSync(filepath);
|
||||
return PNG.sync.read(data);
|
||||
}
|
||||
|
||||
async function captureScreenshots(page, pagePath, viewportName, outputDir) {
|
||||
const fullUrl = `${TARGET_URL}${pagePath}`;
|
||||
const filename = `${slugify(pagePath)}_${viewportName}.png`;
|
||||
const filepath = path.join(outputDir, filename);
|
||||
|
||||
await page.setViewportSize(VIEWPORTS[viewportName]);
|
||||
await page.goto(fullUrl, { waitUntil: 'networkidle', timeout: 30000 });
|
||||
await page.waitForTimeout(500);
|
||||
await page.screenshot({ path: filepath, fullPage: false });
|
||||
|
||||
return { pagePath, viewport: viewportName, file: filename };
|
||||
}
|
||||
|
||||
async function extractBBoxes(page) {
|
||||
return page.evaluate(() => {
|
||||
const els = document.querySelectorAll('[data-testid], [id], [class]');
|
||||
return Array.from(els).slice(0, 50).map(el => {
|
||||
const rect = el.getBoundingClientRect();
|
||||
return {
|
||||
tag: el.tagName,
|
||||
id: el.id || null,
|
||||
testid: el.getAttribute('data-testid') || null,
|
||||
class: el.getAttribute('class')?.slice(0, 60) || null,
|
||||
x: rect.x, y: rect.y, width: rect.width, height: rect.height,
|
||||
visible: rect.width > 0 && rect.height > 0,
|
||||
};
|
||||
}).filter(b => b.visible);
|
||||
});
|
||||
}
|
||||
|
||||
function compareImages(baselinePath, currentPath, diffPath) {
|
||||
const img1 = readPNG(baselinePath);
|
||||
const img2 = readPNG(currentPath);
|
||||
|
||||
if (img1.width !== img2.width || img1.height !== img2.height) {
|
||||
return {
|
||||
mismatchedPixels: -1,
|
||||
mismatchPercent: -1,
|
||||
diffCreated: false,
|
||||
dimensionMismatch: true,
|
||||
baselineDimensions: { width: img1.width, height: img1.height },
|
||||
currentDimensions: { width: img2.width, height: img2.height },
|
||||
};
|
||||
}
|
||||
|
||||
const diff = new PNG({ width: img1.width, height: img1.height });
|
||||
|
||||
const mismatchedPixels = pixelmatch(img1.data, img2.data, diff.data, img1.width, img1.height, {
|
||||
threshold: THRESHOLD,
|
||||
});
|
||||
|
||||
const mismatchPercent = (mismatchedPixels / (img1.width * img1.height)) * 100;
|
||||
|
||||
if (mismatchedPixels > 0) {
|
||||
fs.mkdirSync(path.dirname(diffPath), { recursive: true });
|
||||
fs.writeFileSync(diffPath, PNG.sync.write(diff));
|
||||
}
|
||||
|
||||
return { mismatchedPixels, mismatchPercent, diffCreated: mismatchedPixels > 0 };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!TARGET_URL) {
|
||||
console.error('TARGET_URL environment variable is required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let hasBaselines = false;
|
||||
try {
|
||||
hasBaselines = fs.existsSync(BASELINE_DIR) && fs.statSync(BASELINE_DIR).isDirectory() && fs.readdirSync(BASELINE_DIR).some(f => f.endsWith('.png'));
|
||||
} catch {}
|
||||
const isBaselineMode = !hasBaselines;
|
||||
|
||||
if (isBaselineMode) {
|
||||
console.log('No baselines found. Creating baseline screenshots...');
|
||||
}
|
||||
|
||||
const outputDir = isBaselineMode ? BASELINE_DIR : CURRENT_DIR;
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
|
||||
const screenshots = [];
|
||||
const allConsoleErrors = [];
|
||||
const allNetworkErrors = [];
|
||||
const allBBoxes = [];
|
||||
|
||||
let iterationConsoleErrors = [];
|
||||
let iterationNetworkErrors = [];
|
||||
page.on('console', msg => {
|
||||
if (msg.type() === 'error') {
|
||||
iterationConsoleErrors.push({ type: 'console_error', text: msg.text(), location: msg.location() });
|
||||
}
|
||||
});
|
||||
page.on('pageerror', err => {
|
||||
iterationConsoleErrors.push({ type: 'page_error', text: err.message, stack: err.stack });
|
||||
});
|
||||
page.on('requestfailed', req => {
|
||||
iterationNetworkErrors.push({
|
||||
type: 'network_error',
|
||||
url: req.url(),
|
||||
method: req.method(),
|
||||
failure: req.failure()?.errorText || 'unknown',
|
||||
});
|
||||
});
|
||||
page.on('response', res => {
|
||||
if (res.status() >= 400) {
|
||||
iterationNetworkErrors.push({
|
||||
type: 'http_error',
|
||||
url: res.url(),
|
||||
status: res.status(),
|
||||
method: res.request().method(),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
for (const pagePath of PAGES) {
|
||||
for (const vpName of Object.keys(VIEWPORTS)) {
|
||||
try {
|
||||
iterationConsoleErrors = [];
|
||||
iterationNetworkErrors = [];
|
||||
|
||||
const result = await captureScreenshots(page, pagePath, vpName, outputDir);
|
||||
screenshots.push(result);
|
||||
|
||||
const bboxes = await extractBBoxes(page);
|
||||
allBBoxes.push({ pagePath, viewport: vpName, elements: bboxes });
|
||||
|
||||
await page.waitForTimeout(200);
|
||||
allConsoleErrors.push(...iterationConsoleErrors);
|
||||
allNetworkErrors.push(...iterationNetworkErrors);
|
||||
|
||||
console.log(`Captured ${pagePath} @ ${vpName}`);
|
||||
} catch (err) {
|
||||
screenshots.push({ pagePath, viewport: vpName, error: err.message });
|
||||
console.error(`Failed ${pagePath} @ ${vpName}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
const comparisons = [];
|
||||
|
||||
if (!isBaselineMode) {
|
||||
const baselineFiles = fs.readdirSync(BASELINE_DIR).filter(f => f.endsWith('.png'));
|
||||
const currentFiles = fs.readdirSync(CURRENT_DIR).filter(f => f.endsWith('.png'));
|
||||
const allFiles = [...new Set([...baselineFiles, ...currentFiles])];
|
||||
|
||||
for (const file of allFiles) {
|
||||
const baselinePath = path.join(BASELINE_DIR, file);
|
||||
const currentPath = path.join(CURRENT_DIR, file);
|
||||
|
||||
if (!fs.existsSync(baselinePath)) {
|
||||
comparisons.push({ file, status: 'missing_baseline' });
|
||||
continue;
|
||||
}
|
||||
if (!fs.existsSync(currentPath)) {
|
||||
comparisons.push({ file, status: 'missing_current' });
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const diffPath = path.join(DIFF_DIR, file);
|
||||
const result = compareImages(baselinePath, currentPath, diffPath);
|
||||
const status = result.dimensionMismatch ? 'error' : (result.mismatchedPixels === 0 ? 'pass' : 'fail');
|
||||
comparisons.push({ file, status, ...result });
|
||||
} catch (err) {
|
||||
comparisons.push({ file, status: 'error', error: err.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const passed = comparisons.filter(c => c.status === 'pass').length;
|
||||
const failed = comparisons.filter(c => c.status === 'fail').length;
|
||||
const errors = comparisons.filter(c => c.status === 'error').length;
|
||||
const missing = comparisons.filter(c => c.status === 'missing_baseline' || c.status === 'missing_current').length;
|
||||
|
||||
const report = {
|
||||
timestamp: new Date().toISOString(),
|
||||
mode: isBaselineMode ? 'baseline_creation' : 'comparison',
|
||||
targetUrl: TARGET_URL,
|
||||
pages: PAGES,
|
||||
screenshots,
|
||||
bboxes: allBBoxes,
|
||||
bboxCount: allBBoxes.reduce((sum, b) => sum + b.elements.length, 0),
|
||||
consoleErrors: allConsoleErrors,
|
||||
networkErrors: allNetworkErrors,
|
||||
visualComparison: comparisons.length > 0 ? {
|
||||
threshold: THRESHOLD,
|
||||
summary: { total: comparisons.length, passed, failed, errors, missing },
|
||||
comparisons,
|
||||
} : null,
|
||||
};
|
||||
|
||||
fs.mkdirSync(REPORTS_DIR, { recursive: true });
|
||||
const reportPath = path.join(REPORTS_DIR, 'visual-test-report.json');
|
||||
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
|
||||
console.log(`Report saved to ${reportPath}`);
|
||||
|
||||
if (allConsoleErrors.length > 0) {
|
||||
console.warn(`Found ${allConsoleErrors.length} console errors`);
|
||||
}
|
||||
if (allNetworkErrors.length > 0) {
|
||||
console.warn(`Found ${allNetworkErrors.length} network errors`);
|
||||
}
|
||||
if (failed > 0 || errors > 0 || missing > 0) {
|
||||
console.error(`Visual diffs: ${failed} failed, ${errors} errors, ${missing} missing`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user