feat: add AI-powered visual regression testing infrastructure

- Add docker-compose.web-testing.yml with vlmkit service
- Add 5 test scripts: capture, compare, pipeline, console-monitor, link-checker
- Add vrt.config.json with 3 viewports + maskSelectors
- Add package.json with @mizchi/vlmkit and Node >=24
- Add tests/README.md with VRT documentation
- Update .gitignore for test artifacts

Based on APAW issue #144 / milestone #106

tests/scripts/capture-screenshots.js

@@ -0,0 +1,84 @@
+const { chromium } = require('playwright');
+const path = require('path');
+const fs = require('fs');
+
+const VIEWPORTS = {
+  mobile: { width: 375, height: 667 },
+  tablet: { width: 768, height: 1024 },
+  desktop: { width: 1280, height: 720 },
+};
+
+function slugify(pagePath) {
+  return pagePath.replace(/^\//, '').replace(/[^a-zA-Z0-9_-]/g, '_') || 'home';
+}
+
+async function capturePage(page, pagePath, viewportName, outputDir) {
+  const fullUrl = `${process.env.TARGET_URL}${pagePath}`;
+  const filename = `${slugify(pagePath)}_${viewportName}.png`;
+  const filepath = path.join(outputDir, filename);
+
+  await page.setViewportSize(VIEWPORTS[viewportName]);
+  await page.goto(fullUrl, { waitUntil: 'networkidle', timeout: 30000 });
+  await page.waitForTimeout(500);
+  await page.screenshot({ path: filepath, fullPage: false });
+
+  return { pagePath, viewport: viewportName, file: filename, saved: true };
+}
+
+async function main() {
+  const mode = process.argv[2];
+  if (!['baseline', 'current'].includes(mode)) {
+    console.error('Usage: node capture-screenshots.js <baseline|current>');
+    process.exit(1);
+  }
+
+  const targetUrl = process.env.TARGET_URL;
+  const pages = (process.env.PAGES || '/').split(',').map(p => p.trim()).filter(p => /^\/[A-Za-z0-9._\/-]*$/.test(p));
+  if (pages.length === 0) {
+    console.error('No valid pages in PAGES environment variable');
+    process.exit(1);
+  }
+  const outputDir = mode === 'baseline'
+    ? (process.env.BASELINE_DIR || './screenshots/baseline')
+    : (process.env.CURRENT_DIR || './screenshots/current');
+
+  if (!targetUrl) {
+    console.error('TARGET_URL environment variable is required');
+    process.exit(1);
+  }
+
+  fs.mkdirSync(outputDir, { recursive: true });
+
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  const results = [];
+
+  try {
+    for (const pagePath of pages) {
+      for (const vpName of Object.keys(VIEWPORTS)) {
+        try {
+          const result = await capturePage(page, pagePath, vpName, outputDir);
+          results.push(result);
+          console.log(`Captured ${pagePath} @ ${vpName} -> ${result.file}`);
+        } catch (err) {
+          results.push({ pagePath, viewport: vpName, error: err.message, saved: false });
+          console.error(`Failed ${pagePath} @ ${vpName}: ${err.message}`);
+        }
+      }
+    }
+  } finally {
+    await browser.close();
+  }
+
+  const report = { mode, targetUrl, pages, outputDir, results };
+  const reportPath = path.join(outputDir, '..', `${mode}-capture-report.json`);
+  fs.mkdirSync(path.dirname(reportPath), { recursive: true });
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+  console.log(`Report saved to ${reportPath}`);
+}
+
+main().catch(err => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

tests/scripts/compare-screenshots.js

@@ -0,0 +1,107 @@
+const pixelmatch = require('pixelmatch');
+const { PNG } = require('pngjs');
+const fs = require('fs');
+const path = require('path');
+
+const THRESHOLD = parseFloat(process.env.PIXELMATCH_THRESHOLD) || 0.05;
+const BASELINE_DIR = process.env.BASELINE_DIR || './screenshots/baseline';
+const CURRENT_DIR = process.env.CURRENT_DIR || './screenshots/current';
+const DIFF_DIR = process.env.DIFF_DIR || './screenshots/diffs';
+const REPORTS_DIR = process.env.REPORTS_DIR || './reports';
+
+function readPNG(filepath) {
+  const data = fs.readFileSync(filepath);
+  return PNG.sync.read(data);
+}
+
+function compareImages(baselinePath, currentPath, diffPath) {
+  const img1 = readPNG(baselinePath);
+  const img2 = readPNG(currentPath);
+
+  const { width, height } = img1;
+  const diff = new PNG({ width, height });
+
+  const mismatchedPixels = pixelmatch(img1.data, img2.data, diff.data, width, height, {
+    threshold: THRESHOLD,
+  });
+
+  const totalPixels = width * height;
+  const mismatchPercent = (mismatchedPixels / totalPixels) * 100;
+
+  if (mismatchedPixels > 0) {
+    fs.mkdirSync(path.dirname(diffPath), { recursive: true });
+    fs.writeFileSync(diffPath, PNG.sync.write(diff));
+  }
+
+  return { mismatchedPixels, totalPixels, mismatchPercent, diffCreated: mismatchedPixels > 0 };
+}
+
+function findPNGFiles(dir) {
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir).filter(f => f.endsWith('.png'));
+}
+
+async function main() {
+  const baselineFiles = findPNGFiles(BASELINE_DIR);
+  const currentFiles = findPNGFiles(CURRENT_DIR);
+
+  if (baselineFiles.length === 0) {
+    console.error(`No baseline screenshots found in ${BASELINE_DIR}`);
+    process.exit(1);
+  }
+
+  if (currentFiles.length === 0) {
+    console.error(`No current screenshots found in ${CURRENT_DIR}`);
+    process.exit(1);
+  }
+
+  const allFiles = [...new Set([...baselineFiles, ...currentFiles])];
+  const comparisons = [];
+
+  for (const file of allFiles) {
+    const baselinePath = path.join(BASELINE_DIR, file);
+    const currentPath = path.join(CURRENT_DIR, file);
+
+    if (!fs.existsSync(baselinePath)) {
+      comparisons.push({ file, status: 'missing_baseline' });
+      continue;
+    }
+    if (!fs.existsSync(currentPath)) {
+      comparisons.push({ file, status: 'missing_current' });
+      continue;
+    }
+
+    try {
+      const diffPath = path.join(DIFF_DIR, file);
+      const result = compareImages(baselinePath, currentPath, diffPath);
+      comparisons.push({ file, status: result.mismatchedPixels === 0 ? 'pass' : 'fail', ...result });
+    } catch (err) {
+      comparisons.push({ file, status: 'error', error: err.message });
+    }
+  }
+
+  const passed = comparisons.filter(c => c.status === 'pass').length;
+  const failed = comparisons.filter(c => c.status === 'fail').length;
+  const errors = comparisons.filter(c => c.status === 'error').length;
+  const missing = comparisons.filter(c => c.status === 'missing_baseline' || c.status === 'missing_current').length;
+
+  const report = {
+    timestamp: new Date().toISOString(),
+    threshold: THRESHOLD,
+    summary: { total: comparisons.length, passed, failed, errors, missing },
+    comparisons,
+  };
+
+  fs.mkdirSync(REPORTS_DIR, { recursive: true });
+  const reportPath = path.join(REPORTS_DIR, 'visual-test-report.json');
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+  console.log(`Report saved to ${reportPath}`);
+  console.log(`Passed: ${passed}, Failed: ${failed}, Errors: ${errors}, Missing: ${missing}`);
+
+  if (failed > 0 || errors > 0) process.exit(1);
+}
+
+main().catch(err => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

tests/scripts/console-error-monitor-standalone.js

@@ -0,0 +1,127 @@
+const { chromium } = require('playwright');
+const fs = require('fs');
+const path = require('path');
+
+const TARGET_URL = process.env.TARGET_URL;
+const PAGES = (process.env.PAGES || '/').split(',').map(p => p.trim());
+const REPORTS_DIR = process.env.REPORTS_DIR || './reports';
+
+async function monitorPage(context, pagePath) {
+  const page = await context.newPage();
+  const fullUrl = `${TARGET_URL}${pagePath}`;
+  const consoleErrors = [];
+  const networkErrors = [];
+
+  page.on('console', msg => {
+    if (msg.type() === 'error') {
+      consoleErrors.push({
+        type: 'console_error',
+        text: msg.text(),
+        location: msg.location(),
+        timestamp: new Date().toISOString(),
+      });
+    }
+  });
+
+  page.on('pageerror', err => {
+    consoleErrors.push({
+      type: 'page_error',
+      text: err.message,
+      stack: err.stack,
+      timestamp: new Date().toISOString(),
+    });
+  });
+
+  page.on('requestfailed', req => {
+    networkErrors.push({
+      type: 'network_failure',
+      url: req.url(),
+      method: req.method(),
+      failure: req.failure()?.errorText || 'unknown',
+      resourceType: req.resourceType(),
+      timestamp: new Date().toISOString(),
+    });
+  });
+
+  page.on('response', res => {
+    if (res.status() >= 400) {
+      networkErrors.push({
+        type: 'http_error',
+        url: res.url(),
+        status: res.status(),
+        statusText: res.statusText(),
+        method: res.request().method(),
+        resourceType: res.request().resourceType(),
+        timestamp: new Date().toISOString(),
+      });
+    }
+  });
+
+  try {
+    await page.goto(fullUrl, { waitUntil: 'networkidle', timeout: 30000 });
+    await page.waitForTimeout(1000);
+  } finally {
+    await page.close();
+  }
+
+  return { pagePath, consoleErrors, networkErrors };
+}
+
+async function main() {
+  if (!TARGET_URL) {
+    console.error('TARGET_URL environment variable is required');
+    process.exit(1);
+  }
+
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const results = [];
+
+  try {
+    for (const pagePath of PAGES) {
+      try {
+        const result = await monitorPage(context, pagePath);
+        results.push(result);
+        console.log(`Checked ${pagePath}: ${result.consoleErrors.length} console errors, ${result.networkErrors.length} network errors`);
+      } catch (err) {
+        results.push({ pagePath, error: err.message, consoleErrors: [], networkErrors: [] });
+        console.error(`Failed ${pagePath}: ${err.message}`);
+      }
+    }
+  } finally {
+    await browser.close();
+  }
+
+  const totalConsoleErrors = results.reduce((sum, r) => sum + r.consoleErrors.length, 0);
+  const totalNetworkErrors = results.reduce((sum, r) => sum + r.networkErrors.length, 0);
+  const pagesWithErrors = results.filter(r => r.consoleErrors.length > 0 || r.networkErrors.length > 0).length;
+
+  const report = {
+    timestamp: new Date().toISOString(),
+    targetUrl: TARGET_URL,
+    pages: PAGES,
+    summary: {
+      totalPages: results.length,
+      pagesWithErrors,
+      totalConsoleErrors,
+      totalNetworkErrors,
+      hasErrors: totalConsoleErrors > 0 || totalNetworkErrors > 0,
+    },
+    results,
+  };
+
+  fs.mkdirSync(REPORTS_DIR, { recursive: true });
+  const reportPath = path.join(REPORTS_DIR, 'console-error-report.json');
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+  console.log(`Report saved to ${reportPath}`);
+
+  if (totalConsoleErrors > 0 || totalNetworkErrors > 0) {
+    console.error(`Found ${totalConsoleErrors} console errors and ${totalNetworkErrors} network errors across ${pagesWithErrors} page(s)`);
+    process.exit(1);
+  }
+}
+
+main().catch(err => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

tests/scripts/link-checker.js

@@ -0,0 +1,83 @@
+const { chromium } = require('playwright');
+const fs = require('fs');
+const path = require('path');
+
+const TARGET_URL = process.env.TARGET_URL;
+const PAGES = (process.env.PAGES || '/').split(',').map(p => p.trim());
+const REPORTS_DIR = process.env.REPORTS_DIR || './reports';
+
+async function collectLinks(page, pagePath) {
+  const fullUrl = `${TARGET_URL}${pagePath}`;
+  await page.goto(fullUrl, { waitUntil: 'networkidle', timeout: 30000 });
+  return page.evaluate(() => Array.from(document.querySelectorAll('a[href]'))
+    .map(a => a.href));
+}
+
+async function checkLink(context, url) {
+  try {
+    const resp = await context.request.fetch(url, { method: 'HEAD', timeout: 15000 });
+    return { url, status: resp.status(), ok: resp.ok() };
+  } catch (err) {
+    return { url, status: 0, ok: false, error: err.message };
+  }
+}
+
+async function main() {
+  if (!TARGET_URL) {
+    console.error('TARGET_URL environment variable is required');
+    process.exit(1);
+  }
+
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const results = [];
+
+  try {
+    for (const pagePath of PAGES) {
+      const page = await context.newPage();
+      try {
+        const links = await collectLinks(page, pagePath);
+        await page.close();
+        const checked = [];
+        for (const url of [...new Set(links)]) {
+          if (url.startsWith('mailto:') || url.startsWith('tel:')) continue;
+          checked.push(await checkLink(context, url));
+        }
+        const broken = checked.filter(c => !c.ok);
+        results.push({ pagePath, total: checked.length, broken: broken.length, brokenLinks: broken });
+        console.log(`${pagePath}: ${checked.length} links, ${broken.length} broken`);
+      } catch (err) {
+        results.push({ pagePath, error: err.message, total: 0, broken: 0, brokenLinks: [] });
+        console.error(`Failed ${pagePath}: ${err.message}`);
+      } finally {
+        await page.close().catch(() => {});
+      }
+    }
+  } finally {
+    await browser.close();
+  }
+
+  const totalBroken = results.reduce((s, r) => s + r.broken, 0);
+  const report = {
+    timestamp: new Date().toISOString(),
+    targetUrl: TARGET_URL,
+    pages: PAGES,
+    summary: { totalPages: results.length, totalBroken, hasBroken: totalBroken > 0 },
+    results,
+  };
+
+  fs.mkdirSync(REPORTS_DIR, { recursive: true });
+  const reportPath = path.join(REPORTS_DIR, 'link-check-report.json');
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+  console.log(`Report saved to ${reportPath}`);
+
+  if (totalBroken > 0) {
+    console.error(`Found ${totalBroken} broken links`);
+    process.exit(1);
+  }
+}
+
+main().catch(err => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});

tests/scripts/visual-test-pipeline.js

@@ -0,0 +1,249 @@
+const { chromium } = require('playwright');
+const pixelmatch = require('pixelmatch');
+const { PNG } = require('pngjs');
+const fs = require('fs');
+const path = require('path');
+
+const VIEWPORTS = {
+  mobile: { width: 375, height: 667 },
+  tablet: { width: 768, height: 1024 },
+  desktop: { width: 1280, height: 720 },
+};
+
+const TARGET_URL = process.env.TARGET_URL;
+const PAGES = (process.env.PAGES || '/').split(',').map(p => p.trim());
+const BASELINE_DIR = process.env.BASELINE_DIR || './screenshots/baseline';
+const CURRENT_DIR = process.env.CURRENT_DIR || './screenshots/current';
+const DIFF_DIR = process.env.DIFF_DIR || './screenshots/diffs';
+const REPORTS_DIR = process.env.REPORTS_DIR || './reports';
+const THRESHOLD = parseFloat(process.env.PIXELMATCH_THRESHOLD) || 0.05;
+
+function slugify(pagePath) {
+  return pagePath.replace(/^\//, '').replace(/[^a-zA-Z0-9_-]/g, '_') || 'home';
+}
+
+function readPNG(filepath) {
+  const data = fs.readFileSync(filepath);
+  return PNG.sync.read(data);
+}
+
+async function captureScreenshots(page, pagePath, viewportName, outputDir) {
+  const fullUrl = `${TARGET_URL}${pagePath}`;
+  const filename = `${slugify(pagePath)}_${viewportName}.png`;
+  const filepath = path.join(outputDir, filename);
+
+  await page.setViewportSize(VIEWPORTS[viewportName]);
+  await page.goto(fullUrl, { waitUntil: 'networkidle', timeout: 30000 });
+  await page.waitForTimeout(500);
+  await page.screenshot({ path: filepath, fullPage: false });
+
+  return { pagePath, viewport: viewportName, file: filename };
+}
+
+async function extractBBoxes(page) {
+  return page.evaluate(() => {
+    const els = document.querySelectorAll('[data-testid], [id], [class]');
+    return Array.from(els).slice(0, 50).map(el => {
+      const rect = el.getBoundingClientRect();
+      return {
+        tag: el.tagName,
+        id: el.id || null,
+        testid: el.getAttribute('data-testid') || null,
+        class: el.getAttribute('class')?.slice(0, 60) || null,
+        x: rect.x, y: rect.y, width: rect.width, height: rect.height,
+        visible: rect.width > 0 && rect.height > 0,
+      };
+    }).filter(b => b.visible);
+  });
+}
+
+function compareImages(baselinePath, currentPath, diffPath) {
+  const img1 = readPNG(baselinePath);
+  const img2 = readPNG(currentPath);
+
+  if (img1.width !== img2.width || img1.height !== img2.height) {
+    return {
+      mismatchedPixels: -1,
+      mismatchPercent: -1,
+      diffCreated: false,
+      dimensionMismatch: true,
+      baselineDimensions: { width: img1.width, height: img1.height },
+      currentDimensions: { width: img2.width, height: img2.height },
+    };
+  }
+
+  const diff = new PNG({ width: img1.width, height: img1.height });
+
+  const mismatchedPixels = pixelmatch(img1.data, img2.data, diff.data, img1.width, img1.height, {
+    threshold: THRESHOLD,
+  });
+
+  const mismatchPercent = (mismatchedPixels / (img1.width * img1.height)) * 100;
+
+  if (mismatchedPixels > 0) {
+    fs.mkdirSync(path.dirname(diffPath), { recursive: true });
+    fs.writeFileSync(diffPath, PNG.sync.write(diff));
+  }
+
+  return { mismatchedPixels, mismatchPercent, diffCreated: mismatchedPixels > 0 };
+}
+
+async function main() {
+  if (!TARGET_URL) {
+    console.error('TARGET_URL environment variable is required');
+    process.exit(1);
+  }
+
+  let hasBaselines = false;
+  try {
+    hasBaselines = fs.existsSync(BASELINE_DIR) && fs.statSync(BASELINE_DIR).isDirectory() && fs.readdirSync(BASELINE_DIR).some(f => f.endsWith('.png'));
+  } catch {}
+  const isBaselineMode = !hasBaselines;
+
+  if (isBaselineMode) {
+    console.log('No baselines found. Creating baseline screenshots...');
+  }
+
+  const outputDir = isBaselineMode ? BASELINE_DIR : CURRENT_DIR;
+  fs.mkdirSync(outputDir, { recursive: true });
+
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  const screenshots = [];
+  const allConsoleErrors = [];
+  const allNetworkErrors = [];
+  const allBBoxes = [];
+
+  let iterationConsoleErrors = [];
+  let iterationNetworkErrors = [];
+  page.on('console', msg => {
+    if (msg.type() === 'error') {
+      iterationConsoleErrors.push({ type: 'console_error', text: msg.text(), location: msg.location() });
+    }
+  });
+  page.on('pageerror', err => {
+    iterationConsoleErrors.push({ type: 'page_error', text: err.message, stack: err.stack });
+  });
+  page.on('requestfailed', req => {
+    iterationNetworkErrors.push({
+      type: 'network_error',
+      url: req.url(),
+      method: req.method(),
+      failure: req.failure()?.errorText || 'unknown',
+    });
+  });
+  page.on('response', res => {
+    if (res.status() >= 400) {
+      iterationNetworkErrors.push({
+        type: 'http_error',
+        url: res.url(),
+        status: res.status(),
+        method: res.request().method(),
+      });
+    }
+  });
+
+  try {
+    for (const pagePath of PAGES) {
+      for (const vpName of Object.keys(VIEWPORTS)) {
+        try {
+          iterationConsoleErrors = [];
+          iterationNetworkErrors = [];
+
+          const result = await captureScreenshots(page, pagePath, vpName, outputDir);
+          screenshots.push(result);
+
+          const bboxes = await extractBBoxes(page);
+          allBBoxes.push({ pagePath, viewport: vpName, elements: bboxes });
+
+          await page.waitForTimeout(200);
+          allConsoleErrors.push(...iterationConsoleErrors);
+          allNetworkErrors.push(...iterationNetworkErrors);
+
+          console.log(`Captured ${pagePath} @ ${vpName}`);
+        } catch (err) {
+          screenshots.push({ pagePath, viewport: vpName, error: err.message });
+          console.error(`Failed ${pagePath} @ ${vpName}: ${err.message}`);
+        }
+      }
+    }
+  } finally {
+    await browser.close();
+  }
+
+  const comparisons = [];
+
+  if (!isBaselineMode) {
+    const baselineFiles = fs.readdirSync(BASELINE_DIR).filter(f => f.endsWith('.png'));
+    const currentFiles = fs.readdirSync(CURRENT_DIR).filter(f => f.endsWith('.png'));
+    const allFiles = [...new Set([...baselineFiles, ...currentFiles])];
+
+    for (const file of allFiles) {
+      const baselinePath = path.join(BASELINE_DIR, file);
+      const currentPath = path.join(CURRENT_DIR, file);
+
+      if (!fs.existsSync(baselinePath)) {
+        comparisons.push({ file, status: 'missing_baseline' });
+        continue;
+      }
+      if (!fs.existsSync(currentPath)) {
+        comparisons.push({ file, status: 'missing_current' });
+        continue;
+      }
+
+      try {
+        const diffPath = path.join(DIFF_DIR, file);
+        const result = compareImages(baselinePath, currentPath, diffPath);
+        const status = result.dimensionMismatch ? 'error' : (result.mismatchedPixels === 0 ? 'pass' : 'fail');
+        comparisons.push({ file, status, ...result });
+      } catch (err) {
+        comparisons.push({ file, status: 'error', error: err.message });
+      }
+    }
+  }
+
+  const passed = comparisons.filter(c => c.status === 'pass').length;
+  const failed = comparisons.filter(c => c.status === 'fail').length;
+  const errors = comparisons.filter(c => c.status === 'error').length;
+  const missing = comparisons.filter(c => c.status === 'missing_baseline' || c.status === 'missing_current').length;
+
+  const report = {
+    timestamp: new Date().toISOString(),
+    mode: isBaselineMode ? 'baseline_creation' : 'comparison',
+    targetUrl: TARGET_URL,
+    pages: PAGES,
+    screenshots,
+    bboxes: allBBoxes,
+    bboxCount: allBBoxes.reduce((sum, b) => sum + b.elements.length, 0),
+    consoleErrors: allConsoleErrors,
+    networkErrors: allNetworkErrors,
+    visualComparison: comparisons.length > 0 ? {
+      threshold: THRESHOLD,
+      summary: { total: comparisons.length, passed, failed, errors, missing },
+      comparisons,
+    } : null,
+  };
+
+  fs.mkdirSync(REPORTS_DIR, { recursive: true });
+  const reportPath = path.join(REPORTS_DIR, 'visual-test-report.json');
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+  console.log(`Report saved to ${reportPath}`);
+
+  if (allConsoleErrors.length > 0) {
+    console.warn(`Found ${allConsoleErrors.length} console errors`);
+  }
+  if (allNetworkErrors.length > 0) {
+    console.warn(`Found ${allNetworkErrors.length} network errors`);
+  }
+  if (failed > 0 || errors > 0 || missing > 0) {
+    console.error(`Visual diffs: ${failed} failed, ${errors} errors, ${missing} missing`);
+    process.exit(1);
+  }
+}
+
+main().catch(err => {
+  console.error('Fatal error:', err);
+  process.exit(1);
+});