#!/usr/bin/env node /** * Flaky Test Detection Script * * Analyzes Playwright JSON results to detect flaky tests (tests that passed on retry). * Usage: node scripts/flaky-detection.mjs [results-dir] * * Output: Markdown report to stdout, suitable for piping to a file or GitHub comment. */ import { readFileSync, readdirSync, existsSync } from 'fs'; import { join } from 'path'; const resultsDir = process.argv[2] || 'tests/e2e/test-results'; const resultsFile = process.argv[3] || 'tests/e2e/test-results/results.json'; function analyzeResults(filePath) { if (!existsSync(filePath)) { console.error(`Results file not found: ${filePath}`); return null; } const raw = JSON.parse(readFileSync(filePath, 'utf8')); const suites = raw.suites || []; const flaky = []; const failed = []; const slow = []; function walkSpecs(specs, suitePath = '') { for (const spec of specs) { const fullTitle = suitePath ? `${suitePath} > ${spec.title}` : spec.title; for (const test of spec.tests || []) { const results = test.results || []; // Flaky: passed eventually but had retries if (test.status === 'expected' && results.length > 1) { flaky.push({ title: fullTitle, retries: results.length - 1, file: spec.file || 'unknown', }); } // Failed if (test.status === 'unexpected') { failed.push({ title: fullTitle, file: spec.file || 'unknown', error: results[results.length - 1]?.error?.message?.slice(0, 200) || 'unknown', }); } // Slow (> 30s) const duration = results.reduce((sum, r) => sum + (r.duration || 0), 0); if (duration > 30000) { slow.push({ title: fullTitle, duration: Math.round(duration / 1000), file: spec.file || 'unknown', }); } } } } function walkSuites(suites, path = '') { for (const suite of suites) { const suitePath = path ? `${path} > ${suite.title}` : suite.title; walkSpecs(suite.specs || [], suitePath); walkSuites(suite.suites || [], suitePath); } } walkSuites(suites); return { flaky, failed, slow }; } function generateReport(analysis) { if (!analysis) return '# Flaky Test Report\n\nNo results file found.\n'; const { flaky, failed, slow } = analysis; const lines = ['# Flaky Test Report', '']; lines.push(`Generated: ${new Date().toISOString()}`, ''); // Flaky tests lines.push(`## Flaky Tests (${flaky.length})`, ''); if (flaky.length === 0) { lines.push('No flaky tests detected.', ''); } else { lines.push('| Test | Retries | File |'); lines.push('|------|---------|------|'); for (const t of flaky.sort((a, b) => b.retries - a.retries)) { lines.push(`| ${t.title} | ${t.retries} | \`${t.file}\` |`); } lines.push(''); } // Failed tests lines.push(`## Failed Tests (${failed.length})`, ''); if (failed.length === 0) { lines.push('No failed tests.', ''); } else { lines.push('| Test | Error | File |'); lines.push('|------|-------|------|'); for (const t of failed) { const safeError = t.error.replace(/\|/g, '\\|').replace(/\n/g, ' '); lines.push(`| ${t.title} | ${safeError} | \`${t.file}\` |`); } lines.push(''); } // Slow tests lines.push(`## Slow Tests (> 30s) (${slow.length})`, ''); if (slow.length === 0) { lines.push('No slow tests.', ''); } else { lines.push('| Test | Duration | File |'); lines.push('|------|----------|------|'); for (const t of slow.sort((a, b) => b.duration - a.duration)) { lines.push(`| ${t.title} | ${t.duration}s | \`${t.file}\` |`); } lines.push(''); } return lines.join('\n'); } const analysis = analyzeResults(resultsFile); console.log(generateReport(analysis)); if (analysis && analysis.flaky.length > 0) { process.exit(0); // Flaky tests are warnings, not failures }