veza/scripts/flaky-detection.mjs

#!/usr/bin/env node

/**
 * Flaky Test Detection Script
 *
 * Analyzes Playwright JSON results to detect flaky tests (tests that passed on retry).
 * Usage: node scripts/flaky-detection.mjs [results-dir]
 *
 * Output: Markdown report to stdout, suitable for piping to a file or GitHub comment.
 */

import { readFileSync, readdirSync, existsSync } from 'fs';
import { join } from 'path';

const resultsDir = process.argv[2] || 'tests/e2e/test-results';
const resultsFile = process.argv[3] || 'tests/e2e/test-results/results.json';

function analyzeResults(filePath) {
  if (!existsSync(filePath)) {
    console.error(`Results file not found: ${filePath}`);
    return null;
  }

  const raw = JSON.parse(readFileSync(filePath, 'utf8'));
  const suites = raw.suites || [];
  const flaky = [];
  const failed = [];
  const slow = [];

  function walkSpecs(specs, suitePath = '') {
    for (const spec of specs) {
      const fullTitle = suitePath ? `${suitePath} > ${spec.title}` : spec.title;

      for (const test of spec.tests || []) {
        const results = test.results || [];

        // Flaky: passed eventually but had retries
        if (test.status === 'expected' && results.length > 1) {
          flaky.push({
            title: fullTitle,
            retries: results.length - 1,
            file: spec.file || 'unknown',
          });
        }

        // Failed
        if (test.status === 'unexpected') {
          failed.push({
            title: fullTitle,
            file: spec.file || 'unknown',
            error: results[results.length - 1]?.error?.message?.slice(0, 200) || 'unknown',
          });
        }

        // Slow (> 30s)
        const duration = results.reduce((sum, r) => sum + (r.duration || 0), 0);
        if (duration > 30000) {
          slow.push({
            title: fullTitle,
            duration: Math.round(duration / 1000),
            file: spec.file || 'unknown',
          });
        }
      }
    }
  }

  function walkSuites(suites, path = '') {
    for (const suite of suites) {
      const suitePath = path ? `${path} > ${suite.title}` : suite.title;
      walkSpecs(suite.specs || [], suitePath);
      walkSuites(suite.suites || [], suitePath);
    }
  }

  walkSuites(suites);
  return { flaky, failed, slow };
}

function generateReport(analysis) {
  if (!analysis) return '# Flaky Test Report\n\nNo results file found.\n';

  const { flaky, failed, slow } = analysis;
  const lines = ['# Flaky Test Report', ''];
  lines.push(`Generated: ${new Date().toISOString()}`, '');

  // Flaky tests
  lines.push(`## Flaky Tests (${flaky.length})`, '');
  if (flaky.length === 0) {
    lines.push('No flaky tests detected.', '');
  } else {
    lines.push('| Test | Retries | File |');
    lines.push('|------|---------|------|');
    for (const t of flaky.sort((a, b) => b.retries - a.retries)) {
      lines.push(`| ${t.title} | ${t.retries} | \`${t.file}\` |`);
    }
    lines.push('');
  }

  // Failed tests
  lines.push(`## Failed Tests (${failed.length})`, '');
  if (failed.length === 0) {
    lines.push('No failed tests.', '');
  } else {
    lines.push('| Test | Error | File |');
    lines.push('|------|-------|------|');
    for (const t of failed) {
      const safeError = t.error.replace(/\|/g, '\\|').replace(/\n/g, ' ');
      lines.push(`| ${t.title} | ${safeError} | \`${t.file}\` |`);
    }
    lines.push('');
  }

  // Slow tests
  lines.push(`## Slow Tests (> 30s) (${slow.length})`, '');
  if (slow.length === 0) {
    lines.push('No slow tests.', '');
  } else {
    lines.push('| Test | Duration | File |');
    lines.push('|------|----------|------|');
    for (const t of slow.sort((a, b) => b.duration - a.duration)) {
      lines.push(`| ${t.title} | ${t.duration}s | \`${t.file}\` |`);
    }
    lines.push('');
  }

  return lines.join('\n');
}

const analysis = analyzeResults(resultsFile);
console.log(generateReport(analysis));

if (analysis && analysis.flaky.length > 0) {
  process.exit(0); // Flaky tests are warnings, not failures
}