veza/apps/web/scripts/audit-storybook.js

import { chromium } from 'playwright';
import fs from 'fs';
import path from 'path';

const BATCH_SIZE = 50;
const NAVIGATION_TIMEOUT_MS = 30000;   // 30s per story navigation
const DEFAULT_TIMEOUT_MS = 300000;     // 300s global default for Playwright
const MAX_RETRIES = 2;                 // 2 retries = 3 attempts total
const RETRY_DELAY_MS = 1500;
const POST_LOAD_WAIT_MS = 600;  // Allow MSW and async requests to settle

console.log("Script started");

/** Console errors that are intentional or from Storybook internals (non-blocking). */
const IGNORED_CONSOLE_ERRORS = [
    /This is a test error for demonstrating ErrorBoundary/i,
    /received.*but was unable to determine the source of the event/i,
];

function isIgnoredConsoleError(text, locationUrl = '') {
    if (IGNORED_CONSOLE_ERRORS.some((re) => re.test(text))) return true;
    if (/sb-manager\/|sb-addons\//.test(locationUrl || '')) return true;
    return false;
}

/** Ignore Storybook manager/addon requests; only count app and API failures. */
function isAppRelevantFailure(url) {
    try {
        const pathname = new URL(url).pathname;
        if (pathname.startsWith('/sb-manager/') || pathname.startsWith('/sb-addons/') || pathname.startsWith('/sb-common-assets/')) return false;
        if (pathname === '/index.json' || pathname === '/project.json') return false;
        return true;
    } catch {
        return true;
    }
}

/** Process a single story: navigate with retries and collect errors. */
async function processStory(page, storyId, report) {
    const storyUrl = `http://localhost:6007/iframe.html?id=${storyId}&viewMode=story`;
    const storyDetails = {
        console: [],
        pageErrors: [],
        network: [],
        navigation: null
    };

    const onConsole = (msg) => {
        if (msg.type() === 'error' || msg.type() === 'warning') {
            const location = msg.location();
            storyDetails.console.push({
                type: msg.type(),
                text: msg.text(),
                url: location.url,
                location: `${location.url}:${location.lineNumber}:${location.columnNumber}`
            });
        }
    };
    const onPageError = (err) => {
        storyDetails.pageErrors.push({ message: err.message, stack: err.stack });
    };
    const onRequestFailed = (request) => {
        const url = request.url();
        if (!isAppRelevantFailure(url)) return;
        const failure = request.failure();
        const errorText = failure ? failure.errorText : 'Unknown network error';
        if (errorText === 'net::ERR_ABORTED' && /\/iframe\.html$|\/iframe$/.test(new URL(url).pathname || '')) return;
        // ERR_ABORTED often occurs when navigating away before requests complete; not a blocking error
        if (errorText === 'net::ERR_ABORTED') return;
        storyDetails.network.push({ url, method: request.method(), errorText });
    };

    page.on('console', onConsole);
    page.on('pageerror', onPageError);
    page.on('requestfailed', onRequestFailed);

    let lastError = null;
    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
        try {
            await page.goto(storyUrl, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT_MS });
            await page.waitForTimeout(POST_LOAD_WAIT_MS);
            lastError = null;
            break;
        } catch (e) {
            lastError = e;
            if (attempt < MAX_RETRIES) {
                await page.waitForTimeout(RETRY_DELAY_MS);
            } else {
                storyDetails.navigation = { message: e.message, stack: e.stack };
            }
        }
    }

    page.removeAllListeners('console');
    page.removeAllListeners('pageerror');
    page.removeAllListeners('requestfailed');

    const blockingConsoleErrors = storyDetails.console.filter(
        (c) => c.type === 'error' && !isIgnoredConsoleError(c.text, c.url)
    );
    const errorCount = blockingConsoleErrors.length +
        storyDetails.pageErrors.filter((e) => !isIgnoredConsoleError(e.message)).length +
        storyDetails.network.length +
        (storyDetails.navigation ? 1 : 0);

    if (errorCount > 0) {
        console.log(`[FAIL] ${storyId}: ${errorCount} errors`);
        report.failures[storyId] = storyDetails;
        report.metadata.storiesWithErrors++;
        report.metadata.totalErrors += errorCount;
    }

    return errorCount;
}

async function audit() {
    console.log("Entering audit function");

    let browser;
    try {
        console.log("Launching browser...");
        browser = await chromium.launch({ headless: true });
        console.log("Browser launched");
    } catch (e) {
        console.error("Failed to launch browser:", e);
        process.exit(1);
    }

    const storybookStaticPath = path.resolve(process.cwd(), 'storybook-static/index.json');
    console.log(`Reading index from ${storybookStaticPath}`);

    let stories = [];
    try {
        if (!fs.existsSync(storybookStaticPath)) {
            throw new Error(`File not found: ${storybookStaticPath}`);
        }
        const indexJson = JSON.parse(fs.readFileSync(storybookStaticPath, 'utf8'));
        let allStories = Object.values(indexJson.entries).map(e => e.id);
        const limit = parseInt(process.env.STORYBOOK_AUDIT_LIMIT || '0', 10);
        stories = limit > 0 ? allStories.slice(0, limit) : allStories;
        if (limit > 0) console.log(`Limiting audit to first ${limit} stories`);
    } catch (e) {
        console.error(`Failed to read local index.json: ${e.message}`);
        process.exit(1);
    }

    console.log(`Found ${stories.length} stories. (Batch size: ${BATCH_SIZE}, nav timeout: ${NAVIGATION_TIMEOUT_MS}ms, max retries: ${MAX_RETRIES})`);

    const report = {
        metadata: {
            timestamp: new Date().toISOString(),
            totalStories: stories.length,
            storiesWithErrors: 0,
            totalErrors: 0
        },
        failures: {}
    };

    const batches = [];
    for (let i = 0; i < stories.length; i += BATCH_SIZE) {
        batches.push(stories.slice(i, i + BATCH_SIZE));
    }

    for (let b = 0; b < batches.length; b++) {
        const batch = batches[b];
        const batchStart = b * BATCH_SIZE;
        let page;
        try {
            page = await browser.newPage();
            page.setDefaultTimeout(DEFAULT_TIMEOUT_MS);
        } catch (e) {
            console.error(`Failed to create page for batch ${b + 1}:`, e.message);
            process.exit(1);
        }

        for (let i = 0; i < batch.length; i++) {
            const storyId = batch[i];
            const globalIndex = batchStart + i;
            await processStory(page, storyId, report);
            if (globalIndex % 50 === 0) {
                console.log(`Processed ${globalIndex}/${stories.length}...`);
            }
        }

        await page.close().catch(() => {});
    }

    console.log("Audit complete.");
    console.log(`Stories with errors: ${report.metadata.storiesWithErrors}`);
    console.log(`Total errors captured: ${report.metadata.totalErrors}`);

    const outputPath = path.resolve(process.cwd(), 'storybook_audit_detailed.json');
    fs.writeFileSync(outputPath, JSON.stringify(report, null, 2));
    console.log(`Detailed report saved to: ${outputPath}`);

    await browser.close();

    if (report.metadata.storiesWithErrors > 0 || report.metadata.totalErrors > 0) {
        process.exit(1);
    }
}

audit().catch(e => {
    console.error("Top level error:", e);
    process.exit(1);
});