An open source design system that's fully customizable and agent ready
TypeScript
fix(vibe-tests): deploy screenshots to gh-pages + scoring markdown output (#1042)
Cindy Zhangdcdd49c72648
diff --git a/.github/workflows/cleanup-previews.yml b/.github/workflows/cleanup-previews.ymlindex 54cafea13e..6d2ae24be2 100644--- a/.github/workflows/cleanup-previews.yml+++ b/.github/workflows/cleanup-previews.yml@@ -110,6 +110,38 @@ jobs:echo ""fi+ # ----- Clean old report screenshots -----+ # Screenshots are large (~30 per report). Purge them from reports+ # older than 30 days to control gh-pages size. The report HTML+ # (index.html) and previews are tiny — keep those forever.+ if [ -d "reports" ]; then+ SCREENSHOT_PURGED=0+ CUTOFF_DATE=$(date -d "30 days ago" +%s 2>/dev/null || date -v-30d +%s 2>/dev/null || echo "0")+ if [ "$CUTOFF_DATE" != "0" ]; then+ for report_dir in reports/*/; do+ screenshots_dir="${report_dir}screenshots"+ if [ -d "$screenshots_dir" ]; then+ # Use the git commit date for the screenshots directory+ COMMIT_DATE=$(git log -1 --format="%ct" -- "$screenshots_dir" 2>/dev/null || echo "0")+ if [ "$COMMIT_DATE" != "0" ] && [ "$COMMIT_DATE" -lt "$CUTOFF_DATE" ]; then+ REPORT_NAME=$(basename "$report_dir")+ PNG_COUNT=$(find "$screenshots_dir" -name '*.png' 2>/dev/null | wc -l)+ echo " DELETE ${screenshots_dir} — ${PNG_COUNT} screenshots older than 30 days"+ if [ "$DRY_RUN" != "true" ]; then+ git rm -rf --quiet "$screenshots_dir"+ fi+ SCREENSHOT_PURGED=$((SCREENSHOT_PURGED + 1))+ DELETED=$((DELETED + 1))+ fi+ fi+ done+ if [ "$SCREENSHOT_PURGED" -gt 0 ]; then+ echo "Purged screenshots from ${SCREENSHOT_PURGED} old reports"+ echo ""+ fi+ fi+ fi+# ----- Summary -----echo "Summary: ${DELETED} items deleted"diff --git a/.github/workflows/vibe-screenshots.yml b/.github/workflows/vibe-screenshots.ymlindex 59e03665c9..109825606d 100644--- a/.github/workflows/vibe-screenshots.yml+++ b/.github/workflows/vibe-screenshots.yml@@ -10,11 +10,26 @@ on:description: 'Comma-separated prompt IDs (optional, defaults to all)'required: falsedefault: ''+ deploy_to_gh_pages:+ description: 'Deploy screenshots to gh-pages (default: true)'+ required: false+ default: 'true'+ type: choice+ options:+ - 'true'+ - 'false'+ report_iteration:+ description: 'Primary iteration ID for report path (default: first in iterations list)'+ required: false+ default: ''concurrency:group: vibe-screenshots-${{ github.run_id }}cancel-in-progress: false+permissions:+ contents: write+jobs:build-previews:name: Build Previews@@ -98,3 +113,92 @@ jobs:name: screenshot-manifestpath: internal/vibe-tests/results/*/screenshots/manifest.jsonretention-days: 90++ deploy-screenshots:+ name: Deploy Screenshots to gh-pages+ needs: screenshot+ if: ${{ github.event.inputs.deploy_to_gh_pages != 'false' }}+ runs-on: ubuntu-latest+ steps:+ - name: Checkout gh-pages+ uses: actions/checkout@v4+ with:+ ref: gh-pages+ fetch-depth: 1++ - name: Determine report iteration+ id: report-id+ run: |+ # Use explicit report_iteration input, or fall back to first in the list+ REPORT_ID="${{ github.event.inputs.report_iteration }}"+ if [ -z "$REPORT_ID" ]; then+ REPORT_ID=$(echo "${{ github.event.inputs.iterations }}" | cut -d',' -f1)+ fi+ echo "id=$REPORT_ID" >> "$GITHUB_OUTPUT"+ echo "Report iteration: $REPORT_ID"++ - name: Download screenshots+ uses: actions/download-artifact@v4+ with:+ name: vibe-test-screenshots+ path: /tmp/screenshots++ - name: Download screenshot manifest+ uses: actions/download-artifact@v4+ with:+ name: screenshot-manifest+ path: /tmp/manifests++ - name: Deploy screenshots to report directory+ env:+ REPORT_ID: ${{ steps.report-id.outputs.id }}+ run: |+ REPORT_DIR="reports/${REPORT_ID}"++ # Create screenshots directory in the report+ mkdir -p "${REPORT_DIR}/screenshots"++ # Copy all screenshot PNGs from all iterations+ find /tmp/screenshots -name '*.png' -exec cp {} "${REPORT_DIR}/screenshots/" \;++ # Merge all manifest.json files into one+ # Each iteration has its own manifest — merge them+ node -e "+ const fs = require('fs');+ const path = require('path');+ const merged = {};+ const manifests = [];+ const walkDir = (dir) => {+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {+ const fullPath = path.join(dir, entry.name);+ if (entry.isDirectory()) walkDir(fullPath);+ else if (entry.name === 'manifest.json') manifests.push(fullPath);+ }+ };+ walkDir('/tmp/manifests');+ for (const f of manifests) {+ const data = JSON.parse(fs.readFileSync(f, 'utf-8'));+ Object.assign(merged, data);+ }+ const outDir = 'reports/${REPORT_ID}/screenshots';+ fs.mkdirSync(outDir, { recursive: true });+ fs.writeFileSync(path.join(outDir, 'manifest.json'), JSON.stringify(merged, null, 2));+ const pngCount = fs.readdirSync(outDir).filter(f => f.endsWith('.png')).length;+ console.log('Deployed ' + pngCount + ' screenshots + manifest to ' + outDir);+ "++ - name: Commit and push+ env:+ REPORT_ID: ${{ steps.report-id.outputs.id }}+ run: |+ git config user.name "github-actions[bot]"+ git config user.email "github-actions[bot]@users.noreply.github.com"+ git add "reports/${REPORT_ID}/screenshots/"+ if git diff --cached --quiet; then+ echo "No new screenshots to deploy"+ else+ SCREENSHOT_COUNT=$(find "reports/${REPORT_ID}/screenshots" -name '*.png' | wc -l)+ git commit -m "screenshots: ${REPORT_ID} (${SCREENSHOT_COUNT} images)"+ git push origin gh-pages+ echo "✅ Deployed ${SCREENSHOT_COUNT} screenshots to reports/${REPORT_ID}/screenshots/"+ fidiff --git a/internal/vibe-tests/package.json b/internal/vibe-tests/package.jsonindex 60b5b8e989..08b3c6ef8c 100644--- a/internal/vibe-tests/package.json+++ b/internal/vibe-tests/package.json@@ -22,6 +22,7 @@"generate-a11y-manifest": "tsx src/generate-a11y-manifest.ts","universal": "tsx src/universal-aggregate.ts","universal:compare": "tsx src/universal-compare.ts",+ "universal:compare:md": "tsx src/universal-compare.ts --markdown","report:build": "tsx src/build-report.ts","report:dev": "tsx src/build-report.ts --dev","report:deploy": "tsx src/deploy-report.ts",diff --git a/internal/vibe-tests/src/deploy-report.ts b/internal/vibe-tests/src/deploy-report.tsindex f0b809a06e..3203ff3450 100644--- a/internal/vibe-tests/src/deploy-report.ts+++ b/internal/vibe-tests/src/deploy-report.ts@@ -162,19 +162,48 @@ async function main() {console.log(` ✓ ${previewCount} preview pages copied`);}- // Deploy screenshots if they exist (from GHA vibe-screenshots workflow)- // Check all iterations for screenshots and merge into one directory+ // Deploy screenshots if they exist locally (from GHA artifact download)+ // Also check if screenshots were already deployed to gh-pages by the+ // vibe-screenshots workflow's deploy-screenshots job.const targetScreenshotsDir = path.join(targetDir, 'screenshots');let totalScreenshots = 0;++ // First: check if gh-pages already has screenshots (from GHA deploy-screenshots job)+ if (+ fs.existsSync(targetScreenshotsDir) &&+ countFiles(targetScreenshotsDir, '.png') > 0+ ) {+ totalScreenshots = countFiles(targetScreenshotsDir, '.png');+ console.log(+ ` ✓ ${totalScreenshots} screenshots found on gh-pages (from GHA)`,+ );+ }++ // Second: copy any local screenshots (may add to or overwrite gh-pages ones)for (const id of [iteration, baseline, html].filter(Boolean) as string[]) {const screenshotsDir = path.join(VIBE_DIR, 'results', id, 'screenshots');if (fs.existsSync(screenshotsDir)) {copyDirRecursive(screenshotsDir, targetScreenshotsDir);- totalScreenshots += countFiles(screenshotsDir, '.png');+ const localCount = countFiles(screenshotsDir, '.png');+ totalScreenshots = countFiles(targetScreenshotsDir, '.png');+ console.log(` ✓ ${localCount} screenshots copied from local results`);}}- if (totalScreenshots > 0) {- console.log(` ✓ ${totalScreenshots} screenshots copied`);++ // If screenshots exist (from either source), inject manifest into report HTML+ // so the report app can render them. The report HTML may have been built+ // without screenshot data if screenshots weren't available locally during build.+ if (+ totalScreenshots > 0 &&+ fs.existsSync(path.join(targetDir, 'index.html'))+ ) {+ const manifestPath = path.join(targetScreenshotsDir, 'manifest.json');+ if (fs.existsSync(manifestPath)) {+ injectScreenshotsIntoReport(+ path.join(targetDir, 'index.html'),+ manifestPath,+ );+ }}updateReportsIndex(tmpDir);@@ -206,6 +235,69 @@ async function main() {}}+/**+ * Inject screenshot manifest into a deployed report HTML file.+ *+ * When the report was built without local screenshots (common in sandbox+ * environments that can't download GHA artifacts), the __REPORT_DATA__+ * in the HTML will have `screenshots: null`. This function reads the+ * screenshot manifest from gh-pages and patches the report data so the+ * ScreenshotGallery component renders properly.+ */+function injectScreenshotsIntoReport(+ htmlPath: string,+ manifestPath: string,+): void {+ try {+ const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));+ const screenshots: Record<string, string> = {};++ // Flatten the nested manifest (promptId → target → viewport → theme → filename)+ // into a flat map (filename → relative URL)+ for (const [, targets] of Object.entries(manifest)) {+ for (const [, viewports] of Object.entries(+ targets as Record<string, Record<string, Record<string, string>>>,+ )) {+ for (const [, themes] of Object.entries(+ viewports as Record<string, Record<string, string>>,+ )) {+ for (const [, filename] of Object.entries(+ themes as Record<string, string>,+ )) {+ screenshots[filename] = `screenshots/${filename}`;+ }+ }+ }+ }++ if (Object.keys(screenshots).length === 0) return;++ let html = fs.readFileSync(htmlPath, 'utf-8');++ // Find the __REPORT_DATA__ script tag and patch the screenshots field+ const dataMatch = html.match(+ /window\.__REPORT_DATA__=(\{.+?\});<\/script>/,+ );+ if (dataMatch) {+ const data = JSON.parse(dataMatch[1]);+ if (!data.screenshots || Object.keys(data.screenshots).length === 0) {+ data.screenshots = screenshots;+ const newScript = `window.__REPORT_DATA__=${JSON.stringify(data)};</script>`;+ html = html.replace(+ /window\.__REPORT_DATA__=\{.+?\};<\/script>/,+ newScript,+ );+ fs.writeFileSync(htmlPath, html);+ console.log(+ ` ✓ Injected ${Object.keys(screenshots).length} screenshot references into report`,+ );+ }+ }+ } catch (err) {+ console.warn(` ⚠️ Failed to inject screenshots into report: ${err}`);+ }+}+function copyDirRecursive(src: string, dest: string): void {fs.mkdirSync(dest, {recursive: true});for (const entry of fs.readdirSync(src, {withFileTypes: true})) {diff --git a/internal/vibe-tests/src/universal-compare.ts b/internal/vibe-tests/src/universal-compare.tsindex c27e006386..775846982c 100644--- a/internal/vibe-tests/src/universal-compare.ts+++ b/internal/vibe-tests/src/universal-compare.ts@@ -6,6 +6,7 @@* tsx src/universal-compare.ts --xds abc123 --baseline def456* tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789* tsx src/universal-compare.ts --xds abc123 --baseline def456 --json+ * tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789 --markdown*/import * as fs from 'node:fs';@@ -85,12 +86,14 @@ function parseArgs(): {baseline: string;html?: string;json: boolean;+ markdown: boolean;} {const args = process.argv.slice(2);let xds = '';let baseline = '';let html: string | undefined;let json = false;+ let markdown = false;for (let i = 0; i < args.length; i++) {if ((args[i] === '--xds' || args[i] === '-x') && args[i + 1]) {@@ -101,21 +104,138 @@ function parseArgs(): {html = args[++i];} else if (args[i] === '--json') {json = true;+ } else if (args[i] === '--markdown' || args[i] === '--md') {+ markdown = true;}}if (!xds || !baseline) {console.error(- 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--json]',+ 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--json] [--markdown]',);process.exit(1);}- return {xds, baseline, html, json};+ return {xds, baseline, html, json, markdown};+}++/**+ * Generate a GitHub-flavored markdown summary table.+ * Designed for pasting into GitHub issue bodies.+ */+function toMarkdown(opts: {+ comparison: UniversalComparison;+ xdsId: string;+ baselineId: string;+ htmlId?: string;+ byPrompt: UniversalComparison['byPrompt'];+}): string {+ const {comparison, xdsId, baselineId, htmlId, byPrompt} = opts;+ const {xds, baseline, html: htmlData, winners} = comparison;+ const dimensions = getDimensionNames();+ const isThreeWay = !!htmlData;+ const lines: string[] = [];++ // Summary table+ if (isThreeWay) {+ lines.push(+ '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',+ );+ lines.push(+ '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',+ );+ } else {+ lines.push(+ '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',+ );+ lines.push(+ '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',+ );+ }++ const dimOrder: UniversalDimension[] = [+ 'correctness',+ 'accessibility',+ 'codeQuality',+ 'efficiency',+ 'maintainability',+ ];++ const xdsRow = dimOrder.map(d => xds.averages[d]).join(' | ');+ lines.push(`| **XDS** | \`${xdsId}\` | ${xds.overall} | ${xdsRow} |`);++ const baseRow = dimOrder.map(d => baseline.averages[d]).join(' | ');+ lines.push(+ `| **Baseline** | \`${baselineId}\` | ${baseline.overall} | ${baseRow} |`,+ );++ if (isThreeWay && htmlData) {+ const htmlRow = dimOrder.map(d => htmlData.averages[d]).join(' | ');+ lines.push(+ `| **HTML** | \`${htmlId}\` | ${htmlData.overall} | ${htmlRow} |`,+ );+ }++ lines.push('');++ // Per-prompt winners+ const promptEntries = Object.entries(byPrompt);+ if (promptEntries.length > 0) {+ let xWins = 0;+ let bWins = 0;+ let hWins = 0;+ let ties = 0;+ for (const [, data] of promptEntries) {+ if (data.winner === 'xds') xWins++;+ else if (data.winner === 'baseline') bWins++;+ else if (data.winner === 'html') hWins++;+ else ties++;+ }+ if (isThreeWay) {+ lines.push(+ `**Per-prompt wins:** XDS ${xWins} · Baseline ${bWins} · HTML ${hWins} · Tie ${ties} (${promptEntries.length} prompts)`,+ );+ } else {+ lines.push(+ `**Per-prompt wins:** XDS ${xWins} · Baseline ${bWins} · Tie ${ties} (${promptEntries.length} prompts)`,+ );+ }+ lines.push('');+ }++ // Dark mode+ if (isThreeWay && htmlData) {+ lines.push(+ `**Dark mode:** XDS ${xds.darkModeRate}% · Baseline ${baseline.darkModeRate}% · HTML ${htmlData.darkModeRate}%`,+ );+ } else {+ lines.push(+ `**Dark mode:** XDS ${xds.darkModeRate}% · Baseline ${baseline.darkModeRate}%`,+ );+ }++ // Dimension winners+ lines.push('');+ lines.push('**Dimension winners:**');+ for (const d of dimensions) {+ const w = winners[d];+ const icon =+ w === 'xds' ? '🟢' : w === 'baseline' ? '🔵' : w === 'html' ? '🟡' : '⚪';+ const label = DIMENSION_LABELS[d] || d;+ lines.push(`- ${label}: ${icon} ${w === 'tie' ? 'Tie' : w.toUpperCase()}`);+ }++ return lines.join('\n');}async function main() {- const {xds: xdsId, baseline: baselineId, html: htmlId, json} = parseArgs();+ const {+ xds: xdsId,+ baseline: baselineId,+ html: htmlId,+ json,+ markdown,+ } = parseArgs();const xds = loadOrGenerate(xdsId);const baseline = loadOrGenerate(baselineId);@@ -180,6 +300,19 @@ async function main() {return;}+ if (markdown) {+ console.log(+ toMarkdown({+ comparison,+ xdsId,+ baselineId,+ htmlId,+ byPrompt,+ }),+ );+ return;+ }+// --- Print report ---const title = isThreeWay