Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(vibe-tests): deploy screenshots to gh-pages + scoring markdown output (#1042)

Cindy Zhangdcdd49c72648
diff --git a/.github/workflows/cleanup-previews.yml b/.github/workflows/cleanup-previews.yml
index 54cafea13e..6d2ae24be2 100644
--- a/.github/workflows/cleanup-previews.yml
+++ b/.github/workflows/cleanup-previews.yml
@@ -110,6 +110,38 @@ jobs:
echo ""
fi
+ # ----- Clean old report screenshots -----
+ # Screenshots are large (~30 per report). Purge them from reports
+ # older than 30 days to control gh-pages size. The report HTML
+ # (index.html) and previews are tiny — keep those forever.
+ if [ -d "reports" ]; then
+ SCREENSHOT_PURGED=0
+ CUTOFF_DATE=$(date -d "30 days ago" +%s 2>/dev/null || date -v-30d +%s 2>/dev/null || echo "0")
+ if [ "$CUTOFF_DATE" != "0" ]; then
+ for report_dir in reports/*/; do
+ screenshots_dir="${report_dir}screenshots"
+ if [ -d "$screenshots_dir" ]; then
+ # Use the git commit date for the screenshots directory
+ COMMIT_DATE=$(git log -1 --format="%ct" -- "$screenshots_dir" 2>/dev/null || echo "0")
+ if [ "$COMMIT_DATE" != "0" ] && [ "$COMMIT_DATE" -lt "$CUTOFF_DATE" ]; then
+ REPORT_NAME=$(basename "$report_dir")
+ PNG_COUNT=$(find "$screenshots_dir" -name '*.png' 2>/dev/null | wc -l)
+ echo " DELETE ${screenshots_dir} — ${PNG_COUNT} screenshots older than 30 days"
+ if [ "$DRY_RUN" != "true" ]; then
+ git rm -rf --quiet "$screenshots_dir"
+ fi
+ SCREENSHOT_PURGED=$((SCREENSHOT_PURGED + 1))
+ DELETED=$((DELETED + 1))
+ fi
+ fi
+ done
+ if [ "$SCREENSHOT_PURGED" -gt 0 ]; then
+ echo "Purged screenshots from ${SCREENSHOT_PURGED} old reports"
+ echo ""
+ fi
+ fi
+ fi
+
# ----- Summary -----
echo "Summary: ${DELETED} items deleted"
diff --git a/.github/workflows/vibe-screenshots.yml b/.github/workflows/vibe-screenshots.yml
index 59e03665c9..109825606d 100644
--- a/.github/workflows/vibe-screenshots.yml
+++ b/.github/workflows/vibe-screenshots.yml
@@ -10,11 +10,26 @@ on:
description: 'Comma-separated prompt IDs (optional, defaults to all)'
required: false
default: ''
+ deploy_to_gh_pages:
+ description: 'Deploy screenshots to gh-pages (default: true)'
+ required: false
+ default: 'true'
+ type: choice
+ options:
+ - 'true'
+ - 'false'
+ report_iteration:
+ description: 'Primary iteration ID for report path (default: first in iterations list)'
+ required: false
+ default: ''
concurrency:
group: vibe-screenshots-${{ github.run_id }}
cancel-in-progress: false
+permissions:
+ contents: write
+
jobs:
build-previews:
name: Build Previews
@@ -98,3 +113,92 @@ jobs:
name: screenshot-manifest
path: internal/vibe-tests/results/*/screenshots/manifest.json
retention-days: 90
+
+ deploy-screenshots:
+ name: Deploy Screenshots to gh-pages
+ needs: screenshot
+ if: ${{ github.event.inputs.deploy_to_gh_pages != 'false' }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout gh-pages
+ uses: actions/checkout@v4
+ with:
+ ref: gh-pages
+ fetch-depth: 1
+
+ - name: Determine report iteration
+ id: report-id
+ run: |
+ # Use explicit report_iteration input, or fall back to first in the list
+ REPORT_ID="${{ github.event.inputs.report_iteration }}"
+ if [ -z "$REPORT_ID" ]; then
+ REPORT_ID=$(echo "${{ github.event.inputs.iterations }}" | cut -d',' -f1)
+ fi
+ echo "id=$REPORT_ID" >> "$GITHUB_OUTPUT"
+ echo "Report iteration: $REPORT_ID"
+
+ - name: Download screenshots
+ uses: actions/download-artifact@v4
+ with:
+ name: vibe-test-screenshots
+ path: /tmp/screenshots
+
+ - name: Download screenshot manifest
+ uses: actions/download-artifact@v4
+ with:
+ name: screenshot-manifest
+ path: /tmp/manifests
+
+ - name: Deploy screenshots to report directory
+ env:
+ REPORT_ID: ${{ steps.report-id.outputs.id }}
+ run: |
+ REPORT_DIR="reports/${REPORT_ID}"
+
+ # Create screenshots directory in the report
+ mkdir -p "${REPORT_DIR}/screenshots"
+
+ # Copy all screenshot PNGs from all iterations
+ find /tmp/screenshots -name '*.png' -exec cp {} "${REPORT_DIR}/screenshots/" \;
+
+ # Merge all manifest.json files into one
+ # Each iteration has its own manifest — merge them
+ node -e "
+ const fs = require('fs');
+ const path = require('path');
+ const merged = {};
+ const manifests = [];
+ const walkDir = (dir) => {
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+ const fullPath = path.join(dir, entry.name);
+ if (entry.isDirectory()) walkDir(fullPath);
+ else if (entry.name === 'manifest.json') manifests.push(fullPath);
+ }
+ };
+ walkDir('/tmp/manifests');
+ for (const f of manifests) {
+ const data = JSON.parse(fs.readFileSync(f, 'utf-8'));
+ Object.assign(merged, data);
+ }
+ const outDir = 'reports/${REPORT_ID}/screenshots';
+ fs.mkdirSync(outDir, { recursive: true });
+ fs.writeFileSync(path.join(outDir, 'manifest.json'), JSON.stringify(merged, null, 2));
+ const pngCount = fs.readdirSync(outDir).filter(f => f.endsWith('.png')).length;
+ console.log('Deployed ' + pngCount + ' screenshots + manifest to ' + outDir);
+ "
+
+ - name: Commit and push
+ env:
+ REPORT_ID: ${{ steps.report-id.outputs.id }}
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+ git add "reports/${REPORT_ID}/screenshots/"
+ if git diff --cached --quiet; then
+ echo "No new screenshots to deploy"
+ else
+ SCREENSHOT_COUNT=$(find "reports/${REPORT_ID}/screenshots" -name '*.png' | wc -l)
+ git commit -m "screenshots: ${REPORT_ID} (${SCREENSHOT_COUNT} images)"
+ git push origin gh-pages
+ echo "✅ Deployed ${SCREENSHOT_COUNT} screenshots to reports/${REPORT_ID}/screenshots/"
+ fi
diff --git a/internal/vibe-tests/package.json b/internal/vibe-tests/package.json
index 60b5b8e989..08b3c6ef8c 100644
--- a/internal/vibe-tests/package.json
+++ b/internal/vibe-tests/package.json
@@ -22,6 +22,7 @@
"generate-a11y-manifest": "tsx src/generate-a11y-manifest.ts",
"universal": "tsx src/universal-aggregate.ts",
"universal:compare": "tsx src/universal-compare.ts",
+ "universal:compare:md": "tsx src/universal-compare.ts --markdown",
"report:build": "tsx src/build-report.ts",
"report:dev": "tsx src/build-report.ts --dev",
"report:deploy": "tsx src/deploy-report.ts",
diff --git a/internal/vibe-tests/src/deploy-report.ts b/internal/vibe-tests/src/deploy-report.ts
index f0b809a06e..3203ff3450 100644
--- a/internal/vibe-tests/src/deploy-report.ts
+++ b/internal/vibe-tests/src/deploy-report.ts
@@ -162,19 +162,48 @@ async function main() {
console.log(` ✓ ${previewCount} preview pages copied`);
}
- // Deploy screenshots if they exist (from GHA vibe-screenshots workflow)
- // Check all iterations for screenshots and merge into one directory
+ // Deploy screenshots if they exist locally (from GHA artifact download)
+ // Also check if screenshots were already deployed to gh-pages by the
+ // vibe-screenshots workflow's deploy-screenshots job.
const targetScreenshotsDir = path.join(targetDir, 'screenshots');
let totalScreenshots = 0;
+
+ // First: check if gh-pages already has screenshots (from GHA deploy-screenshots job)
+ if (
+ fs.existsSync(targetScreenshotsDir) &&
+ countFiles(targetScreenshotsDir, '.png') > 0
+ ) {
+ totalScreenshots = countFiles(targetScreenshotsDir, '.png');
+ console.log(
+ ` ✓ ${totalScreenshots} screenshots found on gh-pages (from GHA)`,
+ );
+ }
+
+ // Second: copy any local screenshots (may add to or overwrite gh-pages ones)
for (const id of [iteration, baseline, html].filter(Boolean) as string[]) {
const screenshotsDir = path.join(VIBE_DIR, 'results', id, 'screenshots');
if (fs.existsSync(screenshotsDir)) {
copyDirRecursive(screenshotsDir, targetScreenshotsDir);
- totalScreenshots += countFiles(screenshotsDir, '.png');
+ const localCount = countFiles(screenshotsDir, '.png');
+ totalScreenshots = countFiles(targetScreenshotsDir, '.png');
+ console.log(` ✓ ${localCount} screenshots copied from local results`);
}
}
- if (totalScreenshots > 0) {
- console.log(` ✓ ${totalScreenshots} screenshots copied`);
+
+ // If screenshots exist (from either source), inject manifest into report HTML
+ // so the report app can render them. The report HTML may have been built
+ // without screenshot data if screenshots weren't available locally during build.
+ if (
+ totalScreenshots > 0 &&
+ fs.existsSync(path.join(targetDir, 'index.html'))
+ ) {
+ const manifestPath = path.join(targetScreenshotsDir, 'manifest.json');
+ if (fs.existsSync(manifestPath)) {
+ injectScreenshotsIntoReport(
+ path.join(targetDir, 'index.html'),
+ manifestPath,
+ );
+ }
}
updateReportsIndex(tmpDir);
@@ -206,6 +235,69 @@ async function main() {
}
}
+/**
+ * Inject screenshot manifest into a deployed report HTML file.
+ *
+ * When the report was built without local screenshots (common in sandbox
+ * environments that can't download GHA artifacts), the __REPORT_DATA__
+ * in the HTML will have `screenshots: null`. This function reads the
+ * screenshot manifest from gh-pages and patches the report data so the
+ * ScreenshotGallery component renders properly.
+ */
+function injectScreenshotsIntoReport(
+ htmlPath: string,
+ manifestPath: string,
+): void {
+ try {
+ const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+ const screenshots: Record<string, string> = {};
+
+ // Flatten the nested manifest (promptId → target → viewport → theme → filename)
+ // into a flat map (filename → relative URL)
+ for (const [, targets] of Object.entries(manifest)) {
+ for (const [, viewports] of Object.entries(
+ targets as Record<string, Record<string, Record<string, string>>>,
+ )) {
+ for (const [, themes] of Object.entries(
+ viewports as Record<string, Record<string, string>>,
+ )) {
+ for (const [, filename] of Object.entries(
+ themes as Record<string, string>,
+ )) {
+ screenshots[filename] = `screenshots/${filename}`;
+ }
+ }
+ }
+ }
+
+ if (Object.keys(screenshots).length === 0) return;
+
+ let html = fs.readFileSync(htmlPath, 'utf-8');
+
+ // Find the __REPORT_DATA__ script tag and patch the screenshots field
+ const dataMatch = html.match(
+ /window\.__REPORT_DATA__=(\{.+?\});<\/script>/,
+ );
+ if (dataMatch) {
+ const data = JSON.parse(dataMatch[1]);
+ if (!data.screenshots || Object.keys(data.screenshots).length === 0) {
+ data.screenshots = screenshots;
+ const newScript = `window.__REPORT_DATA__=${JSON.stringify(data)};</script>`;
+ html = html.replace(
+ /window\.__REPORT_DATA__=\{.+?\};<\/script>/,
+ newScript,
+ );
+ fs.writeFileSync(htmlPath, html);
+ console.log(
+ ` ✓ Injected ${Object.keys(screenshots).length} screenshot references into report`,
+ );
+ }
+ }
+ } catch (err) {
+ console.warn(` ⚠️ Failed to inject screenshots into report: ${err}`);
+ }
+}
+
function copyDirRecursive(src: string, dest: string): void {
fs.mkdirSync(dest, {recursive: true});
for (const entry of fs.readdirSync(src, {withFileTypes: true})) {
diff --git a/internal/vibe-tests/src/universal-compare.ts b/internal/vibe-tests/src/universal-compare.ts
index c27e006386..775846982c 100644
--- a/internal/vibe-tests/src/universal-compare.ts
+++ b/internal/vibe-tests/src/universal-compare.ts
@@ -6,6 +6,7 @@
* tsx src/universal-compare.ts --xds abc123 --baseline def456
* tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789
* tsx src/universal-compare.ts --xds abc123 --baseline def456 --json
+ * tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789 --markdown
*/
import * as fs from 'node:fs';
@@ -85,12 +86,14 @@ function parseArgs(): {
baseline: string;
html?: string;
json: boolean;
+ markdown: boolean;
} {
const args = process.argv.slice(2);
let xds = '';
let baseline = '';
let html: string | undefined;
let json = false;
+ let markdown = false;
for (let i = 0; i < args.length; i++) {
if ((args[i] === '--xds' || args[i] === '-x') && args[i + 1]) {
@@ -101,21 +104,138 @@ function parseArgs(): {
html = args[++i];
} else if (args[i] === '--json') {
json = true;
+ } else if (args[i] === '--markdown' || args[i] === '--md') {
+ markdown = true;
}
}
if (!xds || !baseline) {
console.error(
- 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--json]',
+ 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--json] [--markdown]',
);
process.exit(1);
}
- return {xds, baseline, html, json};
+ return {xds, baseline, html, json, markdown};
+}
+
+/**
+ * Generate a GitHub-flavored markdown summary table.
+ * Designed for pasting into GitHub issue bodies.
+ */
+function toMarkdown(opts: {
+ comparison: UniversalComparison;
+ xdsId: string;
+ baselineId: string;
+ htmlId?: string;
+ byPrompt: UniversalComparison['byPrompt'];
+}): string {
+ const {comparison, xdsId, baselineId, htmlId, byPrompt} = opts;
+ const {xds, baseline, html: htmlData, winners} = comparison;
+ const dimensions = getDimensionNames();
+ const isThreeWay = !!htmlData;
+ const lines: string[] = [];
+
+ // Summary table
+ if (isThreeWay) {
+ lines.push(
+ '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',
+ );
+ lines.push(
+ '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',
+ );
+ } else {
+ lines.push(
+ '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',
+ );
+ lines.push(
+ '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',
+ );
+ }
+
+ const dimOrder: UniversalDimension[] = [
+ 'correctness',
+ 'accessibility',
+ 'codeQuality',
+ 'efficiency',
+ 'maintainability',
+ ];
+
+ const xdsRow = dimOrder.map(d => xds.averages[d]).join(' | ');
+ lines.push(`| **XDS** | \`${xdsId}\` | ${xds.overall} | ${xdsRow} |`);
+
+ const baseRow = dimOrder.map(d => baseline.averages[d]).join(' | ');
+ lines.push(
+ `| **Baseline** | \`${baselineId}\` | ${baseline.overall} | ${baseRow} |`,
+ );
+
+ if (isThreeWay && htmlData) {
+ const htmlRow = dimOrder.map(d => htmlData.averages[d]).join(' | ');
+ lines.push(
+ `| **HTML** | \`${htmlId}\` | ${htmlData.overall} | ${htmlRow} |`,
+ );
+ }
+
+ lines.push('');
+
+ // Per-prompt winners
+ const promptEntries = Object.entries(byPrompt);
+ if (promptEntries.length > 0) {
+ let xWins = 0;
+ let bWins = 0;
+ let hWins = 0;
+ let ties = 0;
+ for (const [, data] of promptEntries) {
+ if (data.winner === 'xds') xWins++;
+ else if (data.winner === 'baseline') bWins++;
+ else if (data.winner === 'html') hWins++;
+ else ties++;
+ }
+ if (isThreeWay) {
+ lines.push(
+ `**Per-prompt wins:** XDS ${xWins} · Baseline ${bWins} · HTML ${hWins} · Tie ${ties} (${promptEntries.length} prompts)`,
+ );
+ } else {
+ lines.push(
+ `**Per-prompt wins:** XDS ${xWins} · Baseline ${bWins} · Tie ${ties} (${promptEntries.length} prompts)`,
+ );
+ }
+ lines.push('');
+ }
+
+ // Dark mode
+ if (isThreeWay && htmlData) {
+ lines.push(
+ `**Dark mode:** XDS ${xds.darkModeRate}% · Baseline ${baseline.darkModeRate}% · HTML ${htmlData.darkModeRate}%`,
+ );
+ } else {
+ lines.push(
+ `**Dark mode:** XDS ${xds.darkModeRate}% · Baseline ${baseline.darkModeRate}%`,
+ );
+ }
+
+ // Dimension winners
+ lines.push('');
+ lines.push('**Dimension winners:**');
+ for (const d of dimensions) {
+ const w = winners[d];
+ const icon =
+ w === 'xds' ? '🟢' : w === 'baseline' ? '🔵' : w === 'html' ? '🟡' : '⚪';
+ const label = DIMENSION_LABELS[d] || d;
+ lines.push(`- ${label}: ${icon} ${w === 'tie' ? 'Tie' : w.toUpperCase()}`);
+ }
+
+ return lines.join('\n');
}
async function main() {
- const {xds: xdsId, baseline: baselineId, html: htmlId, json} = parseArgs();
+ const {
+ xds: xdsId,
+ baseline: baselineId,
+ html: htmlId,
+ json,
+ markdown,
+ } = parseArgs();
const xds = loadOrGenerate(xdsId);
const baseline = loadOrGenerate(baselineId);
@@ -180,6 +300,19 @@ async function main() {
return;
}
+ if (markdown) {
+ console.log(
+ toMarkdown({
+ comparison,
+ xdsId,
+ baselineId,
+ htmlId,
+ byPrompt,
+ }),
+ );
+ return;
+ }
+
// --- Print report ---
const title = isThreeWay