Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(vibe-tests): add design judge — vision LLM evaluation dimension (#863)

Erneste9f64dc426b7
diff --git a/internal/vibe-tests/README.md b/internal/vibe-tests/README.md
index 11664a680a..8997fe3ed1 100644
--- a/internal/vibe-tests/README.md
+++ b/internal/vibe-tests/README.md
@@ -64,25 +64,37 @@ The harness:
```
internal/vibe-tests/
├── src/
-│ ├── cli.ts # Command-line interface
-│ ├── types.ts # TypeScript types
-│ ├── runner.ts # Test orchestration
-│ ├── evaluator.ts # Response evaluation
-│ ├── analyst.ts # Results analysis
-│ ├── sampling.ts # Stratified sampling
-│ └── utils.ts # Utilities
+│ ├── cli.ts # Command-line interface
+│ ├── types.ts # TypeScript types
+│ ├── runner.ts # Test orchestration
+│ ├── evaluator.ts # Response evaluation
+│ ├── analyst.ts # Results analysis
+│ ├── sampling.ts # Stratified sampling
+│ ├── universal-eval.ts # 5-dimension static analysis (code-level)
+│ ├── universal-aggregate.ts # Aggregate scores across prompts
+│ ├── universal-compare.ts # Compare XDS vs baseline vs HTML
+│ ├── design-judge.ts # 6th dimension: vision LLM visual fidelity judge
+│ ├── screenshot-previews.ts # Playwright screenshots of pre-built HTML
+│ ├── build-previews.ts # Build self-contained HTML from results
+│ ├── build-report.ts # Build static report HTML + inject data
+│ ├── deploy-report.ts # Deploy report + screenshots to gh-pages
+│ └── utils.ts # Utilities
+├── ideals/ # Human-provided reference images for design judge
+│ └── {promptId}[-{viewport}[-{theme}]].png
├── prompts/
-│ ├── evaluator.md # Evaluator agent prompt
-│ ├── analyst.md # Analyst agent prompt
-│ └── personas/ # User personas
+│ ├── evaluator.md # Evaluator agent prompt
+│ ├── analyst.md # Analyst agent prompt
+│ └── personas/ # User personas
├── test-sets/
-│ └── default.json # Test prompt battery
-├── results/ # Generated (gitignored)
+│ └── default.json # Test prompt battery
+├── results/ # Generated (gitignored)
│ └── {iteration-id}/
│ ├── runs.jsonl
│ ├── analysis.json
+│ ├── universal.json # 5+1 dimension scores
+│ ├── design-scores.json # Design dimension raw judge output
│ └── report.html
-└── iterations.json # Iteration lineage
+└── iterations.json # Iteration lineage
```
## Test Protocols
@@ -255,6 +267,27 @@ gh workflow run vibe-screenshots.yml -f iterations="<id1>,<id2>,<id3>"
**No API keys required.** Code generation is handled by Navi sub-agents before this workflow runs.
+### Design Judge (`src/design-judge.ts`)
+
+Vision LLM evaluation comparing rendered screenshots against human-provided ideal reference images.
+
+```bash
+# Dry run — shows what would be evaluated without API calls
+yarn workspace @xds/vibe-tests design-judge --iteration <id> --dry-run
+
+# Full run — 3 passes per screenshot, desktop + light
+yarn workspace @xds/vibe-tests design-judge --iteration <id>
+
+# Custom: specific prompts, 5 passes, mobile dark
+yarn workspace @xds/vibe-tests design-judge --iteration <id> --prompts cwm-1,ty-3 --passes 5 --viewport mobile --theme dark
+```
+
+**Requires:** `ANTHROPIC_API_KEY` environment variable. Uses Claude Sonnet for vision analysis.
+
+**Prerequisite:** Ideal reference images in `internal/vibe-tests/ideals/`. Only prompts with both screenshots and ideals are evaluated.
+
+**Output:** `results/<iteration>/design-scores.json` — scores are automatically merged into `universal.json` when the aggregate runs.
+
### Direct API Mode (legacy)
The `test:xds:sample` script calls Anthropic directly and requires `ANTHROPIC_API_KEY`. This is the old all-in-one flow — prefer the Navi-orchestrated pipeline for nightly runs.
diff --git a/internal/vibe-tests/ideals/README.md b/internal/vibe-tests/ideals/README.md
new file mode 100644
index 0000000000..7359c77032
--- /dev/null
+++ b/internal/vibe-tests/ideals/README.md
@@ -0,0 +1,40 @@
+# Ideal Reference Images
+
+Reference designs for the Night Watch Design dimension. These are the
+"target" that generated screenshots are compared against.
+
+## Naming Convention
+
+Uses `__` (double underscore) as separator between promptId and viewport/theme.
+This supports multi-segment prompt IDs like `feature-card-1`.
+
+```
+{promptId}__{viewport}__{theme}.png — specific (preferred)
+{promptId}__{viewport}.png — viewport-only (used for both themes)
+{promptId}.png — generic fallback (used for all combos)
+```
+
+Examples:
+
+- `cwm-1__desktop__light.png` — specific to desktop + light
+- `ty-3__desktop.png` — desktop only, used for both light and dark
+- `dd-3.png` — fallback for all viewport/theme combos
+
+## Requirements
+
+- **Format:** PNG
+- **Viewport:** 1280×800 for desktop, 375×812 for mobile (matching screenshot-previews.ts)
+- **Source:** Figma mockup or hand-built HTML — NOT generated by any of the 3 targets (XDS, baseline, HTML)
+- **Review:** At least one designer should review the ideal before committing
+
+## Current Ideals
+
+_None yet — see [issue #733](https://github.com/facebookexperimental/xds/issues/733) for the selection process._
+
+Recommended candidates (from the issue):
+
+- `ty-3` — Metrics dashboard card
+- `dd-3` — Product card grid
+- `cwm-1` — Stripe pricing table
+- `ps-1` — Settings dashboard
+- `ty-8` — Profile card
diff --git a/internal/vibe-tests/package.json b/internal/vibe-tests/package.json
index a2a8951a78..60b5b8e989 100644
--- a/internal/vibe-tests/package.json
+++ b/internal/vibe-tests/package.json
@@ -27,7 +27,9 @@
"report:deploy": "tsx src/deploy-report.ts",
"preview:dev": "vite --config app/vite.config.preview.ts",
"preview:build": "vite build --config app/vite.config.preview.ts",
- "screenshot:previews": "tsx src/screenshot-previews.ts"
+ "screenshot:previews": "tsx src/screenshot-previews.ts",
+ "design-judge": "tsx src/design-judge.ts",
+ "design-judge:dry": "tsx src/design-judge.ts --dry-run"
},
"dependencies": {
"@xds/cli": "*",
@@ -39,4 +41,4 @@
"tailwindcss": "^4.2.1",
"tsx": "^4.19.0"
}
-}
+}
\ No newline at end of file
diff --git a/internal/vibe-tests/src/design-judge.ts b/internal/vibe-tests/src/design-judge.ts
new file mode 100644
index 0000000000..18529e8828
--- /dev/null
+++ b/internal/vibe-tests/src/design-judge.ts
@@ -0,0 +1,620 @@
+#!/usr/bin/env node
+/**
+ * @file Design Judge — vision LLM evaluation of visual fidelity
+ *
+ * Compares rendered screenshots against human-provided ideal reference
+ * images using a vision LLM (Anthropic Claude). Scores across 5 visual
+ * sub-signals on a 0-100 scale, producing a weighted overall Design score.
+ *
+ * Sub-signals:
+ * 1. Layout Fidelity (25%) — structural regions, grid, stacking
+ * 2. Visual Hierarchy (25%) — relative sizing, weight, eye flow
+ * 3. Spacing & Alignment(20%) — consistency, grid alignment, proportions
+ * 4. Component Fidelity (15%) — interactive affordances, borders, radii
+ * 5. Color & Theming (15%) — palette match, surface/accent usage
+ *
+ * Runs 3 judge passes per screenshot and takes the median for stability.
+ *
+ * Usage:
+ * tsx src/design-judge.ts --iteration <id> [--prompts <p1,p2>] [--passes 3]
+ *
+ * @input Screenshot PNGs from screenshot-previews.ts + ideal PNGs from ideals/
+ * @output Design scores in results/<iteration>/design-scores.json
+ * @position internal/vibe-tests/src/design-judge.ts
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {getResultsDir, ensureDir, writeJson, readJson} from './utils.js';
+
+// ============================================================
+// Types
+// ============================================================
+
+export interface DesignSubScores {
+ layout: number;
+ hierarchy: number;
+ spacing: number;
+ components: number;
+ color: number;
+}
+
+export interface DesignJudgment {
+ scores: DesignSubScores;
+ overall: number;
+ notes: string;
+}
+
+export interface DesignResult {
+ promptId: string;
+ target: string;
+ viewport: string;
+ theme: string;
+ /** Individual pass results (typically 3) */
+ passes: DesignJudgment[];
+ /** Median scores across passes */
+ median: DesignJudgment;
+ idealPath: string;
+ screenshotPath: string;
+}
+
+export interface DesignScores {
+ iterationId: string;
+ timestamp: string;
+ model: string;
+ passCount: number;
+ results: DesignResult[];
+ /** Aggregated scores by prompt × target */
+ summary: Record<
+ string,
+ Record<string, {overall: number; sub: DesignSubScores}>
+ >;
+}
+
+// ============================================================
+// Sub-signal weights (must sum to 1.0)
+// ============================================================
+
+const WEIGHTS: Record<keyof DesignSubScores, number> = {
+ layout: 0.25,
+ hierarchy: 0.25,
+ spacing: 0.2,
+ components: 0.15,
+ color: 0.15,
+};
+
+// ============================================================
+// Judge prompt
+// ============================================================
+
+function buildJudgePrompt(promptText: string): string {
+ return `You are a design review judge. You will see two images:
+
+IMAGE 1: The ideal reference design (the target)
+IMAGE 2: A generated UI screenshot (the output)
+
+The original UI prompt was: "${promptText}"
+
+Score the generated output's visual fidelity to the ideal on these sub-signals (0-100 each):
+
+1. LAYOUT FIDELITY (25%): Do major content regions match? Correct stacking, grid/flex structure, sidebar/main/header placement. Focus on structural agreement, not pixel precision.
+2. VISUAL HIERARCHY (25%): Correct relative sizing of headings, body, labels? Eye drawn to right places? Proper weight differentiation?
+3. SPACING & ALIGNMENT (20%): Consistent spacing? Aligned to grid? Proportional padding/margins compared to ideal?
+4. COMPONENT FIDELITY (15%): Do buttons, inputs, toggles look correct? Right affordances, borders, radii? Interactive elements look clickable/tappable?
+5. COLOR & THEMING (15%): Palette broadly matches? Correct surface/accent/text color usage? Appropriate contrast?
+
+IGNORE: placeholder text differences, exact pixel match, content length, interactive states (hover/focus), browser chrome, scrollbars, favicon.
+PENALIZE: layout mismatch, broken hierarchy, gross spacing violations, missing interactive affordances, wrong color scheme.
+
+Return ONLY valid JSON (no markdown, no explanation outside the JSON):
+{
+ "layout": <0-100>,
+ "hierarchy": <0-100>,
+ "spacing": <0-100>,
+ "components": <0-100>,
+ "color": <0-100>,
+ "overall": <weighted average using weights 25/25/20/15/15>,
+ "notes": "brief explanation of major deviations (1-2 sentences)"
+}`;
+}
+
+// ============================================================
+// Anthropic Vision API
+// ============================================================
+
+async function callVisionJudge(
+ idealImagePath: string,
+ screenshotPath: string,
+ promptText: string,
+ model: string,
+): Promise<DesignJudgment> {
+ const apiKey = process.env.ANTHROPIC_API_KEY;
+ if (!apiKey) {
+ throw new Error(
+ 'ANTHROPIC_API_KEY environment variable is required.\n' +
+ 'Set it in GitHub Actions secrets or export it locally.',
+ );
+ }
+
+ const idealBase64 = fs.readFileSync(idealImagePath).toString('base64');
+ const screenshotBase64 = fs.readFileSync(screenshotPath).toString('base64');
+
+ const idealMediaType = idealImagePath.endsWith('.png')
+ ? 'image/png'
+ : 'image/jpeg';
+ const screenshotMediaType = screenshotPath.endsWith('.png')
+ ? 'image/png'
+ : 'image/jpeg';
+
+ const response = await fetch('https://api.anthropic.com/v1/messages', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ 'x-api-key': apiKey,
+ 'anthropic-version': '2023-06-01',
+ },
+ body: JSON.stringify({
+ model,
+ max_tokens: 1024,
+ messages: [
+ {
+ role: 'user',
+ content: [
+ {
+ type: 'image',
+ source: {
+ type: 'base64',
+ media_type: idealMediaType,
+ data: idealBase64,
+ },
+ },
+ {
+ type: 'image',
+ source: {
+ type: 'base64',
+ media_type: screenshotMediaType,
+ data: screenshotBase64,
+ },
+ },
+ {
+ type: 'text',
+ text: buildJudgePrompt(promptText),
+ },
+ ],
+ },
+ ],
+ }),
+ });
+
+ if (!response.ok) {
+ const errorText = await response.text();
+ throw new Error(`Anthropic API error (${response.status}): ${errorText}`);
+ }
+
+ const data = (await response.json()) as {
+ content: Array<{type: string; text?: string}>;
+ };
+ const textBlock = data.content.find(b => b.type === 'text');
+ if (!textBlock?.text) {
+ throw new Error('No text response from Anthropic API');
+ }
+
+ // Parse JSON from response (handle potential markdown wrapping)
+ let jsonStr = textBlock.text.trim();
+ const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
+ if (jsonMatch) {
+ jsonStr = jsonMatch[0];
+ }
+
+ const parsed = JSON.parse(jsonStr) as {
+ layout: number;
+ hierarchy: number;
+ spacing: number;
+ components: number;
+ color: number;
+ overall: number;
+ notes: string;
+ };
+
+ // Recompute overall to ensure correct weighting
+ const overall = Math.round(
+ parsed.layout * WEIGHTS.layout +
+ parsed.hierarchy * WEIGHTS.hierarchy +
+ parsed.spacing * WEIGHTS.spacing +
+ parsed.components * WEIGHTS.components +
+ parsed.color * WEIGHTS.color,
+ );
+
+ return {
+ scores: {
+ layout: clamp(parsed.layout),
+ hierarchy: clamp(parsed.hierarchy),
+ spacing: clamp(parsed.spacing),
+ components: clamp(parsed.components),
+ color: clamp(parsed.color),
+ },
+ overall: clamp(overall),
+ notes: parsed.notes || '',
+ };
+}
+
+// ============================================================
+// Statistics
+// ============================================================
+
+function clamp(n: number): number {
+ return Math.max(0, Math.min(100, Math.round(n)));
+}
+
+function median(values: number[]): number {
+ const sorted = [...values].sort((a, b) => a - b);
+ const mid = Math.floor(sorted.length / 2);
+ if (sorted.length % 2 === 0) {
+ return Math.round((sorted[mid - 1] + sorted[mid]) / 2);
+ }
+ return sorted[mid];
+}
+
+function medianJudgment(passes: DesignJudgment[]): DesignJudgment {
+ const sub: DesignSubScores = {
+ layout: median(passes.map(p => p.scores.layout)),
+ hierarchy: median(passes.map(p => p.scores.hierarchy)),
+ spacing: median(passes.map(p => p.scores.spacing)),
+ components: median(passes.map(p => p.scores.components)),
+ color: median(passes.map(p => p.scores.color)),
+ };
+
+ const overall = Math.round(
+ sub.layout * WEIGHTS.layout +
+ sub.hierarchy * WEIGHTS.hierarchy +
+ sub.spacing * WEIGHTS.spacing +
+ sub.components * WEIGHTS.components +
+ sub.color * WEIGHTS.color,
+ );
+
+ // Pick the notes from the pass closest to the median overall
+ const overallMedian = median(passes.map(p => p.overall));
+ const closestPass = passes.reduce((best, p) =>
+ Math.abs(p.overall - overallMedian) < Math.abs(best.overall - overallMedian)
+ ? p
+ : best,
+ );
+
+ return {scores: sub, overall: clamp(overall), notes: closestPass.notes};
+}
+
+// ============================================================
+// Ideals discovery
+// ============================================================
+
+function getIdealsDir(): string {
+ return path.join(import.meta.dirname, '..', 'ideals');
+}
+
+/**
+ * Find ideal images for a prompt. Uses `__` (double underscore) as separator
+ * between promptId and viewport/theme to support multi-segment prompt IDs
+ * like `feature-card-1`. Looks for files matching:
+ * ideals/{promptId}__{viewport}__{theme}.png
+ * ideals/{promptId}__{viewport}.png
+ * ideals/{promptId}.png (fallback — used for all viewport/theme combos)
+ */
+function findIdeal(
+ promptId: string,
+ viewport: string,
+ theme: string,
+): string | null {
+ const idealsDir = getIdealsDir();
+
+ // Specific match: cwm-1__desktop__light.png
+ const specific = path.join(
+ idealsDir,
+ `${promptId}__${viewport}__${theme}.png`,
+ );
+ if (fs.existsSync(specific)) return specific;
+
+ // Viewport match: cwm-1__desktop.png
+ const viewportMatch = path.join(idealsDir, `${promptId}__${viewport}.png`);
+ if (fs.existsSync(viewportMatch)) return viewportMatch;
+
+ // Generic fallback: cwm-1.png
+ const generic = path.join(idealsDir, `${promptId}.png`);
+ if (fs.existsSync(generic)) return generic;
+
+ return null;
+}
+
+/**
+ * List all prompt IDs that have ideals available.
+ */
+export function listAvailableIdeals(): string[] {
+ const idealsDir = getIdealsDir();
+ if (!fs.existsSync(idealsDir)) return [];
+
+ const files = fs.readdirSync(idealsDir).filter(f => f.endsWith('.png'));
+ const promptIds = new Set<string>();
+
+ for (const file of files) {
+ // Extract prompt ID using `__` separator.
+ // "cwm-1__desktop__light.png" → "cwm-1"
+ // "feature-card-1__desktop.png" → "feature-card-1"
+ // "cwm-1.png" → "cwm-1" (no viewport/theme suffix)
+ const base = file.replace('.png', '');
+ const promptId = base.split('__')[0];
+ if (promptId) {
+ promptIds.add(promptId);
+ }
+ }
+
+ return Array.from(promptIds);
+}
+
+// ============================================================
+// Prompt text lookup
+// ============================================================
+
+function loadPromptText(promptId: string): string {
+ // Try loading from the test set
+ const testSetPath = path.join(
+ import.meta.dirname,
+ '..',
+ 'test-sets',
+ 'default.json',
+ );
+ if (fs.existsSync(testSetPath)) {
+ const testSet = readJson<{prompts: Array<{id: string; prompt: string}>}>(
+ testSetPath,
+ );
+ const match = testSet.prompts.find(p => p.id === promptId);
+ if (match) return match.prompt;
+ }
+
+ return `UI component identified as ${promptId}`;
+}
+
+// ============================================================
+// CLI
+// ============================================================
+
+function parseArgs(): {
+ iteration: string;
+ prompts?: string[];
+ passes: number;
+ model: string;
+ dryRun: boolean;
+ viewport: string;
+ theme: string;
+} {
+ const args = process.argv.slice(2);
+ let iteration = '';
+ let prompts: string[] | undefined;
+ let passes = 3;
+ let model = 'claude-sonnet-4-20250514';
+ let dryRun = false;
+ let viewport = 'desktop';
+ let theme = 'light';
+
+ for (let i = 0; i < args.length; i++) {
+ if (args[i] === '--iteration' && args[i + 1]) {
+ iteration = args[++i];
+ } else if (args[i] === '--prompts' && args[i + 1]) {
+ prompts = args[++i].split(',');
+ } else if (args[i] === '--passes' && args[i + 1]) {
+ passes = parseInt(args[++i], 10);
+ } else if (args[i] === '--model' && args[i + 1]) {
+ model = args[++i];
+ } else if (args[i] === '--viewport' && args[i + 1]) {
+ viewport = args[++i];
+ } else if (args[i] === '--theme' && args[i + 1]) {
+ theme = args[++i];
+ } else if (args[i] === '--dry-run') {
+ dryRun = true;
+ }
+ }
+
+ if (!iteration) {
+ console.error(
+ 'Usage: tsx src/design-judge.ts --iteration <id> [--prompts <p1,p2>] [--passes 3] [--model <model>] [--viewport desktop] [--theme light] [--dry-run]',
+ );
+ process.exit(1);
+ }
+
+ return {iteration, prompts, passes, model, dryRun, viewport, theme};
+}
+
+async function main() {
+ const {iteration, prompts, passes, model, dryRun, viewport, theme} =
+ parseArgs();
+ const resultsDir = getResultsDir();
+ const iterDir = path.join(resultsDir, iteration);
+ const screenshotsDir = path.join(iterDir, 'screenshots');
+
+ // Load screenshot manifest
+ const manifestPath = path.join(screenshotsDir, 'manifest.json');
+ if (!fs.existsSync(manifestPath)) {
+ console.error(
+ `No screenshot manifest found at ${manifestPath}.\n` +
+ 'Run screenshot-previews.ts first.',
+ );
+ process.exit(1);
+ }
+
+ const manifest =
+ readJson<
+ Record<string, Record<string, Record<string, Record<string, string>>>>
+ >(manifestPath);
+
+ // Discover which prompts have ideals
+ const availableIdeals = listAvailableIdeals();
+ if (availableIdeals.length === 0) {
+ console.error(
+ 'No ideal reference images found.\n' +
+ `Place PNG files in: ${getIdealsDir()}/\n` +
+ 'Naming: {promptId}-{viewport}-{theme}.png or {promptId}.png',
+ );
+ process.exit(1);
+ }
+
+ console.log(`\n🎨 Design Judge — ${model}`);
+ console.log(` Iteration: ${iteration}`);
+ console.log(` Passes per screenshot: ${passes}`);
+ console.log(` Viewport: ${viewport}, Theme: ${theme}`);
+ console.log(` Available ideals: ${availableIdeals.join(', ')}`);
+
+ // Filter to prompts that have both screenshots and ideals
+ const promptsToEvaluate = Object.keys(manifest).filter(promptId => {
+ if (prompts && !prompts.includes(promptId)) return false;
+ if (!availableIdeals.includes(promptId)) return false;
+ return true;
+ });
+
+ if (promptsToEvaluate.length === 0) {
+ console.error(
+ '\nNo prompts have both screenshots and ideal references.\n' +
+ `Screenshots exist for: ${Object.keys(manifest).join(', ')}\n` +
+ `Ideals exist for: ${availableIdeals.join(', ')}`,
+ );
+ process.exit(1);
+ }
+
+ console.log(` Prompts to evaluate: ${promptsToEvaluate.join(', ')}\n`);
+
+ const results: DesignResult[] = [];
+ const targets = ['xds', 'baseline', 'html'];
+
+ for (const promptId of promptsToEvaluate) {
+ const promptText = loadPromptText(promptId);
+
+ for (const target of targets) {
+ const screenshotFilename =
+ manifest[promptId]?.[target]?.[viewport]?.[theme];
+ if (!screenshotFilename) {
+ console.log(
+ ` ⏭ No screenshot: ${promptId}/${target}/${viewport}/${theme}`,
+ );
+ continue;
+ }
+
+ const screenshotPath = path.join(screenshotsDir, screenshotFilename);
+ if (!fs.existsSync(screenshotPath)) {
+ console.log(` ⏭ Screenshot file missing: ${screenshotFilename}`);
+ continue;
+ }
+
+ const idealPath = findIdeal(promptId, viewport, theme);
+ if (!idealPath) {
+ console.log(` ⏭ No ideal for: ${promptId}/${viewport}/${theme}`);
+ continue;
+ }
+
+ console.log(` 🔍 ${promptId} / ${target} / ${viewport}-${theme}`);
+
+ if (dryRun) {
+ console.log(` [dry-run] Would call ${model} with ${passes} passes`);
+ console.log(` Ideal: ${path.basename(idealPath)}`);
+ console.log(` Screenshot: ${screenshotFilename}`);
+ continue;
+ }
+
+ // Run multiple passes for stability
+ const judgments: DesignJudgment[] = [];
+ for (let pass = 1; pass <= passes; pass++) {
+ try {
+ const judgment = await callVisionJudge(
+ idealPath,
+ screenshotPath,
+ promptText,
+ model,
+ );
+ judgments.push(judgment);
+ console.log(
+ ` Pass ${pass}/${passes}: ${judgment.overall} ` +
+ `(L:${judgment.scores.layout} H:${judgment.scores.hierarchy} ` +
+ `S:${judgment.scores.spacing} C:${judgment.scores.components} ` +
+ `Co:${judgment.scores.color})`,
+ );
+ } catch (err) {
+ console.error(
+ ` Pass ${pass}/${passes} failed: ${err instanceof Error ? err.message : err}`,
+ );
+ }
+
+ // Rate limiting: 500ms between calls
+ if (pass < passes) {
+ await new Promise(r => setTimeout(r, 500));
+ }
+ }
+
+ if (judgments.length === 0) {
+ console.error(` ✗ All passes failed for ${promptId}/${target}`);
+ continue;
+ }
+
+ const med = medianJudgment(judgments);
+ console.log(` ✓ Median: ${med.overall} — ${med.notes}`);
+
+ results.push({
+ promptId,
+ target,
+ viewport,
+ theme,
+ passes: judgments,
+ median: med,
+ idealPath: path.relative(iterDir, idealPath),
+ screenshotPath: screenshotFilename,
+ });
+ }
+ }
+
+ if (dryRun) {
+ console.log('\n🏁 Dry run complete. No API calls made.');
+ return;
+ }
+
+ // Build summary: prompt → target → scores
+ const summary: DesignScores['summary'] = {};
+ for (const result of results) {
+ if (!summary[result.promptId]) {
+ summary[result.promptId] = {};
+ }
+ summary[result.promptId][result.target] = {
+ overall: result.median.overall,
+ sub: result.median.scores,
+ };
+ }
+
+ const output: DesignScores = {
+ iterationId: iteration,
+ timestamp: new Date().toISOString(),
+ model,
+ passCount: passes,
+ results,
+ summary,
+ };
+
+ const outputPath = path.join(iterDir, 'design-scores.json');
+ writeJson(outputPath, output);
+ console.log(`\n✅ Design scores written to ${outputPath}`);
+
+ // Print summary table
+ console.log('\n📊 Summary:\n');
+ const promptIds = Object.keys(summary);
+ const header = ['Prompt', ...targets.map(t => `${t} (overall)`)].join('\t');
+ console.log(header);
+ console.log('-'.repeat(header.length));
+
+ for (const pid of promptIds) {
+ const row = [
+ pid,
+ ...targets.map(t =>
+ summary[pid]?.[t] ? String(summary[pid][t].overall) : '—',
+ ),
+ ];
+ console.log(row.join('\t'));
+ }
+}
+
+main().catch(err => {
+ console.error(err);
+ process.exit(1);
+});
diff --git a/internal/vibe-tests/src/types.ts b/internal/vibe-tests/src/types.ts
index 25661efec0..239eed245a 100644
--- a/internal/vibe-tests/src/types.ts
+++ b/internal/vibe-tests/src/types.ts
@@ -265,7 +265,8 @@ export type UniversalDimension =
| 'accessibility'
| 'codeQuality'
| 'efficiency'
- | 'maintainability';
+ | 'maintainability'
+ | 'design';
export interface UniversalFinding {
rule: string;
@@ -304,12 +305,25 @@ export interface DimensionScore<M = undefined> {
metrics?: M;
}
+export interface DesignMetrics {
+ layout: number;
+ hierarchy: number;
+ spacing: number;
+ components: number;
+ color: number;
+ passCount: number;
+ /** Max variance across passes (stability indicator) */
+ maxVariance: number;
+}
+
export interface UniversalScore {
correctness: DimensionScore;
accessibility: DimensionScore;
codeQuality: DimensionScore;
efficiency: DimensionScore<EfficiencyMetrics>;
maintainability: DimensionScore<MaintainabilityMetrics>;
+ /** Design dimension — null when no ideal reference image exists for the prompt */
+ design?: DimensionScore<DesignMetrics> | null;
}
export interface CostMetrics {
diff --git a/internal/vibe-tests/src/universal-aggregate.ts b/internal/vibe-tests/src/universal-aggregate.ts
index 65e8ac1377..2d06b211c2 100644
--- a/internal/vibe-tests/src/universal-aggregate.ts
+++ b/internal/vibe-tests/src/universal-aggregate.ts
@@ -1,6 +1,6 @@
#!/usr/bin/env node
/**
- * @file Universal aggregate scoring across 5 dimensions
+ * @file Universal aggregate scoring across 5+1 dimensions (design is optional)
*
* Scores an iteration's results using target-neutral evaluation.
*
@@ -19,12 +19,13 @@ import type {
import {getResultsDir, writeJson} from './utils.js';
import {evaluate, getDimensionNames} from './universal-eval.js';
-const DIMENSION_LABELS: Record<UniversalDimension, string> = {
+const DIMENSION_LABELS: Partial<Record<UniversalDimension, string>> = {
correctness: 'Correctness',
accessibility: 'Accessibility',
codeQuality: 'Code Quality',
efficiency: 'Efficiency',
maintainability: 'Maintainability',
+ design: 'Design',
};
function parseArgs(): {iteration: string; json: boolean} {
@@ -134,7 +135,10 @@ async function main() {
}
}
for (const dim of dimensions) {
- categoryScores[category][dim].push(score[dim].score);
+ const dimScore = score[dim];
+ if (dimScore) {
+ categoryScores[category][dim].push(dimScore.score);
+ }
}
// --- Cost data ---
@@ -196,18 +200,82 @@ async function main() {
};
}
+ // --- Merge design scores if available ---
+ const designScoresPath = path.join(iterDir, 'design-scores.json');
+ let hasDesignScores = false;
+ if (fs.existsSync(designScoresPath)) {
+ try {
+ const designData = JSON.parse(fs.readFileSync(designScoresPath, 'utf-8'));
+ const summary = designData.summary as Record<
+ string,
+ Record<string, {overall: number; sub: Record<string, number>}>
+ >;
+ for (const [promptId, targets] of Object.entries(summary)) {
+ const targetScores = targets[target];
+ if (targetScores && byPrompt[promptId]) {
+ // Find the full result for variance calculation
+ const fullResults = (designData.results || []).filter(
+ (r: {promptId: string; target: string}) =>
+ r.promptId === promptId && r.target === target,
+ );
+ const passes = fullResults[0]?.passes || [];
+ const overallValues = passes.map((p: {overall: number}) => p.overall);
+ const maxVariance =
+ overallValues.length > 1
+ ? Math.max(...overallValues) - Math.min(...overallValues)
+ : 0;
+
+ byPrompt[promptId].design = {
+ score: targetScores.overall,
+ metrics: {
+ layout: targetScores.sub.layout,
+ hierarchy: targetScores.sub.hierarchy,
+ spacing: targetScores.sub.spacing,
+ components: targetScores.sub.components,
+ color: targetScores.sub.color,
+ passCount: passes.length || designData.passCount || 3,
+ maxVariance,
+ },
+ };
+ hasDesignScores = true;
+ }
+ }
+ } catch (err) {
+ console.error(`Warning: Failed to load design scores: ${err}`);
+ }
+ }
+
// Compute averages
const promptCount = Object.keys(byPrompt).length;
const averages = {} as Record<UniversalDimension, number>;
for (const dim of dimensions) {
- const scores = Object.values(byPrompt).map(s => s[dim].score);
- averages[dim] = Math.round(
- scores.reduce((a, b) => a + b, 0) / scores.length,
- );
+ const scores = Object.values(byPrompt)
+ .map(s => s[dim]?.score)
+ .filter((s): s is number => s != null);
+ averages[dim] =
+ scores.length > 0
+ ? Math.round(scores.reduce((a, b) => a + b, 0) / scores.length)
+ : 0;
+ }
+ // Design average: only across prompts that have design scores
+ if (hasDesignScores) {
+ const designScores = Object.values(byPrompt)
+ .filter(s => s.design != null)
+ .map(s => s.design!.score);
+ if (designScores.length > 0) {
+ averages.design = Math.round(
+ designScores.reduce((a, b) => a + b, 0) / designScores.length,
+ );
+ }
}
+ const overallDims = [...dimensions];
+ if (hasDesignScores && averages.design != null) {
+ overallDims.push('design' as UniversalDimension);
+ }
const overall = Math.round(
- dimensions.reduce((s, d) => s + averages[d], 0) / dimensions.length,
+ overallDims.reduce((s, d) => s + (averages[d] ?? 0), 0) /
+ overallDims.length,
);
// Category averages
@@ -274,10 +342,18 @@ async function main() {
console.log('│ Dimension │ Score │');
console.log('├─────────────────────┼───────┤');
for (const dim of dimensions) {
- const label = DIMENSION_LABELS[dim].padEnd(19);
+ const label = (DIMENSION_LABELS[dim] || dim).padEnd(19);
const score = String(averages[dim]).padStart(3);
console.log(`│ ${label} │ ${score} │`);
}
+ if (hasDesignScores && averages.design != null) {
+ const designPromptCount = Object.values(byPrompt).filter(
+ s => s.design != null,
+ ).length;
+ const label = `Design (${designPromptCount}p)`.padEnd(19);
+ const score = String(averages.design).padStart(3);
+ console.log(`│ ${label} │ ${score} │`);
+ }
console.log('├─────────────────────┼───────┤');
console.log(`│ ${'Overall'.padEnd(19)} │ ${String(overall).padStart(3)} │`);
console.log('└─────────────────────┴───────┘');
diff --git a/internal/vibe-tests/src/universal-compare.ts b/internal/vibe-tests/src/universal-compare.ts
index 097e67c435..c27e006386 100644
--- a/internal/vibe-tests/src/universal-compare.ts
+++ b/internal/vibe-tests/src/universal-compare.ts
@@ -19,12 +19,13 @@ import type {
import {writeJson, getResultsDir} from './utils.js';
import {getDimensionNames, getAverageScore} from './universal-eval.js';
-const DIMENSION_LABELS: Record<UniversalDimension, string> = {
+const DIMENSION_LABELS: Partial<Record<UniversalDimension, string>> = {
correctness: 'Correctness',
accessibility: 'Accessibility',
codeQuality: 'Code Quality',
efficiency: 'Efficiency',
maintainability: 'Maintainability',
+ design: 'Design',
};
function loadOrGenerate(iterationId: string): UniversalAggregate {
@@ -193,7 +194,7 @@ async function main() {
console.log('│ Dimension │ XDS │ Baseline │ HTML │ Winner │');
console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');
for (const d of dimensions) {
- const label = DIMENSION_LABELS[d].padEnd(19);
+ const label = (DIMENSION_LABELS[d] || d).padEnd(19);
const xScore = String(xds.averages[d]).padStart(3);
const bScore = String(baseline.averages[d]).padStart(3);
const hScore = String(htmlData!.averages[d]).padStart(3);
@@ -219,7 +220,7 @@ async function main() {
console.log('│ Dimension │ XDS │ Baseline │ Winner │');
console.log('├─────────────────────┼───────┼──────────┼──────────┤');
for (const d of dimensions) {
- const label = DIMENSION_LABELS[d].padEnd(19);
+ const label = (DIMENSION_LABELS[d] || d).padEnd(19);
const xScore = String(xds.averages[d]).padStart(3);
const bScore = String(baseline.averages[d]).padStart(3);
const w = winnerIcon(winners[d]).padEnd(8);
diff --git a/internal/vibe-tests/src/universal-eval.ts b/internal/vibe-tests/src/universal-eval.ts
index 54a09bc984..33b11208a2 100644
--- a/internal/vibe-tests/src/universal-eval.ts
+++ b/internal/vibe-tests/src/universal-eval.ts
@@ -1062,7 +1062,7 @@ export function evaluate(code: string, target: string): UniversalScore {
}
/**
- * Get the list of dimension names.
+ * Get the list of code-level dimension names (always available).
*/
export function getDimensionNames(): UniversalDimension[] {
return [
@@ -1074,21 +1074,53 @@ export function getDimensionNames(): UniversalDimension[] {
];
}
+/**
+ * Get all dimension names including design (when available).
+ */
+export function getAllDimensionNames(): UniversalDimension[] {
+ return [
+ 'correctness',
+ 'accessibility',
+ 'codeQuality',
+ 'efficiency',
+ 'maintainability',
+ 'design',
+ ];
+}
+
/**
* Get the score for a specific dimension.
+ * Returns null for 'design' if not available.
*/
export function getDimensionScore(
score: UniversalScore,
dimension: UniversalDimension,
-): number {
+): number | null {
+ if (dimension === 'design') {
+ return score.design?.score ?? null;
+ }
return score[dimension].score;
}
/**
- * Calculate an average score across all dimensions (unweighted).
+ * Calculate an average score across the 5 core dimensions (unweighted).
+ * Does NOT include design — use getFullAverageScore() when all scores
+ * being compared are guaranteed to have design scores.
*/
export function getAverageScore(score: UniversalScore): number {
const dims = getDimensionNames();
- const total = dims.reduce((sum, d) => sum + score[d].score, 0);
+ const total = dims.reduce((sum, d) => sum + (score[d]?.score ?? 0), 0);
return Math.round(total / dims.length);
}
+
+/**
+ * Calculate an average score across all 6 dimensions including design.
+ * Returns null if design score is not present.
+ */
+export function getFullAverageScore(score: UniversalScore): number | null {
+ if (score.design == null) return null;
+ const dims = getDimensionNames();
+ const coreTotal = dims.reduce((sum, d) => sum + (score[d]?.score ?? 0), 0);
+ const total = coreTotal + score.design.score;
+ return Math.round(total / (dims.length + 1));
+}