An open source design system that's fully customizable and agent ready
TypeScript
feat(vibe-tests): 4-way comparison in report and preview pipeline (#1696)
Cindy Zhang0470d9ebd583
diff --git a/internal/vibe-tests/app/src/report/CompareView.tsx b/internal/vibe-tests/app/src/report/CompareView.tsxindex da8afbcc89..a5a5cb98e7 100644--- a/internal/vibe-tests/app/src/report/CompareView.tsx+++ b/internal/vibe-tests/app/src/report/CompareView.tsx@@ -18,7 +18,7 @@ import {} from './utils';import './report.css';-type WinnerType = 'xds' | 'baseline' | 'html' | 'tie';+type WinnerType = 'xds' | 'xds-tailwind' | 'baseline' | 'html' | 'tie';interface CompareViewProps {comparison: UniversalComparison;@@ -30,6 +30,7 @@ interface DimRow extends Record<string, unknown> {xdsScore: number;baselineScore: number;htmlScore?: number;+ xdsTailwindScore?: number;delta: number;winner: string;}@@ -40,6 +41,7 @@ interface CatRow extends Record<string, unknown> {xdsOverall: number;baselineOverall: number;htmlOverall?: number;+ xdsTailwindOverall?: number;delta: number;}@@ -49,6 +51,7 @@ interface CostRow extends Record<string, unknown> {xds: string;baseline: string;html?: string;+ xdsTailwind?: string;winner: string;}@@ -57,24 +60,26 @@ function costWinner(baseVal: number,lowerIsBetter: boolean,htmlVal?: number,+ twVal?: number,): WinnerType {- if (htmlVal != null) {- const vals = [xdsVal, baseVal, htmlVal];- const best = lowerIsBetter ? Math.min(...vals) : Math.max(...vals);- const atBest = vals.filter(v => v === best).length;- if (atBest > 1) return 'tie';- if (xdsVal === best) return 'xds';- if (baseVal === best) return 'baseline';- return 'html';- }- if (xdsVal === baseVal) return 'tie';- if (lowerIsBetter) return xdsVal < baseVal ? 'xds' : 'baseline';- return xdsVal > baseVal ? 'xds' : 'baseline';+ const entries: [WinnerType, number][] = [+ ['xds', xdsVal],+ ['baseline', baseVal],+ ];+ if (htmlVal != null) entries.push(['html', htmlVal]);+ if (twVal != null) entries.push(['xds-tailwind', twVal]);++ const best = lowerIsBetter+ ? Math.min(...entries.map(([, v]) => v))+ : Math.max(...entries.map(([, v]) => v));+ const atBest = entries.filter(([, v]) => v === best);+ if (atBest.length > 1) return 'tie';+ return atBest[0][0];}function winnerBadgeVariant(w: string,-): 'success' | 'error' | 'warning' | 'neutral' {+): 'success' | 'error' | 'warning' | 'neutral' | 'info' {switch (w) {case 'xds':return 'success';@@ -82,6 +87,8 @@ function winnerBadgeVariant(return 'error';case 'html':return 'warning';+ case 'xds-tailwind':+ return 'info';default:return 'neutral';}@@ -95,6 +102,8 @@ function winnerLabel(w: string): string {return 'Baseline';case 'html':return 'HTML';+ case 'xds-tailwind':+ return 'XDS+TW';default:return 'Tie';}@@ -110,12 +119,15 @@ function CostComparisonSection({xdsCost,baselineCost,htmlCost,+ xdsTailwindCost,}: {xdsCost: CostMetrics;baselineCost: CostMetrics;htmlCost?: CostMetrics;+ xdsTailwindCost?: CostMetrics;}) {const isThreeWay = !!htmlCost;+ const isFourWay = !!xdsTailwindCost;const hasDuration =xdsCost.avgDurationMs > 0 || baselineCost.avgDurationMs > 0;@@ -130,11 +142,17 @@ function CostComparisonSection({...(isThreeWay? {html: `${(htmlCost!.avgDurationMs / 1000).toFixed(1)}s`}: {}),+ ...(isFourWay+ ? {+ xdsTailwind: `${(xdsTailwindCost!.avgDurationMs / 1000).toFixed(1)}s`,+ }+ : {}),winner: costWinner(xdsCost.avgDurationMs,baselineCost.avgDurationMs,true,htmlCost?.avgDurationMs,+ xdsTailwindCost?.avgDurationMs,),},]@@ -147,11 +165,17 @@ function CostComparisonSection({...(isThreeWay? {html: `~${htmlCost!.estimatedInputTokens.toLocaleString()}`}: {}),+ ...(isFourWay+ ? {+ xdsTailwind: `~${xdsTailwindCost!.estimatedInputTokens.toLocaleString()}`,+ }+ : {}),winner: costWinner(xdsCost.estimatedInputTokens,baselineCost.estimatedInputTokens,true,htmlCost?.estimatedInputTokens,+ xdsTailwindCost?.estimatedInputTokens,),},{@@ -162,11 +186,17 @@ function CostComparisonSection({...(isThreeWay? {html: `~${htmlCost!.estimatedOutputTokens.toLocaleString()}`}: {}),+ ...(isFourWay+ ? {+ xdsTailwind: `~${xdsTailwindCost!.estimatedOutputTokens.toLocaleString()}`,+ }+ : {}),winner: costWinner(xdsCost.estimatedOutputTokens,baselineCost.estimatedOutputTokens,true,htmlCost?.estimatedOutputTokens,+ xdsTailwindCost?.estimatedOutputTokens,),},{@@ -179,6 +209,11 @@ function CostComparisonSection({html: `~${(htmlCost!.estimatedInputTokens + htmlCost!.estimatedOutputTokens).toLocaleString()}`,}: {}),+ ...(isFourWay+ ? {+ xdsTailwind: `~${(xdsTailwindCost!.estimatedInputTokens + xdsTailwindCost!.estimatedOutputTokens).toLocaleString()}`,+ }+ : {}),winner: costWinner(xdsCost.estimatedInputTokens + xdsCost.estimatedOutputTokens,baselineCost.estimatedInputTokens + baselineCost.estimatedOutputTokens,@@ -186,6 +221,10 @@ function CostComparisonSection({htmlCost? htmlCost.estimatedInputTokens + htmlCost.estimatedOutputTokens: undefined,+ xdsTailwindCost+ ? xdsTailwindCost.estimatedInputTokens ++ xdsTailwindCost.estimatedOutputTokens+ : undefined,),},{@@ -194,11 +233,15 @@ function CostComparisonSection({xds: String(xdsCost.avgOutputLines),baseline: String(baselineCost.avgOutputLines),...(isThreeWay ? {html: String(htmlCost!.avgOutputLines)} : {}),+ ...(isFourWay+ ? {xdsTailwind: String(xdsTailwindCost!.avgOutputLines)}+ : {}),winner: costWinner(xdsCost.avgOutputLines,baselineCost.avgOutputLines,true,htmlCost?.avgOutputLines,+ xdsTailwindCost?.avgOutputLines,),},{@@ -207,6 +250,9 @@ function CostComparisonSection({xds: String(xdsCost.avgDocsRead),baseline: String(baselineCost.avgDocsRead),...(isThreeWay ? {html: String(htmlCost!.avgDocsRead)} : {}),+ ...(isFourWay+ ? {xdsTailwind: String(xdsTailwindCost!.avgDocsRead)}+ : {}),winner: 'tie', // not inherently better or worse},];@@ -234,6 +280,17 @@ function CostComparisonSection({} satisfies XDSTableColumn<CostRow>,]: []),+ ...(isFourWay+ ? [+ {+ key: 'xdsTailwind' as const,+ header: 'XDS+TW',+ renderCell: (row: CostRow) => (+ <XDSText type="body">{row.xdsTailwind ?? '—'}</XDSText>+ ),+ } satisfies XDSTableColumn<CostRow>,+ ]+ : []),{key: 'winner',header: 'Lower Cost',@@ -258,18 +315,21 @@ function CostComparisonSection({}export function CompareView({comparison}: CompareViewProps) {- const {xds, baseline, html, winners} = comparison;+ const {xds, baseline, html, xdsTailwind, winners} = comparison;const isThreeWay = !!html;+ const isFourWay = !!xdsTailwind;let xdsWins = 0;let baselineWins = 0;let htmlWins = 0;+ let xdsTailwindWins = 0;let ties = 0;for (const dim of ALL_DIMENSIONS) {const w = winners[dim];if (w === 'xds') xdsWins++;else if (w === 'baseline') baselineWins++;else if (w === 'html') htmlWins++;+ else if (w === 'xds-tailwind') xdsTailwindWins++;else ties++;}@@ -281,6 +341,7 @@ export function CompareView({comparison}: CompareViewProps) {xdsScore: xds.averages[dim] ?? 0,baselineScore: baseline.averages[dim] ?? 0,...(isThreeWay ? {htmlScore: html!.averages[dim] ?? 0} : {}),+ ...(isFourWay ? {xdsTailwindScore: xdsTailwind!.averages[dim] ?? 0} : {}),delta: (xds.averages[dim] ?? 0) - (baseline.averages[dim] ?? 0),winner: winners[dim],}));@@ -314,6 +375,21 @@ export function CompareView({comparison}: CompareViewProps) {} satisfies XDSTableColumn<DimRow>,]: []),+ ...(isFourWay+ ? [+ {+ key: 'xdsTailwindScore' as const,+ header: 'XDS+TW',+ renderCell: (row: DimRow) => (+ <XDSText type="body">+ {row.xdsTailwindScore != null+ ? formatScore(row.xdsTailwindScore)+ : '—'}+ </XDSText>+ ),+ } satisfies XDSTableColumn<DimRow>,+ ]+ : []),{key: 'delta',header: 'Delta (XDS−Base)',@@ -340,6 +416,7 @@ export function CompareView({comparison}: CompareViewProps) {...Object.keys(xds.byCategory),...Object.keys(baseline.byCategory),...(html ? Object.keys(html.byCategory) : []),+ ...(xdsTailwind ? Object.keys(xdsTailwind.byCategory) : []),]);const catData: CatRow[] = [...allCategories].map(cat => {@@ -347,6 +424,9 @@ export function CompareView({comparison}: CompareViewProps) {const baseCat = baseline.byCategory[cat] ?? {};const htmlCat =html?.byCategory[cat] ?? ({} as Record<UniversalDimension, number>);+ const twCat =+ xdsTailwind?.byCategory[cat] ??+ ({} as Record<UniversalDimension, number>);const xdsAvg =CODE_DIMENSIONS.reduce((s, d) => s + ((xdsCat[d as UniversalDimension] as number) ?? 0),@@ -363,12 +443,19 @@ export function CompareView({comparison}: CompareViewProps) {0,) / CODE_DIMENSIONS.length: undefined;+ const twAvg = isFourWay+ ? CODE_DIMENSIONS.reduce(+ (s, d) => s + ((twCat[d as UniversalDimension] as number) ?? 0),+ 0,+ ) / CODE_DIMENSIONS.length+ : undefined;return {id: cat,category: cat,xdsOverall: xdsAvg,baselineOverall: baseAvg,...(htmlAvg != null ? {htmlOverall: htmlAvg} : {}),+ ...(twAvg != null ? {xdsTailwindOverall: twAvg} : {}),delta: xdsAvg - baseAvg,};});@@ -402,6 +489,21 @@ export function CompareView({comparison}: CompareViewProps) {} satisfies XDSTableColumn<CatRow>,]: []),+ ...(isFourWay+ ? [+ {+ key: 'xdsTailwindOverall' as const,+ header: 'XDS+TW',+ renderCell: (row: CatRow) => (+ <XDSText type="body">+ {row.xdsTailwindOverall != null+ ? formatScore(row.xdsTailwindOverall)+ : '—'}+ </XDSText>+ ),+ } satisfies XDSTableColumn<CatRow>,+ ]+ : []),{key: 'delta',header: 'Delta (XDS−Base)',@@ -414,14 +516,19 @@ export function CompareView({comparison}: CompareViewProps) {},];+ // Determine grid class based on number of win cards+ const winCardCount =+ 2 + (isThreeWay ? 1 : 0) + (isFourWay ? 1 : 0) + 1; // targets + ties+ const summaryGridClass =+ winCardCount >= 5+ ? 'report-compare-summaryGrid5'+ : winCardCount === 4+ ? 'report-compare-summaryGrid4'+ : 'report-compare-summaryGrid';+return (<XDSVStack gap={4}>- <div- className={- isThreeWay- ? 'report-compare-summaryGrid4'- : 'report-compare-summaryGrid'- }>+ <div className={summaryGridClass}><XDSCard><div className="report-compare-winCard"><XDSVStack gap={2}>@@ -454,6 +561,18 @@ export function CompareView({comparison}: CompareViewProps) {</div></XDSCard>)}+ {isFourWay && (+ <XDSCard>+ <div className="report-compare-winCard">+ <XDSVStack gap={2}>+ <XDSText type="label">XDS+TW Wins</XDSText>+ <XDSHeading level={2}>+ <span className="report-color-info">{xdsTailwindWins}</span>+ </XDSHeading>+ </XDSVStack>+ </div>+ </XDSCard>+ )}<XDSCard><div className="report-compare-winCard"><XDSVStack gap={2}>@@ -497,6 +616,7 @@ export function CompareView({comparison}: CompareViewProps) {xdsCost={xds.cost}baselineCost={baseline.cost}htmlCost={html?.cost}+ xdsTailwindCost={xdsTailwind?.cost}/></XDSVStack>)}diff --git a/internal/vibe-tests/app/src/report/PromptDetailCard.tsx b/internal/vibe-tests/app/src/report/PromptDetailCard.tsxindex 7b3c37cccf..15c7ad9453 100644--- a/internal/vibe-tests/app/src/report/PromptDetailCard.tsx+++ b/internal/vibe-tests/app/src/report/PromptDetailCard.tsx@@ -94,10 +94,12 @@ interface PromptDetailCardProps {xdsScore?: UniversalScore;baselineScore?: UniversalScore;htmlScore?: UniversalScore;+ xdsTailwindScore?: UniversalScore;hasXdsCode: boolean;hasBaselineCode: boolean;hasHtmlCode: boolean;- onViewCode: (target: 'xds' | 'baseline' | 'html') => void;+ hasXdsTailwindCode: boolean;+ onViewCode: (target: 'xds' | 'baseline' | 'html' | 'xds-tailwind') => void;/** Relative preview URLs keyed by target (e.g. { xds: "previews/sd-1/xds.html" }) */previewUrls?: Record<string, string>;}@@ -108,23 +110,40 @@ export function PromptDetailCard({xdsScore,baselineScore,htmlScore,+ xdsTailwindScore,hasXdsCode,hasBaselineCode,hasHtmlCode,+ hasXdsTailwindCode,onViewCode,previewUrls,}: PromptDetailCardProps) {const hasBoth = !!(xdsScore && baselineScore);const hasAll = !!(xdsScore && baselineScore && htmlScore);+ const hasFourWay = !!(xdsScore && baselineScore && xdsTailwindScore);const hasAnyPreview =- previewUrls?.xds || previewUrls?.baseline || previewUrls?.html;- const hasAnyCode = hasXdsCode || hasBaselineCode || hasHtmlCode;+ previewUrls?.xds ||+ previewUrls?.baseline ||+ previewUrls?.html ||+ previewUrls?.['xds-tailwind'];+ const hasAnyCode =+ hasXdsCode || hasBaselineCode || hasHtmlCode || hasXdsTailwindCode;- const scoresClassName = hasAll- ? 'report-promptDetail-scoresRow3'- : hasBoth- ? 'report-promptDetail-scoresRow'- : 'report-promptDetail-scoresRowSingle';+ // Count how many score blocks we have+ const scoreCount = [+ xdsScore,+ baselineScore,+ htmlScore,+ xdsTailwindScore,+ ].filter(Boolean).length;+ const scoresClassName =+ scoreCount >= 4+ ? 'report-promptDetail-scoresRow4'+ : scoreCount === 3+ ? 'report-promptDetail-scoresRow3'+ : scoreCount === 2+ ? 'report-promptDetail-scoresRow'+ : 'report-promptDetail-scoresRowSingle';return (<XDSCard>@@ -164,6 +183,16 @@ export function PromptDetailCard({label="HTML Preview"/>)}+ {previewUrls?.['xds-tailwind'] && (+ <XDSButton+ variant="secondary"+ size="sm"+ onClick={() =>+ window.open(previewUrls['xds-tailwind'], '_blank')+ }+ label="XDS+TW Preview"+ />+ )}{hasXdsCode && (<XDSButtonvariant="ghost"@@ -188,18 +217,29 @@ export function PromptDetailCard({label="HTML Code"/>)}+ {hasXdsTailwindCode && (+ <XDSButton+ variant="ghost"+ size="sm"+ onClick={() => onViewCode('xds-tailwind')}+ label="XDS+TW Code"+ />+ )}</div>)}</div>{/* Score summaries in constrained grid */}- {(xdsScore || baselineScore || htmlScore) && (+ {(xdsScore || baselineScore || htmlScore || xdsTailwindScore) && (<div className={scoresClassName}>{xdsScore && <ScoreSummary label="XDS" score={xdsScore} />}{baselineScore && (<ScoreSummary label="Baseline" score={baselineScore} />)}{htmlScore && <ScoreSummary label="HTML" score={htmlScore} />}+ {xdsTailwindScore && (+ <ScoreSummary label="XDS+TW" score={xdsTailwindScore} />+ )}</div>)}@@ -239,6 +279,18 @@ export function PromptDetailCard({</div></>)}++ {xdsTailwindScore && (+ <>+ <XDSDivider />+ <div className="report-promptDetail-section">+ <div className="report-promptDetail-sectionLabel">+ <XDSText type="label">XDS+TW Findings</XDSText>+ </div>+ <Findings score={xdsTailwindScore} />+ </div>+ </>+ )}</XDSVStack></div></XDSCard>diff --git a/internal/vibe-tests/app/src/report/Report.tsx b/internal/vibe-tests/app/src/report/Report.tsxindex 037688b64c..49de78c720 100644--- a/internal/vibe-tests/app/src/report/Report.tsx+++ b/internal/vibe-tests/app/src/report/Report.tsx@@ -159,7 +159,7 @@ export function Report() {const [activeTab, setActiveTab] = useState('overview');const [codeModal, setCodeModal] = useState<{promptId: string;- target: 'xds' | 'baseline' | 'html';+ target: 'xds' | 'baseline' | 'html' | 'xds-tailwind';} | null>(null);const data: ReportData | undefined = window.__REPORT_DATA__;@@ -246,6 +246,14 @@ export function Report() {compareLabel="HTML"/>)}+ {comparison?.xdsTailwind && (+ <ScoreCard+ label="Overall Score vs XDS+TW"+ score={universal.overall}+ compareScore={comparison.xdsTailwind.overall}+ compareLabel="XDS+TW"+ />+ )}{/* Dimension scores grid */}<XDSVStack gap={3}>@@ -274,9 +282,15 @@ export function Report() {{comparison && (<XDSVStack gap={3}><XDSHeading level={3}>- {comparison.html- ? 'XDS vs Baseline vs HTML Comparison'- : 'XDS vs Baseline Comparison'}+ {[+ 'XDS',+ 'Baseline',+ comparison.html ? 'HTML' : null,+ comparison.xdsTailwind ? 'XDS+TW' : null,+ ]+ .filter(Boolean)+ .join(' vs ')}{' '}+ Comparison</XDSHeading><CompareView comparison={comparison} /></XDSVStack>@@ -299,9 +313,15 @@ export function Report() {xdsScore={universal.byPrompt[promptId]}baselineScore={comparison?.baseline.byPrompt[promptId]}htmlScore={comparison?.html?.byPrompt[promptId]}+ xdsTailwindScore={+ comparison?.xdsTailwind?.byPrompt[promptId]+ }hasXdsCode={!!data.sourceCode?.[promptId]}hasBaselineCode={!!data.baselineSourceCode?.[promptId]}hasHtmlCode={!!data.htmlSourceCode?.[promptId]}+ hasXdsTailwindCode={+ !!data.xdsTailwindSourceCode?.[promptId]+ }onViewCode={target => setCodeModal({promptId, target})}previewUrls={data.previews?.[promptId]}/>@@ -318,7 +338,9 @@ export function Report() {? data.sourceCode?.[codeModal.promptId]: codeModal.target === 'baseline'? data.baselineSourceCode?.[codeModal.promptId]- : data.htmlSourceCode?.[codeModal.promptId];+ : codeModal.target === 'xds-tailwind'+ ? data.xdsTailwindSourceCode?.[codeModal.promptId]+ : data.htmlSourceCode?.[codeModal.promptId];return code ? (<CodeModalisOpendiff --git a/internal/vibe-tests/app/src/report/report.css b/internal/vibe-tests/app/src/report/report.cssindex 896a122c4b..d75aa6c378 100644--- a/internal/vibe-tests/app/src/report/report.css+++ b/internal/vibe-tests/app/src/report/report.css@@ -44,6 +44,12 @@gap: var(--spacing-3);}+.report-compare-summaryGrid5 {+ display: grid;+ grid-template-columns: repeat(5, 1fr);+ gap: var(--spacing-3);+}+.report-compare-winCard {padding: var(--spacing-3);text-align: center;@@ -65,6 +71,10 @@color: var(--color-warning);}+.report-color-info {+ color: var(--color-info, #8b5cf6);+}+/* === PromptDetailCard === */.report-promptDetail-card {padding: var(--spacing-4);@@ -105,6 +115,12 @@gap: var(--spacing-3);}+.report-promptDetail-scoresRow4 {+ display: grid;+ grid-template-columns: 1fr 1fr 1fr 1fr;+ gap: var(--spacing-3);+}+.report-promptDetail-scoresRowSingle {display: grid;grid-template-columns: 1fr;diff --git a/internal/vibe-tests/app/src/report/types.ts b/internal/vibe-tests/app/src/report/types.tsindex 49e594068e..e3d532af6b 100644--- a/internal/vibe-tests/app/src/report/types.ts+++ b/internal/vibe-tests/app/src/report/types.ts@@ -29,6 +29,7 @@ export interface ReportData {sourceCode?: Record<string, string>;baselineSourceCode?: Record<string, string>;htmlSourceCode?: Record<string, string>;+ xdsTailwindSourceCode?: Record<string, string>;/** Map of promptId → { target → relative URL } */previews?: Record<string, Record<string, string>>;/** Map of promptId → prompt text */diff --git a/internal/vibe-tests/src/build-report.ts b/internal/vibe-tests/src/build-report.tsindex 3839eaa815..c1a2fc7c97 100644--- a/internal/vibe-tests/src/build-report.ts+++ b/internal/vibe-tests/src/build-report.ts@@ -72,6 +72,7 @@ function parseArgs(): {iteration: string;baseline?: string;html?: string;+ xdsTailwind?: string;withScreenshots: boolean;dev: boolean;} {@@ -79,6 +80,7 @@ function parseArgs(): {let iteration = '';let baseline: string | undefined;let html: string | undefined;+ let xdsTailwind: string | undefined;let withScreenshots = false;let dev = false;@@ -92,6 +94,9 @@ function parseArgs(): {} else if (args[i] === '--html' && args[i + 1]) {html = args[i + 1];i++;+ } else if (args[i] === '--xds-tailwind' && args[i + 1]) {+ xdsTailwind = args[i + 1];+ i++;} else if (args[i] === '--with-screenshots') {withScreenshots = true;} else if (args[i] === '--dev') {@@ -101,12 +106,12 @@ function parseArgs(): {if (!iteration) {console.error(- 'Usage: tsx src/build-report.ts --iteration <id> [--baseline <id>] [--with-screenshots] [--dev]',+ 'Usage: tsx src/build-report.ts --iteration <id> [--baseline <id>] [--html <id>] [--xds-tailwind <id>] [--with-screenshots] [--dev]',);process.exit(1);}- return {iteration, baseline, html, withScreenshots, dev};+ return {iteration, baseline, html, xdsTailwind, withScreenshots, dev};}function ensureUniversalJson(iteration: string): UniversalAggregate {@@ -128,20 +133,24 @@ function ensureComparison(xdsId: string,baselineId: string,htmlId?: string,+ xdsTailwindId?: string,): UniversalComparison {const resultsDir = getResultsDir();- const comparisonFilename = htmlId- ? `comparison-${xdsId}-${baselineId}-${htmlId}.json`- : `comparison-${xdsId}-${baselineId}.json`;+ const idParts = [xdsId, baselineId];+ if (htmlId) idParts.push(htmlId);+ if (xdsTailwindId) idParts.push(xdsTailwindId);+ const comparisonFilename = `comparison-${idParts.join('-')}.json`;const comparisonPath = path.join(resultsDir, comparisonFilename);if (!fs.existsSync(comparisonPath)) {const htmlFlag = htmlId ? ` --html ${htmlId}` : '';- console.log(- `Generating comparison for ${xdsId} vs ${baselineId}${htmlId ? ` vs ${htmlId}` : ''}...`,- );+ const twFlag = xdsTailwindId ? ` --xds-tailwind ${xdsTailwindId}` : '';+ const vsLabel = [baselineId, htmlId, xdsTailwindId]+ .filter(Boolean)+ .join(' vs ');+ console.log(`Generating comparison for ${xdsId} vs ${vsLabel}...`);execSync(- `tsx ${path.join(import.meta.dirname, 'universal-compare.ts')} --xds ${xdsId} --baseline ${baselineId}${htmlFlag}`,+ `tsx ${path.join(import.meta.dirname, 'universal-compare.ts')} --xds ${xdsId} --baseline ${baselineId}${htmlFlag}${twFlag}`,{cwd: path.join(import.meta.dirname, '..'), stdio: 'inherit'},);}@@ -180,6 +189,7 @@ function buildDataScript(opts: {sourceCode?: Record<string, string>;baselineSourceCode?: Record<string, string>;htmlSourceCode?: Record<string, string>;+ xdsTailwindSourceCode?: Record<string, string>;previews?: Record<string, Record<string, string>>;screenshots?: Record<string, string>;prompts?: Record<string, string>;@@ -192,6 +202,7 @@ function buildDataScript(opts: {sourceCode: opts.sourceCode,baselineSourceCode: opts.baselineSourceCode,htmlSourceCode: opts.htmlSourceCode,+ xdsTailwindSourceCode: opts.xdsTailwindSourceCode,previews: opts.previews,screenshots: opts.screenshots,prompts: opts.prompts,@@ -241,6 +252,7 @@ async function main() {iteration,baseline,html,+ xdsTailwind,withScreenshots: _withScreenshots,dev,} = parseArgs();@@ -267,7 +279,10 @@ async function main() {if (html) {ensureUniversalJson(html);}- comparison = ensureComparison(iteration, baseline, html);+ if (xdsTailwind) {+ ensureUniversalJson(xdsTailwind);+ }+ comparison = ensureComparison(iteration, baseline, html, xdsTailwind);}// Step 3: Load source code for per-prompt inspection@@ -320,6 +335,24 @@ async function main() {}}+ let xdsTailwindSourceCode: Record<string, string> | undefined;+ if (xdsTailwind) {+ xdsTailwindSourceCode = {};+ const twCodeDir = path.join(resultsDir, xdsTailwind, 'results');+ if (fs.existsSync(twCodeDir)) {+ ensureTsxFiles(twCodeDir);+ for (const file of fs+ .readdirSync(twCodeDir)+ .filter(f => f.endsWith('.tsx'))) {+ const promptId = path.basename(file, '.tsx');+ xdsTailwindSourceCode[promptId] = fs.readFileSync(+ path.join(twCodeDir, file),+ 'utf-8',+ );+ }+ }+ }+// Load preview manifest if availableconst previewManifestPath = path.join(iterDir, 'previews', 'manifest.json');let previews: Record<string, Record<string, string>> | undefined;@@ -334,7 +367,9 @@ async function main() {// Screenshots use relative paths (screenshots/filename.png) that resolve// when deployed to gh-pages alongside the report.let screenshots: Record<string, string> | undefined;- for (const id of [iteration, baseline, html].filter(Boolean) as string[]) {+ for (const id of [iteration, baseline, html, xdsTailwind].filter(+ Boolean,+ ) as string[]) {const screenshotManifestPath = path.join(resultsDir,id,@@ -388,6 +423,7 @@ async function main() {sourceCode,baselineSourceCode,htmlSourceCode,+ xdsTailwindSourceCode,previews,screenshots,prompts,diff --git a/internal/vibe-tests/src/deploy-report.ts b/internal/vibe-tests/src/deploy-report.tsindex 5f6a120b6d..f3cc0b4ac0 100644--- a/internal/vibe-tests/src/deploy-report.ts+++ b/internal/vibe-tests/src/deploy-report.ts@@ -23,6 +23,7 @@ function parseArgs(): {iteration: string;baseline?: string;html?: string;+ xdsTailwind?: string;dryRun: boolean;skipPreviews: boolean;} {@@ -30,6 +31,7 @@ function parseArgs(): {let iteration = '';let baseline: string | undefined;let html: string | undefined;+ let xdsTailwind: string | undefined;let dryRun = false;let skipPreviews = false;@@ -40,6 +42,8 @@ function parseArgs(): {baseline = args[++i];} else if (args[i] === '--html' && args[i + 1]) {html = args[++i];+ } else if (args[i] === '--xds-tailwind' && args[i + 1]) {+ xdsTailwind = args[++i];} else if (args[i] === '--dry-run') {dryRun = true;} else if (args[i] === '--skip-previews') {@@ -49,12 +53,12 @@ function parseArgs(): {if (!iteration) {console.error(- 'Usage: tsx src/deploy-report.ts --iteration <id> [--baseline <id>] [--html <id>] [--dry-run] [--skip-previews]',+ 'Usage: tsx src/deploy-report.ts --iteration <id> [--baseline <id>] [--html <id>] [--xds-tailwind <id>] [--dry-run] [--skip-previews]',);process.exit(1);}- return {iteration, baseline, html, dryRun, skipPreviews};+ return {iteration, baseline, html, xdsTailwind, dryRun, skipPreviews};}function run(cmd: string, opts?: {cwd?: string; silent?: boolean}): string {@@ -68,7 +72,8 @@ function runSilent(cmd: string, opts?: {cwd?: string}): string {}async function main() {- const {iteration, baseline, html, dryRun, skipPreviews} = parseArgs();+ const {iteration, baseline, html, xdsTailwind, dryRun, skipPreviews} =+ parseArgs();const deployPath = `reports/${iteration}`;@@ -82,7 +87,9 @@ async function main() {// Step 1: Build previews if result files exist (.tsx or .json)if (!skipPreviews) {const iterationsWithCode: string[] = [];- for (const id of [iteration, baseline, html].filter(Boolean) as string[]) {+ for (const id of [iteration, baseline, html, xdsTailwind].filter(+ Boolean,+ ) as string[]) {const codeDir = path.join(VIBE_DIR, 'results', id, 'results');if (fs.existsSync(codeDir) &&@@ -116,6 +123,7 @@ async function main() {const buildArgs = [`--iteration ${iteration}`];if (baseline) buildArgs.push(`--baseline ${baseline}`);if (html) buildArgs.push(`--html ${html}`);+ if (xdsTailwind) buildArgs.push(`--xds-tailwind ${xdsTailwind}`);run(`npx tsx src/build-report.ts ${buildArgs.join(' ')}`, {cwd: VIBE_DIR});@@ -182,7 +190,9 @@ async function main() {}// Second: copy any local screenshots (may add to or overwrite gh-pages ones)- for (const id of [iteration, baseline, html].filter(Boolean) as string[]) {+ for (const id of [iteration, baseline, html, xdsTailwind].filter(+ Boolean,+ ) as string[]) {const screenshotsDir = path.join(VIBE_DIR, 'results', id, 'screenshots');if (fs.existsSync(screenshotsDir)) {copyDirRecursive(screenshotsDir, targetScreenshotsDir);diff --git a/internal/vibe-tests/src/types.ts b/internal/vibe-tests/src/types.tsindex 0e4fcd68d2..a28aeb822e 100644--- a/internal/vibe-tests/src/types.ts+++ b/internal/vibe-tests/src/types.ts@@ -315,18 +315,22 @@ export interface UniversalAggregate {cost?: CostMetrics;}+export type TargetName = 'xds' | 'xds-tailwind' | 'baseline' | 'html';+export interface UniversalComparison {xds: UniversalAggregate;baseline: UniversalAggregate;html?: UniversalAggregate;- winners: Record<UniversalDimension, 'xds' | 'baseline' | 'html' | 'tie'>;+ xdsTailwind?: UniversalAggregate;+ winners: Record<UniversalDimension, TargetName | 'tie'>;byPrompt: Record<string,{xds: UniversalScore;baseline: UniversalScore;html?: UniversalScore;- winner: 'xds' | 'baseline' | 'html' | 'tie';+ xdsTailwind?: UniversalScore;+ winner: TargetName | 'tie';}>;}diff --git a/internal/vibe-tests/src/universal-compare.ts b/internal/vibe-tests/src/universal-compare.tsindex 775846982c..cb6a38972a 100644--- a/internal/vibe-tests/src/universal-compare.ts+++ b/internal/vibe-tests/src/universal-compare.ts@@ -5,6 +5,7 @@* Usage:* tsx src/universal-compare.ts --xds abc123 --baseline def456* tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789+ * tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789 --xds-tailwind jkl012* tsx src/universal-compare.ts --xds abc123 --baseline def456 --json* tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789 --markdown*/@@ -16,6 +17,7 @@ import type {UniversalAggregate,UniversalComparison,UniversalDimension,+ TargetName,} from './types.js';import {writeJson, getResultsDir} from './utils.js';import {getDimensionNames, getAverageScore} from './universal-eval.js';@@ -50,22 +52,25 @@ function loadOrGenerate(iterationId: string): UniversalAggregate {return JSON.parse(fs.readFileSync(universalPath, 'utf-8'));}-type WinnerType = 'xds' | 'baseline' | 'html' | 'tie';--function winner(xdsVal: number, baseVal: number, htmlVal?: number): WinnerType {- if (htmlVal != null) {- const max = Math.max(xdsVal, baseVal, htmlVal);- const atMax = [xdsVal === max, baseVal === max, htmlVal === max].filter(- Boolean,- ).length;- if (atMax > 1) return 'tie';- if (xdsVal === max) return 'xds';- if (baseVal === max) return 'baseline';- return 'html';- }- if (xdsVal > baseVal) return 'xds';- if (baseVal > xdsVal) return 'baseline';- return 'tie';+type WinnerType = TargetName | 'tie';++function winner(+ xdsVal: number,+ baseVal: number,+ htmlVal?: number,+ xdsTailwindVal?: number,+): WinnerType {+ const entries: [TargetName, number][] = [+ ['xds', xdsVal],+ ['baseline', baseVal],+ ];+ if (htmlVal != null) entries.push(['html', htmlVal]);+ if (xdsTailwindVal != null) entries.push(['xds-tailwind', xdsTailwindVal]);++ const max = Math.max(...entries.map(([, v]) => v));+ const atMax = entries.filter(([, v]) => v === max);+ if (atMax.length > 1) return 'tie';+ return atMax[0][0];}function winnerIcon(w: WinnerType): string {@@ -76,6 +81,8 @@ function winnerIcon(w: WinnerType): string {return '🔵 Base';case 'html':return '🟡 HTML';+ case 'xds-tailwind':+ return '🟣 XDS+TW';case 'tie':return '⚪ Tie';}@@ -85,6 +92,7 @@ function parseArgs(): {xds: string;baseline: string;html?: string;+ xdsTailwind?: string;json: boolean;markdown: boolean;} {@@ -92,6 +100,7 @@ function parseArgs(): {let xds = '';let baseline = '';let html: string | undefined;+ let xdsTailwind: string | undefined;let json = false;let markdown = false;@@ -102,6 +111,8 @@ function parseArgs(): {baseline = args[++i];} else if (args[i] === '--html' && args[i + 1]) {html = args[++i];+ } else if (args[i] === '--xds-tailwind' && args[i + 1]) {+ xdsTailwind = args[++i];} else if (args[i] === '--json') {json = true;} else if (args[i] === '--markdown' || args[i] === '--md') {@@ -111,12 +122,12 @@ function parseArgs(): {if (!xds || !baseline) {console.error(- 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--json] [--markdown]',+ 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--xds-tailwind <id>] [--json] [--markdown]',);process.exit(1);}- return {xds, baseline, html, json, markdown};+ return {xds, baseline, html, xdsTailwind, json, markdown};}/**@@ -128,30 +139,22 @@ function toMarkdown(opts: {xdsId: string;baselineId: string;htmlId?: string;+ xdsTailwindId?: string;byPrompt: UniversalComparison['byPrompt'];}): string {- const {comparison, xdsId, baselineId, htmlId, byPrompt} = opts;- const {xds, baseline, html: htmlData, winners} = comparison;+ const {comparison, xdsId, baselineId, htmlId, xdsTailwindId, byPrompt} =+ opts;+ const {xds, baseline, html: htmlData, xdsTailwind: twData, winners} =+ comparison;const dimensions = getDimensionNames();- const isThreeWay = !!htmlData;const lines: string[] = [];- // Summary table- if (isThreeWay) {- lines.push(- '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',- );- lines.push(- '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',- );- } else {- lines.push(- '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',- );- lines.push(- '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',- );- }+ lines.push(+ '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',+ );+ lines.push(+ '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',+ );const dimOrder: UniversalDimension[] = ['correctness',@@ -169,13 +172,20 @@ function toMarkdown(opts: {`| **Baseline** | \`${baselineId}\` | ${baseline.overall} | ${baseRow} |`,);- if (isThreeWay && htmlData) {+ if (htmlData) {const htmlRow = dimOrder.map(d => htmlData.averages[d]).join(' | ');lines.push(`| **HTML** | \`${htmlId}\` | ${htmlData.overall} | ${htmlRow} |`,);}+ if (twData) {+ const twRow = dimOrder.map(d => twData.averages[d]).join(' | ');+ lines.push(+ `| **XDS+TW** | \`${xdsTailwindId}\` | ${twData.overall} | ${twRow} |`,+ );+ }+lines.push('');// Per-prompt winners@@ -184,43 +194,47 @@ function toMarkdown(opts: {let xWins = 0;let bWins = 0;let hWins = 0;+ let twWins = 0;let ties = 0;for (const [, data] of promptEntries) {if (data.winner === 'xds') xWins++;else if (data.winner === 'baseline') bWins++;else if (data.winner === 'html') hWins++;+ else if (data.winner === 'xds-tailwind') twWins++;else ties++;}- if (isThreeWay) {- lines.push(- `**Per-prompt wins:** XDS ${xWins} · Baseline ${bWins} · HTML ${hWins} · Tie ${ties} (${promptEntries.length} prompts)`,- );- } else {- lines.push(- `**Per-prompt wins:** XDS ${xWins} · Baseline ${bWins} · Tie ${ties} (${promptEntries.length} prompts)`,- );- }+ const parts = [`XDS ${xWins}`, `Baseline ${bWins}`];+ if (htmlData) parts.push(`HTML ${hWins}`);+ if (twData) parts.push(`XDS+TW ${twWins}`);+ parts.push(`Tie ${ties}`);+ lines.push(+ `**Per-prompt wins:** ${parts.join(' · ')} (${promptEntries.length} prompts)`,+ );lines.push('');}// Dark mode- if (isThreeWay && htmlData) {- lines.push(- `**Dark mode:** XDS ${xds.darkModeRate}% · Baseline ${baseline.darkModeRate}% · HTML ${htmlData.darkModeRate}%`,- );- } else {- lines.push(- `**Dark mode:** XDS ${xds.darkModeRate}% · Baseline ${baseline.darkModeRate}%`,- );- }+ const dmParts = [+ `XDS ${xds.darkModeRate}%`,+ `Baseline ${baseline.darkModeRate}%`,+ ];+ if (htmlData) dmParts.push(`HTML ${htmlData.darkModeRate}%`);+ if (twData) dmParts.push(`XDS+TW ${twData.darkModeRate}%`);+ lines.push(`**Dark mode:** ${dmParts.join(' · ')}`);// Dimension winnerslines.push('');lines.push('**Dimension winners:**');for (const d of dimensions) {const w = winners[d];- const icon =- w === 'xds' ? '🟢' : w === 'baseline' ? '🔵' : w === 'html' ? '🟡' : '⚪';+ const iconMap: Record<string, string> = {+ xds: '🟢',+ baseline: '🔵',+ html: '🟡',+ 'xds-tailwind': '🟣',+ tie: '⚪',+ };+ const icon = iconMap[w] ?? '⚪';const label = DIMENSION_LABELS[d] || d;lines.push(`- ${label}: ${icon} ${w === 'tie' ? 'Tie' : w.toUpperCase()}`);}@@ -233,6 +247,7 @@ async function main() {xds: xdsId,baseline: baselineId,html: htmlId,+ xdsTailwind: xdsTailwindId,json,markdown,} = parseArgs();@@ -240,9 +255,11 @@ async function main() {const xds = loadOrGenerate(xdsId);const baseline = loadOrGenerate(baselineId);const htmlData = htmlId ? loadOrGenerate(htmlId) : undefined;+ const twData = xdsTailwindId ? loadOrGenerate(xdsTailwindId) : undefined;const dimensions = getDimensionNames();const isThreeWay = !!htmlData;+ const isFourWay = !!twData;// Build comparisonconst winners = {} as Record<UniversalDimension, WinnerType>;@@ -251,6 +268,7 @@ async function main() {xds.averages[d],baseline.averages[d],htmlData?.averages[d],+ twData?.averages[d],);}@@ -259,6 +277,7 @@ async function main() {...Object.keys(xds.byPrompt),...Object.keys(baseline.byPrompt),...(htmlData ? Object.keys(htmlData.byPrompt) : []),+ ...(twData ? Object.keys(twData.byPrompt) : []),]);const byPrompt: UniversalComparison['byPrompt'] = {};@@ -266,15 +285,18 @@ async function main() {const xdsScore = xds.byPrompt[promptId];const baselineScore = baseline.byPrompt[promptId];const htmlScore = htmlData?.byPrompt[promptId];+ const twScore = twData?.byPrompt[promptId];if (xdsScore && baselineScore) {byPrompt[promptId] = {xds: xdsScore,baseline: baselineScore,...(htmlScore ? {html: htmlScore} : {}),+ ...(twScore ? {xdsTailwind: twScore} : {}),winner: winner(getAverageScore(xdsScore),getAverageScore(baselineScore),htmlScore ? getAverageScore(htmlScore) : undefined,+ twScore ? getAverageScore(twScore) : undefined,),};}@@ -284,14 +306,16 @@ async function main() {xds,baseline,...(htmlData ? {html: htmlData} : {}),+ ...(twData ? {xdsTailwind: twData} : {}),winners,byPrompt,};- // Save- const outputFilename = htmlId- ? `comparison-${xdsId}-${baselineId}-${htmlId}.json`- : `comparison-${xdsId}-${baselineId}.json`;+ // Save — include all IDs in filename for uniqueness+ const idParts = [xdsId, baselineId];+ if (htmlId) idParts.push(htmlId);+ if (xdsTailwindId) idParts.push(xdsTailwindId);+ const outputFilename = `comparison-${idParts.join('-')}.json`;const outputPath = path.join(getResultsDir(), outputFilename);writeJson(outputPath, comparison);@@ -307,6 +331,7 @@ async function main() {xdsId,baselineId,htmlId,+ xdsTailwindId,byPrompt,}),);@@ -314,216 +339,153 @@ async function main() {}// --- Print report ---+ const targetNames = ['XDS', 'Baseline'];+ if (isThreeWay) targetNames.push('HTML');+ if (isFourWay) targetNames.push('XDS+TW');- const title = isThreeWay- ? '📊 Universal Comparison: XDS vs Baseline vs HTML'- : '📊 Universal Comparison: XDS vs Baseline';+ const title = `📊 Universal Comparison: ${targetNames.join(' vs ')}`;console.log(`\n${title}`);- console.log('═'.repeat(isThreeWay ? 66 : 52));-- if (isThreeWay) {- // Three-way table- console.log('┌─────────────────────┬───────┬──────────┬──────┬──────────┐');- console.log('│ Dimension │ XDS │ Baseline │ HTML │ Winner │');- console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');- for (const d of dimensions) {- const label = (DIMENSION_LABELS[d] || d).padEnd(19);- const xScore = String(xds.averages[d]).padStart(3);- const bScore = String(baseline.averages[d]).padStart(3);- const hScore = String(htmlData!.averages[d]).padStart(3);- const w = winnerIcon(winners[d]).padEnd(8);- console.log(- `│ ${label} │ ${xScore} │ ${bScore} │ ${hScore} │ ${w} │`,- );- }- console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');- const xOverall = String(xds.overall).padStart(3);- const bOverall = String(baseline.overall).padStart(3);- const hOverall = String(htmlData!.overall).padStart(3);- const overallW = winnerIcon(- winner(xds.overall, baseline.overall, htmlData!.overall),- ).padEnd(8);- console.log(- `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${hOverall} │ ${overallW} │`,- );- console.log('└─────────────────────┴───────┴──────────┴──────┴──────────┘');- } else {- // Two-way table (unchanged)- console.log('┌─────────────────────┬───────┬──────────┬──────────┐');- console.log('│ Dimension │ XDS │ Baseline │ Winner │');- console.log('├─────────────────────┼───────┼──────────┼──────────┤');- for (const d of dimensions) {- const label = (DIMENSION_LABELS[d] || d).padEnd(19);- const xScore = String(xds.averages[d]).padStart(3);- const bScore = String(baseline.averages[d]).padStart(3);- const w = winnerIcon(winners[d]).padEnd(8);- console.log(`│ ${label} │ ${xScore} │ ${bScore} │ ${w} │`);+ console.log('═'.repeat(title.length + 2));++ // Build a generic table for any number of targets+ type TargetEntry = {label: string; data: typeof xds};+ const targets: TargetEntry[] = [+ {label: 'XDS', data: xds},+ {label: 'Baseline', data: baseline},+ ];+ if (isThreeWay) targets.push({label: 'HTML', data: htmlData!});+ if (isFourWay) targets.push({label: 'XDS+TW', data: twData!});++ // Use markdown-style table for CLI (simpler than box-drawing for N targets)+ const header = ['Dimension', ...targets.map(t => t.label), 'Winner'];+ const rows: string[][] = [];+ for (const d of dimensions) {+ const row = [+ DIMENSION_LABELS[d] || d,+ ...targets.map(t => String(t.data.averages[d])),+ winnerIcon(winners[d]),+ ];+ rows.push(row);+ }+ // Overall row+ const overallWinner = winner(+ xds.overall,+ baseline.overall,+ htmlData?.overall,+ twData?.overall,+ );+ rows.push([+ 'Overall',+ ...targets.map(t => String(t.data.overall)),+ winnerIcon(overallWinner),+ ]);++ // Compute column widths and print+ const allRows = [header, ...rows];+ const colWidths = header.map((_, ci) =>+ Math.max(...allRows.map(r => r[ci].length)),+ );+ const sep = colWidths.map(w => '─'.repeat(w + 2)).join('┼');+ console.log('┌' + sep.replaceAll('┼', '┬') + '┐');+ console.log(+ '│' ++ header.map((h, i) => ` ${h.padEnd(colWidths[i])} `).join('│') ++ '│',+ );+ console.log('├' + sep + '┤');+ for (let ri = 0; ri < rows.length; ri++) {+ if (ri === rows.length - 1) {+ console.log('├' + sep + '┤');}- console.log('├─────────────────────┼───────┼──────────┼──────────┤');- const xOverall = String(xds.overall).padStart(3);- const bOverall = String(baseline.overall).padStart(3);- const overallW = winnerIcon(winner(xds.overall, baseline.overall)).padEnd(- 8,- );console.log(- `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${overallW} │`,+ '│' ++ rows[ri].map((c, i) => ` ${c.padEnd(colWidths[i])} `).join('│') ++ '│',);- console.log('└─────────────────────┴───────┴──────────┴──────────┘');}+ console.log('└' + sep.replaceAll('┼', '┴') + '┘');// Dark mode- const darkModeStr = isThreeWay- ? `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}% | HTML ${htmlData!.darkModeRate}%`- : `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}%`;- console.log(darkModeStr);+ const dmParts = targets.map(t => `${t.label} ${t.data.darkModeRate}%`);+ console.log(`\n🌙 Dark Mode: ${dmParts.join(' | ')}`);// Efficiency metrics comparison- const xdsEff = Object.values(xds.byPrompt).map(s => s.efficiency.metrics!);- const baseEff = Object.values(baseline.byPrompt).map(- s => s.efficiency.metrics!,- );- const htmlEff = htmlData- ? Object.values(htmlData.byPrompt).map(s => s.efficiency.metrics!)- : [];- if (xdsEff.length > 0 && baseEff.length > 0) {- const xDpe =- xdsEff.reduce((s, m) => s + m.decisionsPerElement, 0) / xdsEff.length;- const bDpe =- baseEff.reduce((s, m) => s + m.decisionsPerElement, 0) / baseEff.length;- const xLines = xdsEff.reduce((s, m) => s + m.codeLines, 0) / xdsEff.length;- const bLines =- baseEff.reduce((s, m) => s + m.codeLines, 0) / baseEff.length;+ const targetEffMetrics = targets.map(t => ({+ label: t.label,+ metrics: Object.values(t.data.byPrompt).map(s => s.efficiency.metrics!),+ }));+ if (targetEffMetrics.every(t => t.metrics.length > 0)) {+ const avgDpe = targetEffMetrics.map(+ t => t.metrics.reduce((s, m) => s + m.decisionsPerElement, 0) / t.metrics.length,+ );+ const avgLines = targetEffMetrics.map(+ t => t.metrics.reduce((s, m) => s + m.codeLines, 0) / t.metrics.length,+ );console.log(`\n⚡ Efficiency Metrics:`);-- if (isThreeWay && htmlEff.length > 0) {- const hDpe =- htmlEff.reduce((s, m) => s + m.decisionsPerElement, 0) / htmlEff.length;- const hLines =- htmlEff.reduce((s, m) => s + m.codeLines, 0) / htmlEff.length;- console.log(- ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | HTML ${hDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe, hDpe))}`,- );- console.log(- ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | HTML ${Math.round(hLines)} | ${winnerIcon(winner(bLines, xLines, hLines))}`,- );- } else {- console.log(- ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe))}`,- );- console.log(- ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | ${winnerIcon(winner(bLines, xLines))}`,- );- }+ // For decisions/element, lower is better (reverse args for winner)+ const dpeParts = targetEffMetrics.map((t, i) => `${t.label} ${avgDpe[i].toFixed(1)}`);+ console.log(` Decisions/element: ${dpeParts.join(' | ')}`);+ const linesParts = targetEffMetrics.map((t, i) => `${t.label} ${Math.round(avgLines[i])}`);+ console.log(` Avg code lines: ${linesParts.join(' | ')}`);}// Maintainability metrics comparison- const xdsMaint = Object.values(xds.byPrompt).map(- s => s.maintainability.metrics!,- );- const baseMaint = Object.values(baseline.byPrompt).map(- s => s.maintainability.metrics!,- );- const htmlMaint = htmlData- ? Object.values(htmlData.byPrompt).map(s => s.maintainability.metrics!)- : [];- if (xdsMaint.length > 0 && baseMaint.length > 0) {- const xSem =- xdsMaint.reduce((s, m) => s + m.semanticRatio, 0) / xdsMaint.length;- const bSem =- baseMaint.reduce((s, m) => s + m.semanticRatio, 0) / baseMaint.length;- const xMagic = xdsMaint.reduce((s, m) => s + m.magicValueCount, 0);- const bMagic = baseMaint.reduce((s, m) => s + m.magicValueCount, 0);+ const targetMaintMetrics = targets.map(t => ({+ label: t.label,+ metrics: Object.values(t.data.byPrompt).map(s => s.maintainability.metrics!),+ }));+ if (targetMaintMetrics.every(t => t.metrics.length > 0)) {+ const avgSem = targetMaintMetrics.map(+ t => t.metrics.reduce((s, m) => s + m.semanticRatio, 0) / t.metrics.length,+ );+ const totalMagic = targetMaintMetrics.map(+ t => t.metrics.reduce((s, m) => s + m.magicValueCount, 0),+ );console.log(`\n🔧 Maintainability Metrics:`);-- if (isThreeWay && htmlMaint.length > 0) {- const hSem =- htmlMaint.reduce((s, m) => s + m.semanticRatio, 0) / htmlMaint.length;- const hMagic = htmlMaint.reduce((s, m) => s + m.magicValueCount, 0);- console.log(- ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | HTML ${(hSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem, hSem))}`,- );- console.log(- ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | HTML ${hMagic} | ${winnerIcon(winner(bMagic, xMagic, hMagic))}`,- );- } else {- console.log(- ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem))}`,- );- console.log(- ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | ${winnerIcon(winner(bMagic, xMagic))}`,- );- }+ const semParts = targetMaintMetrics.map((t, i) => `${t.label} ${(avgSem[i] * 100).toFixed(0)}%`);+ console.log(` Semantic ratio: ${semParts.join(' | ')}`);+ const magicParts = targetMaintMetrics.map((t, i) => `${t.label} ${totalMagic[i]}`);+ console.log(` Magic values: ${magicParts.join(' | ')}`);}// Cost comparisonif (xds.cost && baseline.cost) {console.log(`\n💰 Cost:`);- const xAvgMs = xds.cost.avgDurationMs;- const bAvgMs = baseline.cost.avgDurationMs;- if (isThreeWay && htmlData?.cost) {- const hAvgMs = htmlData.cost.avgDurationMs;- if (xAvgMs > 0 && bAvgMs > 0) {- console.log(- ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | HTML ${(hAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs, hAvgMs))}`,- );- }- console.log(- ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | HTML ${htmlData.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines, htmlData.cost.avgOutputLines))}`,- );- console.log(- ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead} | HTML ${htmlData.cost.avgDocsRead}`,- );- const xTokens =- xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens;- const bTokens =- baseline.cost.estimatedInputTokens +- baseline.cost.estimatedOutputTokens;- const hTokens =- htmlData.cost.estimatedInputTokens +- htmlData.cost.estimatedOutputTokens;- console.log(- ` Est. tokens: XDS ~${xTokens} | Baseline ~${bTokens} | HTML ~${hTokens} | ${winnerIcon(winner(bTokens, xTokens, hTokens))}`,- );- } else {- if (xAvgMs > 0 && bAvgMs > 0) {- console.log(- ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs))}`,- );- }- console.log(- ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines))}`,- );- console.log(- ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead}`,- );- console.log(- ` Est. tokens: XDS ~${xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens} | Baseline ~${baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens} | ${winnerIcon(winner(baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens, xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens))}`,- );+ const costTargets = targets.filter(t => t.data.cost);+ const hasDuration = costTargets.some(t => t.data.cost!.avgDurationMs > 0);+ if (hasDuration) {+ const durParts = costTargets.map(t => `${t.label} ${(t.data.cost!.avgDurationMs / 1000).toFixed(1)}s`);+ console.log(` Avg duration: ${durParts.join(' | ')}`);}+ const linesParts = costTargets.map(t => `${t.label} ${t.data.cost!.avgOutputLines}`);+ console.log(` Avg output lines: ${linesParts.join(' | ')}`);+ const docsParts = costTargets.map(t => `${t.label} ${t.data.cost!.avgDocsRead}`);+ console.log(` Avg docs read: ${docsParts.join(' | ')}`);+ const tokenParts = costTargets.map(t => {+ const total = t.data.cost!.estimatedInputTokens + t.data.cost!.estimatedOutputTokens;+ return `${t.label} ~${total}`;+ });+ console.log(` Est. tokens: ${tokenParts.join(' | ')}`);}// Per-prompt winsconst promptEntries = Object.entries(byPrompt);if (promptEntries.length > 0) {- let xWins = 0;- let bWins = 0;- let hWins = 0;- let ties = 0;+ const winCounts: Record<string, number> = {};+ for (const t of targets) winCounts[t.label] = 0;+ winCounts['Tie'] = 0;for (const [, data] of promptEntries) {- if (data.winner === 'xds') xWins++;- else if (data.winner === 'baseline') bWins++;- else if (data.winner === 'html') hWins++;- else ties++;- }- if (isThreeWay) {- console.log(- `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | HTML wins ${hWins} | Ties ${ties} (${promptEntries.length} prompts)`,- );- } else {- console.log(- `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | Ties ${ties} (${promptEntries.length} prompts)`,- );+ if (data.winner === 'xds') winCounts['XDS']++;+ else if (data.winner === 'baseline') winCounts['Baseline']++;+ else if (data.winner === 'html') winCounts['HTML']++;+ else if (data.winner === 'xds-tailwind') winCounts['XDS+TW']++;+ else winCounts['Tie']++;}+ const parts = targets.map(t => `${t.label} wins ${winCounts[t.label]}`);+ parts.push(`Ties ${winCounts['Tie']}`);+ console.log(+ `\n📝 Per-Prompt: ${parts.join(' | ')} (${promptEntries.length} prompts)`,+ );}console.log(`\nSaved: ${outputPath}\n`);diff --git a/internal/vibe-tests/src/universal-eval.ts b/internal/vibe-tests/src/universal-eval.tsindex cbac5c8b72..22a527e17f 100644--- a/internal/vibe-tests/src/universal-eval.ts+++ b/internal/vibe-tests/src/universal-eval.ts@@ -104,22 +104,114 @@ function scoreTscErrors(tscResult: TscResult): {}// ============================================================-// 1. Correctness (tsc-based)+// 1. Correctness (tsc + phantom prop detection)// ============================================================+/**+ * Detect "phantom props" — event handlers and props that look valid+ * but silently do nothing in React DOM. These compile fine on targets+ * without strict types (baseline, html) but represent real bugs.+ *+ * Applied equally to ALL targets for fairness.+ */+function analyzePhantomProps(code: string): UniversalFinding[] {+ const findings: UniversalFinding[] = [];+ const lines = code.split('\n');++ for (let i = 0; i < lines.length; i++) {+ const line = lines[i];+ const lineNum = i + 1;++ // Skip comments and imports+ const trimmed = line.trim();+ if (+ trimmed.startsWith('//') ||+ trimmed.startsWith('*') ||+ trimmed.startsWith('import ')+ )+ continue;++ // onPress on any element/component — React Native, not DOM+ if (/\bonPress\s*[={]/.test(line) && !/onPressedChange/.test(line)) {+ findings.push({+ rule: 'phantom-prop',+ severity: 'critical',+ detail:+ 'onPress is React Native — use onClick for React DOM. Handler will never fire.',+ line: lineNum,+ });+ }++ // onLongPress — React Native only+ if (/\bonLongPress\s*[={]/.test(line)) {+ findings.push({+ rule: 'phantom-prop',+ severity: 'critical',+ detail:+ 'onLongPress is React Native — no DOM equivalent. Handler will never fire.',+ line: lineNum,+ });+ }++ // onChange on <button> — buttons don't fire change events+ if (/<button\b/i.test(line) || /<Button\b/.test(line)) {+ const context = lines.slice(i, Math.min(i + 3, lines.length)).join(' ');+ if (+ /\bonChange\s*[={]/.test(context) &&+ !/type\s*=\s*["']/.test(context)+ ) {+ findings.push({+ rule: 'phantom-prop',+ severity: 'moderate',+ detail: 'onChange on button — buttons do not fire change events',+ line: lineNum,+ });+ }+ }++ // onSubmit on non-form elements+ if (/\bonSubmit\s*[={]/.test(line)) {+ const context = lines+ .slice(Math.max(0, i - 2), Math.min(i + 2, lines.length))+ .join(' ');+ if (+ /<(div|section|main|span)\b/.test(context) &&+ !/<form\b/.test(context)+ ) {+ findings.push({+ rule: 'phantom-prop',+ severity: 'moderate',+ detail:+ 'onSubmit on non-form element — only <form> fires submit events',+ line: lineNum,+ });+ }+ }+ }++ return findings;+}+function analyzeCorrectness(code: string,_target: string,tscResult?: TscResult | null,): DimensionScore {- // tsc results are the sole correctness signal+ const findings: UniversalFinding[] = [];++ // Phantom prop detection — runs on ALL targets equally+ findings.push(...analyzePhantomProps(code));++ // tsc results — phantom props already caught by tsc don't double-penalizeif (tscResult) {- const {score, findings} = scoreTscErrors(tscResult);- return {score: clamp(score), findings};+ const tsc = scoreTscErrors(tscResult);+ findings.push(...tsc.findings);+ // tsc already penalizes type errors (which catch phantom props on strict targets).+ // Don't add phantom prop penalty on top — it's already in the tsc score.+ return {score: clamp(tsc.score), findings};}- // No tsc data available — check for missing export only- const findings: UniversalFinding[] = [];+ // No tsc data — phantom props + structural checksif (!/export\s+(default|function|const|class)/.test(code)) {findings.push({rule: 'missing-export',@@ -128,10 +220,13 @@ function analyzeCorrectness(});}- // Without tsc data, assume code is correct (backwards compat for old iterations)let score = 100;for (const f of findings) {- score -= f.severity === 'minor' ? 3 : 0;+ if (f.rule === 'phantom-prop') {+ score -= f.severity === 'critical' ? 15 : 8;+ } else if (f.severity === 'minor') {+ score -= 3;+ }}return {score: clamp(score), findings};}@@ -533,14 +628,20 @@ function analyzeEfficiency(if (stripped === '}' || stripped === '};') inTypeBlock = false;continue;}- if ((target === 'xds' || target === 'xds-tailwind') && stripped.includes('stylex.create'))+ if (+ (target === 'xds' || target === 'xds-tailwind') &&+ stripped.includes('stylex.create')+ )inStyleBlock = true;if (inStyleBlock) {stylingLines++;if (stripped === '});') inStyleBlock = false;continue;}- if ((target === 'baseline' || target === 'xds-tailwind') && stripped.includes('className=')) {+ if (+ (target === 'baseline' || target === 'xds-tailwind') &&+ stripped.includes('className=')+ ) {stylingLines++;continue;}