Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(vibe-tests): 4-way comparison in report and preview pipeline (#1696)

Cindy Zhang0470d9ebd583
diff --git a/internal/vibe-tests/app/src/report/CompareView.tsx b/internal/vibe-tests/app/src/report/CompareView.tsx
index da8afbcc89..a5a5cb98e7 100644
--- a/internal/vibe-tests/app/src/report/CompareView.tsx
+++ b/internal/vibe-tests/app/src/report/CompareView.tsx
@@ -18,7 +18,7 @@ import {
} from './utils';
import './report.css';
-type WinnerType = 'xds' | 'baseline' | 'html' | 'tie';
+type WinnerType = 'xds' | 'xds-tailwind' | 'baseline' | 'html' | 'tie';
interface CompareViewProps {
comparison: UniversalComparison;
@@ -30,6 +30,7 @@ interface DimRow extends Record<string, unknown> {
xdsScore: number;
baselineScore: number;
htmlScore?: number;
+ xdsTailwindScore?: number;
delta: number;
winner: string;
}
@@ -40,6 +41,7 @@ interface CatRow extends Record<string, unknown> {
xdsOverall: number;
baselineOverall: number;
htmlOverall?: number;
+ xdsTailwindOverall?: number;
delta: number;
}
@@ -49,6 +51,7 @@ interface CostRow extends Record<string, unknown> {
xds: string;
baseline: string;
html?: string;
+ xdsTailwind?: string;
winner: string;
}
@@ -57,24 +60,26 @@ function costWinner(
baseVal: number,
lowerIsBetter: boolean,
htmlVal?: number,
+ twVal?: number,
): WinnerType {
- if (htmlVal != null) {
- const vals = [xdsVal, baseVal, htmlVal];
- const best = lowerIsBetter ? Math.min(...vals) : Math.max(...vals);
- const atBest = vals.filter(v => v === best).length;
- if (atBest > 1) return 'tie';
- if (xdsVal === best) return 'xds';
- if (baseVal === best) return 'baseline';
- return 'html';
- }
- if (xdsVal === baseVal) return 'tie';
- if (lowerIsBetter) return xdsVal < baseVal ? 'xds' : 'baseline';
- return xdsVal > baseVal ? 'xds' : 'baseline';
+ const entries: [WinnerType, number][] = [
+ ['xds', xdsVal],
+ ['baseline', baseVal],
+ ];
+ if (htmlVal != null) entries.push(['html', htmlVal]);
+ if (twVal != null) entries.push(['xds-tailwind', twVal]);
+
+ const best = lowerIsBetter
+ ? Math.min(...entries.map(([, v]) => v))
+ : Math.max(...entries.map(([, v]) => v));
+ const atBest = entries.filter(([, v]) => v === best);
+ if (atBest.length > 1) return 'tie';
+ return atBest[0][0];
}
function winnerBadgeVariant(
w: string,
-): 'success' | 'error' | 'warning' | 'neutral' {
+): 'success' | 'error' | 'warning' | 'neutral' | 'info' {
switch (w) {
case 'xds':
return 'success';
@@ -82,6 +87,8 @@ function winnerBadgeVariant(
return 'error';
case 'html':
return 'warning';
+ case 'xds-tailwind':
+ return 'info';
default:
return 'neutral';
}
@@ -95,6 +102,8 @@ function winnerLabel(w: string): string {
return 'Baseline';
case 'html':
return 'HTML';
+ case 'xds-tailwind':
+ return 'XDS+TW';
default:
return 'Tie';
}
@@ -110,12 +119,15 @@ function CostComparisonSection({
xdsCost,
baselineCost,
htmlCost,
+ xdsTailwindCost,
}: {
xdsCost: CostMetrics;
baselineCost: CostMetrics;
htmlCost?: CostMetrics;
+ xdsTailwindCost?: CostMetrics;
}) {
const isThreeWay = !!htmlCost;
+ const isFourWay = !!xdsTailwindCost;
const hasDuration =
xdsCost.avgDurationMs > 0 || baselineCost.avgDurationMs > 0;
@@ -130,11 +142,17 @@ function CostComparisonSection({
...(isThreeWay
? {html: `${(htmlCost!.avgDurationMs / 1000).toFixed(1)}s`}
: {}),
+ ...(isFourWay
+ ? {
+ xdsTailwind: `${(xdsTailwindCost!.avgDurationMs / 1000).toFixed(1)}s`,
+ }
+ : {}),
winner: costWinner(
xdsCost.avgDurationMs,
baselineCost.avgDurationMs,
true,
htmlCost?.avgDurationMs,
+ xdsTailwindCost?.avgDurationMs,
),
},
]
@@ -147,11 +165,17 @@ function CostComparisonSection({
...(isThreeWay
? {html: `~${htmlCost!.estimatedInputTokens.toLocaleString()}`}
: {}),
+ ...(isFourWay
+ ? {
+ xdsTailwind: `~${xdsTailwindCost!.estimatedInputTokens.toLocaleString()}`,
+ }
+ : {}),
winner: costWinner(
xdsCost.estimatedInputTokens,
baselineCost.estimatedInputTokens,
true,
htmlCost?.estimatedInputTokens,
+ xdsTailwindCost?.estimatedInputTokens,
),
},
{
@@ -162,11 +186,17 @@ function CostComparisonSection({
...(isThreeWay
? {html: `~${htmlCost!.estimatedOutputTokens.toLocaleString()}`}
: {}),
+ ...(isFourWay
+ ? {
+ xdsTailwind: `~${xdsTailwindCost!.estimatedOutputTokens.toLocaleString()}`,
+ }
+ : {}),
winner: costWinner(
xdsCost.estimatedOutputTokens,
baselineCost.estimatedOutputTokens,
true,
htmlCost?.estimatedOutputTokens,
+ xdsTailwindCost?.estimatedOutputTokens,
),
},
{
@@ -179,6 +209,11 @@ function CostComparisonSection({
html: `~${(htmlCost!.estimatedInputTokens + htmlCost!.estimatedOutputTokens).toLocaleString()}`,
}
: {}),
+ ...(isFourWay
+ ? {
+ xdsTailwind: `~${(xdsTailwindCost!.estimatedInputTokens + xdsTailwindCost!.estimatedOutputTokens).toLocaleString()}`,
+ }
+ : {}),
winner: costWinner(
xdsCost.estimatedInputTokens + xdsCost.estimatedOutputTokens,
baselineCost.estimatedInputTokens + baselineCost.estimatedOutputTokens,
@@ -186,6 +221,10 @@ function CostComparisonSection({
htmlCost
? htmlCost.estimatedInputTokens + htmlCost.estimatedOutputTokens
: undefined,
+ xdsTailwindCost
+ ? xdsTailwindCost.estimatedInputTokens +
+ xdsTailwindCost.estimatedOutputTokens
+ : undefined,
),
},
{
@@ -194,11 +233,15 @@ function CostComparisonSection({
xds: String(xdsCost.avgOutputLines),
baseline: String(baselineCost.avgOutputLines),
...(isThreeWay ? {html: String(htmlCost!.avgOutputLines)} : {}),
+ ...(isFourWay
+ ? {xdsTailwind: String(xdsTailwindCost!.avgOutputLines)}
+ : {}),
winner: costWinner(
xdsCost.avgOutputLines,
baselineCost.avgOutputLines,
true,
htmlCost?.avgOutputLines,
+ xdsTailwindCost?.avgOutputLines,
),
},
{
@@ -207,6 +250,9 @@ function CostComparisonSection({
xds: String(xdsCost.avgDocsRead),
baseline: String(baselineCost.avgDocsRead),
...(isThreeWay ? {html: String(htmlCost!.avgDocsRead)} : {}),
+ ...(isFourWay
+ ? {xdsTailwind: String(xdsTailwindCost!.avgDocsRead)}
+ : {}),
winner: 'tie', // not inherently better or worse
},
];
@@ -234,6 +280,17 @@ function CostComparisonSection({
} satisfies XDSTableColumn<CostRow>,
]
: []),
+ ...(isFourWay
+ ? [
+ {
+ key: 'xdsTailwind' as const,
+ header: 'XDS+TW',
+ renderCell: (row: CostRow) => (
+ <XDSText type="body">{row.xdsTailwind ?? '—'}</XDSText>
+ ),
+ } satisfies XDSTableColumn<CostRow>,
+ ]
+ : []),
{
key: 'winner',
header: 'Lower Cost',
@@ -258,18 +315,21 @@ function CostComparisonSection({
}
export function CompareView({comparison}: CompareViewProps) {
- const {xds, baseline, html, winners} = comparison;
+ const {xds, baseline, html, xdsTailwind, winners} = comparison;
const isThreeWay = !!html;
+ const isFourWay = !!xdsTailwind;
let xdsWins = 0;
let baselineWins = 0;
let htmlWins = 0;
+ let xdsTailwindWins = 0;
let ties = 0;
for (const dim of ALL_DIMENSIONS) {
const w = winners[dim];
if (w === 'xds') xdsWins++;
else if (w === 'baseline') baselineWins++;
else if (w === 'html') htmlWins++;
+ else if (w === 'xds-tailwind') xdsTailwindWins++;
else ties++;
}
@@ -281,6 +341,7 @@ export function CompareView({comparison}: CompareViewProps) {
xdsScore: xds.averages[dim] ?? 0,
baselineScore: baseline.averages[dim] ?? 0,
...(isThreeWay ? {htmlScore: html!.averages[dim] ?? 0} : {}),
+ ...(isFourWay ? {xdsTailwindScore: xdsTailwind!.averages[dim] ?? 0} : {}),
delta: (xds.averages[dim] ?? 0) - (baseline.averages[dim] ?? 0),
winner: winners[dim],
}));
@@ -314,6 +375,21 @@ export function CompareView({comparison}: CompareViewProps) {
} satisfies XDSTableColumn<DimRow>,
]
: []),
+ ...(isFourWay
+ ? [
+ {
+ key: 'xdsTailwindScore' as const,
+ header: 'XDS+TW',
+ renderCell: (row: DimRow) => (
+ <XDSText type="body">
+ {row.xdsTailwindScore != null
+ ? formatScore(row.xdsTailwindScore)
+ : '—'}
+ </XDSText>
+ ),
+ } satisfies XDSTableColumn<DimRow>,
+ ]
+ : []),
{
key: 'delta',
header: 'Delta (XDS−Base)',
@@ -340,6 +416,7 @@ export function CompareView({comparison}: CompareViewProps) {
...Object.keys(xds.byCategory),
...Object.keys(baseline.byCategory),
...(html ? Object.keys(html.byCategory) : []),
+ ...(xdsTailwind ? Object.keys(xdsTailwind.byCategory) : []),
]);
const catData: CatRow[] = [...allCategories].map(cat => {
@@ -347,6 +424,9 @@ export function CompareView({comparison}: CompareViewProps) {
const baseCat = baseline.byCategory[cat] ?? {};
const htmlCat =
html?.byCategory[cat] ?? ({} as Record<UniversalDimension, number>);
+ const twCat =
+ xdsTailwind?.byCategory[cat] ??
+ ({} as Record<UniversalDimension, number>);
const xdsAvg =
CODE_DIMENSIONS.reduce(
(s, d) => s + ((xdsCat[d as UniversalDimension] as number) ?? 0),
@@ -363,12 +443,19 @@ export function CompareView({comparison}: CompareViewProps) {
0,
) / CODE_DIMENSIONS.length
: undefined;
+ const twAvg = isFourWay
+ ? CODE_DIMENSIONS.reduce(
+ (s, d) => s + ((twCat[d as UniversalDimension] as number) ?? 0),
+ 0,
+ ) / CODE_DIMENSIONS.length
+ : undefined;
return {
id: cat,
category: cat,
xdsOverall: xdsAvg,
baselineOverall: baseAvg,
...(htmlAvg != null ? {htmlOverall: htmlAvg} : {}),
+ ...(twAvg != null ? {xdsTailwindOverall: twAvg} : {}),
delta: xdsAvg - baseAvg,
};
});
@@ -402,6 +489,21 @@ export function CompareView({comparison}: CompareViewProps) {
} satisfies XDSTableColumn<CatRow>,
]
: []),
+ ...(isFourWay
+ ? [
+ {
+ key: 'xdsTailwindOverall' as const,
+ header: 'XDS+TW',
+ renderCell: (row: CatRow) => (
+ <XDSText type="body">
+ {row.xdsTailwindOverall != null
+ ? formatScore(row.xdsTailwindOverall)
+ : '—'}
+ </XDSText>
+ ),
+ } satisfies XDSTableColumn<CatRow>,
+ ]
+ : []),
{
key: 'delta',
header: 'Delta (XDS−Base)',
@@ -414,14 +516,19 @@ export function CompareView({comparison}: CompareViewProps) {
},
];
+ // Determine grid class based on number of win cards
+ const winCardCount =
+ 2 + (isThreeWay ? 1 : 0) + (isFourWay ? 1 : 0) + 1; // targets + ties
+ const summaryGridClass =
+ winCardCount >= 5
+ ? 'report-compare-summaryGrid5'
+ : winCardCount === 4
+ ? 'report-compare-summaryGrid4'
+ : 'report-compare-summaryGrid';
+
return (
<XDSVStack gap={4}>
- <div
- className={
- isThreeWay
- ? 'report-compare-summaryGrid4'
- : 'report-compare-summaryGrid'
- }>
+ <div className={summaryGridClass}>
<XDSCard>
<div className="report-compare-winCard">
<XDSVStack gap={2}>
@@ -454,6 +561,18 @@ export function CompareView({comparison}: CompareViewProps) {
</div>
</XDSCard>
)}
+ {isFourWay && (
+ <XDSCard>
+ <div className="report-compare-winCard">
+ <XDSVStack gap={2}>
+ <XDSText type="label">XDS+TW Wins</XDSText>
+ <XDSHeading level={2}>
+ <span className="report-color-info">{xdsTailwindWins}</span>
+ </XDSHeading>
+ </XDSVStack>
+ </div>
+ </XDSCard>
+ )}
<XDSCard>
<div className="report-compare-winCard">
<XDSVStack gap={2}>
@@ -497,6 +616,7 @@ export function CompareView({comparison}: CompareViewProps) {
xdsCost={xds.cost}
baselineCost={baseline.cost}
htmlCost={html?.cost}
+ xdsTailwindCost={xdsTailwind?.cost}
/>
</XDSVStack>
)}
diff --git a/internal/vibe-tests/app/src/report/PromptDetailCard.tsx b/internal/vibe-tests/app/src/report/PromptDetailCard.tsx
index 7b3c37cccf..15c7ad9453 100644
--- a/internal/vibe-tests/app/src/report/PromptDetailCard.tsx
+++ b/internal/vibe-tests/app/src/report/PromptDetailCard.tsx
@@ -94,10 +94,12 @@ interface PromptDetailCardProps {
xdsScore?: UniversalScore;
baselineScore?: UniversalScore;
htmlScore?: UniversalScore;
+ xdsTailwindScore?: UniversalScore;
hasXdsCode: boolean;
hasBaselineCode: boolean;
hasHtmlCode: boolean;
- onViewCode: (target: 'xds' | 'baseline' | 'html') => void;
+ hasXdsTailwindCode: boolean;
+ onViewCode: (target: 'xds' | 'baseline' | 'html' | 'xds-tailwind') => void;
/** Relative preview URLs keyed by target (e.g. { xds: "previews/sd-1/xds.html" }) */
previewUrls?: Record<string, string>;
}
@@ -108,23 +110,40 @@ export function PromptDetailCard({
xdsScore,
baselineScore,
htmlScore,
+ xdsTailwindScore,
hasXdsCode,
hasBaselineCode,
hasHtmlCode,
+ hasXdsTailwindCode,
onViewCode,
previewUrls,
}: PromptDetailCardProps) {
const hasBoth = !!(xdsScore && baselineScore);
const hasAll = !!(xdsScore && baselineScore && htmlScore);
+ const hasFourWay = !!(xdsScore && baselineScore && xdsTailwindScore);
const hasAnyPreview =
- previewUrls?.xds || previewUrls?.baseline || previewUrls?.html;
- const hasAnyCode = hasXdsCode || hasBaselineCode || hasHtmlCode;
+ previewUrls?.xds ||
+ previewUrls?.baseline ||
+ previewUrls?.html ||
+ previewUrls?.['xds-tailwind'];
+ const hasAnyCode =
+ hasXdsCode || hasBaselineCode || hasHtmlCode || hasXdsTailwindCode;
- const scoresClassName = hasAll
- ? 'report-promptDetail-scoresRow3'
- : hasBoth
- ? 'report-promptDetail-scoresRow'
- : 'report-promptDetail-scoresRowSingle';
+ // Count how many score blocks we have
+ const scoreCount = [
+ xdsScore,
+ baselineScore,
+ htmlScore,
+ xdsTailwindScore,
+ ].filter(Boolean).length;
+ const scoresClassName =
+ scoreCount >= 4
+ ? 'report-promptDetail-scoresRow4'
+ : scoreCount === 3
+ ? 'report-promptDetail-scoresRow3'
+ : scoreCount === 2
+ ? 'report-promptDetail-scoresRow'
+ : 'report-promptDetail-scoresRowSingle';
return (
<XDSCard>
@@ -164,6 +183,16 @@ export function PromptDetailCard({
label="HTML Preview"
/>
)}
+ {previewUrls?.['xds-tailwind'] && (
+ <XDSButton
+ variant="secondary"
+ size="sm"
+ onClick={() =>
+ window.open(previewUrls['xds-tailwind'], '_blank')
+ }
+ label="XDS+TW Preview"
+ />
+ )}
{hasXdsCode && (
<XDSButton
variant="ghost"
@@ -188,18 +217,29 @@ export function PromptDetailCard({
label="HTML Code"
/>
)}
+ {hasXdsTailwindCode && (
+ <XDSButton
+ variant="ghost"
+ size="sm"
+ onClick={() => onViewCode('xds-tailwind')}
+ label="XDS+TW Code"
+ />
+ )}
</div>
)}
</div>
{/* Score summaries in constrained grid */}
- {(xdsScore || baselineScore || htmlScore) && (
+ {(xdsScore || baselineScore || htmlScore || xdsTailwindScore) && (
<div className={scoresClassName}>
{xdsScore && <ScoreSummary label="XDS" score={xdsScore} />}
{baselineScore && (
<ScoreSummary label="Baseline" score={baselineScore} />
)}
{htmlScore && <ScoreSummary label="HTML" score={htmlScore} />}
+ {xdsTailwindScore && (
+ <ScoreSummary label="XDS+TW" score={xdsTailwindScore} />
+ )}
</div>
)}
@@ -239,6 +279,18 @@ export function PromptDetailCard({
</div>
</>
)}
+
+ {xdsTailwindScore && (
+ <>
+ <XDSDivider />
+ <div className="report-promptDetail-section">
+ <div className="report-promptDetail-sectionLabel">
+ <XDSText type="label">XDS+TW Findings</XDSText>
+ </div>
+ <Findings score={xdsTailwindScore} />
+ </div>
+ </>
+ )}
</XDSVStack>
</div>
</XDSCard>
diff --git a/internal/vibe-tests/app/src/report/Report.tsx b/internal/vibe-tests/app/src/report/Report.tsx
index 037688b64c..49de78c720 100644
--- a/internal/vibe-tests/app/src/report/Report.tsx
+++ b/internal/vibe-tests/app/src/report/Report.tsx
@@ -159,7 +159,7 @@ export function Report() {
const [activeTab, setActiveTab] = useState('overview');
const [codeModal, setCodeModal] = useState<{
promptId: string;
- target: 'xds' | 'baseline' | 'html';
+ target: 'xds' | 'baseline' | 'html' | 'xds-tailwind';
} | null>(null);
const data: ReportData | undefined = window.__REPORT_DATA__;
@@ -246,6 +246,14 @@ export function Report() {
compareLabel="HTML"
/>
)}
+ {comparison?.xdsTailwind && (
+ <ScoreCard
+ label="Overall Score vs XDS+TW"
+ score={universal.overall}
+ compareScore={comparison.xdsTailwind.overall}
+ compareLabel="XDS+TW"
+ />
+ )}
{/* Dimension scores grid */}
<XDSVStack gap={3}>
@@ -274,9 +282,15 @@ export function Report() {
{comparison && (
<XDSVStack gap={3}>
<XDSHeading level={3}>
- {comparison.html
- ? 'XDS vs Baseline vs HTML Comparison'
- : 'XDS vs Baseline Comparison'}
+ {[
+ 'XDS',
+ 'Baseline',
+ comparison.html ? 'HTML' : null,
+ comparison.xdsTailwind ? 'XDS+TW' : null,
+ ]
+ .filter(Boolean)
+ .join(' vs ')}{' '}
+ Comparison
</XDSHeading>
<CompareView comparison={comparison} />
</XDSVStack>
@@ -299,9 +313,15 @@ export function Report() {
xdsScore={universal.byPrompt[promptId]}
baselineScore={comparison?.baseline.byPrompt[promptId]}
htmlScore={comparison?.html?.byPrompt[promptId]}
+ xdsTailwindScore={
+ comparison?.xdsTailwind?.byPrompt[promptId]
+ }
hasXdsCode={!!data.sourceCode?.[promptId]}
hasBaselineCode={!!data.baselineSourceCode?.[promptId]}
hasHtmlCode={!!data.htmlSourceCode?.[promptId]}
+ hasXdsTailwindCode={
+ !!data.xdsTailwindSourceCode?.[promptId]
+ }
onViewCode={target => setCodeModal({promptId, target})}
previewUrls={data.previews?.[promptId]}
/>
@@ -318,7 +338,9 @@ export function Report() {
? data.sourceCode?.[codeModal.promptId]
: codeModal.target === 'baseline'
? data.baselineSourceCode?.[codeModal.promptId]
- : data.htmlSourceCode?.[codeModal.promptId];
+ : codeModal.target === 'xds-tailwind'
+ ? data.xdsTailwindSourceCode?.[codeModal.promptId]
+ : data.htmlSourceCode?.[codeModal.promptId];
return code ? (
<CodeModal
isOpen
diff --git a/internal/vibe-tests/app/src/report/report.css b/internal/vibe-tests/app/src/report/report.css
index 896a122c4b..d75aa6c378 100644
--- a/internal/vibe-tests/app/src/report/report.css
+++ b/internal/vibe-tests/app/src/report/report.css
@@ -44,6 +44,12 @@
gap: var(--spacing-3);
}
+.report-compare-summaryGrid5 {
+ display: grid;
+ grid-template-columns: repeat(5, 1fr);
+ gap: var(--spacing-3);
+}
+
.report-compare-winCard {
padding: var(--spacing-3);
text-align: center;
@@ -65,6 +71,10 @@
color: var(--color-warning);
}
+.report-color-info {
+ color: var(--color-info, #8b5cf6);
+}
+
/* === PromptDetailCard === */
.report-promptDetail-card {
padding: var(--spacing-4);
@@ -105,6 +115,12 @@
gap: var(--spacing-3);
}
+.report-promptDetail-scoresRow4 {
+ display: grid;
+ grid-template-columns: 1fr 1fr 1fr 1fr;
+ gap: var(--spacing-3);
+}
+
.report-promptDetail-scoresRowSingle {
display: grid;
grid-template-columns: 1fr;
diff --git a/internal/vibe-tests/app/src/report/types.ts b/internal/vibe-tests/app/src/report/types.ts
index 49e594068e..e3d532af6b 100644
--- a/internal/vibe-tests/app/src/report/types.ts
+++ b/internal/vibe-tests/app/src/report/types.ts
@@ -29,6 +29,7 @@ export interface ReportData {
sourceCode?: Record<string, string>;
baselineSourceCode?: Record<string, string>;
htmlSourceCode?: Record<string, string>;
+ xdsTailwindSourceCode?: Record<string, string>;
/** Map of promptId → { target → relative URL } */
previews?: Record<string, Record<string, string>>;
/** Map of promptId → prompt text */
diff --git a/internal/vibe-tests/src/build-report.ts b/internal/vibe-tests/src/build-report.ts
index 3839eaa815..c1a2fc7c97 100644
--- a/internal/vibe-tests/src/build-report.ts
+++ b/internal/vibe-tests/src/build-report.ts
@@ -72,6 +72,7 @@ function parseArgs(): {
iteration: string;
baseline?: string;
html?: string;
+ xdsTailwind?: string;
withScreenshots: boolean;
dev: boolean;
} {
@@ -79,6 +80,7 @@ function parseArgs(): {
let iteration = '';
let baseline: string | undefined;
let html: string | undefined;
+ let xdsTailwind: string | undefined;
let withScreenshots = false;
let dev = false;
@@ -92,6 +94,9 @@ function parseArgs(): {
} else if (args[i] === '--html' && args[i + 1]) {
html = args[i + 1];
i++;
+ } else if (args[i] === '--xds-tailwind' && args[i + 1]) {
+ xdsTailwind = args[i + 1];
+ i++;
} else if (args[i] === '--with-screenshots') {
withScreenshots = true;
} else if (args[i] === '--dev') {
@@ -101,12 +106,12 @@ function parseArgs(): {
if (!iteration) {
console.error(
- 'Usage: tsx src/build-report.ts --iteration <id> [--baseline <id>] [--with-screenshots] [--dev]',
+ 'Usage: tsx src/build-report.ts --iteration <id> [--baseline <id>] [--html <id>] [--xds-tailwind <id>] [--with-screenshots] [--dev]',
);
process.exit(1);
}
- return {iteration, baseline, html, withScreenshots, dev};
+ return {iteration, baseline, html, xdsTailwind, withScreenshots, dev};
}
function ensureUniversalJson(iteration: string): UniversalAggregate {
@@ -128,20 +133,24 @@ function ensureComparison(
xdsId: string,
baselineId: string,
htmlId?: string,
+ xdsTailwindId?: string,
): UniversalComparison {
const resultsDir = getResultsDir();
- const comparisonFilename = htmlId
- ? `comparison-${xdsId}-${baselineId}-${htmlId}.json`
- : `comparison-${xdsId}-${baselineId}.json`;
+ const idParts = [xdsId, baselineId];
+ if (htmlId) idParts.push(htmlId);
+ if (xdsTailwindId) idParts.push(xdsTailwindId);
+ const comparisonFilename = `comparison-${idParts.join('-')}.json`;
const comparisonPath = path.join(resultsDir, comparisonFilename);
if (!fs.existsSync(comparisonPath)) {
const htmlFlag = htmlId ? ` --html ${htmlId}` : '';
- console.log(
- `Generating comparison for ${xdsId} vs ${baselineId}${htmlId ? ` vs ${htmlId}` : ''}...`,
- );
+ const twFlag = xdsTailwindId ? ` --xds-tailwind ${xdsTailwindId}` : '';
+ const vsLabel = [baselineId, htmlId, xdsTailwindId]
+ .filter(Boolean)
+ .join(' vs ');
+ console.log(`Generating comparison for ${xdsId} vs ${vsLabel}...`);
execSync(
- `tsx ${path.join(import.meta.dirname, 'universal-compare.ts')} --xds ${xdsId} --baseline ${baselineId}${htmlFlag}`,
+ `tsx ${path.join(import.meta.dirname, 'universal-compare.ts')} --xds ${xdsId} --baseline ${baselineId}${htmlFlag}${twFlag}`,
{cwd: path.join(import.meta.dirname, '..'), stdio: 'inherit'},
);
}
@@ -180,6 +189,7 @@ function buildDataScript(opts: {
sourceCode?: Record<string, string>;
baselineSourceCode?: Record<string, string>;
htmlSourceCode?: Record<string, string>;
+ xdsTailwindSourceCode?: Record<string, string>;
previews?: Record<string, Record<string, string>>;
screenshots?: Record<string, string>;
prompts?: Record<string, string>;
@@ -192,6 +202,7 @@ function buildDataScript(opts: {
sourceCode: opts.sourceCode,
baselineSourceCode: opts.baselineSourceCode,
htmlSourceCode: opts.htmlSourceCode,
+ xdsTailwindSourceCode: opts.xdsTailwindSourceCode,
previews: opts.previews,
screenshots: opts.screenshots,
prompts: opts.prompts,
@@ -241,6 +252,7 @@ async function main() {
iteration,
baseline,
html,
+ xdsTailwind,
withScreenshots: _withScreenshots,
dev,
} = parseArgs();
@@ -267,7 +279,10 @@ async function main() {
if (html) {
ensureUniversalJson(html);
}
- comparison = ensureComparison(iteration, baseline, html);
+ if (xdsTailwind) {
+ ensureUniversalJson(xdsTailwind);
+ }
+ comparison = ensureComparison(iteration, baseline, html, xdsTailwind);
}
// Step 3: Load source code for per-prompt inspection
@@ -320,6 +335,24 @@ async function main() {
}
}
+ let xdsTailwindSourceCode: Record<string, string> | undefined;
+ if (xdsTailwind) {
+ xdsTailwindSourceCode = {};
+ const twCodeDir = path.join(resultsDir, xdsTailwind, 'results');
+ if (fs.existsSync(twCodeDir)) {
+ ensureTsxFiles(twCodeDir);
+ for (const file of fs
+ .readdirSync(twCodeDir)
+ .filter(f => f.endsWith('.tsx'))) {
+ const promptId = path.basename(file, '.tsx');
+ xdsTailwindSourceCode[promptId] = fs.readFileSync(
+ path.join(twCodeDir, file),
+ 'utf-8',
+ );
+ }
+ }
+ }
+
// Load preview manifest if available
const previewManifestPath = path.join(iterDir, 'previews', 'manifest.json');
let previews: Record<string, Record<string, string>> | undefined;
@@ -334,7 +367,9 @@ async function main() {
// Screenshots use relative paths (screenshots/filename.png) that resolve
// when deployed to gh-pages alongside the report.
let screenshots: Record<string, string> | undefined;
- for (const id of [iteration, baseline, html].filter(Boolean) as string[]) {
+ for (const id of [iteration, baseline, html, xdsTailwind].filter(
+ Boolean,
+ ) as string[]) {
const screenshotManifestPath = path.join(
resultsDir,
id,
@@ -388,6 +423,7 @@ async function main() {
sourceCode,
baselineSourceCode,
htmlSourceCode,
+ xdsTailwindSourceCode,
previews,
screenshots,
prompts,
diff --git a/internal/vibe-tests/src/deploy-report.ts b/internal/vibe-tests/src/deploy-report.ts
index 5f6a120b6d..f3cc0b4ac0 100644
--- a/internal/vibe-tests/src/deploy-report.ts
+++ b/internal/vibe-tests/src/deploy-report.ts
@@ -23,6 +23,7 @@ function parseArgs(): {
iteration: string;
baseline?: string;
html?: string;
+ xdsTailwind?: string;
dryRun: boolean;
skipPreviews: boolean;
} {
@@ -30,6 +31,7 @@ function parseArgs(): {
let iteration = '';
let baseline: string | undefined;
let html: string | undefined;
+ let xdsTailwind: string | undefined;
let dryRun = false;
let skipPreviews = false;
@@ -40,6 +42,8 @@ function parseArgs(): {
baseline = args[++i];
} else if (args[i] === '--html' && args[i + 1]) {
html = args[++i];
+ } else if (args[i] === '--xds-tailwind' && args[i + 1]) {
+ xdsTailwind = args[++i];
} else if (args[i] === '--dry-run') {
dryRun = true;
} else if (args[i] === '--skip-previews') {
@@ -49,12 +53,12 @@ function parseArgs(): {
if (!iteration) {
console.error(
- 'Usage: tsx src/deploy-report.ts --iteration <id> [--baseline <id>] [--html <id>] [--dry-run] [--skip-previews]',
+ 'Usage: tsx src/deploy-report.ts --iteration <id> [--baseline <id>] [--html <id>] [--xds-tailwind <id>] [--dry-run] [--skip-previews]',
);
process.exit(1);
}
- return {iteration, baseline, html, dryRun, skipPreviews};
+ return {iteration, baseline, html, xdsTailwind, dryRun, skipPreviews};
}
function run(cmd: string, opts?: {cwd?: string; silent?: boolean}): string {
@@ -68,7 +72,8 @@ function runSilent(cmd: string, opts?: {cwd?: string}): string {
}
async function main() {
- const {iteration, baseline, html, dryRun, skipPreviews} = parseArgs();
+ const {iteration, baseline, html, xdsTailwind, dryRun, skipPreviews} =
+ parseArgs();
const deployPath = `reports/${iteration}`;
@@ -82,7 +87,9 @@ async function main() {
// Step 1: Build previews if result files exist (.tsx or .json)
if (!skipPreviews) {
const iterationsWithCode: string[] = [];
- for (const id of [iteration, baseline, html].filter(Boolean) as string[]) {
+ for (const id of [iteration, baseline, html, xdsTailwind].filter(
+ Boolean,
+ ) as string[]) {
const codeDir = path.join(VIBE_DIR, 'results', id, 'results');
if (
fs.existsSync(codeDir) &&
@@ -116,6 +123,7 @@ async function main() {
const buildArgs = [`--iteration ${iteration}`];
if (baseline) buildArgs.push(`--baseline ${baseline}`);
if (html) buildArgs.push(`--html ${html}`);
+ if (xdsTailwind) buildArgs.push(`--xds-tailwind ${xdsTailwind}`);
run(`npx tsx src/build-report.ts ${buildArgs.join(' ')}`, {cwd: VIBE_DIR});
@@ -182,7 +190,9 @@ async function main() {
}
// Second: copy any local screenshots (may add to or overwrite gh-pages ones)
- for (const id of [iteration, baseline, html].filter(Boolean) as string[]) {
+ for (const id of [iteration, baseline, html, xdsTailwind].filter(
+ Boolean,
+ ) as string[]) {
const screenshotsDir = path.join(VIBE_DIR, 'results', id, 'screenshots');
if (fs.existsSync(screenshotsDir)) {
copyDirRecursive(screenshotsDir, targetScreenshotsDir);
diff --git a/internal/vibe-tests/src/types.ts b/internal/vibe-tests/src/types.ts
index 0e4fcd68d2..a28aeb822e 100644
--- a/internal/vibe-tests/src/types.ts
+++ b/internal/vibe-tests/src/types.ts
@@ -315,18 +315,22 @@ export interface UniversalAggregate {
cost?: CostMetrics;
}
+export type TargetName = 'xds' | 'xds-tailwind' | 'baseline' | 'html';
+
export interface UniversalComparison {
xds: UniversalAggregate;
baseline: UniversalAggregate;
html?: UniversalAggregate;
- winners: Record<UniversalDimension, 'xds' | 'baseline' | 'html' | 'tie'>;
+ xdsTailwind?: UniversalAggregate;
+ winners: Record<UniversalDimension, TargetName | 'tie'>;
byPrompt: Record<
string,
{
xds: UniversalScore;
baseline: UniversalScore;
html?: UniversalScore;
- winner: 'xds' | 'baseline' | 'html' | 'tie';
+ xdsTailwind?: UniversalScore;
+ winner: TargetName | 'tie';
}
>;
}
diff --git a/internal/vibe-tests/src/universal-compare.ts b/internal/vibe-tests/src/universal-compare.ts
index 775846982c..cb6a38972a 100644
--- a/internal/vibe-tests/src/universal-compare.ts
+++ b/internal/vibe-tests/src/universal-compare.ts
@@ -5,6 +5,7 @@
* Usage:
* tsx src/universal-compare.ts --xds abc123 --baseline def456
* tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789
+ * tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789 --xds-tailwind jkl012
* tsx src/universal-compare.ts --xds abc123 --baseline def456 --json
* tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789 --markdown
*/
@@ -16,6 +17,7 @@ import type {
UniversalAggregate,
UniversalComparison,
UniversalDimension,
+ TargetName,
} from './types.js';
import {writeJson, getResultsDir} from './utils.js';
import {getDimensionNames, getAverageScore} from './universal-eval.js';
@@ -50,22 +52,25 @@ function loadOrGenerate(iterationId: string): UniversalAggregate {
return JSON.parse(fs.readFileSync(universalPath, 'utf-8'));
}
-type WinnerType = 'xds' | 'baseline' | 'html' | 'tie';
-
-function winner(xdsVal: number, baseVal: number, htmlVal?: number): WinnerType {
- if (htmlVal != null) {
- const max = Math.max(xdsVal, baseVal, htmlVal);
- const atMax = [xdsVal === max, baseVal === max, htmlVal === max].filter(
- Boolean,
- ).length;
- if (atMax > 1) return 'tie';
- if (xdsVal === max) return 'xds';
- if (baseVal === max) return 'baseline';
- return 'html';
- }
- if (xdsVal > baseVal) return 'xds';
- if (baseVal > xdsVal) return 'baseline';
- return 'tie';
+type WinnerType = TargetName | 'tie';
+
+function winner(
+ xdsVal: number,
+ baseVal: number,
+ htmlVal?: number,
+ xdsTailwindVal?: number,
+): WinnerType {
+ const entries: [TargetName, number][] = [
+ ['xds', xdsVal],
+ ['baseline', baseVal],
+ ];
+ if (htmlVal != null) entries.push(['html', htmlVal]);
+ if (xdsTailwindVal != null) entries.push(['xds-tailwind', xdsTailwindVal]);
+
+ const max = Math.max(...entries.map(([, v]) => v));
+ const atMax = entries.filter(([, v]) => v === max);
+ if (atMax.length > 1) return 'tie';
+ return atMax[0][0];
}
function winnerIcon(w: WinnerType): string {
@@ -76,6 +81,8 @@ function winnerIcon(w: WinnerType): string {
return '🔵 Base';
case 'html':
return '🟡 HTML';
+ case 'xds-tailwind':
+ return '🟣 XDS+TW';
case 'tie':
return '⚪ Tie';
}
@@ -85,6 +92,7 @@ function parseArgs(): {
xds: string;
baseline: string;
html?: string;
+ xdsTailwind?: string;
json: boolean;
markdown: boolean;
} {
@@ -92,6 +100,7 @@ function parseArgs(): {
let xds = '';
let baseline = '';
let html: string | undefined;
+ let xdsTailwind: string | undefined;
let json = false;
let markdown = false;
@@ -102,6 +111,8 @@ function parseArgs(): {
baseline = args[++i];
} else if (args[i] === '--html' && args[i + 1]) {
html = args[++i];
+ } else if (args[i] === '--xds-tailwind' && args[i + 1]) {
+ xdsTailwind = args[++i];
} else if (args[i] === '--json') {
json = true;
} else if (args[i] === '--markdown' || args[i] === '--md') {
@@ -111,12 +122,12 @@ function parseArgs(): {
if (!xds || !baseline) {
console.error(
- 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--json] [--markdown]',
+ 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--xds-tailwind <id>] [--json] [--markdown]',
);
process.exit(1);
}
- return {xds, baseline, html, json, markdown};
+ return {xds, baseline, html, xdsTailwind, json, markdown};
}
/**
@@ -128,30 +139,22 @@ function toMarkdown(opts: {
xdsId: string;
baselineId: string;
htmlId?: string;
+ xdsTailwindId?: string;
byPrompt: UniversalComparison['byPrompt'];
}): string {
- const {comparison, xdsId, baselineId, htmlId, byPrompt} = opts;
- const {xds, baseline, html: htmlData, winners} = comparison;
+ const {comparison, xdsId, baselineId, htmlId, xdsTailwindId, byPrompt} =
+ opts;
+ const {xds, baseline, html: htmlData, xdsTailwind: twData, winners} =
+ comparison;
const dimensions = getDimensionNames();
- const isThreeWay = !!htmlData;
const lines: string[] = [];
- // Summary table
- if (isThreeWay) {
- lines.push(
- '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',
- );
- lines.push(
- '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',
- );
- } else {
- lines.push(
- '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',
- );
- lines.push(
- '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',
- );
- }
+ lines.push(
+ '| Target | Iteration | Overall | Correctness | Accessibility | Code Quality | Efficiency | Maintainability |',
+ );
+ lines.push(
+ '|--------|-----------|---------|-------------|---------------|--------------|------------|-----------------|',
+ );
const dimOrder: UniversalDimension[] = [
'correctness',
@@ -169,13 +172,20 @@ function toMarkdown(opts: {
`| **Baseline** | \`${baselineId}\` | ${baseline.overall} | ${baseRow} |`,
);
- if (isThreeWay && htmlData) {
+ if (htmlData) {
const htmlRow = dimOrder.map(d => htmlData.averages[d]).join(' | ');
lines.push(
`| **HTML** | \`${htmlId}\` | ${htmlData.overall} | ${htmlRow} |`,
);
}
+ if (twData) {
+ const twRow = dimOrder.map(d => twData.averages[d]).join(' | ');
+ lines.push(
+ `| **XDS+TW** | \`${xdsTailwindId}\` | ${twData.overall} | ${twRow} |`,
+ );
+ }
+
lines.push('');
// Per-prompt winners
@@ -184,43 +194,47 @@ function toMarkdown(opts: {
let xWins = 0;
let bWins = 0;
let hWins = 0;
+ let twWins = 0;
let ties = 0;
for (const [, data] of promptEntries) {
if (data.winner === 'xds') xWins++;
else if (data.winner === 'baseline') bWins++;
else if (data.winner === 'html') hWins++;
+ else if (data.winner === 'xds-tailwind') twWins++;
else ties++;
}
- if (isThreeWay) {
- lines.push(
- `**Per-prompt wins:** XDS ${xWins} · Baseline ${bWins} · HTML ${hWins} · Tie ${ties} (${promptEntries.length} prompts)`,
- );
- } else {
- lines.push(
- `**Per-prompt wins:** XDS ${xWins} · Baseline ${bWins} · Tie ${ties} (${promptEntries.length} prompts)`,
- );
- }
+ const parts = [`XDS ${xWins}`, `Baseline ${bWins}`];
+ if (htmlData) parts.push(`HTML ${hWins}`);
+ if (twData) parts.push(`XDS+TW ${twWins}`);
+ parts.push(`Tie ${ties}`);
+ lines.push(
+ `**Per-prompt wins:** ${parts.join(' · ')} (${promptEntries.length} prompts)`,
+ );
lines.push('');
}
// Dark mode
- if (isThreeWay && htmlData) {
- lines.push(
- `**Dark mode:** XDS ${xds.darkModeRate}% · Baseline ${baseline.darkModeRate}% · HTML ${htmlData.darkModeRate}%`,
- );
- } else {
- lines.push(
- `**Dark mode:** XDS ${xds.darkModeRate}% · Baseline ${baseline.darkModeRate}%`,
- );
- }
+ const dmParts = [
+ `XDS ${xds.darkModeRate}%`,
+ `Baseline ${baseline.darkModeRate}%`,
+ ];
+ if (htmlData) dmParts.push(`HTML ${htmlData.darkModeRate}%`);
+ if (twData) dmParts.push(`XDS+TW ${twData.darkModeRate}%`);
+ lines.push(`**Dark mode:** ${dmParts.join(' · ')}`);
// Dimension winners
lines.push('');
lines.push('**Dimension winners:**');
for (const d of dimensions) {
const w = winners[d];
- const icon =
- w === 'xds' ? '🟢' : w === 'baseline' ? '🔵' : w === 'html' ? '🟡' : '⚪';
+ const iconMap: Record<string, string> = {
+ xds: '🟢',
+ baseline: '🔵',
+ html: '🟡',
+ 'xds-tailwind': '🟣',
+ tie: '⚪',
+ };
+ const icon = iconMap[w] ?? '⚪';
const label = DIMENSION_LABELS[d] || d;
lines.push(`- ${label}: ${icon} ${w === 'tie' ? 'Tie' : w.toUpperCase()}`);
}
@@ -233,6 +247,7 @@ async function main() {
xds: xdsId,
baseline: baselineId,
html: htmlId,
+ xdsTailwind: xdsTailwindId,
json,
markdown,
} = parseArgs();
@@ -240,9 +255,11 @@ async function main() {
const xds = loadOrGenerate(xdsId);
const baseline = loadOrGenerate(baselineId);
const htmlData = htmlId ? loadOrGenerate(htmlId) : undefined;
+ const twData = xdsTailwindId ? loadOrGenerate(xdsTailwindId) : undefined;
const dimensions = getDimensionNames();
const isThreeWay = !!htmlData;
+ const isFourWay = !!twData;
// Build comparison
const winners = {} as Record<UniversalDimension, WinnerType>;
@@ -251,6 +268,7 @@ async function main() {
xds.averages[d],
baseline.averages[d],
htmlData?.averages[d],
+ twData?.averages[d],
);
}
@@ -259,6 +277,7 @@ async function main() {
...Object.keys(xds.byPrompt),
...Object.keys(baseline.byPrompt),
...(htmlData ? Object.keys(htmlData.byPrompt) : []),
+ ...(twData ? Object.keys(twData.byPrompt) : []),
]);
const byPrompt: UniversalComparison['byPrompt'] = {};
@@ -266,15 +285,18 @@ async function main() {
const xdsScore = xds.byPrompt[promptId];
const baselineScore = baseline.byPrompt[promptId];
const htmlScore = htmlData?.byPrompt[promptId];
+ const twScore = twData?.byPrompt[promptId];
if (xdsScore && baselineScore) {
byPrompt[promptId] = {
xds: xdsScore,
baseline: baselineScore,
...(htmlScore ? {html: htmlScore} : {}),
+ ...(twScore ? {xdsTailwind: twScore} : {}),
winner: winner(
getAverageScore(xdsScore),
getAverageScore(baselineScore),
htmlScore ? getAverageScore(htmlScore) : undefined,
+ twScore ? getAverageScore(twScore) : undefined,
),
};
}
@@ -284,14 +306,16 @@ async function main() {
xds,
baseline,
...(htmlData ? {html: htmlData} : {}),
+ ...(twData ? {xdsTailwind: twData} : {}),
winners,
byPrompt,
};
- // Save
- const outputFilename = htmlId
- ? `comparison-${xdsId}-${baselineId}-${htmlId}.json`
- : `comparison-${xdsId}-${baselineId}.json`;
+ // Save — include all IDs in filename for uniqueness
+ const idParts = [xdsId, baselineId];
+ if (htmlId) idParts.push(htmlId);
+ if (xdsTailwindId) idParts.push(xdsTailwindId);
+ const outputFilename = `comparison-${idParts.join('-')}.json`;
const outputPath = path.join(getResultsDir(), outputFilename);
writeJson(outputPath, comparison);
@@ -307,6 +331,7 @@ async function main() {
xdsId,
baselineId,
htmlId,
+ xdsTailwindId,
byPrompt,
}),
);
@@ -314,216 +339,153 @@ async function main() {
}
// --- Print report ---
+ const targetNames = ['XDS', 'Baseline'];
+ if (isThreeWay) targetNames.push('HTML');
+ if (isFourWay) targetNames.push('XDS+TW');
- const title = isThreeWay
- ? '📊 Universal Comparison: XDS vs Baseline vs HTML'
- : '📊 Universal Comparison: XDS vs Baseline';
+ const title = `📊 Universal Comparison: ${targetNames.join(' vs ')}`;
console.log(`\n${title}`);
- console.log('═'.repeat(isThreeWay ? 66 : 52));
-
- if (isThreeWay) {
- // Three-way table
- console.log('┌─────────────────────┬───────┬──────────┬──────┬──────────┐');
- console.log('│ Dimension │ XDS │ Baseline │ HTML │ Winner │');
- console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');
- for (const d of dimensions) {
- const label = (DIMENSION_LABELS[d] || d).padEnd(19);
- const xScore = String(xds.averages[d]).padStart(3);
- const bScore = String(baseline.averages[d]).padStart(3);
- const hScore = String(htmlData!.averages[d]).padStart(3);
- const w = winnerIcon(winners[d]).padEnd(8);
- console.log(
- `│ ${label} │ ${xScore} │ ${bScore} │ ${hScore} │ ${w} │`,
- );
- }
- console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');
- const xOverall = String(xds.overall).padStart(3);
- const bOverall = String(baseline.overall).padStart(3);
- const hOverall = String(htmlData!.overall).padStart(3);
- const overallW = winnerIcon(
- winner(xds.overall, baseline.overall, htmlData!.overall),
- ).padEnd(8);
- console.log(
- `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${hOverall} │ ${overallW} │`,
- );
- console.log('└─────────────────────┴───────┴──────────┴──────┴──────────┘');
- } else {
- // Two-way table (unchanged)
- console.log('┌─────────────────────┬───────┬──────────┬──────────┐');
- console.log('│ Dimension │ XDS │ Baseline │ Winner │');
- console.log('├─────────────────────┼───────┼──────────┼──────────┤');
- for (const d of dimensions) {
- const label = (DIMENSION_LABELS[d] || d).padEnd(19);
- const xScore = String(xds.averages[d]).padStart(3);
- const bScore = String(baseline.averages[d]).padStart(3);
- const w = winnerIcon(winners[d]).padEnd(8);
- console.log(`│ ${label} │ ${xScore} │ ${bScore} │ ${w} │`);
+ console.log('═'.repeat(title.length + 2));
+
+ // Build a generic table for any number of targets
+ type TargetEntry = {label: string; data: typeof xds};
+ const targets: TargetEntry[] = [
+ {label: 'XDS', data: xds},
+ {label: 'Baseline', data: baseline},
+ ];
+ if (isThreeWay) targets.push({label: 'HTML', data: htmlData!});
+ if (isFourWay) targets.push({label: 'XDS+TW', data: twData!});
+
+ // Use markdown-style table for CLI (simpler than box-drawing for N targets)
+ const header = ['Dimension', ...targets.map(t => t.label), 'Winner'];
+ const rows: string[][] = [];
+ for (const d of dimensions) {
+ const row = [
+ DIMENSION_LABELS[d] || d,
+ ...targets.map(t => String(t.data.averages[d])),
+ winnerIcon(winners[d]),
+ ];
+ rows.push(row);
+ }
+ // Overall row
+ const overallWinner = winner(
+ xds.overall,
+ baseline.overall,
+ htmlData?.overall,
+ twData?.overall,
+ );
+ rows.push([
+ 'Overall',
+ ...targets.map(t => String(t.data.overall)),
+ winnerIcon(overallWinner),
+ ]);
+
+ // Compute column widths and print
+ const allRows = [header, ...rows];
+ const colWidths = header.map((_, ci) =>
+ Math.max(...allRows.map(r => r[ci].length)),
+ );
+ const sep = colWidths.map(w => '─'.repeat(w + 2)).join('┼');
+ console.log('┌' + sep.replaceAll('┼', '┬') + '┐');
+ console.log(
+ '│' +
+ header.map((h, i) => ` ${h.padEnd(colWidths[i])} `).join('│') +
+ '│',
+ );
+ console.log('├' + sep + '┤');
+ for (let ri = 0; ri < rows.length; ri++) {
+ if (ri === rows.length - 1) {
+ console.log('├' + sep + '┤');
}
- console.log('├─────────────────────┼───────┼──────────┼──────────┤');
- const xOverall = String(xds.overall).padStart(3);
- const bOverall = String(baseline.overall).padStart(3);
- const overallW = winnerIcon(winner(xds.overall, baseline.overall)).padEnd(
- 8,
- );
console.log(
- `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${overallW} │`,
+ '│' +
+ rows[ri].map((c, i) => ` ${c.padEnd(colWidths[i])} `).join('│') +
+ '│',
);
- console.log('└─────────────────────┴───────┴──────────┴──────────┘');
}
+ console.log('└' + sep.replaceAll('┼', '┴') + '┘');
// Dark mode
- const darkModeStr = isThreeWay
- ? `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}% | HTML ${htmlData!.darkModeRate}%`
- : `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}%`;
- console.log(darkModeStr);
+ const dmParts = targets.map(t => `${t.label} ${t.data.darkModeRate}%`);
+ console.log(`\n🌙 Dark Mode: ${dmParts.join(' | ')}`);
// Efficiency metrics comparison
- const xdsEff = Object.values(xds.byPrompt).map(s => s.efficiency.metrics!);
- const baseEff = Object.values(baseline.byPrompt).map(
- s => s.efficiency.metrics!,
- );
- const htmlEff = htmlData
- ? Object.values(htmlData.byPrompt).map(s => s.efficiency.metrics!)
- : [];
- if (xdsEff.length > 0 && baseEff.length > 0) {
- const xDpe =
- xdsEff.reduce((s, m) => s + m.decisionsPerElement, 0) / xdsEff.length;
- const bDpe =
- baseEff.reduce((s, m) => s + m.decisionsPerElement, 0) / baseEff.length;
- const xLines = xdsEff.reduce((s, m) => s + m.codeLines, 0) / xdsEff.length;
- const bLines =
- baseEff.reduce((s, m) => s + m.codeLines, 0) / baseEff.length;
+ const targetEffMetrics = targets.map(t => ({
+ label: t.label,
+ metrics: Object.values(t.data.byPrompt).map(s => s.efficiency.metrics!),
+ }));
+ if (targetEffMetrics.every(t => t.metrics.length > 0)) {
+ const avgDpe = targetEffMetrics.map(
+ t => t.metrics.reduce((s, m) => s + m.decisionsPerElement, 0) / t.metrics.length,
+ );
+ const avgLines = targetEffMetrics.map(
+ t => t.metrics.reduce((s, m) => s + m.codeLines, 0) / t.metrics.length,
+ );
console.log(`\n⚡ Efficiency Metrics:`);
-
- if (isThreeWay && htmlEff.length > 0) {
- const hDpe =
- htmlEff.reduce((s, m) => s + m.decisionsPerElement, 0) / htmlEff.length;
- const hLines =
- htmlEff.reduce((s, m) => s + m.codeLines, 0) / htmlEff.length;
- console.log(
- ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | HTML ${hDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe, hDpe))}`,
- );
- console.log(
- ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | HTML ${Math.round(hLines)} | ${winnerIcon(winner(bLines, xLines, hLines))}`,
- );
- } else {
- console.log(
- ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe))}`,
- );
- console.log(
- ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | ${winnerIcon(winner(bLines, xLines))}`,
- );
- }
+ // For decisions/element, lower is better (reverse args for winner)
+ const dpeParts = targetEffMetrics.map((t, i) => `${t.label} ${avgDpe[i].toFixed(1)}`);
+ console.log(` Decisions/element: ${dpeParts.join(' | ')}`);
+ const linesParts = targetEffMetrics.map((t, i) => `${t.label} ${Math.round(avgLines[i])}`);
+ console.log(` Avg code lines: ${linesParts.join(' | ')}`);
}
// Maintainability metrics comparison
- const xdsMaint = Object.values(xds.byPrompt).map(
- s => s.maintainability.metrics!,
- );
- const baseMaint = Object.values(baseline.byPrompt).map(
- s => s.maintainability.metrics!,
- );
- const htmlMaint = htmlData
- ? Object.values(htmlData.byPrompt).map(s => s.maintainability.metrics!)
- : [];
- if (xdsMaint.length > 0 && baseMaint.length > 0) {
- const xSem =
- xdsMaint.reduce((s, m) => s + m.semanticRatio, 0) / xdsMaint.length;
- const bSem =
- baseMaint.reduce((s, m) => s + m.semanticRatio, 0) / baseMaint.length;
- const xMagic = xdsMaint.reduce((s, m) => s + m.magicValueCount, 0);
- const bMagic = baseMaint.reduce((s, m) => s + m.magicValueCount, 0);
+ const targetMaintMetrics = targets.map(t => ({
+ label: t.label,
+ metrics: Object.values(t.data.byPrompt).map(s => s.maintainability.metrics!),
+ }));
+ if (targetMaintMetrics.every(t => t.metrics.length > 0)) {
+ const avgSem = targetMaintMetrics.map(
+ t => t.metrics.reduce((s, m) => s + m.semanticRatio, 0) / t.metrics.length,
+ );
+ const totalMagic = targetMaintMetrics.map(
+ t => t.metrics.reduce((s, m) => s + m.magicValueCount, 0),
+ );
console.log(`\n🔧 Maintainability Metrics:`);
-
- if (isThreeWay && htmlMaint.length > 0) {
- const hSem =
- htmlMaint.reduce((s, m) => s + m.semanticRatio, 0) / htmlMaint.length;
- const hMagic = htmlMaint.reduce((s, m) => s + m.magicValueCount, 0);
- console.log(
- ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | HTML ${(hSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem, hSem))}`,
- );
- console.log(
- ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | HTML ${hMagic} | ${winnerIcon(winner(bMagic, xMagic, hMagic))}`,
- );
- } else {
- console.log(
- ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem))}`,
- );
- console.log(
- ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | ${winnerIcon(winner(bMagic, xMagic))}`,
- );
- }
+ const semParts = targetMaintMetrics.map((t, i) => `${t.label} ${(avgSem[i] * 100).toFixed(0)}%`);
+ console.log(` Semantic ratio: ${semParts.join(' | ')}`);
+ const magicParts = targetMaintMetrics.map((t, i) => `${t.label} ${totalMagic[i]}`);
+ console.log(` Magic values: ${magicParts.join(' | ')}`);
}
// Cost comparison
if (xds.cost && baseline.cost) {
console.log(`\n💰 Cost:`);
- const xAvgMs = xds.cost.avgDurationMs;
- const bAvgMs = baseline.cost.avgDurationMs;
- if (isThreeWay && htmlData?.cost) {
- const hAvgMs = htmlData.cost.avgDurationMs;
- if (xAvgMs > 0 && bAvgMs > 0) {
- console.log(
- ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | HTML ${(hAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs, hAvgMs))}`,
- );
- }
- console.log(
- ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | HTML ${htmlData.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines, htmlData.cost.avgOutputLines))}`,
- );
- console.log(
- ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead} | HTML ${htmlData.cost.avgDocsRead}`,
- );
- const xTokens =
- xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens;
- const bTokens =
- baseline.cost.estimatedInputTokens +
- baseline.cost.estimatedOutputTokens;
- const hTokens =
- htmlData.cost.estimatedInputTokens +
- htmlData.cost.estimatedOutputTokens;
- console.log(
- ` Est. tokens: XDS ~${xTokens} | Baseline ~${bTokens} | HTML ~${hTokens} | ${winnerIcon(winner(bTokens, xTokens, hTokens))}`,
- );
- } else {
- if (xAvgMs > 0 && bAvgMs > 0) {
- console.log(
- ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs))}`,
- );
- }
- console.log(
- ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines))}`,
- );
- console.log(
- ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead}`,
- );
- console.log(
- ` Est. tokens: XDS ~${xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens} | Baseline ~${baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens} | ${winnerIcon(winner(baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens, xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens))}`,
- );
+ const costTargets = targets.filter(t => t.data.cost);
+ const hasDuration = costTargets.some(t => t.data.cost!.avgDurationMs > 0);
+ if (hasDuration) {
+ const durParts = costTargets.map(t => `${t.label} ${(t.data.cost!.avgDurationMs / 1000).toFixed(1)}s`);
+ console.log(` Avg duration: ${durParts.join(' | ')}`);
}
+ const linesParts = costTargets.map(t => `${t.label} ${t.data.cost!.avgOutputLines}`);
+ console.log(` Avg output lines: ${linesParts.join(' | ')}`);
+ const docsParts = costTargets.map(t => `${t.label} ${t.data.cost!.avgDocsRead}`);
+ console.log(` Avg docs read: ${docsParts.join(' | ')}`);
+ const tokenParts = costTargets.map(t => {
+ const total = t.data.cost!.estimatedInputTokens + t.data.cost!.estimatedOutputTokens;
+ return `${t.label} ~${total}`;
+ });
+ console.log(` Est. tokens: ${tokenParts.join(' | ')}`);
}
// Per-prompt wins
const promptEntries = Object.entries(byPrompt);
if (promptEntries.length > 0) {
- let xWins = 0;
- let bWins = 0;
- let hWins = 0;
- let ties = 0;
+ const winCounts: Record<string, number> = {};
+ for (const t of targets) winCounts[t.label] = 0;
+ winCounts['Tie'] = 0;
for (const [, data] of promptEntries) {
- if (data.winner === 'xds') xWins++;
- else if (data.winner === 'baseline') bWins++;
- else if (data.winner === 'html') hWins++;
- else ties++;
- }
- if (isThreeWay) {
- console.log(
- `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | HTML wins ${hWins} | Ties ${ties} (${promptEntries.length} prompts)`,
- );
- } else {
- console.log(
- `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | Ties ${ties} (${promptEntries.length} prompts)`,
- );
+ if (data.winner === 'xds') winCounts['XDS']++;
+ else if (data.winner === 'baseline') winCounts['Baseline']++;
+ else if (data.winner === 'html') winCounts['HTML']++;
+ else if (data.winner === 'xds-tailwind') winCounts['XDS+TW']++;
+ else winCounts['Tie']++;
}
+ const parts = targets.map(t => `${t.label} wins ${winCounts[t.label]}`);
+ parts.push(`Ties ${winCounts['Tie']}`);
+ console.log(
+ `\n📝 Per-Prompt: ${parts.join(' | ')} (${promptEntries.length} prompts)`,
+ );
}
console.log(`\nSaved: ${outputPath}\n`);
diff --git a/internal/vibe-tests/src/universal-eval.ts b/internal/vibe-tests/src/universal-eval.ts
index cbac5c8b72..22a527e17f 100644
--- a/internal/vibe-tests/src/universal-eval.ts
+++ b/internal/vibe-tests/src/universal-eval.ts
@@ -104,22 +104,114 @@ function scoreTscErrors(tscResult: TscResult): {
}
// ============================================================
-// 1. Correctness (tsc-based)
+// 1. Correctness (tsc + phantom prop detection)
// ============================================================
+/**
+ * Detect "phantom props" — event handlers and props that look valid
+ * but silently do nothing in React DOM. These compile fine on targets
+ * without strict types (baseline, html) but represent real bugs.
+ *
+ * Applied equally to ALL targets for fairness.
+ */
+function analyzePhantomProps(code: string): UniversalFinding[] {
+ const findings: UniversalFinding[] = [];
+ const lines = code.split('\n');
+
+ for (let i = 0; i < lines.length; i++) {
+ const line = lines[i];
+ const lineNum = i + 1;
+
+ // Skip comments and imports
+ const trimmed = line.trim();
+ if (
+ trimmed.startsWith('//') ||
+ trimmed.startsWith('*') ||
+ trimmed.startsWith('import ')
+ )
+ continue;
+
+ // onPress on any element/component — React Native, not DOM
+ if (/\bonPress\s*[={]/.test(line) && !/onPressedChange/.test(line)) {
+ findings.push({
+ rule: 'phantom-prop',
+ severity: 'critical',
+ detail:
+ 'onPress is React Native — use onClick for React DOM. Handler will never fire.',
+ line: lineNum,
+ });
+ }
+
+ // onLongPress — React Native only
+ if (/\bonLongPress\s*[={]/.test(line)) {
+ findings.push({
+ rule: 'phantom-prop',
+ severity: 'critical',
+ detail:
+ 'onLongPress is React Native — no DOM equivalent. Handler will never fire.',
+ line: lineNum,
+ });
+ }
+
+ // onChange on <button> — buttons don't fire change events
+ if (/<button\b/i.test(line) || /<Button\b/.test(line)) {
+ const context = lines.slice(i, Math.min(i + 3, lines.length)).join(' ');
+ if (
+ /\bonChange\s*[={]/.test(context) &&
+ !/type\s*=\s*["']/.test(context)
+ ) {
+ findings.push({
+ rule: 'phantom-prop',
+ severity: 'moderate',
+ detail: 'onChange on button — buttons do not fire change events',
+ line: lineNum,
+ });
+ }
+ }
+
+ // onSubmit on non-form elements
+ if (/\bonSubmit\s*[={]/.test(line)) {
+ const context = lines
+ .slice(Math.max(0, i - 2), Math.min(i + 2, lines.length))
+ .join(' ');
+ if (
+ /<(div|section|main|span)\b/.test(context) &&
+ !/<form\b/.test(context)
+ ) {
+ findings.push({
+ rule: 'phantom-prop',
+ severity: 'moderate',
+ detail:
+ 'onSubmit on non-form element — only <form> fires submit events',
+ line: lineNum,
+ });
+ }
+ }
+ }
+
+ return findings;
+}
+
function analyzeCorrectness(
code: string,
_target: string,
tscResult?: TscResult | null,
): DimensionScore {
- // tsc results are the sole correctness signal
+ const findings: UniversalFinding[] = [];
+
+ // Phantom prop detection — runs on ALL targets equally
+ findings.push(...analyzePhantomProps(code));
+
+ // tsc results — phantom props already caught by tsc don't double-penalize
if (tscResult) {
- const {score, findings} = scoreTscErrors(tscResult);
- return {score: clamp(score), findings};
+ const tsc = scoreTscErrors(tscResult);
+ findings.push(...tsc.findings);
+ // tsc already penalizes type errors (which catch phantom props on strict targets).
+ // Don't add phantom prop penalty on top — it's already in the tsc score.
+ return {score: clamp(tsc.score), findings};
}
- // No tsc data available — check for missing export only
- const findings: UniversalFinding[] = [];
+ // No tsc data — phantom props + structural checks
if (!/export\s+(default|function|const|class)/.test(code)) {
findings.push({
rule: 'missing-export',
@@ -128,10 +220,13 @@ function analyzeCorrectness(
});
}
- // Without tsc data, assume code is correct (backwards compat for old iterations)
let score = 100;
for (const f of findings) {
- score -= f.severity === 'minor' ? 3 : 0;
+ if (f.rule === 'phantom-prop') {
+ score -= f.severity === 'critical' ? 15 : 8;
+ } else if (f.severity === 'minor') {
+ score -= 3;
+ }
}
return {score: clamp(score), findings};
}
@@ -533,14 +628,20 @@ function analyzeEfficiency(
if (stripped === '}' || stripped === '};') inTypeBlock = false;
continue;
}
- if ((target === 'xds' || target === 'xds-tailwind') && stripped.includes('stylex.create'))
+ if (
+ (target === 'xds' || target === 'xds-tailwind') &&
+ stripped.includes('stylex.create')
+ )
inStyleBlock = true;
if (inStyleBlock) {
stylingLines++;
if (stripped === '});') inStyleBlock = false;
continue;
}
- if ((target === 'baseline' || target === 'xds-tailwind') && stripped.includes('className=')) {
+ if (
+ (target === 'baseline' || target === 'xds-tailwind') &&
+ stripped.includes('className=')
+ ) {
stylingLines++;
continue;
}