Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(vibe-tests): three-way comparison (XDS vs Baseline vs HTML) (#367)

Cindy Zhangecc7d385a165
diff --git a/internal/vibe-tests/app/src/report/CompareView.tsx b/internal/vibe-tests/app/src/report/CompareView.tsx
index 875a1a30c6..ef4ce1c636 100644
--- a/internal/vibe-tests/app/src/report/CompareView.tsx
+++ b/internal/vibe-tests/app/src/report/CompareView.tsx
@@ -20,6 +20,11 @@ const styles = stylex.create({
gridTemplateColumns: 'repeat(3, 1fr)',
gap: spacingVars['--spacing-3'],
},
+ summaryGrid4: {
+ display: 'grid',
+ gridTemplateColumns: 'repeat(4, 1fr)',
+ gap: spacingVars['--spacing-3'],
+ },
winCard: {
padding: spacingVars['--spacing-3'],
textAlign: 'center',
@@ -33,6 +38,9 @@ const styles = stylex.create({
neutral: {
color: colorVars['--color-text-secondary'],
},
+ warning: {
+ color: colorVars['--color-warning'],
+ },
costGrid: {
display: 'grid',
gridTemplateColumns: 'repeat(auto-fill, minmax(200px, 1fr))',
@@ -51,6 +59,8 @@ const styles = stylex.create({
},
});
+type WinnerType = 'xds' | 'baseline' | 'html' | 'tie';
+
interface CompareViewProps {
comparison: UniversalComparison;
}
@@ -60,6 +70,7 @@ interface DimRow extends Record<string, unknown> {
dimension: string;
xdsScore: number;
baselineScore: number;
+ htmlScore?: number;
delta: number;
winner: string;
}
@@ -69,6 +80,7 @@ interface CatRow extends Record<string, unknown> {
category: string;
xdsOverall: number;
baselineOverall: number;
+ htmlOverall?: number;
delta: number;
}
@@ -77,6 +89,7 @@ interface CostRow extends Record<string, unknown> {
metric: string;
xds: string;
baseline: string;
+ html?: string;
winner: string;
}
@@ -84,19 +97,60 @@ function costWinner(
xdsVal: number,
baseVal: number,
lowerIsBetter: boolean,
-): 'xds' | 'baseline' | 'tie' {
+ htmlVal?: number,
+): WinnerType {
+ if (htmlVal != null) {
+ const vals = [xdsVal, baseVal, htmlVal];
+ const best = lowerIsBetter ? Math.min(...vals) : Math.max(...vals);
+ const atBest = vals.filter(v => v === best).length;
+ if (atBest > 1) return 'tie';
+ if (xdsVal === best) return 'xds';
+ if (baseVal === best) return 'baseline';
+ return 'html';
+ }
if (xdsVal === baseVal) return 'tie';
if (lowerIsBetter) return xdsVal < baseVal ? 'xds' : 'baseline';
return xdsVal > baseVal ? 'xds' : 'baseline';
}
+function winnerBadgeVariant(
+ w: string,
+): 'success' | 'error' | 'warning' | 'neutral' {
+ switch (w) {
+ case 'xds':
+ return 'success';
+ case 'baseline':
+ return 'error';
+ case 'html':
+ return 'warning';
+ default:
+ return 'neutral';
+ }
+}
+
+function winnerLabel(w: string): string {
+ switch (w) {
+ case 'xds':
+ return 'XDS';
+ case 'baseline':
+ return 'Baseline';
+ case 'html':
+ return 'HTML';
+ default:
+ return 'Tie';
+ }
+}
+
function CostComparisonSection({
xdsCost,
baselineCost,
+ htmlCost,
}: {
xdsCost: CostMetrics;
baselineCost: CostMetrics;
+ htmlCost?: CostMetrics;
}) {
+ const isThreeWay = !!htmlCost;
const hasDuration =
xdsCost.avgDurationMs > 0 || baselineCost.avgDurationMs > 0;
@@ -108,10 +162,14 @@ function CostComparisonSection({
metric: 'Avg Duration',
xds: `${(xdsCost.avgDurationMs / 1000).toFixed(1)}s`,
baseline: `${(baselineCost.avgDurationMs / 1000).toFixed(1)}s`,
+ ...(isThreeWay
+ ? {html: `${(htmlCost!.avgDurationMs / 1000).toFixed(1)}s`}
+ : {}),
winner: costWinner(
xdsCost.avgDurationMs,
baselineCost.avgDurationMs,
true,
+ htmlCost?.avgDurationMs,
),
},
]
@@ -121,10 +179,14 @@ function CostComparisonSection({
metric: 'Input Tokens',
xds: `~${xdsCost.estimatedInputTokens.toLocaleString()}`,
baseline: `~${baselineCost.estimatedInputTokens.toLocaleString()}`,
+ ...(isThreeWay
+ ? {html: `~${htmlCost!.estimatedInputTokens.toLocaleString()}`}
+ : {}),
winner: costWinner(
xdsCost.estimatedInputTokens,
baselineCost.estimatedInputTokens,
true,
+ htmlCost?.estimatedInputTokens,
),
},
{
@@ -132,10 +194,14 @@ function CostComparisonSection({
metric: 'Output Tokens',
xds: `~${xdsCost.estimatedOutputTokens.toLocaleString()}`,
baseline: `~${baselineCost.estimatedOutputTokens.toLocaleString()}`,
+ ...(isThreeWay
+ ? {html: `~${htmlCost!.estimatedOutputTokens.toLocaleString()}`}
+ : {}),
winner: costWinner(
xdsCost.estimatedOutputTokens,
baselineCost.estimatedOutputTokens,
true,
+ htmlCost?.estimatedOutputTokens,
),
},
{
@@ -143,10 +209,18 @@ function CostComparisonSection({
metric: 'Total Tokens',
xds: `~${(xdsCost.estimatedInputTokens + xdsCost.estimatedOutputTokens).toLocaleString()}`,
baseline: `~${(baselineCost.estimatedInputTokens + baselineCost.estimatedOutputTokens).toLocaleString()}`,
+ ...(isThreeWay
+ ? {
+ html: `~${(htmlCost!.estimatedInputTokens + htmlCost!.estimatedOutputTokens).toLocaleString()}`,
+ }
+ : {}),
winner: costWinner(
xdsCost.estimatedInputTokens + xdsCost.estimatedOutputTokens,
baselineCost.estimatedInputTokens + baselineCost.estimatedOutputTokens,
true,
+ htmlCost
+ ? htmlCost.estimatedInputTokens + htmlCost.estimatedOutputTokens
+ : undefined,
),
},
{
@@ -154,10 +228,12 @@ function CostComparisonSection({
metric: 'Avg Output Lines',
xds: String(xdsCost.avgOutputLines),
baseline: String(baselineCost.avgOutputLines),
+ ...(isThreeWay ? {html: String(htmlCost!.avgOutputLines)} : {}),
winner: costWinner(
xdsCost.avgOutputLines,
baselineCost.avgOutputLines,
true,
+ htmlCost?.avgOutputLines,
),
},
{
@@ -165,6 +241,7 @@ function CostComparisonSection({
metric: 'Avg Docs Read',
xds: String(xdsCost.avgDocsRead),
baseline: String(baselineCost.avgDocsRead),
+ ...(isThreeWay ? {html: String(htmlCost!.avgDocsRead)} : {}),
winner: 'tie', // not inherently better or worse
},
];
@@ -181,23 +258,23 @@ function CostComparisonSection({
header: 'Baseline',
renderCell: row => <XDSText type="body">{row.baseline}</XDSText>,
},
+ ...(isThreeWay
+ ? [
+ {
+ key: 'html' as const,
+ header: 'HTML',
+ renderCell: (row: CostRow) => (
+ <XDSText type="body">{row.html ?? '—'}</XDSText>
+ ),
+ } satisfies XDSTableColumn<CostRow>,
+ ]
+ : []),
{
key: 'winner',
header: 'Lower Cost',
renderCell: row => (
- <XDSBadge
- variant={
- row.winner === 'xds'
- ? 'success'
- : row.winner === 'baseline'
- ? 'error'
- : 'neutral'
- }>
- {row.winner === 'xds'
- ? 'XDS'
- : row.winner === 'baseline'
- ? 'Baseline'
- : '—'}
+ <XDSBadge variant={winnerBadgeVariant(row.winner)}>
+ {row.winner === 'tie' ? '—' : winnerLabel(row.winner)}
</XDSBadge>
),
},
@@ -215,15 +292,18 @@ function CostComparisonSection({
}
export function CompareView({comparison}: CompareViewProps) {
- const {xds, baseline, winners} = comparison;
+ const {xds, baseline, html, winners} = comparison;
+ const isThreeWay = !!html;
let xdsWins = 0;
let baselineWins = 0;
+ let htmlWins = 0;
let ties = 0;
for (const dim of ALL_DIMENSIONS) {
const w = winners[dim];
if (w === 'xds') xdsWins++;
else if (w === 'baseline') baselineWins++;
+ else if (w === 'html') htmlWins++;
else ties++;
}
@@ -232,6 +312,7 @@ export function CompareView({comparison}: CompareViewProps) {
dimension: DIMENSION_LABELS[dim],
xdsScore: xds.averages[dim],
baselineScore: baseline.averages[dim],
+ ...(isThreeWay ? {htmlScore: html!.averages[dim]} : {}),
delta: xds.averages[dim] - baseline.averages[dim],
winner: winners[dim],
}));
@@ -252,9 +333,22 @@ export function CompareView({comparison}: CompareViewProps) {
<XDSText type="body">{formatScore(row.baselineScore)}</XDSText>
),
},
+ ...(isThreeWay
+ ? [
+ {
+ key: 'htmlScore' as const,
+ header: 'HTML',
+ renderCell: (row: DimRow) => (
+ <XDSText type="body">
+ {row.htmlScore != null ? formatScore(row.htmlScore) : '—'}
+ </XDSText>
+ ),
+ } satisfies XDSTableColumn<DimRow>,
+ ]
+ : []),
{
key: 'delta',
- header: 'Delta',
+ header: 'Delta (XDS−Base)',
renderCell: row => (
<XDSText
type="body"
@@ -274,19 +368,8 @@ export function CompareView({comparison}: CompareViewProps) {
key: 'winner',
header: 'Winner',
renderCell: row => (
- <XDSBadge
- variant={
- row.winner === 'xds'
- ? 'success'
- : row.winner === 'baseline'
- ? 'error'
- : 'neutral'
- }>
- {row.winner === 'xds'
- ? 'XDS'
- : row.winner === 'baseline'
- ? 'Baseline'
- : 'Tie'}
+ <XDSBadge variant={winnerBadgeVariant(row.winner)}>
+ {winnerLabel(row.winner)}
</XDSBadge>
),
},
@@ -295,11 +378,14 @@ export function CompareView({comparison}: CompareViewProps) {
const allCategories = new Set([
...Object.keys(xds.byCategory),
...Object.keys(baseline.byCategory),
+ ...(html ? Object.keys(html.byCategory) : []),
]);
const catData: CatRow[] = [...allCategories].map(cat => {
const xdsCat = xds.byCategory[cat] ?? {};
const baseCat = baseline.byCategory[cat] ?? {};
+ const htmlCat =
+ html?.byCategory[cat] ?? ({} as Record<UniversalDimension, number>);
const xdsAvg =
ALL_DIMENSIONS.reduce(
(s, d) => s + ((xdsCat[d as UniversalDimension] as number) ?? 0),
@@ -310,11 +396,18 @@ export function CompareView({comparison}: CompareViewProps) {
(s, d) => s + ((baseCat[d as UniversalDimension] as number) ?? 0),
0,
) / ALL_DIMENSIONS.length;
+ const htmlAvg = isThreeWay
+ ? ALL_DIMENSIONS.reduce(
+ (s, d) => s + ((htmlCat[d as UniversalDimension] as number) ?? 0),
+ 0,
+ ) / ALL_DIMENSIONS.length
+ : undefined;
return {
id: cat,
category: cat,
xdsOverall: xdsAvg,
baselineOverall: baseAvg,
+ ...(htmlAvg != null ? {htmlOverall: htmlAvg} : {}),
delta: xdsAvg - baseAvg,
};
});
@@ -335,9 +428,22 @@ export function CompareView({comparison}: CompareViewProps) {
<XDSText type="body">{formatScore(row.baselineOverall)}</XDSText>
),
},
+ ...(isThreeWay
+ ? [
+ {
+ key: 'htmlOverall' as const,
+ header: 'HTML',
+ renderCell: (row: CatRow) => (
+ <XDSText type="body">
+ {row.htmlOverall != null ? formatScore(row.htmlOverall) : '—'}
+ </XDSText>
+ ),
+ } satisfies XDSTableColumn<CatRow>,
+ ]
+ : []),
{
key: 'delta',
- header: 'Delta',
+ header: 'Delta (XDS−Base)',
renderCell: row => (
<XDSText
type="body"
@@ -357,7 +463,10 @@ export function CompareView({comparison}: CompareViewProps) {
return (
<XDSVStack gap="space4">
- <div {...stylex.props(styles.summaryGrid)}>
+ <div
+ {...stylex.props(
+ isThreeWay ? styles.summaryGrid4 : styles.summaryGrid,
+ )}>
<XDSCard>
<div {...stylex.props(styles.winCard)}>
<XDSVStack gap="space2">
@@ -378,6 +487,18 @@ export function CompareView({comparison}: CompareViewProps) {
</XDSVStack>
</div>
</XDSCard>
+ {isThreeWay && (
+ <XDSCard>
+ <div {...stylex.props(styles.winCard)}>
+ <XDSVStack gap="space2">
+ <XDSText type="label">HTML Wins</XDSText>
+ <XDSHeading level={2}>
+ <span {...stylex.props(styles.warning)}>{htmlWins}</span>
+ </XDSHeading>
+ </XDSVStack>
+ </div>
+ </XDSCard>
+ )}
<XDSCard>
<div {...stylex.props(styles.winCard)}>
<XDSVStack gap="space2">
@@ -420,6 +541,7 @@ export function CompareView({comparison}: CompareViewProps) {
<CostComparisonSection
xdsCost={xds.cost}
baselineCost={baseline.cost}
+ htmlCost={html?.cost}
/>
</XDSVStack>
)}
diff --git a/internal/vibe-tests/app/src/report/PromptDetailCard.tsx b/internal/vibe-tests/app/src/report/PromptDetailCard.tsx
index 55aa633294..45e1fa0b3f 100644
--- a/internal/vibe-tests/app/src/report/PromptDetailCard.tsx
+++ b/internal/vibe-tests/app/src/report/PromptDetailCard.tsx
@@ -44,6 +44,11 @@ const styles = stylex.create({
gridTemplateColumns: '1fr 1fr',
gap: spacingVars['--spacing-3'],
},
+ scoresRow3: {
+ display: 'grid',
+ gridTemplateColumns: '1fr 1fr 1fr',
+ gap: spacingVars['--spacing-3'],
+ },
scoresRowSingle: {
display: 'grid',
gridTemplateColumns: '1fr',
@@ -149,6 +154,7 @@ interface PromptDetailCardProps {
promptText?: string;
xdsScore?: UniversalScore;
baselineScore?: UniversalScore;
+ htmlScore?: UniversalScore;
hasXdsCode: boolean;
hasBaselineCode: boolean;
hasHtmlCode: boolean;
@@ -162,6 +168,7 @@ export function PromptDetailCard({
promptText,
xdsScore,
baselineScore,
+ htmlScore,
hasXdsCode,
hasBaselineCode,
hasHtmlCode,
@@ -169,10 +176,17 @@ export function PromptDetailCard({
previewUrls,
}: PromptDetailCardProps) {
const hasBoth = !!(xdsScore && baselineScore);
+ const hasAll = !!(xdsScore && baselineScore && htmlScore);
const hasAnyPreview =
previewUrls?.xds || previewUrls?.baseline || previewUrls?.html;
const hasAnyCode = hasXdsCode || hasBaselineCode || hasHtmlCode;
+ const scoresStyle = hasAll
+ ? styles.scoresRow3
+ : hasBoth
+ ? styles.scoresRow
+ : styles.scoresRowSingle;
+
return (
<XDSCard>
<div {...stylex.props(styles.card)}>
@@ -240,15 +254,13 @@ export function PromptDetailCard({
</div>
{/* Score summaries in constrained grid */}
- {(xdsScore || baselineScore) && (
- <div
- {...stylex.props(
- hasBoth ? styles.scoresRow : styles.scoresRowSingle,
- )}>
+ {(xdsScore || baselineScore || htmlScore) && (
+ <div {...stylex.props(scoresStyle)}>
{xdsScore && <ScoreSummary label="XDS" score={xdsScore} />}
{baselineScore && (
<ScoreSummary label="Baseline" score={baselineScore} />
)}
+ {htmlScore && <ScoreSummary label="HTML" score={htmlScore} />}
</div>
)}
@@ -276,6 +288,18 @@ export function PromptDetailCard({
</div>
</>
)}
+
+ {htmlScore && (
+ <>
+ <XDSDivider />
+ <div {...stylex.props(styles.section)}>
+ <div {...stylex.props(styles.sectionLabel)}>
+ <XDSText type="label">HTML Findings</XDSText>
+ </div>
+ <Findings score={htmlScore} />
+ </div>
+ </>
+ )}
</XDSVStack>
</div>
</XDSCard>
diff --git a/internal/vibe-tests/app/src/report/Report.tsx b/internal/vibe-tests/app/src/report/Report.tsx
index beb2029be6..f29033dd03 100644
--- a/internal/vibe-tests/app/src/report/Report.tsx
+++ b/internal/vibe-tests/app/src/report/Report.tsx
@@ -277,6 +277,14 @@ export function Report() {
compareScore={comparison?.baseline.overall}
compareLabel="Baseline"
/>
+ {comparison?.html && (
+ <ScoreCard
+ label="Overall Score vs HTML"
+ score={universal.overall}
+ compareScore={comparison.html.overall}
+ compareLabel="HTML"
+ />
+ )}
{/* Dimension scores grid */}
<XDSVStack gap="space3">
@@ -303,7 +311,9 @@ export function Report() {
{comparison && (
<XDSVStack gap="space3">
<XDSHeading level={3}>
- XDS vs Baseline Comparison
+ {comparison.html
+ ? 'XDS vs Baseline vs HTML Comparison'
+ : 'XDS vs Baseline Comparison'}
</XDSHeading>
<CompareView comparison={comparison} />
</XDSVStack>
@@ -325,6 +335,7 @@ export function Report() {
promptText={data.prompts?.[promptId]}
xdsScore={universal.byPrompt[promptId]}
baselineScore={comparison?.baseline.byPrompt[promptId]}
+ htmlScore={comparison?.html?.byPrompt[promptId]}
hasXdsCode={!!data.sourceCode?.[promptId]}
hasBaselineCode={!!data.baselineSourceCode?.[promptId]}
hasHtmlCode={!!data.htmlSourceCode?.[promptId]}
diff --git a/internal/vibe-tests/src/build-report.ts b/internal/vibe-tests/src/build-report.ts
index 0d94fe58db..0954d19a3d 100644
--- a/internal/vibe-tests/src/build-report.ts
+++ b/internal/vibe-tests/src/build-report.ts
@@ -59,7 +59,7 @@ function parseArgs(): {
process.exit(1);
}
- return {iteration, baseline, withScreenshots, dev};
+ return {iteration, baseline, html, withScreenshots, dev};
}
function ensureUniversalJson(iteration: string): UniversalAggregate {
@@ -80,17 +80,21 @@ function ensureUniversalJson(iteration: string): UniversalAggregate {
function ensureComparison(
xdsId: string,
baselineId: string,
+ htmlId?: string,
): UniversalComparison {
const resultsDir = getResultsDir();
- const comparisonPath = path.join(
- resultsDir,
- `comparison-${xdsId}-${baselineId}.json`,
- );
+ const comparisonFilename = htmlId
+ ? `comparison-${xdsId}-${baselineId}-${htmlId}.json`
+ : `comparison-${xdsId}-${baselineId}.json`;
+ const comparisonPath = path.join(resultsDir, comparisonFilename);
if (!fs.existsSync(comparisonPath)) {
- console.log(`Generating comparison for ${xdsId} vs ${baselineId}...`);
+ const htmlFlag = htmlId ? ` --html ${htmlId}` : '';
+ console.log(
+ `Generating comparison for ${xdsId} vs ${baselineId}${htmlId ? ` vs ${htmlId}` : ''}...`,
+ );
execSync(
- `tsx ${path.join(import.meta.dirname, 'universal-compare.ts')} --xds ${xdsId} --baseline ${baselineId}`,
+ `tsx ${path.join(import.meta.dirname, 'universal-compare.ts')} --xds ${xdsId} --baseline ${baselineId}${htmlFlag}`,
{cwd: path.join(import.meta.dirname, '..'), stdio: 'inherit'},
);
}
@@ -184,7 +188,13 @@ function inlineCss(htmlPath: string, distDir: string): void {
}
async function main() {
- const {iteration, baseline, html, withScreenshots, dev} = parseArgs();
+ const {
+ iteration,
+ baseline,
+ html,
+ withScreenshots: _withScreenshots,
+ dev,
+ } = parseArgs();
const resultsDir = getResultsDir();
const iterDir = path.join(resultsDir, iteration);
const manifestPath = path.join(iterDir, 'manifest.json');
@@ -205,7 +215,10 @@ async function main() {
if (baseline) {
console.log('📊 Step 2: Ensuring comparison data...');
ensureUniversalJson(baseline);
- comparison = ensureComparison(iteration, baseline);
+ if (html) {
+ ensureUniversalJson(html);
+ }
+ comparison = ensureComparison(iteration, baseline, html);
}
// Step 3: Load source code for per-prompt inspection
diff --git a/internal/vibe-tests/src/types.ts b/internal/vibe-tests/src/types.ts
index b2e33bff99..25661efec0 100644
--- a/internal/vibe-tests/src/types.ts
+++ b/internal/vibe-tests/src/types.ts
@@ -345,13 +345,15 @@ export interface UniversalAggregate {
export interface UniversalComparison {
xds: UniversalAggregate;
baseline: UniversalAggregate;
- winners: Record<UniversalDimension, 'xds' | 'baseline' | 'tie'>;
+ html?: UniversalAggregate;
+ winners: Record<UniversalDimension, 'xds' | 'baseline' | 'html' | 'tie'>;
byPrompt: Record<
string,
{
xds: UniversalScore;
baseline: UniversalScore;
- winner: 'xds' | 'baseline' | 'tie';
+ html?: UniversalScore;
+ winner: 'xds' | 'baseline' | 'html' | 'tie';
}
>;
}
diff --git a/internal/vibe-tests/src/universal-compare.ts b/internal/vibe-tests/src/universal-compare.ts
index f7b285f097..097e67c435 100644
--- a/internal/vibe-tests/src/universal-compare.ts
+++ b/internal/vibe-tests/src/universal-compare.ts
@@ -1,9 +1,10 @@
#!/usr/bin/env node
/**
- * @file Universal Compare — side-by-side comparison of two iterations
+ * @file Universal Compare — side-by-side comparison of two or three iterations
*
* Usage:
* tsx src/universal-compare.ts --xds abc123 --baseline def456
+ * tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789
* tsx src/universal-compare.ts --xds abc123 --baseline def456 --json
*/
@@ -47,27 +48,47 @@ function loadOrGenerate(iterationId: string): UniversalAggregate {
return JSON.parse(fs.readFileSync(universalPath, 'utf-8'));
}
-function winner(a: number, b: number): 'xds' | 'baseline' | 'tie' {
- if (a > b) return 'xds';
- if (b > a) return 'baseline';
+type WinnerType = 'xds' | 'baseline' | 'html' | 'tie';
+
+function winner(xdsVal: number, baseVal: number, htmlVal?: number): WinnerType {
+ if (htmlVal != null) {
+ const max = Math.max(xdsVal, baseVal, htmlVal);
+ const atMax = [xdsVal === max, baseVal === max, htmlVal === max].filter(
+ Boolean,
+ ).length;
+ if (atMax > 1) return 'tie';
+ if (xdsVal === max) return 'xds';
+ if (baseVal === max) return 'baseline';
+ return 'html';
+ }
+ if (xdsVal > baseVal) return 'xds';
+ if (baseVal > xdsVal) return 'baseline';
return 'tie';
}
-function winnerIcon(w: 'xds' | 'baseline' | 'tie'): string {
+function winnerIcon(w: WinnerType): string {
switch (w) {
case 'xds':
return '🟢 XDS';
case 'baseline':
return '🔵 Base';
+ case 'html':
+ return '🟡 HTML';
case 'tie':
return '⚪ Tie';
}
}
-function parseArgs(): {xds: string; baseline: string; json: boolean} {
+function parseArgs(): {
+ xds: string;
+ baseline: string;
+ html?: string;
+ json: boolean;
+} {
const args = process.argv.slice(2);
let xds = '';
let baseline = '';
+ let html: string | undefined;
let json = false;
for (let i = 0; i < args.length; i++) {
@@ -75,6 +96,8 @@ function parseArgs(): {xds: string; baseline: string; json: boolean} {
xds = args[++i];
} else if ((args[i] === '--baseline' || args[i] === '-b') && args[i + 1]) {
baseline = args[++i];
+ } else if (args[i] === '--html' && args[i + 1]) {
+ html = args[++i];
} else if (args[i] === '--json') {
json = true;
}
@@ -82,57 +105,73 @@ function parseArgs(): {xds: string; baseline: string; json: boolean} {
if (!xds || !baseline) {
console.error(
- 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--json]',
+ 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--json]',
);
process.exit(1);
}
- return {xds, baseline, json};
+ return {xds, baseline, html, json};
}
async function main() {
- const {xds: xdsId, baseline: baselineId, json} = parseArgs();
+ const {xds: xdsId, baseline: baselineId, html: htmlId, json} = parseArgs();
const xds = loadOrGenerate(xdsId);
const baseline = loadOrGenerate(baselineId);
+ const htmlData = htmlId ? loadOrGenerate(htmlId) : undefined;
const dimensions = getDimensionNames();
+ const isThreeWay = !!htmlData;
// Build comparison
- const winners = {} as Record<UniversalDimension, 'xds' | 'baseline' | 'tie'>;
+ const winners = {} as Record<UniversalDimension, WinnerType>;
for (const d of dimensions) {
- winners[d] = winner(xds.averages[d], baseline.averages[d]);
+ winners[d] = winner(
+ xds.averages[d],
+ baseline.averages[d],
+ htmlData?.averages[d],
+ );
}
// Per-prompt comparison
const allPromptIds = new Set([
...Object.keys(xds.byPrompt),
...Object.keys(baseline.byPrompt),
+ ...(htmlData ? Object.keys(htmlData.byPrompt) : []),
]);
const byPrompt: UniversalComparison['byPrompt'] = {};
for (const promptId of allPromptIds) {
const xdsScore = xds.byPrompt[promptId];
const baselineScore = baseline.byPrompt[promptId];
+ const htmlScore = htmlData?.byPrompt[promptId];
if (xdsScore && baselineScore) {
byPrompt[promptId] = {
xds: xdsScore,
baseline: baselineScore,
+ ...(htmlScore ? {html: htmlScore} : {}),
winner: winner(
getAverageScore(xdsScore),
getAverageScore(baselineScore),
+ htmlScore ? getAverageScore(htmlScore) : undefined,
),
};
}
}
- const comparison: UniversalComparison = {xds, baseline, winners, byPrompt};
+ const comparison: UniversalComparison = {
+ xds,
+ baseline,
+ ...(htmlData ? {html: htmlData} : {}),
+ winners,
+ byPrompt,
+ };
// Save
- const outputPath = path.join(
- getResultsDir(),
- `comparison-${xdsId}-${baselineId}.json`,
- );
+ const outputFilename = htmlId
+ ? `comparison-${xdsId}-${baselineId}-${htmlId}.json`
+ : `comparison-${xdsId}-${baselineId}.json`;
+ const outputPath = path.join(getResultsDir(), outputFilename);
writeJson(outputPath, comparison);
if (json) {
@@ -142,39 +181,76 @@ async function main() {
// --- Print report ---
- console.log(`\n📊 Universal Comparison: XDS vs Baseline`);
- console.log('═'.repeat(52));
+ const title = isThreeWay
+ ? '📊 Universal Comparison: XDS vs Baseline vs HTML'
+ : '📊 Universal Comparison: XDS vs Baseline';
+ console.log(`\n${title}`);
+ console.log('═'.repeat(isThreeWay ? 66 : 52));
- // Dimension table
- console.log('┌─────────────────────┬───────┬──────────┬──────────┐');
- console.log('│ Dimension │ XDS │ Baseline │ Winner │');
- console.log('├─────────────────────┼───────┼──────────┼──────────┤');
- for (const d of dimensions) {
- const label = DIMENSION_LABELS[d].padEnd(19);
- const xScore = String(xds.averages[d]).padStart(3);
- const bScore = String(baseline.averages[d]).padStart(3);
- const w = winnerIcon(winners[d]).padEnd(8);
- console.log(`│ ${label} │ ${xScore} │ ${bScore} │ ${w} │`);
+ if (isThreeWay) {
+ // Three-way table
+ console.log('┌─────────────────────┬───────┬──────────┬──────┬──────────┐');
+ console.log('│ Dimension │ XDS │ Baseline │ HTML │ Winner │');
+ console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');
+ for (const d of dimensions) {
+ const label = DIMENSION_LABELS[d].padEnd(19);
+ const xScore = String(xds.averages[d]).padStart(3);
+ const bScore = String(baseline.averages[d]).padStart(3);
+ const hScore = String(htmlData!.averages[d]).padStart(3);
+ const w = winnerIcon(winners[d]).padEnd(8);
+ console.log(
+ `│ ${label} │ ${xScore} │ ${bScore} │ ${hScore} │ ${w} │`,
+ );
+ }
+ console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');
+ const xOverall = String(xds.overall).padStart(3);
+ const bOverall = String(baseline.overall).padStart(3);
+ const hOverall = String(htmlData!.overall).padStart(3);
+ const overallW = winnerIcon(
+ winner(xds.overall, baseline.overall, htmlData!.overall),
+ ).padEnd(8);
+ console.log(
+ `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${hOverall} │ ${overallW} │`,
+ );
+ console.log('└─────────────────────┴───────┴──────────┴──────┴──────────┘');
+ } else {
+ // Two-way table (unchanged)
+ console.log('┌─────────────────────┬───────┬──────────┬──────────┐');
+ console.log('│ Dimension │ XDS │ Baseline │ Winner │');
+ console.log('├─────────────────────┼───────┼──────────┼──────────┤');
+ for (const d of dimensions) {
+ const label = DIMENSION_LABELS[d].padEnd(19);
+ const xScore = String(xds.averages[d]).padStart(3);
+ const bScore = String(baseline.averages[d]).padStart(3);
+ const w = winnerIcon(winners[d]).padEnd(8);
+ console.log(`│ ${label} │ ${xScore} │ ${bScore} │ ${w} │`);
+ }
+ console.log('├─────────────────────┼───────┼──────────┼──────────┤');
+ const xOverall = String(xds.overall).padStart(3);
+ const bOverall = String(baseline.overall).padStart(3);
+ const overallW = winnerIcon(winner(xds.overall, baseline.overall)).padEnd(
+ 8,
+ );
+ console.log(
+ `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${overallW} │`,
+ );
+ console.log('└─────────────────────┴───────┴──────────┴──────────┘');
}
- console.log('├─────────────────────┼───────┼──────────┼──────────┤');
- const xOverall = String(xds.overall).padStart(3);
- const bOverall = String(baseline.overall).padStart(3);
- const overallW = winnerIcon(winner(xds.overall, baseline.overall)).padEnd(8);
- console.log(
- `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${overallW} │`,
- );
- console.log('└─────────────────────┴───────┴──────────┴──────────┘');
// Dark mode
- console.log(
- `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}%`,
- );
+ const darkModeStr = isThreeWay
+ ? `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}% | HTML ${htmlData!.darkModeRate}%`
+ : `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}%`;
+ console.log(darkModeStr);
// Efficiency metrics comparison
const xdsEff = Object.values(xds.byPrompt).map(s => s.efficiency.metrics!);
const baseEff = Object.values(baseline.byPrompt).map(
s => s.efficiency.metrics!,
);
+ const htmlEff = htmlData
+ ? Object.values(htmlData.byPrompt).map(s => s.efficiency.metrics!)
+ : [];
if (xdsEff.length > 0 && baseEff.length > 0) {
const xDpe =
xdsEff.reduce((s, m) => s + m.decisionsPerElement, 0) / xdsEff.length;
@@ -184,12 +260,26 @@ async function main() {
const bLines =
baseEff.reduce((s, m) => s + m.codeLines, 0) / baseEff.length;
console.log(`\n⚡ Efficiency Metrics:`);
- console.log(
- ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe))}`,
- );
- console.log(
- ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | ${winnerIcon(winner(bLines, xLines))}`,
- );
+
+ if (isThreeWay && htmlEff.length > 0) {
+ const hDpe =
+ htmlEff.reduce((s, m) => s + m.decisionsPerElement, 0) / htmlEff.length;
+ const hLines =
+ htmlEff.reduce((s, m) => s + m.codeLines, 0) / htmlEff.length;
+ console.log(
+ ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | HTML ${hDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe, hDpe))}`,
+ );
+ console.log(
+ ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | HTML ${Math.round(hLines)} | ${winnerIcon(winner(bLines, xLines, hLines))}`,
+ );
+ } else {
+ console.log(
+ ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe))}`,
+ );
+ console.log(
+ ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | ${winnerIcon(winner(bLines, xLines))}`,
+ );
+ }
}
// Maintainability metrics comparison
@@ -199,6 +289,9 @@ async function main() {
const baseMaint = Object.values(baseline.byPrompt).map(
s => s.maintainability.metrics!,
);
+ const htmlMaint = htmlData
+ ? Object.values(htmlData.byPrompt).map(s => s.maintainability.metrics!)
+ : [];
if (xdsMaint.length > 0 && baseMaint.length > 0) {
const xSem =
xdsMaint.reduce((s, m) => s + m.semanticRatio, 0) / xdsMaint.length;
@@ -207,12 +300,25 @@ async function main() {
const xMagic = xdsMaint.reduce((s, m) => s + m.magicValueCount, 0);
const bMagic = baseMaint.reduce((s, m) => s + m.magicValueCount, 0);
console.log(`\n🔧 Maintainability Metrics:`);
- console.log(
- ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem))}`,
- );
- console.log(
- ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | ${winnerIcon(winner(bMagic, xMagic))}`,
- );
+
+ if (isThreeWay && htmlMaint.length > 0) {
+ const hSem =
+ htmlMaint.reduce((s, m) => s + m.semanticRatio, 0) / htmlMaint.length;
+ const hMagic = htmlMaint.reduce((s, m) => s + m.magicValueCount, 0);
+ console.log(
+ ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | HTML ${(hSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem, hSem))}`,
+ );
+ console.log(
+ ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | HTML ${hMagic} | ${winnerIcon(winner(bMagic, xMagic, hMagic))}`,
+ );
+ } else {
+ console.log(
+ ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem))}`,
+ );
+ console.log(
+ ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | ${winnerIcon(winner(bMagic, xMagic))}`,
+ );
+ }
}
// Cost comparison
@@ -220,20 +326,46 @@ async function main() {
console.log(`\n💰 Cost:`);
const xAvgMs = xds.cost.avgDurationMs;
const bAvgMs = baseline.cost.avgDurationMs;
- if (xAvgMs > 0 && bAvgMs > 0) {
+ if (isThreeWay && htmlData?.cost) {
+ const hAvgMs = htmlData.cost.avgDurationMs;
+ if (xAvgMs > 0 && bAvgMs > 0) {
+ console.log(
+ ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | HTML ${(hAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs, hAvgMs))}`,
+ );
+ }
+ console.log(
+ ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | HTML ${htmlData.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines, htmlData.cost.avgOutputLines))}`,
+ );
+ console.log(
+ ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead} | HTML ${htmlData.cost.avgDocsRead}`,
+ );
+ const xTokens =
+ xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens;
+ const bTokens =
+ baseline.cost.estimatedInputTokens +
+ baseline.cost.estimatedOutputTokens;
+ const hTokens =
+ htmlData.cost.estimatedInputTokens +
+ htmlData.cost.estimatedOutputTokens;
+ console.log(
+ ` Est. tokens: XDS ~${xTokens} | Baseline ~${bTokens} | HTML ~${hTokens} | ${winnerIcon(winner(bTokens, xTokens, hTokens))}`,
+ );
+ } else {
+ if (xAvgMs > 0 && bAvgMs > 0) {
+ console.log(
+ ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs))}`,
+ );
+ }
console.log(
- ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs))}`,
+ ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines))}`,
+ );
+ console.log(
+ ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead}`,
+ );
+ console.log(
+ ` Est. tokens: XDS ~${xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens} | Baseline ~${baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens} | ${winnerIcon(winner(baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens, xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens))}`,
);
}
- console.log(
- ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines))}`,
- );
- console.log(
- ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead}`,
- );
- console.log(
- ` Est. tokens: XDS ~${xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens} | Baseline ~${baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens} | ${winnerIcon(winner(baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens, xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens))}`,
- );
}
// Per-prompt wins
@@ -241,15 +373,23 @@ async function main() {
if (promptEntries.length > 0) {
let xWins = 0;
let bWins = 0;
+ let hWins = 0;
let ties = 0;
for (const [, data] of promptEntries) {
if (data.winner === 'xds') xWins++;
else if (data.winner === 'baseline') bWins++;
+ else if (data.winner === 'html') hWins++;
else ties++;
}
- console.log(
- `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | Ties ${ties} (${promptEntries.length} prompts)`,
- );
+ if (isThreeWay) {
+ console.log(
+ `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | HTML wins ${hWins} | Ties ${ties} (${promptEntries.length} prompts)`,
+ );
+ } else {
+ console.log(
+ `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | Ties ${ties} (${promptEntries.length} prompts)`,
+ );
+ }
}
console.log(`\nSaved: ${outputPath}\n`);