An open source design system that's fully customizable and agent ready
TypeScript
feat(vibe-tests): three-way comparison (XDS vs Baseline vs HTML) (#367)
Cindy Zhangecc7d385a165
diff --git a/internal/vibe-tests/app/src/report/CompareView.tsx b/internal/vibe-tests/app/src/report/CompareView.tsxindex 875a1a30c6..ef4ce1c636 100644--- a/internal/vibe-tests/app/src/report/CompareView.tsx+++ b/internal/vibe-tests/app/src/report/CompareView.tsx@@ -20,6 +20,11 @@ const styles = stylex.create({gridTemplateColumns: 'repeat(3, 1fr)',gap: spacingVars['--spacing-3'],},+ summaryGrid4: {+ display: 'grid',+ gridTemplateColumns: 'repeat(4, 1fr)',+ gap: spacingVars['--spacing-3'],+ },winCard: {padding: spacingVars['--spacing-3'],textAlign: 'center',@@ -33,6 +38,9 @@ const styles = stylex.create({neutral: {color: colorVars['--color-text-secondary'],},+ warning: {+ color: colorVars['--color-warning'],+ },costGrid: {display: 'grid',gridTemplateColumns: 'repeat(auto-fill, minmax(200px, 1fr))',@@ -51,6 +59,8 @@ const styles = stylex.create({},});+type WinnerType = 'xds' | 'baseline' | 'html' | 'tie';+interface CompareViewProps {comparison: UniversalComparison;}@@ -60,6 +70,7 @@ interface DimRow extends Record<string, unknown> {dimension: string;xdsScore: number;baselineScore: number;+ htmlScore?: number;delta: number;winner: string;}@@ -69,6 +80,7 @@ interface CatRow extends Record<string, unknown> {category: string;xdsOverall: number;baselineOverall: number;+ htmlOverall?: number;delta: number;}@@ -77,6 +89,7 @@ interface CostRow extends Record<string, unknown> {metric: string;xds: string;baseline: string;+ html?: string;winner: string;}@@ -84,19 +97,60 @@ function costWinner(xdsVal: number,baseVal: number,lowerIsBetter: boolean,-): 'xds' | 'baseline' | 'tie' {+ htmlVal?: number,+): WinnerType {+ if (htmlVal != null) {+ const vals = [xdsVal, baseVal, htmlVal];+ const best = lowerIsBetter ? Math.min(...vals) : Math.max(...vals);+ const atBest = vals.filter(v => v === best).length;+ if (atBest > 1) return 'tie';+ if (xdsVal === best) return 'xds';+ if (baseVal === best) return 'baseline';+ return 'html';+ }if (xdsVal === baseVal) return 'tie';if (lowerIsBetter) return xdsVal < baseVal ? 'xds' : 'baseline';return xdsVal > baseVal ? 'xds' : 'baseline';}+function winnerBadgeVariant(+ w: string,+): 'success' | 'error' | 'warning' | 'neutral' {+ switch (w) {+ case 'xds':+ return 'success';+ case 'baseline':+ return 'error';+ case 'html':+ return 'warning';+ default:+ return 'neutral';+ }+}++function winnerLabel(w: string): string {+ switch (w) {+ case 'xds':+ return 'XDS';+ case 'baseline':+ return 'Baseline';+ case 'html':+ return 'HTML';+ default:+ return 'Tie';+ }+}+function CostComparisonSection({xdsCost,baselineCost,+ htmlCost,}: {xdsCost: CostMetrics;baselineCost: CostMetrics;+ htmlCost?: CostMetrics;}) {+ const isThreeWay = !!htmlCost;const hasDuration =xdsCost.avgDurationMs > 0 || baselineCost.avgDurationMs > 0;@@ -108,10 +162,14 @@ function CostComparisonSection({metric: 'Avg Duration',xds: `${(xdsCost.avgDurationMs / 1000).toFixed(1)}s`,baseline: `${(baselineCost.avgDurationMs / 1000).toFixed(1)}s`,+ ...(isThreeWay+ ? {html: `${(htmlCost!.avgDurationMs / 1000).toFixed(1)}s`}+ : {}),winner: costWinner(xdsCost.avgDurationMs,baselineCost.avgDurationMs,true,+ htmlCost?.avgDurationMs,),},]@@ -121,10 +179,14 @@ function CostComparisonSection({metric: 'Input Tokens',xds: `~${xdsCost.estimatedInputTokens.toLocaleString()}`,baseline: `~${baselineCost.estimatedInputTokens.toLocaleString()}`,+ ...(isThreeWay+ ? {html: `~${htmlCost!.estimatedInputTokens.toLocaleString()}`}+ : {}),winner: costWinner(xdsCost.estimatedInputTokens,baselineCost.estimatedInputTokens,true,+ htmlCost?.estimatedInputTokens,),},{@@ -132,10 +194,14 @@ function CostComparisonSection({metric: 'Output Tokens',xds: `~${xdsCost.estimatedOutputTokens.toLocaleString()}`,baseline: `~${baselineCost.estimatedOutputTokens.toLocaleString()}`,+ ...(isThreeWay+ ? {html: `~${htmlCost!.estimatedOutputTokens.toLocaleString()}`}+ : {}),winner: costWinner(xdsCost.estimatedOutputTokens,baselineCost.estimatedOutputTokens,true,+ htmlCost?.estimatedOutputTokens,),},{@@ -143,10 +209,18 @@ function CostComparisonSection({metric: 'Total Tokens',xds: `~${(xdsCost.estimatedInputTokens + xdsCost.estimatedOutputTokens).toLocaleString()}`,baseline: `~${(baselineCost.estimatedInputTokens + baselineCost.estimatedOutputTokens).toLocaleString()}`,+ ...(isThreeWay+ ? {+ html: `~${(htmlCost!.estimatedInputTokens + htmlCost!.estimatedOutputTokens).toLocaleString()}`,+ }+ : {}),winner: costWinner(xdsCost.estimatedInputTokens + xdsCost.estimatedOutputTokens,baselineCost.estimatedInputTokens + baselineCost.estimatedOutputTokens,true,+ htmlCost+ ? htmlCost.estimatedInputTokens + htmlCost.estimatedOutputTokens+ : undefined,),},{@@ -154,10 +228,12 @@ function CostComparisonSection({metric: 'Avg Output Lines',xds: String(xdsCost.avgOutputLines),baseline: String(baselineCost.avgOutputLines),+ ...(isThreeWay ? {html: String(htmlCost!.avgOutputLines)} : {}),winner: costWinner(xdsCost.avgOutputLines,baselineCost.avgOutputLines,true,+ htmlCost?.avgOutputLines,),},{@@ -165,6 +241,7 @@ function CostComparisonSection({metric: 'Avg Docs Read',xds: String(xdsCost.avgDocsRead),baseline: String(baselineCost.avgDocsRead),+ ...(isThreeWay ? {html: String(htmlCost!.avgDocsRead)} : {}),winner: 'tie', // not inherently better or worse},];@@ -181,23 +258,23 @@ function CostComparisonSection({header: 'Baseline',renderCell: row => <XDSText type="body">{row.baseline}</XDSText>,},+ ...(isThreeWay+ ? [+ {+ key: 'html' as const,+ header: 'HTML',+ renderCell: (row: CostRow) => (+ <XDSText type="body">{row.html ?? '—'}</XDSText>+ ),+ } satisfies XDSTableColumn<CostRow>,+ ]+ : []),{key: 'winner',header: 'Lower Cost',renderCell: row => (- <XDSBadge- variant={- row.winner === 'xds'- ? 'success'- : row.winner === 'baseline'- ? 'error'- : 'neutral'- }>- {row.winner === 'xds'- ? 'XDS'- : row.winner === 'baseline'- ? 'Baseline'- : '—'}+ <XDSBadge variant={winnerBadgeVariant(row.winner)}>+ {row.winner === 'tie' ? '—' : winnerLabel(row.winner)}</XDSBadge>),},@@ -215,15 +292,18 @@ function CostComparisonSection({}export function CompareView({comparison}: CompareViewProps) {- const {xds, baseline, winners} = comparison;+ const {xds, baseline, html, winners} = comparison;+ const isThreeWay = !!html;let xdsWins = 0;let baselineWins = 0;+ let htmlWins = 0;let ties = 0;for (const dim of ALL_DIMENSIONS) {const w = winners[dim];if (w === 'xds') xdsWins++;else if (w === 'baseline') baselineWins++;+ else if (w === 'html') htmlWins++;else ties++;}@@ -232,6 +312,7 @@ export function CompareView({comparison}: CompareViewProps) {dimension: DIMENSION_LABELS[dim],xdsScore: xds.averages[dim],baselineScore: baseline.averages[dim],+ ...(isThreeWay ? {htmlScore: html!.averages[dim]} : {}),delta: xds.averages[dim] - baseline.averages[dim],winner: winners[dim],}));@@ -252,9 +333,22 @@ export function CompareView({comparison}: CompareViewProps) {<XDSText type="body">{formatScore(row.baselineScore)}</XDSText>),},+ ...(isThreeWay+ ? [+ {+ key: 'htmlScore' as const,+ header: 'HTML',+ renderCell: (row: DimRow) => (+ <XDSText type="body">+ {row.htmlScore != null ? formatScore(row.htmlScore) : '—'}+ </XDSText>+ ),+ } satisfies XDSTableColumn<DimRow>,+ ]+ : []),{key: 'delta',- header: 'Delta',+ header: 'Delta (XDS−Base)',renderCell: row => (<XDSTexttype="body"@@ -274,19 +368,8 @@ export function CompareView({comparison}: CompareViewProps) {key: 'winner',header: 'Winner',renderCell: row => (- <XDSBadge- variant={- row.winner === 'xds'- ? 'success'- : row.winner === 'baseline'- ? 'error'- : 'neutral'- }>- {row.winner === 'xds'- ? 'XDS'- : row.winner === 'baseline'- ? 'Baseline'- : 'Tie'}+ <XDSBadge variant={winnerBadgeVariant(row.winner)}>+ {winnerLabel(row.winner)}</XDSBadge>),},@@ -295,11 +378,14 @@ export function CompareView({comparison}: CompareViewProps) {const allCategories = new Set([...Object.keys(xds.byCategory),...Object.keys(baseline.byCategory),+ ...(html ? Object.keys(html.byCategory) : []),]);const catData: CatRow[] = [...allCategories].map(cat => {const xdsCat = xds.byCategory[cat] ?? {};const baseCat = baseline.byCategory[cat] ?? {};+ const htmlCat =+ html?.byCategory[cat] ?? ({} as Record<UniversalDimension, number>);const xdsAvg =ALL_DIMENSIONS.reduce((s, d) => s + ((xdsCat[d as UniversalDimension] as number) ?? 0),@@ -310,11 +396,18 @@ export function CompareView({comparison}: CompareViewProps) {(s, d) => s + ((baseCat[d as UniversalDimension] as number) ?? 0),0,) / ALL_DIMENSIONS.length;+ const htmlAvg = isThreeWay+ ? ALL_DIMENSIONS.reduce(+ (s, d) => s + ((htmlCat[d as UniversalDimension] as number) ?? 0),+ 0,+ ) / ALL_DIMENSIONS.length+ : undefined;return {id: cat,category: cat,xdsOverall: xdsAvg,baselineOverall: baseAvg,+ ...(htmlAvg != null ? {htmlOverall: htmlAvg} : {}),delta: xdsAvg - baseAvg,};});@@ -335,9 +428,22 @@ export function CompareView({comparison}: CompareViewProps) {<XDSText type="body">{formatScore(row.baselineOverall)}</XDSText>),},+ ...(isThreeWay+ ? [+ {+ key: 'htmlOverall' as const,+ header: 'HTML',+ renderCell: (row: CatRow) => (+ <XDSText type="body">+ {row.htmlOverall != null ? formatScore(row.htmlOverall) : '—'}+ </XDSText>+ ),+ } satisfies XDSTableColumn<CatRow>,+ ]+ : []),{key: 'delta',- header: 'Delta',+ header: 'Delta (XDS−Base)',renderCell: row => (<XDSTexttype="body"@@ -357,7 +463,10 @@ export function CompareView({comparison}: CompareViewProps) {return (<XDSVStack gap="space4">- <div {...stylex.props(styles.summaryGrid)}>+ <div+ {...stylex.props(+ isThreeWay ? styles.summaryGrid4 : styles.summaryGrid,+ )}><XDSCard><div {...stylex.props(styles.winCard)}><XDSVStack gap="space2">@@ -378,6 +487,18 @@ export function CompareView({comparison}: CompareViewProps) {</XDSVStack></div></XDSCard>+ {isThreeWay && (+ <XDSCard>+ <div {...stylex.props(styles.winCard)}>+ <XDSVStack gap="space2">+ <XDSText type="label">HTML Wins</XDSText>+ <XDSHeading level={2}>+ <span {...stylex.props(styles.warning)}>{htmlWins}</span>+ </XDSHeading>+ </XDSVStack>+ </div>+ </XDSCard>+ )}<XDSCard><div {...stylex.props(styles.winCard)}><XDSVStack gap="space2">@@ -420,6 +541,7 @@ export function CompareView({comparison}: CompareViewProps) {<CostComparisonSectionxdsCost={xds.cost}baselineCost={baseline.cost}+ htmlCost={html?.cost}/></XDSVStack>)}diff --git a/internal/vibe-tests/app/src/report/PromptDetailCard.tsx b/internal/vibe-tests/app/src/report/PromptDetailCard.tsxindex 55aa633294..45e1fa0b3f 100644--- a/internal/vibe-tests/app/src/report/PromptDetailCard.tsx+++ b/internal/vibe-tests/app/src/report/PromptDetailCard.tsx@@ -44,6 +44,11 @@ const styles = stylex.create({gridTemplateColumns: '1fr 1fr',gap: spacingVars['--spacing-3'],},+ scoresRow3: {+ display: 'grid',+ gridTemplateColumns: '1fr 1fr 1fr',+ gap: spacingVars['--spacing-3'],+ },scoresRowSingle: {display: 'grid',gridTemplateColumns: '1fr',@@ -149,6 +154,7 @@ interface PromptDetailCardProps {promptText?: string;xdsScore?: UniversalScore;baselineScore?: UniversalScore;+ htmlScore?: UniversalScore;hasXdsCode: boolean;hasBaselineCode: boolean;hasHtmlCode: boolean;@@ -162,6 +168,7 @@ export function PromptDetailCard({promptText,xdsScore,baselineScore,+ htmlScore,hasXdsCode,hasBaselineCode,hasHtmlCode,@@ -169,10 +176,17 @@ export function PromptDetailCard({previewUrls,}: PromptDetailCardProps) {const hasBoth = !!(xdsScore && baselineScore);+ const hasAll = !!(xdsScore && baselineScore && htmlScore);const hasAnyPreview =previewUrls?.xds || previewUrls?.baseline || previewUrls?.html;const hasAnyCode = hasXdsCode || hasBaselineCode || hasHtmlCode;+ const scoresStyle = hasAll+ ? styles.scoresRow3+ : hasBoth+ ? styles.scoresRow+ : styles.scoresRowSingle;+return (<XDSCard><div {...stylex.props(styles.card)}>@@ -240,15 +254,13 @@ export function PromptDetailCard({</div>{/* Score summaries in constrained grid */}- {(xdsScore || baselineScore) && (- <div- {...stylex.props(- hasBoth ? styles.scoresRow : styles.scoresRowSingle,- )}>+ {(xdsScore || baselineScore || htmlScore) && (+ <div {...stylex.props(scoresStyle)}>{xdsScore && <ScoreSummary label="XDS" score={xdsScore} />}{baselineScore && (<ScoreSummary label="Baseline" score={baselineScore} />)}+ {htmlScore && <ScoreSummary label="HTML" score={htmlScore} />}</div>)}@@ -276,6 +288,18 @@ export function PromptDetailCard({</div></>)}++ {htmlScore && (+ <>+ <XDSDivider />+ <div {...stylex.props(styles.section)}>+ <div {...stylex.props(styles.sectionLabel)}>+ <XDSText type="label">HTML Findings</XDSText>+ </div>+ <Findings score={htmlScore} />+ </div>+ </>+ )}</XDSVStack></div></XDSCard>diff --git a/internal/vibe-tests/app/src/report/Report.tsx b/internal/vibe-tests/app/src/report/Report.tsxindex beb2029be6..f29033dd03 100644--- a/internal/vibe-tests/app/src/report/Report.tsx+++ b/internal/vibe-tests/app/src/report/Report.tsx@@ -277,6 +277,14 @@ export function Report() {compareScore={comparison?.baseline.overall}compareLabel="Baseline"/>+ {comparison?.html && (+ <ScoreCard+ label="Overall Score vs HTML"+ score={universal.overall}+ compareScore={comparison.html.overall}+ compareLabel="HTML"+ />+ )}{/* Dimension scores grid */}<XDSVStack gap="space3">@@ -303,7 +311,9 @@ export function Report() {{comparison && (<XDSVStack gap="space3"><XDSHeading level={3}>- XDS vs Baseline Comparison+ {comparison.html+ ? 'XDS vs Baseline vs HTML Comparison'+ : 'XDS vs Baseline Comparison'}</XDSHeading><CompareView comparison={comparison} /></XDSVStack>@@ -325,6 +335,7 @@ export function Report() {promptText={data.prompts?.[promptId]}xdsScore={universal.byPrompt[promptId]}baselineScore={comparison?.baseline.byPrompt[promptId]}+ htmlScore={comparison?.html?.byPrompt[promptId]}hasXdsCode={!!data.sourceCode?.[promptId]}hasBaselineCode={!!data.baselineSourceCode?.[promptId]}hasHtmlCode={!!data.htmlSourceCode?.[promptId]}diff --git a/internal/vibe-tests/src/build-report.ts b/internal/vibe-tests/src/build-report.tsindex 0d94fe58db..0954d19a3d 100644--- a/internal/vibe-tests/src/build-report.ts+++ b/internal/vibe-tests/src/build-report.ts@@ -59,7 +59,7 @@ function parseArgs(): {process.exit(1);}- return {iteration, baseline, withScreenshots, dev};+ return {iteration, baseline, html, withScreenshots, dev};}function ensureUniversalJson(iteration: string): UniversalAggregate {@@ -80,17 +80,21 @@ function ensureUniversalJson(iteration: string): UniversalAggregate {function ensureComparison(xdsId: string,baselineId: string,+ htmlId?: string,): UniversalComparison {const resultsDir = getResultsDir();- const comparisonPath = path.join(- resultsDir,- `comparison-${xdsId}-${baselineId}.json`,- );+ const comparisonFilename = htmlId+ ? `comparison-${xdsId}-${baselineId}-${htmlId}.json`+ : `comparison-${xdsId}-${baselineId}.json`;+ const comparisonPath = path.join(resultsDir, comparisonFilename);if (!fs.existsSync(comparisonPath)) {- console.log(`Generating comparison for ${xdsId} vs ${baselineId}...`);+ const htmlFlag = htmlId ? ` --html ${htmlId}` : '';+ console.log(+ `Generating comparison for ${xdsId} vs ${baselineId}${htmlId ? ` vs ${htmlId}` : ''}...`,+ );execSync(- `tsx ${path.join(import.meta.dirname, 'universal-compare.ts')} --xds ${xdsId} --baseline ${baselineId}`,+ `tsx ${path.join(import.meta.dirname, 'universal-compare.ts')} --xds ${xdsId} --baseline ${baselineId}${htmlFlag}`,{cwd: path.join(import.meta.dirname, '..'), stdio: 'inherit'},);}@@ -184,7 +188,13 @@ function inlineCss(htmlPath: string, distDir: string): void {}async function main() {- const {iteration, baseline, html, withScreenshots, dev} = parseArgs();+ const {+ iteration,+ baseline,+ html,+ withScreenshots: _withScreenshots,+ dev,+ } = parseArgs();const resultsDir = getResultsDir();const iterDir = path.join(resultsDir, iteration);const manifestPath = path.join(iterDir, 'manifest.json');@@ -205,7 +215,10 @@ async function main() {if (baseline) {console.log('📊 Step 2: Ensuring comparison data...');ensureUniversalJson(baseline);- comparison = ensureComparison(iteration, baseline);+ if (html) {+ ensureUniversalJson(html);+ }+ comparison = ensureComparison(iteration, baseline, html);}// Step 3: Load source code for per-prompt inspectiondiff --git a/internal/vibe-tests/src/types.ts b/internal/vibe-tests/src/types.tsindex b2e33bff99..25661efec0 100644--- a/internal/vibe-tests/src/types.ts+++ b/internal/vibe-tests/src/types.ts@@ -345,13 +345,15 @@ export interface UniversalAggregate {export interface UniversalComparison {xds: UniversalAggregate;baseline: UniversalAggregate;- winners: Record<UniversalDimension, 'xds' | 'baseline' | 'tie'>;+ html?: UniversalAggregate;+ winners: Record<UniversalDimension, 'xds' | 'baseline' | 'html' | 'tie'>;byPrompt: Record<string,{xds: UniversalScore;baseline: UniversalScore;- winner: 'xds' | 'baseline' | 'tie';+ html?: UniversalScore;+ winner: 'xds' | 'baseline' | 'html' | 'tie';}>;}diff --git a/internal/vibe-tests/src/universal-compare.ts b/internal/vibe-tests/src/universal-compare.tsindex f7b285f097..097e67c435 100644--- a/internal/vibe-tests/src/universal-compare.ts+++ b/internal/vibe-tests/src/universal-compare.ts@@ -1,9 +1,10 @@#!/usr/bin/env node/**- * @file Universal Compare — side-by-side comparison of two iterations+ * @file Universal Compare — side-by-side comparison of two or three iterations** Usage:* tsx src/universal-compare.ts --xds abc123 --baseline def456+ * tsx src/universal-compare.ts --xds abc123 --baseline def456 --html ghi789* tsx src/universal-compare.ts --xds abc123 --baseline def456 --json*/@@ -47,27 +48,47 @@ function loadOrGenerate(iterationId: string): UniversalAggregate {return JSON.parse(fs.readFileSync(universalPath, 'utf-8'));}-function winner(a: number, b: number): 'xds' | 'baseline' | 'tie' {- if (a > b) return 'xds';- if (b > a) return 'baseline';+type WinnerType = 'xds' | 'baseline' | 'html' | 'tie';++function winner(xdsVal: number, baseVal: number, htmlVal?: number): WinnerType {+ if (htmlVal != null) {+ const max = Math.max(xdsVal, baseVal, htmlVal);+ const atMax = [xdsVal === max, baseVal === max, htmlVal === max].filter(+ Boolean,+ ).length;+ if (atMax > 1) return 'tie';+ if (xdsVal === max) return 'xds';+ if (baseVal === max) return 'baseline';+ return 'html';+ }+ if (xdsVal > baseVal) return 'xds';+ if (baseVal > xdsVal) return 'baseline';return 'tie';}-function winnerIcon(w: 'xds' | 'baseline' | 'tie'): string {+function winnerIcon(w: WinnerType): string {switch (w) {case 'xds':return '🟢 XDS';case 'baseline':return '🔵 Base';+ case 'html':+ return '🟡 HTML';case 'tie':return '⚪ Tie';}}-function parseArgs(): {xds: string; baseline: string; json: boolean} {+function parseArgs(): {+ xds: string;+ baseline: string;+ html?: string;+ json: boolean;+} {const args = process.argv.slice(2);let xds = '';let baseline = '';+ let html: string | undefined;let json = false;for (let i = 0; i < args.length; i++) {@@ -75,6 +96,8 @@ function parseArgs(): {xds: string; baseline: string; json: boolean} {xds = args[++i];} else if ((args[i] === '--baseline' || args[i] === '-b') && args[i + 1]) {baseline = args[++i];+ } else if (args[i] === '--html' && args[i + 1]) {+ html = args[++i];} else if (args[i] === '--json') {json = true;}@@ -82,57 +105,73 @@ function parseArgs(): {xds: string; baseline: string; json: boolean} {if (!xds || !baseline) {console.error(- 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--json]',+ 'Usage: tsx src/universal-compare.ts --xds <id> --baseline <id> [--html <id>] [--json]',);process.exit(1);}- return {xds, baseline, json};+ return {xds, baseline, html, json};}async function main() {- const {xds: xdsId, baseline: baselineId, json} = parseArgs();+ const {xds: xdsId, baseline: baselineId, html: htmlId, json} = parseArgs();const xds = loadOrGenerate(xdsId);const baseline = loadOrGenerate(baselineId);+ const htmlData = htmlId ? loadOrGenerate(htmlId) : undefined;const dimensions = getDimensionNames();+ const isThreeWay = !!htmlData;// Build comparison- const winners = {} as Record<UniversalDimension, 'xds' | 'baseline' | 'tie'>;+ const winners = {} as Record<UniversalDimension, WinnerType>;for (const d of dimensions) {- winners[d] = winner(xds.averages[d], baseline.averages[d]);+ winners[d] = winner(+ xds.averages[d],+ baseline.averages[d],+ htmlData?.averages[d],+ );}// Per-prompt comparisonconst allPromptIds = new Set([...Object.keys(xds.byPrompt),...Object.keys(baseline.byPrompt),+ ...(htmlData ? Object.keys(htmlData.byPrompt) : []),]);const byPrompt: UniversalComparison['byPrompt'] = {};for (const promptId of allPromptIds) {const xdsScore = xds.byPrompt[promptId];const baselineScore = baseline.byPrompt[promptId];+ const htmlScore = htmlData?.byPrompt[promptId];if (xdsScore && baselineScore) {byPrompt[promptId] = {xds: xdsScore,baseline: baselineScore,+ ...(htmlScore ? {html: htmlScore} : {}),winner: winner(getAverageScore(xdsScore),getAverageScore(baselineScore),+ htmlScore ? getAverageScore(htmlScore) : undefined,),};}}- const comparison: UniversalComparison = {xds, baseline, winners, byPrompt};+ const comparison: UniversalComparison = {+ xds,+ baseline,+ ...(htmlData ? {html: htmlData} : {}),+ winners,+ byPrompt,+ };// Save- const outputPath = path.join(- getResultsDir(),- `comparison-${xdsId}-${baselineId}.json`,- );+ const outputFilename = htmlId+ ? `comparison-${xdsId}-${baselineId}-${htmlId}.json`+ : `comparison-${xdsId}-${baselineId}.json`;+ const outputPath = path.join(getResultsDir(), outputFilename);writeJson(outputPath, comparison);if (json) {@@ -142,39 +181,76 @@ async function main() {// --- Print report ---- console.log(`\n📊 Universal Comparison: XDS vs Baseline`);- console.log('═'.repeat(52));+ const title = isThreeWay+ ? '📊 Universal Comparison: XDS vs Baseline vs HTML'+ : '📊 Universal Comparison: XDS vs Baseline';+ console.log(`\n${title}`);+ console.log('═'.repeat(isThreeWay ? 66 : 52));- // Dimension table- console.log('┌─────────────────────┬───────┬──────────┬──────────┐');- console.log('│ Dimension │ XDS │ Baseline │ Winner │');- console.log('├─────────────────────┼───────┼──────────┼──────────┤');- for (const d of dimensions) {- const label = DIMENSION_LABELS[d].padEnd(19);- const xScore = String(xds.averages[d]).padStart(3);- const bScore = String(baseline.averages[d]).padStart(3);- const w = winnerIcon(winners[d]).padEnd(8);- console.log(`│ ${label} │ ${xScore} │ ${bScore} │ ${w} │`);+ if (isThreeWay) {+ // Three-way table+ console.log('┌─────────────────────┬───────┬──────────┬──────┬──────────┐');+ console.log('│ Dimension │ XDS │ Baseline │ HTML │ Winner │');+ console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');+ for (const d of dimensions) {+ const label = DIMENSION_LABELS[d].padEnd(19);+ const xScore = String(xds.averages[d]).padStart(3);+ const bScore = String(baseline.averages[d]).padStart(3);+ const hScore = String(htmlData!.averages[d]).padStart(3);+ const w = winnerIcon(winners[d]).padEnd(8);+ console.log(+ `│ ${label} │ ${xScore} │ ${bScore} │ ${hScore} │ ${w} │`,+ );+ }+ console.log('├─────────────────────┼───────┼──────────┼──────┼──────────┤');+ const xOverall = String(xds.overall).padStart(3);+ const bOverall = String(baseline.overall).padStart(3);+ const hOverall = String(htmlData!.overall).padStart(3);+ const overallW = winnerIcon(+ winner(xds.overall, baseline.overall, htmlData!.overall),+ ).padEnd(8);+ console.log(+ `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${hOverall} │ ${overallW} │`,+ );+ console.log('└─────────────────────┴───────┴──────────┴──────┴──────────┘');+ } else {+ // Two-way table (unchanged)+ console.log('┌─────────────────────┬───────┬──────────┬──────────┐');+ console.log('│ Dimension │ XDS │ Baseline │ Winner │');+ console.log('├─────────────────────┼───────┼──────────┼──────────┤');+ for (const d of dimensions) {+ const label = DIMENSION_LABELS[d].padEnd(19);+ const xScore = String(xds.averages[d]).padStart(3);+ const bScore = String(baseline.averages[d]).padStart(3);+ const w = winnerIcon(winners[d]).padEnd(8);+ console.log(`│ ${label} │ ${xScore} │ ${bScore} │ ${w} │`);+ }+ console.log('├─────────────────────┼───────┼──────────┼──────────┤');+ const xOverall = String(xds.overall).padStart(3);+ const bOverall = String(baseline.overall).padStart(3);+ const overallW = winnerIcon(winner(xds.overall, baseline.overall)).padEnd(+ 8,+ );+ console.log(+ `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${overallW} │`,+ );+ console.log('└─────────────────────┴───────┴──────────┴──────────┘');}- console.log('├─────────────────────┼───────┼──────────┼──────────┤');- const xOverall = String(xds.overall).padStart(3);- const bOverall = String(baseline.overall).padStart(3);- const overallW = winnerIcon(winner(xds.overall, baseline.overall)).padEnd(8);- console.log(- `│ ${'Overall'.padEnd(19)} │ ${xOverall} │ ${bOverall} │ ${overallW} │`,- );- console.log('└─────────────────────┴───────┴──────────┴──────────┘');// Dark mode- console.log(- `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}%`,- );+ const darkModeStr = isThreeWay+ ? `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}% | HTML ${htmlData!.darkModeRate}%`+ : `\n🌙 Dark Mode: XDS ${xds.darkModeRate}% | Baseline ${baseline.darkModeRate}%`;+ console.log(darkModeStr);// Efficiency metrics comparisonconst xdsEff = Object.values(xds.byPrompt).map(s => s.efficiency.metrics!);const baseEff = Object.values(baseline.byPrompt).map(s => s.efficiency.metrics!,);+ const htmlEff = htmlData+ ? Object.values(htmlData.byPrompt).map(s => s.efficiency.metrics!)+ : [];if (xdsEff.length > 0 && baseEff.length > 0) {const xDpe =xdsEff.reduce((s, m) => s + m.decisionsPerElement, 0) / xdsEff.length;@@ -184,12 +260,26 @@ async function main() {const bLines =baseEff.reduce((s, m) => s + m.codeLines, 0) / baseEff.length;console.log(`\n⚡ Efficiency Metrics:`);- console.log(- ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe))}`,- );- console.log(- ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | ${winnerIcon(winner(bLines, xLines))}`,- );++ if (isThreeWay && htmlEff.length > 0) {+ const hDpe =+ htmlEff.reduce((s, m) => s + m.decisionsPerElement, 0) / htmlEff.length;+ const hLines =+ htmlEff.reduce((s, m) => s + m.codeLines, 0) / htmlEff.length;+ console.log(+ ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | HTML ${hDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe, hDpe))}`,+ );+ console.log(+ ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | HTML ${Math.round(hLines)} | ${winnerIcon(winner(bLines, xLines, hLines))}`,+ );+ } else {+ console.log(+ ` Decisions/element: XDS ${xDpe.toFixed(1)} | Baseline ${bDpe.toFixed(1)} | ${winnerIcon(winner(bDpe, xDpe))}`,+ );+ console.log(+ ` Avg code lines: XDS ${Math.round(xLines)} | Baseline ${Math.round(bLines)} | ${winnerIcon(winner(bLines, xLines))}`,+ );+ }}// Maintainability metrics comparison@@ -199,6 +289,9 @@ async function main() {const baseMaint = Object.values(baseline.byPrompt).map(s => s.maintainability.metrics!,);+ const htmlMaint = htmlData+ ? Object.values(htmlData.byPrompt).map(s => s.maintainability.metrics!)+ : [];if (xdsMaint.length > 0 && baseMaint.length > 0) {const xSem =xdsMaint.reduce((s, m) => s + m.semanticRatio, 0) / xdsMaint.length;@@ -207,12 +300,25 @@ async function main() {const xMagic = xdsMaint.reduce((s, m) => s + m.magicValueCount, 0);const bMagic = baseMaint.reduce((s, m) => s + m.magicValueCount, 0);console.log(`\n🔧 Maintainability Metrics:`);- console.log(- ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem))}`,- );- console.log(- ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | ${winnerIcon(winner(bMagic, xMagic))}`,- );++ if (isThreeWay && htmlMaint.length > 0) {+ const hSem =+ htmlMaint.reduce((s, m) => s + m.semanticRatio, 0) / htmlMaint.length;+ const hMagic = htmlMaint.reduce((s, m) => s + m.magicValueCount, 0);+ console.log(+ ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | HTML ${(hSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem, hSem))}`,+ );+ console.log(+ ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | HTML ${hMagic} | ${winnerIcon(winner(bMagic, xMagic, hMagic))}`,+ );+ } else {+ console.log(+ ` Semantic ratio: XDS ${(xSem * 100).toFixed(0)}% | Baseline ${(bSem * 100).toFixed(0)}% | ${winnerIcon(winner(xSem, bSem))}`,+ );+ console.log(+ ` Magic values: XDS ${xMagic} | Baseline ${bMagic} | ${winnerIcon(winner(bMagic, xMagic))}`,+ );+ }}// Cost comparison@@ -220,20 +326,46 @@ async function main() {console.log(`\n💰 Cost:`);const xAvgMs = xds.cost.avgDurationMs;const bAvgMs = baseline.cost.avgDurationMs;- if (xAvgMs > 0 && bAvgMs > 0) {+ if (isThreeWay && htmlData?.cost) {+ const hAvgMs = htmlData.cost.avgDurationMs;+ if (xAvgMs > 0 && bAvgMs > 0) {+ console.log(+ ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | HTML ${(hAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs, hAvgMs))}`,+ );+ }+ console.log(+ ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | HTML ${htmlData.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines, htmlData.cost.avgOutputLines))}`,+ );+ console.log(+ ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead} | HTML ${htmlData.cost.avgDocsRead}`,+ );+ const xTokens =+ xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens;+ const bTokens =+ baseline.cost.estimatedInputTokens ++ baseline.cost.estimatedOutputTokens;+ const hTokens =+ htmlData.cost.estimatedInputTokens ++ htmlData.cost.estimatedOutputTokens;+ console.log(+ ` Est. tokens: XDS ~${xTokens} | Baseline ~${bTokens} | HTML ~${hTokens} | ${winnerIcon(winner(bTokens, xTokens, hTokens))}`,+ );+ } else {+ if (xAvgMs > 0 && bAvgMs > 0) {+ console.log(+ ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs))}`,+ );+ }console.log(- ` Avg duration: XDS ${(xAvgMs / 1000).toFixed(1)}s | Baseline ${(bAvgMs / 1000).toFixed(1)}s | ${winnerIcon(winner(bAvgMs, xAvgMs))}`,+ ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines))}`,+ );+ console.log(+ ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead}`,+ );+ console.log(+ ` Est. tokens: XDS ~${xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens} | Baseline ~${baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens} | ${winnerIcon(winner(baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens, xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens))}`,);}- console.log(- ` Avg output lines: XDS ${xds.cost.avgOutputLines} | Baseline ${baseline.cost.avgOutputLines} | ${winnerIcon(winner(baseline.cost.avgOutputLines, xds.cost.avgOutputLines))}`,- );- console.log(- ` Avg docs read: XDS ${xds.cost.avgDocsRead} | Baseline ${baseline.cost.avgDocsRead}`,- );- console.log(- ` Est. tokens: XDS ~${xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens} | Baseline ~${baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens} | ${winnerIcon(winner(baseline.cost.estimatedInputTokens + baseline.cost.estimatedOutputTokens, xds.cost.estimatedInputTokens + xds.cost.estimatedOutputTokens))}`,- );}// Per-prompt wins@@ -241,15 +373,23 @@ async function main() {if (promptEntries.length > 0) {let xWins = 0;let bWins = 0;+ let hWins = 0;let ties = 0;for (const [, data] of promptEntries) {if (data.winner === 'xds') xWins++;else if (data.winner === 'baseline') bWins++;+ else if (data.winner === 'html') hWins++;else ties++;}- console.log(- `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | Ties ${ties} (${promptEntries.length} prompts)`,- );+ if (isThreeWay) {+ console.log(+ `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | HTML wins ${hWins} | Ties ${ties} (${promptEntries.length} prompts)`,+ );+ } else {+ console.log(+ `\n📝 Per-Prompt: XDS wins ${xWins} | Baseline wins ${bWins} | Ties ${ties} (${promptEntries.length} prompts)`,+ );+ }}console.log(`\nSaved: ${outputPath}\n`);