Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(vibe-tests): report deploy pipeline with gh-pages publishing (#266)

Cindy Zhangd1fcf26185c5
diff --git a/internal/vibe-tests/app/package.json b/internal/vibe-tests/app/package.json
index 4ec97509ed..cb2f5ba300 100644
--- a/internal/vibe-tests/app/package.json
+++ b/internal/vibe-tests/app/package.json
@@ -16,6 +16,7 @@
"devDependencies": {
"@vitejs/plugin-react": "^4.0.0",
"@stylexjs/unplugin": "*",
- "vite": "^5.0.0"
+ "vite": "^5.0.0",
+ "vite-plugin-singlefile": "^2.0.0"
}
}
diff --git a/internal/vibe-tests/app/src/main.tsx b/internal/vibe-tests/app/src/main.tsx
index 8e3c8389c1..b20b578b8c 100644
--- a/internal/vibe-tests/app/src/main.tsx
+++ b/internal/vibe-tests/app/src/main.tsx
@@ -2,12 +2,14 @@ import React, {Suspense, lazy} from 'react';
import {createRoot} from 'react-dom/client';
const params = new URLSearchParams(window.location.search);
-const mode = params.get('mode') ?? 'preview';
+const mode = params.get('mode') ?? 'report';
const theme = params.get('theme') ?? 'light';
function App() {
if (mode === 'report') {
- const Report = lazy(() => import('./report/Report'));
+ const Report = lazy(() =>
+ import('./report/Report').then(m => ({default: m.Report})),
+ );
return (
<Suspense fallback={<div>Loading report...</div>}>
<Report />
diff --git a/internal/vibe-tests/app/src/preview.tsx b/internal/vibe-tests/app/src/preview.tsx
new file mode 100644
index 0000000000..94c4aa80af
--- /dev/null
+++ b/internal/vibe-tests/app/src/preview.tsx
@@ -0,0 +1,3 @@
+export default function Preview({theme}: {theme: string}) {
+ return <div>No component loaded. Use the screenshot script.</div>;
+}
diff --git a/internal/vibe-tests/app/src/report/CompareView.tsx b/internal/vibe-tests/app/src/report/CompareView.tsx
index 16e830260a..30edd68511 100644
--- a/internal/vibe-tests/app/src/report/CompareView.tsx
+++ b/internal/vibe-tests/app/src/report/CompareView.tsx
@@ -8,20 +8,21 @@ import {XDSBadge} from '@xds/core/Badge';
import {XDSTable} from '@xds/core/Table';
import type {XDSTableColumn} from '@xds/core/Table/types';
import {spacingVars, colorVars} from '@xds/core/theme/tokens.stylex';
-import type {UniversalComparison, UniversalDimension} from './types';
+import type {
+ UniversalComparison,
+ UniversalDimension,
+ CostMetrics,
+} from './types';
import {ALL_DIMENSIONS, DIMENSION_LABELS, formatScore} from './utils';
const styles = stylex.create({
- section: {
- padding: spacingVars['--spacing-4'],
- },
summaryGrid: {
display: 'grid',
gridTemplateColumns: 'repeat(3, 1fr)',
- gap: spacingVars['--spacing-4'],
+ gap: spacingVars['--spacing-3'],
},
winCard: {
- padding: spacingVars['--spacing-4'],
+ padding: spacingVars['--spacing-3'],
textAlign: 'center',
},
positive: {
@@ -33,6 +34,22 @@ const styles = stylex.create({
neutral: {
color: colorVars['--color-text-secondary'],
},
+ costGrid: {
+ display: 'grid',
+ gridTemplateColumns: 'repeat(auto-fill, minmax(200px, 1fr))',
+ gap: spacingVars['--spacing-3'],
+ },
+ costCard: {
+ padding: spacingVars['--spacing-3'],
+ },
+ costLabel: {
+ marginBlockEnd: spacingVars['--spacing-1'],
+ },
+ costValues: {
+ display: 'flex',
+ gap: spacingVars['--spacing-3'],
+ alignItems: 'baseline',
+ },
});
interface CompareViewProps {
@@ -56,10 +73,151 @@ interface CatRow extends Record<string, unknown> {
delta: number;
}
+interface CostRow extends Record<string, unknown> {
+ id: string;
+ metric: string;
+ xds: string;
+ baseline: string;
+ winner: string;
+}
+
+function costWinner(
+ xdsVal: number,
+ baseVal: number,
+ lowerIsBetter: boolean,
+): 'xds' | 'baseline' | 'tie' {
+ if (xdsVal === baseVal) return 'tie';
+ if (lowerIsBetter) return xdsVal < baseVal ? 'xds' : 'baseline';
+ return xdsVal > baseVal ? 'xds' : 'baseline';
+}
+
+function CostComparisonSection({
+ xdsCost,
+ baselineCost,
+}: {
+ xdsCost: CostMetrics;
+ baselineCost: CostMetrics;
+}) {
+ const hasDuration =
+ xdsCost.avgDurationMs > 0 || baselineCost.avgDurationMs > 0;
+
+ const costData: CostRow[] = [
+ ...(hasDuration
+ ? [
+ {
+ id: 'duration',
+ metric: 'Avg Duration',
+ xds: `${(xdsCost.avgDurationMs / 1000).toFixed(1)}s`,
+ baseline: `${(baselineCost.avgDurationMs / 1000).toFixed(1)}s`,
+ winner: costWinner(
+ xdsCost.avgDurationMs,
+ baselineCost.avgDurationMs,
+ true,
+ ),
+ },
+ ]
+ : []),
+ {
+ id: 'input-tokens',
+ metric: 'Input Tokens',
+ xds: `~${xdsCost.estimatedInputTokens.toLocaleString()}`,
+ baseline: `~${baselineCost.estimatedInputTokens.toLocaleString()}`,
+ winner: costWinner(
+ xdsCost.estimatedInputTokens,
+ baselineCost.estimatedInputTokens,
+ true,
+ ),
+ },
+ {
+ id: 'output-tokens',
+ metric: 'Output Tokens',
+ xds: `~${xdsCost.estimatedOutputTokens.toLocaleString()}`,
+ baseline: `~${baselineCost.estimatedOutputTokens.toLocaleString()}`,
+ winner: costWinner(
+ xdsCost.estimatedOutputTokens,
+ baselineCost.estimatedOutputTokens,
+ true,
+ ),
+ },
+ {
+ id: 'total-tokens',
+ metric: 'Total Tokens',
+ xds: `~${(xdsCost.estimatedInputTokens + xdsCost.estimatedOutputTokens).toLocaleString()}`,
+ baseline: `~${(baselineCost.estimatedInputTokens + baselineCost.estimatedOutputTokens).toLocaleString()}`,
+ winner: costWinner(
+ xdsCost.estimatedInputTokens + xdsCost.estimatedOutputTokens,
+ baselineCost.estimatedInputTokens + baselineCost.estimatedOutputTokens,
+ true,
+ ),
+ },
+ {
+ id: 'output-lines',
+ metric: 'Avg Output Lines',
+ xds: String(xdsCost.avgOutputLines),
+ baseline: String(baselineCost.avgOutputLines),
+ winner: costWinner(
+ xdsCost.avgOutputLines,
+ baselineCost.avgOutputLines,
+ true,
+ ),
+ },
+ {
+ id: 'docs-read',
+ metric: 'Avg Docs Read',
+ xds: String(xdsCost.avgDocsRead),
+ baseline: String(baselineCost.avgDocsRead),
+ winner: 'tie', // not inherently better or worse
+ },
+ ];
+
+ const costColumns: XDSTableColumn<CostRow>[] = [
+ {key: 'metric', header: 'Metric'},
+ {
+ key: 'xds',
+ header: 'XDS',
+ renderCell: row => <XDSText type="body">{row.xds}</XDSText>,
+ },
+ {
+ key: 'baseline',
+ header: 'Baseline',
+ renderCell: row => <XDSText type="body">{row.baseline}</XDSText>,
+ },
+ {
+ key: 'winner',
+ header: 'Lower Cost',
+ renderCell: row => (
+ <XDSBadge
+ variant={
+ row.winner === 'xds'
+ ? 'success'
+ : row.winner === 'baseline'
+ ? 'error'
+ : 'neutral'
+ }>
+ {row.winner === 'xds'
+ ? 'XDS'
+ : row.winner === 'baseline'
+ ? 'Baseline'
+ : '—'}
+ </XDSBadge>
+ ),
+ },
+ ];
+
+ return (
+ <XDSTable<CostRow>
+ data={costData}
+ columns={costColumns}
+ idKey="id"
+ density="balanced"
+ dividers="rows"
+ />
+ );
+}
+
export function CompareView({comparison}: CompareViewProps) {
const {xds, baseline, winners} = comparison;
- // Count wins
let xdsWins = 0;
let baselineWins = 0;
let ties = 0;
@@ -70,7 +228,6 @@ export function CompareView({comparison}: CompareViewProps) {
else ties++;
}
- // Dimension comparison rows
const dimData: DimRow[] = ALL_DIMENSIONS.map(dim => ({
id: dim,
dimension: DIMENSION_LABELS[dim],
@@ -136,7 +293,6 @@ export function CompareView({comparison}: CompareViewProps) {
},
];
- // Category breakdown
const allCategories = new Set([
...Object.keys(xds.byCategory),
...Object.keys(baseline.byCategory),
@@ -201,7 +357,7 @@ export function CompareView({comparison}: CompareViewProps) {
];
return (
- <XDSVStack gap="space6">
+ <XDSVStack gap="space4">
<div {...stylex.props(styles.summaryGrid)}>
<XDSCard>
<div {...stylex.props(styles.winCard)}>
@@ -258,6 +414,16 @@ export function CompareView({comparison}: CompareViewProps) {
/>
</XDSVStack>
)}
+
+ {xds.cost && baseline.cost && (
+ <XDSVStack gap="space3">
+ <XDSHeading level={3}>Cost Comparison</XDSHeading>
+ <CostComparisonSection
+ xdsCost={xds.cost}
+ baselineCost={baseline.cost}
+ />
+ </XDSVStack>
+ )}
</XDSVStack>
);
}
diff --git a/internal/vibe-tests/app/src/report/DimensionTable.tsx b/internal/vibe-tests/app/src/report/DimensionTable.tsx
index 2776e8c18e..2a15f7fabb 100644
--- a/internal/vibe-tests/app/src/report/DimensionTable.tsx
+++ b/internal/vibe-tests/app/src/report/DimensionTable.tsx
@@ -21,12 +21,11 @@ interface RowData extends Record<string, unknown> {
id: string;
promptId: string;
category: string;
+ correctness: number;
accessibility: number;
codeQuality: number;
- repetition: number;
- conciseness: number;
- themeAdherence: number;
- correctness: number;
+ efficiency: number;
+ maintainability: number;
overall: number;
}
@@ -48,12 +47,11 @@ export function DimensionTable({byPrompt, categories}: DimensionTableProps) {
id: promptId,
promptId,
category: categories?.[promptId] ?? '—',
+ correctness: score.correctness.score,
accessibility: score.accessibility.score,
codeQuality: score.codeQuality.score,
- repetition: score.repetition.score,
- conciseness: score.conciseness.score,
- themeAdherence: score.themeAdherence.score,
- correctness: score.correctness.score,
+ efficiency: score.efficiency.score,
+ maintainability: score.maintainability.score,
overall: computeOverall(score),
}));
diff --git a/internal/vibe-tests/app/src/report/Report.tsx b/internal/vibe-tests/app/src/report/Report.tsx
index 27ca61e141..9baa64898d 100644
--- a/internal/vibe-tests/app/src/report/Report.tsx
+++ b/internal/vibe-tests/app/src/report/Report.tsx
@@ -1,4 +1,4 @@
-import {useState, useMemo} from 'react';
+import {useState} from 'react';
import * as stylex from '@stylexjs/stylex';
import {XDSTheme} from '@xds/core/theme';
import {XDSVStack} from '@xds/core/Stack';
@@ -12,7 +12,7 @@ import {XDSButton} from '@xds/core/Button';
import {defaultTheme} from '@xds/theme/default';
import {spacingVars, colorVars} from '@xds/core/theme/tokens.stylex';
import type {ReportData} from './types';
-import {ALL_DIMENSIONS, DIMENSION_LABELS, formatScore} from './utils';
+import {ALL_DIMENSIONS, DIMENSION_LABELS} from './utils';
import {ScoreCard} from './ScoreCard';
import {DimensionTable} from './DimensionTable';
import {CompareView} from './CompareView';
@@ -26,18 +26,18 @@ const styles = stylex.create({
container: {
maxWidth: '1200px',
marginInline: 'auto',
- padding: spacingVars['--spacing-6'],
+ padding: spacingVars['--spacing-5'],
},
header: {
- paddingBlock: spacingVars['--spacing-4'],
+ paddingBlock: spacingVars['--spacing-3'],
},
scoreGrid: {
display: 'grid',
gridTemplateColumns: 'repeat(auto-fill, minmax(180px, 1fr))',
- gap: spacingVars['--spacing-4'],
+ gap: spacingVars['--spacing-3'],
},
metricsCard: {
- padding: spacingVars['--spacing-4'],
+ padding: spacingVars['--spacing-3'],
},
metricsGrid: {
display: 'grid',
@@ -48,7 +48,7 @@ const styles = stylex.create({
textAlign: 'center',
},
tabContent: {
- paddingBlock: spacingVars['--spacing-6'],
+ paddingBlockStart: spacingVars['--spacing-3'],
},
emptyState: {
padding: spacingVars['--spacing-8'],
@@ -56,68 +56,134 @@ const styles = stylex.create({
},
});
-function ConcisenessMetricsCard({
+function MetricValue({label, value}: {label: string; value: string}) {
+ return (
+ <div {...stylex.props(styles.metricItem)}>
+ <XDSVStack gap="space1">
+ <XDSText type="label">{label}</XDSText>
+ <XDSHeading level={3}>{value}</XDSHeading>
+ </XDSVStack>
+ </div>
+ );
+}
+
+function EfficiencyMetricsCard({
byPrompt,
}: {
byPrompt: ReportData['universal']['byPrompt'];
}) {
- // Aggregate conciseness metrics across prompts
const entries = Object.values(byPrompt);
if (entries.length === 0) return null;
- const totals = entries.reduce(
- (acc, score) => {
- const m = score.conciseness.metrics;
- return {
- totalLines: acc.totalLines + m.totalLines,
- codeLines: acc.codeLines + m.codeLines,
- stylingRatio: acc.stylingRatio + m.stylingRatio,
- boilerplateRatio: acc.boilerplateRatio + m.boilerplateRatio,
- count: acc.count + 1,
- };
- },
- {
- totalLines: 0,
- codeLines: 0,
- stylingRatio: 0,
- boilerplateRatio: 0,
- count: 0,
- },
+ const metrics = entries.map(s => s.efficiency.metrics).filter(Boolean);
+ if (metrics.length === 0) return null;
+
+ const avgDecisions =
+ metrics.reduce((s, m) => s + (m?.decisionsPerElement ?? 0), 0) /
+ metrics.length;
+ const avgLines =
+ metrics.reduce((s, m) => s + (m?.codeLines ?? 0), 0) / metrics.length;
+ const avgStylingRatio =
+ metrics.reduce((s, m) => s + (m?.stylingRatio ?? 0), 0) / metrics.length;
+ const avgBoilerplate =
+ metrics.reduce((s, m) => s + (m?.boilerplateRatio ?? 0), 0) /
+ metrics.length;
+
+ return (
+ <XDSCard>
+ <div {...stylex.props(styles.metricsCard)}>
+ <XDSVStack gap="space2">
+ <XDSHeading level={4}>Efficiency Metrics</XDSHeading>
+ <div {...stylex.props(styles.metricsGrid)}>
+ <MetricValue
+ label="Decisions / Element"
+ value={avgDecisions.toFixed(1)}
+ />
+ <MetricValue
+ label="Avg Code Lines"
+ value={String(Math.round(avgLines))}
+ />
+ <MetricValue
+ label="Styling Ratio"
+ value={`${(avgStylingRatio * 100).toFixed(1)}%`}
+ />
+ <MetricValue
+ label="Boilerplate"
+ value={`${(avgBoilerplate * 100).toFixed(1)}%`}
+ />
+ </div>
+ </XDSVStack>
+ </div>
+ </XDSCard>
);
+}
+
+function MaintainabilityMetricsCard({
+ byPrompt,
+}: {
+ byPrompt: ReportData['universal']['byPrompt'];
+}) {
+ const entries = Object.values(byPrompt);
+ if (entries.length === 0) return null;
+
+ const metrics = entries.map(s => s.maintainability.metrics).filter(Boolean);
+ if (metrics.length === 0) return null;
- const avgStyling = (totals.stylingRatio / totals.count) * 100;
- const avgBoilerplate = (totals.boilerplateRatio / totals.count) * 100;
+ const avgSemantic =
+ metrics.reduce((s, m) => s + (m?.semanticRatio ?? 0), 0) / metrics.length;
+ const totalMagic = metrics.reduce((s, m) => s + (m?.magicValueCount ?? 0), 0);
+ const darkModeCount = metrics.filter(m => m?.darkModeSupport).length;
return (
<XDSCard>
<div {...stylex.props(styles.metricsCard)}>
- <XDSVStack gap="space3">
- <XDSHeading level={4}>Conciseness Metrics</XDSHeading>
+ <XDSVStack gap="space2">
+ <XDSHeading level={4}>Maintainability Metrics</XDSHeading>
<div {...stylex.props(styles.metricsGrid)}>
- <div {...stylex.props(styles.metricItem)}>
- <XDSVStack gap="space1">
- <XDSText type="label">Total Lines</XDSText>
- <XDSHeading level={3}>{totals.totalLines}</XDSHeading>
- </XDSVStack>
- </div>
- <div {...stylex.props(styles.metricItem)}>
- <XDSVStack gap="space1">
- <XDSText type="label">Code Lines</XDSText>
- <XDSHeading level={3}>{totals.codeLines}</XDSHeading>
- </XDSVStack>
- </div>
- <div {...stylex.props(styles.metricItem)}>
- <XDSVStack gap="space1">
- <XDSText type="label">Styling Ratio</XDSText>
- <XDSHeading level={3}>{avgStyling.toFixed(1)}%</XDSHeading>
- </XDSVStack>
- </div>
- <div {...stylex.props(styles.metricItem)}>
- <XDSVStack gap="space1">
- <XDSText type="label">Boilerplate Ratio</XDSText>
- <XDSHeading level={3}>{avgBoilerplate.toFixed(1)}%</XDSHeading>
- </XDSVStack>
- </div>
+ <MetricValue
+ label="Semantic Ratio"
+ value={`${(avgSemantic * 100).toFixed(0)}%`}
+ />
+ <MetricValue label="Magic Values" value={String(totalMagic)} />
+ <MetricValue
+ label="Dark Mode"
+ value={`${darkModeCount}/${entries.length}`}
+ />
+ </div>
+ </XDSVStack>
+ </div>
+ </XDSCard>
+ );
+}
+
+function CostMetricsCard({cost}: {cost: ReportData['universal']['cost']}) {
+ if (!cost) return null;
+
+ return (
+ <XDSCard>
+ <div {...stylex.props(styles.metricsCard)}>
+ <XDSVStack gap="space2">
+ <XDSHeading level={4}>Cost</XDSHeading>
+ <div {...stylex.props(styles.metricsGrid)}>
+ {cost.avgDurationMs > 0 && (
+ <MetricValue
+ label="Avg Duration"
+ value={`${(cost.avgDurationMs / 1000).toFixed(1)}s`}
+ />
+ )}
+ <MetricValue
+ label="Avg Output"
+ value={`${cost.avgOutputLines} lines`}
+ />
+ <MetricValue label="Docs Read" value={String(cost.avgDocsRead)} />
+ <MetricValue
+ label="Input Tokens"
+ value={`~${cost.estimatedInputTokens.toLocaleString()}`}
+ />
+ <MetricValue
+ label="Output Tokens"
+ value={`~${cost.estimatedOutputTokens.toLocaleString()}`}
+ />
</div>
</XDSVStack>
</div>
@@ -134,10 +200,6 @@ export function Report() {
const hasScreenshots =
data?.screenshots && Object.keys(data.screenshots).length > 0;
- const toggleTheme = () => {
- setThemeMode(m => (m === 'light' ? 'dark' : 'light'));
- };
-
if (!data) {
return (
<XDSTheme theme={defaultTheme} mode={themeMode}>
@@ -164,7 +226,7 @@ export function Report() {
<XDSTheme theme={defaultTheme} mode={themeMode}>
<div {...stylex.props(styles.root)}>
<div {...stylex.props(styles.container)}>
- <XDSVStack gap="space6">
+ <XDSVStack gap="space5">
{/* Header */}
<div {...stylex.props(styles.header)}>
<XDSHStack gap="space4" hAlign="between" vAlign="center">
@@ -179,7 +241,11 @@ export function Report() {
</XDSText>
)}
</XDSVStack>
- <XDSButton variant="secondary" onClick={toggleTheme}>
+ <XDSButton
+ variant="secondary"
+ onClick={() =>
+ setThemeMode(m => (m === 'light' ? 'dark' : 'light'))
+ }>
{themeMode === 'light' ? '🌙 Dark' : '☀️ Light'}
</XDSButton>
</XDSHStack>
@@ -197,7 +263,7 @@ export function Report() {
{/* Tab Content */}
<div {...stylex.props(styles.tabContent)}>
{activeTab === 'overview' && (
- <XDSVStack gap="space6">
+ <XDSVStack gap="space4">
{/* Overall score */}
<ScoreCard
label="Overall Score"
@@ -222,20 +288,10 @@ export function Report() {
</div>
</XDSVStack>
- {/* Dark mode rate */}
- <XDSCard>
- <div {...stylex.props(styles.metricsCard)}>
- <XDSHStack gap="space4" hAlign="center">
- <XDSText type="label">Dark Mode Support Rate</XDSText>
- <XDSHeading level={3}>
- {(universal.darkModeRate * 100).toFixed(0)}%
- </XDSHeading>
- </XDSHStack>
- </div>
- </XDSCard>
-
- {/* Conciseness metrics */}
- <ConcisenessMetricsCard byPrompt={universal.byPrompt} />
+ {/* Sub-metrics */}
+ <EfficiencyMetricsCard byPrompt={universal.byPrompt} />
+ <MaintainabilityMetricsCard byPrompt={universal.byPrompt} />
+ <CostMetricsCard cost={universal.cost} />
{/* Comparison view */}
{comparison && (
@@ -250,25 +306,7 @@ export function Report() {
)}
{activeTab === 'byPrompt' && (
- <DimensionTable
- byPrompt={universal.byPrompt}
- categories={
- Object.keys(universal.byCategory).length > 0
- ? Object.entries(universal.byPrompt).reduce(
- (acc, [promptId]) => {
- // Find category for this prompt by checking byCategory
- for (const [cat, dims] of Object.entries(
- universal.byCategory,
- )) {
- acc[promptId] = acc[promptId] ?? cat;
- }
- return acc;
- },
- {} as Record<string, string>,
- )
- : undefined
- }
- />
+ <DimensionTable byPrompt={universal.byPrompt} />
)}
{activeTab === 'screenshots' && screenshots && (
diff --git a/internal/vibe-tests/app/src/report/ScoreCard.tsx b/internal/vibe-tests/app/src/report/ScoreCard.tsx
index 94dd13dc14..ab8a1ec882 100644
--- a/internal/vibe-tests/app/src/report/ScoreCard.tsx
+++ b/internal/vibe-tests/app/src/report/ScoreCard.tsx
@@ -10,7 +10,7 @@ import {formatScore, scoreToProgressVariant} from './utils';
const styles = stylex.create({
card: {
- padding: spacingVars['--spacing-4'],
+ padding: spacingVars['--spacing-3'],
},
deltaPositive: {
color: colorVars['--color-positive'],
diff --git a/internal/vibe-tests/app/src/report/types.ts b/internal/vibe-tests/app/src/report/types.ts
index 00990121fa..e6e3d48bfd 100644
--- a/internal/vibe-tests/app/src/report/types.ts
+++ b/internal/vibe-tests/app/src/report/types.ts
@@ -3,7 +3,9 @@ import type {
UniversalAggregate,
UniversalComparison,
UniversalDimension,
- ConcisenessMetrics,
+ EfficiencyMetrics,
+ MaintainabilityMetrics,
+ CostMetrics,
DimensionScore,
UniversalFinding,
} from '../../../src/types';
@@ -13,7 +15,9 @@ export type {
UniversalAggregate,
UniversalComparison,
UniversalDimension,
- ConcisenessMetrics,
+ EfficiencyMetrics,
+ MaintainabilityMetrics,
+ CostMetrics,
DimensionScore,
UniversalFinding,
};
diff --git a/internal/vibe-tests/app/src/report/utils.ts b/internal/vibe-tests/app/src/report/utils.ts
index 8d0c8b3357..4b272de496 100644
--- a/internal/vibe-tests/app/src/report/utils.ts
+++ b/internal/vibe-tests/app/src/report/utils.ts
@@ -1,21 +1,19 @@
import type {UniversalDimension, UniversalScore} from './types';
export const ALL_DIMENSIONS: UniversalDimension[] = [
+ 'correctness',
'accessibility',
'codeQuality',
- 'repetition',
- 'conciseness',
- 'themeAdherence',
- 'correctness',
+ 'efficiency',
+ 'maintainability',
];
export const DIMENSION_LABELS: Record<UniversalDimension, string> = {
+ correctness: 'Correctness',
accessibility: 'Accessibility',
codeQuality: 'Code Quality',
- repetition: 'DRYness',
- conciseness: 'Conciseness',
- themeAdherence: 'Theme Adherence',
- correctness: 'Correctness',
+ efficiency: 'Efficiency',
+ maintainability: 'Maintainability',
};
export function scoreToStatusVariant(
@@ -37,10 +35,10 @@ export function scoreToProgressVariant(
}
export function computeOverall(score: UniversalScore): number {
- const values = ALL_DIMENSIONS.map(d => score[d].score);
- return values.reduce((sum, v) => sum + v, 0) / values.length;
+ const total = ALL_DIMENSIONS.reduce((sum, d) => sum + score[d].score, 0);
+ return Math.round(total / ALL_DIMENSIONS.length);
}
export function formatScore(n: number): string {
- return n.toFixed(1);
+ return String(Math.round(n));
}
diff --git a/internal/vibe-tests/app/vite.config.ts b/internal/vibe-tests/app/vite.config.ts
index e72f04a0e1..0d88aa0e46 100644
--- a/internal/vibe-tests/app/vite.config.ts
+++ b/internal/vibe-tests/app/vite.config.ts
@@ -3,6 +3,7 @@ import {fileURLToPath} from 'url';
import {defineConfig} from 'vite';
import react from '@vitejs/plugin-react';
import stylex from '@stylexjs/unplugin';
+import {viteSingleFile} from 'vite-plugin-singlefile';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(__dirname, '../../..');
@@ -18,18 +19,38 @@ export default defineConfig({
rootDir: repoRoot,
},
aliases: {
- '@xds/core/theme/tokens.stylex': path.resolve(repoRoot, 'packages/core/src/theme/tokens.stylex.ts'),
+ '@xds/core/theme/tokens.stylex': path.resolve(
+ repoRoot,
+ 'packages/core/src/theme/tokens.stylex.ts',
+ ),
},
}),
react(),
+ viteSingleFile(),
],
resolve: {
alias: {
- '@xds/core/theme/tokens.stylex': path.resolve(repoRoot, 'packages/core/src/theme/tokens.stylex.ts'),
+ '@xds/core/theme/tokens.stylex': path.resolve(
+ repoRoot,
+ 'packages/core/src/theme/tokens.stylex.ts',
+ ),
'@xds/core': path.resolve(repoRoot, 'packages/core/src'),
- '@xds/theme/default': path.resolve(repoRoot, 'packages/theme/src/default'),
- '@xds/theme/neutral': path.resolve(repoRoot, 'packages/theme/src/neutral'),
- '@xds/theme': path.resolve(repoRoot, 'packages/theme/src'),
+ '@xds/theme-default': path.resolve(
+ repoRoot,
+ 'packages/themes/default/src',
+ ),
+ '@xds/theme/default': path.resolve(
+ repoRoot,
+ 'packages/themes/default/src',
+ ),
+ '@xds/theme-neutral': path.resolve(
+ repoRoot,
+ 'packages/themes/neutral/src',
+ ),
+ '@xds/theme/neutral': path.resolve(
+ repoRoot,
+ 'packages/themes/neutral/src',
+ ),
},
},
server: {port: 5173, strictPort: true},
diff --git a/internal/vibe-tests/package.json b/internal/vibe-tests/package.json
index 07db19870d..a056471c6d 100644
--- a/internal/vibe-tests/package.json
+++ b/internal/vibe-tests/package.json
@@ -24,7 +24,12 @@
"analyze": "tsx src/cli.ts analyze",
"quality": "tsx src/quality.ts",
"compare": "tsx src/compare.ts",
- "generate-a11y-manifest": "tsx src/generate-a11y-manifest.ts"
+ "generate-a11y-manifest": "tsx src/generate-a11y-manifest.ts",
+ "universal": "tsx src/universal-aggregate.ts",
+ "universal:compare": "tsx src/universal-compare.ts",
+ "report:build": "tsx src/build-report.ts",
+ "report:dev": "tsx src/build-report.ts --dev",
+ "report:deploy": "tsx src/deploy-report.ts"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.52.0",
diff --git a/internal/vibe-tests/scripts/publish-report.sh b/internal/vibe-tests/scripts/publish-report.sh
new file mode 100644
index 0000000000..d2856faff4
--- /dev/null
+++ b/internal/vibe-tests/scripts/publish-report.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Publish a vibe test report to GitHub Pages.
+#
+# Usage:
+# ./scripts/publish-report.sh <xds-iteration> [baseline-iteration]
+#
+# The report is always published under reports/<xds-iteration>/ on gh-pages.
+# No --slug, no --name, no override — just the hash.
+#
+# Examples:
+# ./scripts/publish-report.sh a87fa578
+# ./scripts/publish-report.sh a87fa578 446469c7
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+VIBE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+if [ $# -lt 1 ]; then
+ echo "Usage: $0 <xds-iteration> [baseline-iteration]"
+ echo ""
+ echo "Builds and deploys a vibe test report to GitHub Pages."
+ echo "Report URL: https://facebookexperimental.github.io/xds/reports/<xds-iteration>/"
+ exit 1
+fi
+
+XDS_ITER="$1"
+BASELINE_ITER="${2:-}"
+
+ARGS="--iteration $XDS_ITER"
+if [ -n "$BASELINE_ITER" ]; then
+ ARGS="$ARGS --baseline $BASELINE_ITER"
+fi
+
+echo ""
+echo "📊 Publishing vibe test report"
+echo " XDS iteration: $XDS_ITER"
+if [ -n "$BASELINE_ITER" ]; then
+ echo " Baseline iteration: $BASELINE_ITER"
+fi
+echo " Deploy to: reports/$XDS_ITER/"
+echo ""
+
+cd "$VIBE_DIR"
+exec npx tsx src/deploy-report.ts $ARGS
diff --git a/internal/vibe-tests/src/build-report.ts b/internal/vibe-tests/src/build-report.ts
index 70dcb381ba..19c7819532 100644
--- a/internal/vibe-tests/src/build-report.ts
+++ b/internal/vibe-tests/src/build-report.ts
@@ -1,8 +1,10 @@
#!/usr/bin/env node
/**
- * @file Build pipeline: eval → data injection → Vite build → static HTML
+ * @file Build pipeline: eval → Vite build → inject data into HTML
*
- * Orchestrates the full report generation pipeline.
+ * Builds the report app, then injects evaluation data directly into
+ * the HTML as a <script> tag. This guarantees the data is available
+ * before React renders — no module import race conditions.
*
* Usage:
* tsx src/build-report.ts --iteration <id>
@@ -14,7 +16,7 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import {execSync} from 'node:child_process';
-import {getResultsDir, readJson, writeJson, ensureDir} from './utils.js';
+import {getResultsDir, readJson, ensureDir} from './utils.js';
import type {UniversalAggregate, UniversalComparison} from './types.js';
const APP_DIR = path.join(import.meta.dirname, '..', 'app');
@@ -91,49 +93,62 @@ function ensureComparison(
return readJson<UniversalComparison>(comparisonPath);
}
-function generateReportData(opts: {
+/**
+ * Build the report data as a <script> tag to inject into the HTML.
+ * This runs before the app bundle, so window.__REPORT_DATA__ is
+ * guaranteed to exist when React renders.
+ */
+function buildDataScript(opts: {
iteration: string;
iterationData: UniversalAggregate;
baseline?: string;
comparison?: UniversalComparison;
manifest: unknown;
- screenshotPaths?: string[];
}): string {
- const lines: string[] = [
- '// AUTO-GENERATED — do not edit',
- '// Generated by build-report.ts',
- '',
- `export const iterationId = ${JSON.stringify(opts.iteration)};`,
- '',
- `export const iterationData = ${JSON.stringify(opts.iterationData, null, 2)} as const;`,
- '',
- `export const manifest = ${JSON.stringify(opts.manifest, null, 2)} as const;`,
- '',
- ];
-
- if (opts.baseline && opts.comparison) {
- lines.push(`export const baselineId = ${JSON.stringify(opts.baseline)};`);
- lines.push('');
- lines.push(
- `export const comparison = ${JSON.stringify(opts.comparison, null, 2)} as const;`,
- );
- lines.push('');
- } else {
- lines.push('export const baselineId: string | undefined = undefined;');
- lines.push('export const comparison: unknown = undefined;');
- lines.push('');
- }
+ const reportData = {
+ universal: opts.iterationData,
+ comparison: opts.comparison ?? undefined,
+ iterationId: opts.iteration,
+ target: (opts.manifest as {config?: {target?: string}})?.config?.target,
+ };
+
+ return `<script>window.__REPORT_DATA__=${JSON.stringify(reportData)};</script>`;
+}
- if (opts.screenshotPaths && opts.screenshotPaths.length > 0) {
- lines.push(
- `export const screenshotPaths = ${JSON.stringify(opts.screenshotPaths)} as const;`,
- );
+/**
+ * Inject the data script into the built HTML, before the first <script> tag
+ * so it runs before the app bundle.
+ */
+function injectDataIntoHtml(htmlPath: string, dataScript: string): void {
+ let html = fs.readFileSync(htmlPath, 'utf-8');
+ const firstScript = html.indexOf('<script');
+ if (firstScript === -1) {
+ html = html.replace('</body>', `${dataScript}\n</body>`);
} else {
- lines.push('export const screenshotPaths: string[] = [];');
+ html =
+ html.slice(0, firstScript) + dataScript + '\n' + html.slice(firstScript);
}
+ fs.writeFileSync(htmlPath, html);
+}
- lines.push('');
- return lines.join('\n');
+/**
+ * Inline any CSS files from dist/assets/ that vite-plugin-singlefile
+ * didn't handle (e.g. StyleX extracted CSS).
+ */
+function inlineCss(htmlPath: string, distDir: string): void {
+ const assetsDir = path.join(distDir, 'assets');
+ if (!fs.existsSync(assetsDir)) return;
+
+ const cssFiles = fs.readdirSync(assetsDir).filter(f => f.endsWith('.css'));
+ if (cssFiles.length === 0) return;
+
+ console.log(` Inlining ${cssFiles.length} CSS file(s)...`);
+ let html = fs.readFileSync(htmlPath, 'utf-8');
+ for (const cssFile of cssFiles) {
+ const css = fs.readFileSync(path.join(assetsDir, cssFile), 'utf-8');
+ html = html.replace('</head>', `<style>${css}</style>\n</head>`);
+ }
+ fs.writeFileSync(htmlPath, html);
}
async function main() {
@@ -161,37 +176,33 @@ async function main() {
comparison = ensureComparison(iteration, baseline);
}
- // Step 3: Collect screenshot paths
- let screenshotPaths: string[] = [];
- if (withScreenshots) {
- const screenshotsDir = path.join(iterDir, 'screenshots');
- if (fs.existsSync(screenshotsDir)) {
- screenshotPaths = fs
- .readdirSync(screenshotsDir)
- .filter(f => f.endsWith('.png'))
- .sort();
- }
- }
-
- // Step 4: Generate report data module
- console.log('📝 Step 3: Generating report data module...');
- const dataDir = path.join(APP_DIR, 'src', 'data');
- ensureDir(dataDir);
-
- const reportDataContent = generateReportData({
+ // Step 3: Build the data script
+ console.log('📝 Step 3: Preparing report data...');
+ const dataScript = buildDataScript({
iteration,
iterationData,
baseline,
comparison,
manifest,
- screenshotPaths,
});
+ console.log(` ✓ ${(dataScript.length / 1024).toFixed(0)} KB of report data`);
- fs.writeFileSync(path.join(dataDir, 'report-data.ts'), reportDataContent);
- console.log(' ✓ app/src/data/report-data.ts');
-
- // Step 5: Dev mode or build
+ // Step 4: Dev mode or build
if (dev) {
+ // For dev mode, write data to a module file so Vite can serve it
+ const dataDir = path.join(APP_DIR, 'src', 'data');
+ ensureDir(dataDir);
+ const reportData = {
+ universal: iterationData,
+ comparison: comparison ?? undefined,
+ iterationId: iteration,
+ target: (manifest as {config?: {target?: string}})?.config?.target,
+ };
+ fs.writeFileSync(
+ path.join(dataDir, 'report-data.ts'),
+ `// AUTO-GENERATED for dev mode\n(window as any).__REPORT_DATA__ = ${JSON.stringify(reportData, null, 2)};\n`,
+ );
+
console.log('\n🚀 Starting Vite dev server...');
const {createServer} = await import('vite');
const server = await createServer({
@@ -209,10 +220,14 @@ async function main() {
stdio: 'inherit',
});
- // Copy built report to results dir
+ // Step 5: Inject data and styles into the built HTML
const distDir = path.join(APP_DIR, 'dist');
const reportHtml = path.join(distDir, 'index.html');
if (fs.existsSync(reportHtml)) {
+ console.log('💉 Step 5: Injecting report data and styles...');
+ injectDataIntoHtml(reportHtml, dataScript);
+ inlineCss(reportHtml, distDir);
+
const destPath = path.join(iterDir, 'report.html');
fs.copyFileSync(reportHtml, destPath);
console.log(`\n✅ Report saved to: ${destPath}`);
diff --git a/internal/vibe-tests/src/deploy-report.ts b/internal/vibe-tests/src/deploy-report.ts
new file mode 100644
index 0000000000..e8970867cb
--- /dev/null
+++ b/internal/vibe-tests/src/deploy-report.ts
@@ -0,0 +1,214 @@
+#!/usr/bin/env node
+/**
+ * @file Deploy report to GitHub Pages
+ *
+ * Builds a static report HTML and pushes it to the gh-pages branch
+ * under reports/{iteration-id}/. The path is always the iteration hash —
+ * no overrides, no custom slugs.
+ *
+ * Usage:
+ * tsx src/deploy-report.ts --iteration <id>
+ * tsx src/deploy-report.ts --iteration <id> --baseline <id>
+ * tsx src/deploy-report.ts --iteration <id> --dry-run
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {execSync} from 'node:child_process';
+
+const REPO_ROOT = path.resolve(import.meta.dirname, '../../..');
+const VIBE_DIR = path.resolve(import.meta.dirname, '..');
+
+function parseArgs(): {
+ iteration: string;
+ baseline?: string;
+ dryRun: boolean;
+} {
+ const args = process.argv.slice(2);
+ let iteration = '';
+ let baseline: string | undefined;
+ let dryRun = false;
+
+ for (let i = 0; i < args.length; i++) {
+ if (args[i] === '--iteration' && args[i + 1]) {
+ iteration = args[++i];
+ } else if (args[i] === '--baseline' && args[i + 1]) {
+ baseline = args[++i];
+ } else if (args[i] === '--dry-run') {
+ dryRun = true;
+ }
+ }
+
+ if (!iteration) {
+ console.error(
+ 'Usage: tsx src/deploy-report.ts --iteration <id> [--baseline <id>] [--dry-run]',
+ );
+ process.exit(1);
+ }
+
+ return {iteration, baseline, dryRun};
+}
+
+function run(cmd: string, opts?: {cwd?: string; silent?: boolean}): string {
+ const cwd = opts?.cwd ?? REPO_ROOT;
+ const stdio = opts?.silent ? 'pipe' : 'inherit';
+ return execSync(cmd, {cwd, stdio, encoding: 'utf-8'}) ?? '';
+}
+
+function runSilent(cmd: string, opts?: {cwd?: string}): string {
+ return run(cmd, {...opts, silent: true}).trim();
+}
+
+async function main() {
+ const {iteration, baseline, dryRun} = parseArgs();
+
+ const deployPath = `reports/${iteration}`;
+
+ console.log(`\n📦 Deploy Report Pipeline`);
+ console.log(` Iteration: ${iteration}`);
+ if (baseline) console.log(` Baseline: ${baseline}`);
+ console.log(` Deploy to: ${deployPath}`);
+ if (dryRun) console.log(` ⚠️ DRY RUN — will build but not push`);
+ console.log('');
+
+ // Step 1: Build the report
+ console.log('🏗️ Step 1: Building report...');
+ const buildArgs = [`--iteration ${iteration}`];
+ if (baseline) buildArgs.push(`--baseline ${baseline}`);
+
+ run(`npx tsx src/build-report.ts ${buildArgs.join(' ')}`, {cwd: VIBE_DIR});
+
+ const reportHtml = path.join(VIBE_DIR, 'results', iteration, 'report.html');
+ if (!fs.existsSync(reportHtml)) {
+ console.error('❌ Report build failed — no report.html found');
+ process.exit(1);
+ }
+
+ const reportSize = fs.statSync(reportHtml).size;
+ console.log(` ✓ report.html (${(reportSize / 1024).toFixed(0)} KB)\n`);
+
+ if (dryRun) {
+ console.log('✅ Dry run complete. Report built but not deployed.');
+ console.log(` File: ${reportHtml}`);
+ return;
+ }
+
+ // Step 2: Deploy to gh-pages
+ console.log('🚀 Step 2: Deploying to gh-pages...');
+
+ const tmpDir = path.join(REPO_ROOT, '.deploy-tmp');
+ if (fs.existsSync(tmpDir)) {
+ fs.rmSync(tmpDir, {recursive: true});
+ }
+ fs.mkdirSync(tmpDir, {recursive: true});
+
+ try {
+ const remoteUrl = runSilent('git remote get-url origin', {cwd: REPO_ROOT});
+
+ run(
+ `git clone --depth 1 --branch gh-pages --single-branch ${remoteUrl} ${tmpDir}`,
+ {cwd: REPO_ROOT},
+ );
+
+ const targetDir = path.join(tmpDir, deployPath);
+ fs.mkdirSync(targetDir, {recursive: true});
+ fs.copyFileSync(reportHtml, path.join(targetDir, 'index.html'));
+
+ updateReportsIndex(tmpDir);
+
+ run('git add .', {cwd: tmpDir});
+
+ const status = runSilent('git status --porcelain', {cwd: tmpDir});
+ if (!status) {
+ console.log(' ℹ️ No changes to deploy (report already exists)');
+ return;
+ }
+
+ const commitMsg = baseline
+ ? `report: ${iteration} (vs ${baseline})`
+ : `report: ${iteration}`;
+
+ run(`git commit -m "${commitMsg}"`, {cwd: tmpDir});
+ run('git push origin gh-pages', {cwd: tmpDir});
+
+ const repoUrl = getRepoUrl();
+ const reportUrl = repoUrl
+ ? `${repoUrl}/${deployPath}/`
+ : `(gh-pages)/${deployPath}/`;
+
+ console.log(`\n✅ Deployed!`);
+ console.log(` 📊 ${reportUrl}`);
+ } finally {
+ fs.rmSync(tmpDir, {recursive: true, force: true});
+ }
+}
+
+function updateReportsIndex(ghPagesDir: string): void {
+ const reportsDir = path.join(ghPagesDir, 'reports');
+ if (!fs.existsSync(reportsDir)) return;
+
+ const entries = fs
+ .readdirSync(reportsDir, {withFileTypes: true})
+ .filter(e => e.isDirectory())
+ .map(e => e.name)
+ .sort()
+ .reverse();
+
+ const html = `<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>XDS Vibe Test Reports</title>
+ <style>
+ body {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+ max-width: 600px;
+ margin: 40px auto;
+ padding: 0 20px;
+ color: #1a1a1a;
+ }
+ h1 { font-size: 1.5rem; margin-bottom: 0.5rem; }
+ p { color: #666; margin-bottom: 2rem; }
+ ul { list-style: none; padding: 0; }
+ li { margin-bottom: 0.5rem; }
+ a {
+ color: #0969da;
+ text-decoration: none;
+ padding: 8px 12px;
+ display: inline-block;
+ border-radius: 6px;
+ transition: background 0.15s;
+ }
+ a:hover { background: #f0f4f8; }
+ </style>
+</head>
+<body>
+ <h1>📊 XDS Vibe Test Reports</h1>
+ <p>Evaluation reports comparing XDS, baseline, and raw HTML targets.</p>
+ <ul>
+ ${entries.map(name => `<li><a href="${name}/">${name}</a></li>`).join('\n ')}
+ </ul>
+</body>
+</html>`;
+
+ fs.writeFileSync(path.join(reportsDir, 'index.html'), html);
+}
+
+function getRepoUrl(): string | null {
+ try {
+ const remote = runSilent('git remote get-url origin', {cwd: REPO_ROOT});
+ const match = remote.match(/github\.com[:/]([^/]+)\/([^/.]+)/);
+ if (match) {
+ return `https://${match[1]}.github.io/${match[2]}`;
+ }
+ } catch {
+ // ignore
+ }
+ return null;
+}
+
+main().catch(err => {
+ console.error(err);
+ process.exit(1);
+});
diff --git a/internal/vibe-tests/src/interactive.ts b/internal/vibe-tests/src/interactive.ts
index baf23e2a20..a31411ae88 100644
--- a/internal/vibe-tests/src/interactive.ts
+++ b/internal/vibe-tests/src/interactive.ts
@@ -904,7 +904,7 @@ function createTaskManifest(
resultsDir,
);
- const task: AgentTask & {subagentPrompt: string} = {
+ const task: AgentTask & {subagentPrompt: string; createdAt: string} = {
promptId: prompt.id,
category: prompt.category,
prompt: prompt.prompt,
@@ -914,6 +914,7 @@ function createTaskManifest(
degradation: config.degradation,
target: config.target,
subagentPrompt,
+ createdAt: new Date().toISOString(),
};
writeJson(path.join(tasksDir, `${prompt.id}.json`), task);
}
@@ -1023,7 +1024,8 @@ For each test, the subagent should:
5. Track which doc files were read during the task
6. Write result to individual file: ${resultsDir}/results/{promptId}.json
- Include a "docsRead" array listing all doc files read (e.g., ["AGENTS.md", "Button.md", "tokens.md"])
- - This is required for accurate token usage tracking
+ - Include "completedAt" with the current ISO timestamp (new Date().toISOString())
+ - These are required for accurate cost tracking
(Use individual files to avoid parallel write conflicts)
### After All Tests Complete
diff --git a/internal/vibe-tests/src/types.ts b/internal/vibe-tests/src/types.ts
index 513bd41f69..b2e33bff99 100644
--- a/internal/vibe-tests/src/types.ts
+++ b/internal/vibe-tests/src/types.ts
@@ -255,3 +255,103 @@ export interface QualityAssessment {
overallScore: QualityScore;
summary: string;
}
+
+// ============================================================
+// Universal Evaluation Types (target-neutral, 5 dimensions)
+// ============================================================
+
+export type UniversalDimension =
+ | 'correctness'
+ | 'accessibility'
+ | 'codeQuality'
+ | 'efficiency'
+ | 'maintainability';
+
+export interface UniversalFinding {
+ rule: string;
+ severity?: 'critical' | 'moderate' | 'minor';
+ detail: string;
+ line?: number;
+ count?: number;
+ example?: string;
+}
+
+export interface EfficiencyMetrics {
+ totalLines: number;
+ codeLines: number;
+ stylingLines: number;
+ boilerplateLines: number;
+ logicLines: number;
+ stylingRatio: number;
+ boilerplateRatio: number;
+ decisionsPerElement: number;
+ elementCount: number;
+ stylingDecisionCount: number;
+ duplicateRatio: number;
+}
+
+export interface MaintainabilityMetrics {
+ semanticRatio: number;
+ magicValueCount: number;
+ semanticValueCount: number;
+ avgStateSpread: number;
+ darkModeSupport: boolean;
+}
+
+export interface DimensionScore<M = undefined> {
+ score: number;
+ findings?: UniversalFinding[];
+ metrics?: M;
+}
+
+export interface UniversalScore {
+ correctness: DimensionScore;
+ accessibility: DimensionScore;
+ codeQuality: DimensionScore;
+ efficiency: DimensionScore<EfficiencyMetrics>;
+ maintainability: DimensionScore<MaintainabilityMetrics>;
+}
+
+export interface CostMetrics {
+ totalDurationMs: number;
+ avgDurationMs: number;
+ avgOutputChars: number;
+ avgOutputLines: number;
+ avgDocsRead: number;
+ estimatedInputTokens: number;
+ estimatedOutputTokens: number;
+ byPrompt: Record<
+ string,
+ {
+ durationMs: number;
+ outputChars: number;
+ outputLines: number;
+ docsRead: string[];
+ estimatedInputTokens: number;
+ estimatedOutputTokens: number;
+ }
+ >;
+}
+
+export interface UniversalAggregate {
+ averages: Record<UniversalDimension, number>;
+ overall: number;
+ byPrompt: Record<string, UniversalScore>;
+ byCategory: Record<string, Record<UniversalDimension, number>>;
+ darkModeRate: number;
+ cost?: CostMetrics;
+}
+
+export interface UniversalComparison {
+ xds: UniversalAggregate;
+ baseline: UniversalAggregate;
+ winners: Record<UniversalDimension, 'xds' | 'baseline' | 'tie'>;
+ byPrompt: Record<
+ string,
+ {
+ xds: UniversalScore;
+ baseline: UniversalScore;
+ winner: 'xds' | 'baseline' | 'tie';
+ }
+ >;
+}
diff --git a/internal/vibe-tests/src/universal-aggregate.ts b/internal/vibe-tests/src/universal-aggregate.ts
index a7a2a2e73b..65e8ac1377 100644
--- a/internal/vibe-tests/src/universal-aggregate.ts
+++ b/internal/vibe-tests/src/universal-aggregate.ts
@@ -138,28 +138,45 @@ async function main() {
}
// --- Cost data ---
- // Duration: infer from file timestamps (task creation → result write)
let durationMs = 0;
- const taskPath = path.join(iterDir, 'tasks', `${promptId}.json`);
- if (fs.existsSync(taskPath)) {
- const taskStat = fs.statSync(taskPath);
- const resultStat = fs.statSync(codePath);
- const inferred = resultStat.mtimeMs - taskStat.mtimeMs;
- if (inferred > 0) durationMs = Math.round(inferred);
- }
-
- // Docs read: from companion .json file
let docsRead: string[] = [];
+
+ // Read result metadata (docsRead, completedAt)
const jsonPath = path.join(codeDir, `${promptId}.json`);
+ let completedAt: string | undefined;
if (fs.existsSync(jsonPath)) {
try {
const meta = JSON.parse(fs.readFileSync(jsonPath, 'utf-8'));
docsRead = meta.docsRead || [];
+ completedAt = meta.completedAt;
} catch {
// ignore parse errors
}
}
+ // Duration: prefer createdAt/completedAt timestamps, fall back to file mtimes
+ const taskPath = path.join(iterDir, 'tasks', `${promptId}.json`);
+ if (fs.existsSync(taskPath)) {
+ try {
+ const taskMeta = JSON.parse(fs.readFileSync(taskPath, 'utf-8'));
+ if (taskMeta.createdAt && completedAt) {
+ const start = new Date(taskMeta.createdAt).getTime();
+ const end = new Date(completedAt).getTime();
+ if (end > start) durationMs = Math.round(end - start);
+ }
+ } catch {
+ // ignore
+ }
+
+ // Fall back to file timestamps if no explicit timestamps
+ if (durationMs === 0) {
+ const taskStat = fs.statSync(taskPath);
+ const resultStat = fs.statSync(codePath);
+ const inferred = resultStat.mtimeMs - taskStat.mtimeMs;
+ if (inferred > 0) durationMs = Math.round(inferred);
+ }
+ }
+
// Estimate input tokens from docs read
let estimatedInputChars = 0;
for (const doc of docsRead) {
diff --git a/package.json b/package.json
index a1ba5b94fe..7a5c12a596 100644
--- a/package.json
+++ b/package.json
@@ -46,6 +46,7 @@
"tsup": "^8.3.0",
"typescript": "^5.7.0",
"typescript-eslint": "^8.53.1",
+ "vite-plugin-singlefile": "^2.3.0",
"vitest": "^2.1.0"
},
"resolutions": {
diff --git a/yarn.lock b/yarn.lock
index 59925ad9fc..27d5f72bd0 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5116,6 +5116,13 @@ [email protected]:
pathe "^1.1.2"
vite "^5.0.0"
+vite-plugin-singlefile@^2.3.0:
+ version "2.3.0"
+ resolved "https://registry.yarnpkg.com/vite-plugin-singlefile/-/vite-plugin-singlefile-2.3.0.tgz#71f7d8f8958cc019295e159355284ca3ecca97f1"
+ integrity sha512-DAcHzYypM0CasNLSz/WG0VdKOCxGHErfrjOoyIPiNxTPTGmO6rRD/te93n1YL/s+miXq66ipF1brMBikf99c6A==
+ dependencies:
+ micromatch "^4.0.8"
+
vite@^5.0.0, vite@^5.4.0:
version "5.4.21"
resolved "https://registry.yarnpkg.com/vite/-/vite-5.4.21.tgz#84a4f7c5d860b071676d39ba513c0d598fdc7027"