An open source design system that's fully customizable and agent ready
TypeScript
// Copyright (c) Meta Platforms, Inc. and affiliates.
/**
* @file setup-eval.ts
* @input Pristine and post-executor measurements from setup-measure.mjs
* @output Deterministic damage, task-completion, integrity, and acceptance results
* @position internal/vibe-tests/setup-test — the scoring half
*/
export type GeometrySnapshot = {
x: number;
y: number;
top: number;
right: number;
bottom: number;
left: number;
width: number;
height: number;
};
export type ProbeReading =
| {missing: true}
| {
style: Record<string, string>;
geometry: GeometrySnapshot;
text: string;
descendantText?: string;
contrast: number | null;
};
export type TaskResultReading = {
count: number;
visible: boolean;
focusable: boolean;
text: string;
style: Record<string, string>;
geometry: GeometrySnapshot | null;
};
export type LayerSurfaceReading =
| {kind: string; missing: true; styleReference?: string}
| {
kind: string;
styleReference?: string;
visible: boolean;
display: string;
visibility: string;
opacity: string;
position: string;
zIndex: string;
style: Record<string, string>;
bounds: GeometrySnapshot;
intersectsViewport: boolean;
clippingAncestor: string | null;
centerHitSelf: boolean;
centerHitProbe: string | null;
/**
* The host token-scoping selectors this surface is inside, nearest first,
* discovered from the app's own emitted CSS. Absent on measurements taken
* before this was recorded, which compare as an empty list.
*/
tokenScopes?: string[];
topLayer: {
tagName: string;
role: string | null;
open: boolean;
popover: string | null;
ariaModal: string | null;
inTopLayer: boolean;
portalChild: boolean;
};
};
export type InteractionReading = {
id?: string;
direction?: 'host-baseline' | 'astryx-in-host' | 'host-in-astryx';
opened: boolean;
keyboardReached?: Record<string, boolean>;
error?: string | null;
surfaces: Record<string, LayerSurfaceReading>;
};
export type SchemeReading = {
probes: Record<string, ProbeReading>;
variables: Record<string, string>;
colorScheme: string;
consoleErrors: string[];
pageErrors: string[];
failedRequests: string[];
interaction?: InteractionReading;
taskResults?: Record<string, TaskResultReading>;
taskInteractions?: Record<string, InteractionReading>;
};
export type ResultContract = {
name: string;
exact: number;
visible?: boolean;
focusable?: boolean;
text?: string;
preserveTextFromHostProbe?: string;
};
export type InteractionContract = {
id: string;
direction: 'astryx-in-host' | 'host-in-astryx';
open: Array<{
name: string;
source: 'probe' | 'result';
method: 'click' | 'keyboard-focus' | 'keyboard-activate';
key?: string;
}>;
surfaces: Array<{
name: string;
marker?: string;
source: 'probe' | 'result';
kind: string;
styleReference?: string;
}>;
};
export type AllowedHostChange = {
fixture: string;
probe: string;
fields: string[];
/**
* Whether this probe's accumulated text may *gain* content.
*
* A task that mandates inserting a control into an existing host container
* necessarily adds that control's text to the container's text, so comparing
* the container's text exactly reports the mandated insertion as host damage.
* This permits added text and nothing else: every baseline word must still be
* present, in its original order. Losing a word, changing one, or reordering
* them is still a regression, so the container's existing content stays as
* protected as it was. It is deliberately separate from `fields` — putting
* `text` in `fields` would exempt the comparison outright and let an executor
* delete host copy.
*/
textInsertionOnly?: boolean;
};
export type ReplacedHostProbe = {
fixture: string;
probe: string;
result: string;
};
export type AllowedOverlayChange = {
fixture: string;
surface: string;
fields: string[];
};
export type TaskContract = {
allowedHostChanges: AllowedHostChange[];
replacedHostProbes: ReplacedHostProbe[];
allowedOverlayChanges: AllowedOverlayChange[];
results: ResultContract[];
interactions: InteractionContract[];
};
export type SetupIntegrity = {
diffSha256: string;
attestedDiffSha256: string | null;
diffMatchesAttestation: boolean;
usesAstryx: boolean;
changedFiles: string[];
escapeHatches: string[];
};
export type Measurement = {
label: string;
fixture?: string;
build: {
ok: boolean;
status: number;
ms: number;
stdout: string;
stderr: string;
};
measurementErrors?: string[];
layerOrder?: string[];
task?: {id: string; kind: string; contract: TaskContract};
executionStatus?: string;
integrity?: SetupIntegrity;
schemes: Record<'light' | 'dark', SchemeReading>;
};
export type Scheme = 'light' | 'dark';
export type Category = 'typography' | 'color' | 'geometry' | 'spacing';
const CATEGORY_OF: Record<string, Category> = {
color: 'color',
backgroundColor: 'color',
boxShadow: 'color',
fontFamily: 'typography',
fontSize: 'typography',
fontWeight: 'typography',
lineHeight: 'typography',
letterSpacing: 'typography',
paddingTop: 'spacing',
paddingRight: 'spacing',
paddingBottom: 'spacing',
paddingLeft: 'spacing',
marginTop: 'spacing',
marginRight: 'spacing',
marginBottom: 'spacing',
marginLeft: 'spacing',
gap: 'spacing',
rowGap: 'spacing',
columnGap: 'spacing',
};
export const categoryOf = (property: string): Category => {
if (/^border.*Color$/.test(property)) {
return 'color';
}
if (property.startsWith('geometry.')) {
return 'geometry';
}
return CATEGORY_OF[property] ?? 'geometry';
};
export type Regression = {
scheme?: Scheme;
probe: string;
property: string;
category: Category;
before: string;
after: string;
};
const stringValue = (value: unknown) =>
value === undefined ? '(missing)' : String(value);
/**
* Whether `after` is `before` with text added and nothing removed.
*
* Words are compared in order: every baseline word must still appear, in the
* same sequence, with new words allowed between them. A deletion, a
* substitution, and a reordering all fail, so a container whose text may grow
* still cannot lose or rewrite the host's own copy.
*/
export function isTextInsertionOnly(before: string, after: string): boolean {
const baseline = before.trim().split(/\s+/).filter(Boolean);
const current = after.trim().split(/\s+/).filter(Boolean);
if (current.length < baseline.length) {
return false;
}
let index = 0;
for (const word of current) {
if (index < baseline.length && word === baseline[index]) {
index += 1;
}
}
return index === baseline.length;
}
/** Compare the complete captured style and geometry key union exactly. */
export function regressions(
before: SchemeReading,
after: SchemeReading,
allowedFields: ReadonlyMap<string, ReadonlySet<string>> = new Map(),
replacedProbes: ReadonlySet<string> = new Set(),
textInsertionProbes: ReadonlySet<string> = new Set(),
): {changed: Regression[]; missing: string[]} {
const changed: Regression[] = [];
const missing: string[] = [];
for (const [probe, baselineReading] of Object.entries(before.probes)) {
if (replacedProbes.has(probe)) {
continue;
}
const allowed = allowedFields.get(probe) ?? new Set<string>();
const armReading = after.probes[probe];
if (!armReading || 'missing' in armReading) {
if (!('missing' in baselineReading)) {
missing.push(probe);
}
continue;
}
if ('missing' in baselineReading) {
continue;
}
const beforeValues: Record<string, unknown> = {
...baselineReading.style,
...Object.fromEntries(
Object.entries(baselineReading.geometry).map(([key, value]) => [
`geometry.${key}`,
value,
]),
),
text: baselineReading.text,
};
const afterValues: Record<string, unknown> = {
...armReading.style,
...Object.fromEntries(
Object.entries(armReading.geometry).map(([key, value]) => [
`geometry.${key}`,
value,
]),
),
text: armReading.text,
};
for (const property of new Set([
...Object.keys(beforeValues),
...Object.keys(afterValues),
])) {
if (beforeValues[property] === afterValues[property]) {
continue;
}
if (allowed.has(property)) {
continue;
}
if (
property === 'text' &&
textInsertionProbes.has(probe) &&
isTextInsertionOnly(
stringValue(beforeValues.text),
stringValue(afterValues.text),
)
) {
continue;
}
changed.push({
probe,
property,
category: categoryOf(property),
before: stringValue(beforeValues[property]),
after: stringValue(afterValues[property]),
});
}
}
return {changed, missing};
}
export function countByCategory(
changed: Regression[],
): Record<Category, number> {
const counts: Record<Category, number> = {
typography: 0,
color: 0,
geometry: 0,
spacing: 0,
};
for (const regression of changed) {
counts[regression.category] += 1;
}
return counts;
}
export type ContrastFinding = {probe: string; before: number; after: number};
export function contrastFailures(
before: SchemeReading,
after: SchemeReading,
floor = 4.5,
replacedProbes: ReadonlySet<string> = new Set(),
): ContrastFinding[] {
const findings: ContrastFinding[] = [];
for (const [probe, baselineReading] of Object.entries(before.probes)) {
if (replacedProbes.has(probe)) {
continue;
}
const armReading = after.probes[probe];
if (
'missing' in baselineReading ||
!armReading ||
'missing' in armReading
) {
continue;
}
const beforeContrast = baselineReading.contrast;
const afterContrast = armReading.contrast;
if (
beforeContrast != null &&
afterContrast != null &&
beforeContrast >= floor &&
afterContrast < floor
) {
findings.push({
probe,
before: beforeContrast,
after: afterContrast,
});
}
}
return findings;
}
export function modeDependence(
baseline: Measurement,
arm: Measurement,
replacedProbes: ReadonlySet<string> = new Set(),
): string[] {
const probes = Object.keys(arm.schemes.light.probes);
return probes.filter(
probe =>
!replacedProbes.has(probe) &&
stableAcrossSchemes(baseline, probe) &&
!stableAcrossSchemes(arm, probe),
);
}
function stableAcrossSchemes(measurement: Measurement, probe: string): boolean {
const light = measurement.schemes.light.probes[probe];
const dark = measurement.schemes.dark.probes[probe];
if (!light || !dark || 'missing' in light || 'missing' in dark) {
return true;
}
return (
JSON.stringify(light.style) === JSON.stringify(dark.style) &&
JSON.stringify(light.geometry) === JSON.stringify(dark.geometry)
);
}
export function cascadeInverted(
layerOrder: string[] | undefined,
{systemPrefix = 'astryx', utilityLayer = 'utilities'} = {},
): boolean {
if (!layerOrder?.length) {
return false;
}
const utilities = layerOrder.indexOf(utilityLayer);
if (utilities === -1) {
return false;
}
return layerOrder.some(
(layer, index) => index > utilities && layer.startsWith(systemPrefix),
);
}
export function layerOrderFailures(layerOrder: string[] | undefined): string[] {
const order = layerOrder ?? [];
const failures = [];
for (const required of ['astryx-base', 'astryx-theme', 'utilities']) {
if (!order.includes(required)) {
failures.push(`missing-${required}`);
}
}
if (cascadeInverted(order)) {
failures.push('cascade-inverted');
}
return failures;
}
export type VariableFinding = {name: string; before: string; after: string};
export function variableCapture(
before: SchemeReading,
after: SchemeReading,
): VariableFinding[] {
const findings: VariableFinding[] = [];
for (const [name, value] of Object.entries(before.variables)) {
const armValue = after.variables[name];
if (armValue !== undefined && armValue !== value) {
findings.push({name, before: value, after: armValue});
}
}
return findings;
}
export type LayeringFailure = {
interaction?: string;
direction?: string;
surface: string;
problems: string[];
centerHitProbe: string | null;
};
const numericZIndex = (surface: LayerSurfaceReading | undefined) => {
if (
!surface ||
'missing' in surface ||
!/^-?\d+(?:\.\d+)?$/.test(surface.zIndex)
) {
return null;
}
return Number(surface.zIndex);
};
function failuresForInteraction(
interaction: InteractionReading,
): LayeringFailure[] {
const context = {
...(interaction.id ? {interaction: interaction.id} : {}),
...(interaction.direction ? {direction: interaction.direction} : {}),
};
if (!interaction.opened) {
return [
{
...context,
surface: 'interaction',
problems: [interaction.error ? 'open-error' : 'not-opened'],
centerHitProbe: null,
},
];
}
const backdrop = Object.values(interaction.surfaces).find(
surface => surface.kind === 'backdrop',
);
const dialog = Object.values(interaction.surfaces).find(
surface => surface.kind === 'dialog',
);
const backdropZ = numericZIndex(backdrop);
const dialogZ = numericZIndex(dialog);
const failures: LayeringFailure[] = [];
for (const [name, surface] of Object.entries(interaction.surfaces)) {
if ('missing' in surface) {
failures.push({
...context,
surface: name,
problems: ['missing'],
centerHitProbe: null,
});
continue;
}
const isNested = surface.kind === 'tooltip' || surface.kind === 'popover';
const problems: string[] = [];
if (!surface.visible) {
problems.push('hidden');
}
if (surface.bounds.width <= 0 || surface.bounds.height <= 0) {
problems.push('empty-bounds');
}
if (!surface.intersectsViewport) {
problems.push('outside-viewport');
}
if (surface.clippingAncestor) {
problems.push('clipped');
}
if (isNested && !surface.centerHitSelf) {
problems.push('center-occluded');
}
if (isNested && !surface.topLayer.inTopLayer) {
const nestedZ = numericZIndex(surface);
if (nestedZ == null) {
problems.push('unresolved-stack-order');
} else {
if (backdropZ == null || nestedZ <= backdropZ) {
problems.push('below-backdrop');
}
if (dialogZ == null || nestedZ <= dialogZ) {
problems.push('below-dialog');
}
}
}
if (problems.length > 0) {
failures.push({
...context,
surface: name,
problems,
centerHitProbe: surface.centerHitProbe,
});
}
}
return failures;
}
export function nestedLayerFailures(reading: SchemeReading): LayeringFailure[] {
return [
...(reading.interaction ? failuresForInteraction(reading.interaction) : []),
...Object.values(reading.taskInteractions ?? {}).flatMap(
failuresForInteraction,
),
];
}
function overlayStyleRegressions(
before: SchemeReading,
after: SchemeReading,
allowedFields: ReadonlyMap<string, ReadonlySet<string>> = new Map(),
): Regression[] {
if (!before.interaction || !after.interaction) {
return [];
}
const changed: Regression[] = [];
for (const [name, baselineSurface] of Object.entries(
before.interaction.surfaces,
)) {
const allowed = allowedFields.get(name) ?? new Set<string>();
const armSurface = after.interaction.surfaces[name];
if (
!armSurface ||
'missing' in armSurface ||
'missing' in baselineSurface
) {
continue;
}
const styleProperties = new Set([
...Object.keys(baselineSurface.style),
...Object.keys(armSurface.style),
]);
if (baselineSurface.kind === 'backdrop') {
styleProperties.delete('color');
styleProperties.delete('fontFamily');
styleProperties.delete('fontSize');
styleProperties.delete('fontWeight');
styleProperties.delete('lineHeight');
styleProperties.delete('letterSpacing');
for (const side of ['Top', 'Right', 'Bottom', 'Left']) {
if (
baselineSurface.style[`border${side}Width`] === '0px' &&
armSurface.style[`border${side}Width`] === '0px'
) {
styleProperties.delete(`border${side}Color`);
}
}
}
for (const property of styleProperties) {
if (
baselineSurface.style[property] !== armSurface.style[property] &&
!allowed.has(property)
) {
changed.push({
probe: `overlay:${name}`,
property,
category: categoryOf(property),
before: stringValue(baselineSurface.style[property]),
after: stringValue(armSurface.style[property]),
});
}
}
const baselineValues: Record<string, unknown> = {
visible: baselineSurface.visible,
display: baselineSurface.display,
visibility: baselineSurface.visibility,
opacity: baselineSurface.opacity,
position: baselineSurface.position,
zIndex: baselineSurface.zIndex,
...Object.fromEntries(
Object.entries(baselineSurface.bounds).map(([key, value]) => [
`geometry.${key}`,
value,
]),
),
};
const armValues: Record<string, unknown> = {
visible: armSurface.visible,
display: armSurface.display,
visibility: armSurface.visibility,
opacity: armSurface.opacity,
position: armSurface.position,
zIndex: armSurface.zIndex,
...Object.fromEntries(
Object.entries(armSurface.bounds).map(([key, value]) => [
`geometry.${key}`,
value,
]),
),
};
for (const property of Object.keys(baselineValues)) {
if (
baselineValues[property] !== armValues[property] &&
!allowed.has(property)
) {
changed.push({
probe: `overlay:${name}`,
property,
category: categoryOf(property),
before: stringValue(baselineValues[property]),
after: stringValue(armValues[property]),
});
}
}
}
return changed;
}
function taskContractFailures(
baseline: Measurement,
arm: Measurement,
): string[] {
if (!arm.task) {
return ['missing-task-contract'];
}
const failures: string[] = [];
for (const scheme of ['light', 'dark'] as const) {
const reading = arm.schemes[scheme];
for (const replaced of arm.task.contract.replacedHostProbes) {
if (replaced.fixture !== arm.fixture) {
continue;
}
const original = reading.probes[replaced.probe];
if (original && !('missing' in original)) {
failures.push(`${scheme}:${replaced.probe}:not-replaced`);
}
}
for (const result of arm.task.contract.results) {
const measured = reading.taskResults?.[result.name];
if (!measured || measured.count !== result.exact) {
failures.push(`${scheme}:${result.name}:count`);
continue;
}
if (result.visible && !measured.visible) {
failures.push(`${scheme}:${result.name}:hidden`);
}
if (result.focusable && !measured.focusable) {
failures.push(`${scheme}:${result.name}:not-focusable`);
}
if (result.text !== undefined && measured.text !== result.text) {
failures.push(`${scheme}:${result.name}:text`);
}
if (result.preserveTextFromHostProbe) {
const reference =
baseline.schemes[scheme].probes[result.preserveTextFromHostProbe];
if (
!reference ||
'missing' in reference ||
measured.text !== reference.text
) {
failures.push(`${scheme}:${result.name}:host-text`);
}
}
}
for (const interaction of arm.task.contract.interactions) {
const measured = reading.taskInteractions?.[interaction.id];
if (!measured) {
failures.push(`${scheme}:${interaction.id}:missing-interaction`);
continue;
}
for (const step of interaction.open.filter(step =>
step.method.startsWith('keyboard'),
)) {
if (measured.keyboardReached?.[step.name] !== true) {
failures.push(`${scheme}:${interaction.id}:${step.name}:keyboard`);
}
}
for (const surfaceContract of interaction.surfaces) {
if (!surfaceContract.styleReference) {
continue;
}
const surface = measured.surfaces[surfaceContract.name];
const reference =
baseline.schemes[scheme].interaction?.surfaces[
surfaceContract.styleReference
];
if (
!surface ||
!reference ||
'missing' in surface ||
'missing' in reference ||
JSON.stringify(surface.style) !== JSON.stringify(reference.style)
) {
failures.push(
`${scheme}:${interaction.id}:${surfaceContract.name}:host-style`,
);
}
/**
* Relocated host UI must still sit inside the host token scopes that
* were painting it.
*
* Reported separately from `host-style` because they are different
* problems with opposite fixes, and a style diff alone cannot tell them
* apart. Losing the boundary means the host's own rules stopped
* reaching the element: the fix is to restore the boundary, and
* restating the colours instead produces a surface that matches today
* and silently desynchronizes the next time the host retunes its
* palette. A style difference *inside* the right scopes is a real
* restyle. Neither is excused by the other, so both are reported.
*/
if (
surface &&
reference &&
!('missing' in surface) &&
!('missing' in reference) &&
JSON.stringify(reference.tokenScopes ?? []) !==
JSON.stringify(surface.tokenScopes ?? [])
) {
failures.push(
`${scheme}:${interaction.id}:${surfaceContract.name}:host-boundary`,
);
}
}
}
}
return [...new Set(failures)];
}
function integrityFailures(arm: Measurement): string[] {
if (!arm.integrity) {
return ['missing-diff-integrity'];
}
const failures = [...arm.integrity.escapeHatches];
if (!arm.integrity.usesAstryx) {
failures.push('missing-astryx-use');
}
if (!arm.integrity.attestedDiffSha256) {
failures.push('missing-agent-diff-attestation');
} else if (!arm.integrity.diffMatchesAttestation) {
failures.push('post-run-manual-edit');
}
return [...new Set(failures)];
}
const uniqueObjects = <T>(values: T[]): T[] => [
...new Map(values.map(value => [JSON.stringify(value), value])).values(),
];
export type SetupScore = {
label: string;
builds: boolean;
clean: boolean;
validRun: boolean;
executionSucceeded: boolean;
consoleErrors: number;
failedRequests: number;
regressions: number;
regressionDetails: Regression[];
byCategory: Record<Category, number>;
baselineFailures: string[];
missingProbes: string[];
contrastFailures: ContrastFinding[];
modeDependent: string[];
variablesCaptured: VariableFinding[];
measurementErrors: string[];
layeringFailures: LayeringFailure[];
layerOrderFailures: string[];
cascadeInverted: boolean;
layerOrder: string[];
taskSuccess: boolean;
taskFailures: string[];
integrityFailures: string[];
};
function baselineValidationFailures(baseline: Measurement): string[] {
const failures: string[] = [];
for (const scheme of ['light', 'dark'] as const) {
const reading = baseline.schemes[scheme];
if (!reading || Object.keys(reading.probes).length === 0) {
failures.push(`${scheme}:no-baseline-probes`);
continue;
}
for (const [name, probe] of Object.entries(reading.probes)) {
if ('missing' in probe) {
failures.push(`${scheme}:${name}:baseline-missing`);
}
}
if (reading.interaction) {
if (!reading.interaction.opened) {
failures.push(`${scheme}:baseline-interaction-not-opened`);
}
for (const [name, surface] of Object.entries(
reading.interaction.surfaces,
)) {
if ('missing' in surface) {
failures.push(`${scheme}:${name}:baseline-missing`);
}
}
for (const [name, reached] of Object.entries(
reading.interaction.keyboardReached ?? {},
)) {
if (!reached) {
failures.push(`${scheme}:${name}:baseline-keyboard`);
}
}
}
}
return failures;
}
function allowedHostFields(arm: Measurement) {
const fields = new Map<string, Set<string>>();
if (!arm.fixture || !arm.task) {
return fields;
}
for (const allowed of arm.task.contract.allowedHostChanges) {
if (allowed.fixture !== arm.fixture) {
continue;
}
const allowedForProbe = fields.get(allowed.probe) ?? new Set<string>();
for (const field of allowed.fields) {
allowedForProbe.add(field);
}
fields.set(allowed.probe, allowedForProbe);
}
return fields;
}
/**
* The probes whose accumulated text may gain the mandated insertion's own text,
* and only gain it. See `AllowedHostChange.textInsertionOnly`.
*/
function textInsertionProbes(arm: Measurement) {
if (!arm.fixture || !arm.task) {
return new Set<string>();
}
return new Set(
arm.task.contract.allowedHostChanges
.filter(
allowed =>
allowed.fixture === arm.fixture && allowed.textInsertionOnly === true,
)
.map(allowed => allowed.probe),
);
}
function replacedHostProbes(arm: Measurement) {
if (!arm.fixture || !arm.task) {
return new Set<string>();
}
return new Set(
arm.task.contract.replacedHostProbes
.filter(replaced => replaced.fixture === arm.fixture)
.map(replaced => replaced.probe),
);
}
function allowedOverlayFields(arm: Measurement) {
const fields = new Map<string, Set<string>>();
if (!arm.fixture || !arm.task) {
return fields;
}
for (const allowed of arm.task.contract.allowedOverlayChanges) {
if (allowed.fixture !== arm.fixture) {
continue;
}
const allowedForSurface = fields.get(allowed.surface) ?? new Set<string>();
for (const field of allowed.fields) {
allowedForSurface.add(field);
}
fields.set(allowed.surface, allowedForSurface);
}
return fields;
}
export function scoreArm(baseline: Measurement, arm: Measurement): SetupScore {
const baselineFailures = baselineValidationFailures(baseline);
const status = arm.executionStatus;
const executionSucceeded = status === 'succeeded';
const validRun = executionSucceeded || status === 'agent-failure';
if (!arm.build.ok) {
return {
label: arm.label,
builds: false,
clean: false,
validRun,
executionSucceeded,
consoleErrors: 0,
failedRequests: 0,
regressions: 0,
regressionDetails: [],
byCategory: {typography: 0, color: 0, geometry: 0, spacing: 0},
baselineFailures,
missingProbes: [],
contrastFailures: [],
modeDependent: [],
variablesCaptured: [],
measurementErrors: arm.measurementErrors ?? [],
layeringFailures: [],
layerOrderFailures: ['build-failed'],
cascadeInverted: false,
layerOrder: arm.layerOrder ?? [],
taskSuccess: false,
taskFailures: ['build-failed'],
integrityFailures: integrityFailures(arm),
};
}
const allowedFields = allowedHostFields(arm);
const replacedProbes = replacedHostProbes(arm);
const overlayFields = allowedOverlayFields(arm);
const insertionProbes = textInsertionProbes(arm);
const regressionDetails = uniqueObjects(
(['light', 'dark'] as const).flatMap(scheme =>
[
...regressions(
baseline.schemes[scheme],
arm.schemes[scheme],
allowedFields,
replacedProbes,
insertionProbes,
).changed,
...overlayStyleRegressions(
baseline.schemes[scheme],
arm.schemes[scheme],
overlayFields,
),
].map(regression => ({...regression, scheme})),
),
);
const missingProbes = [
...new Set(
(['light', 'dark'] as const).flatMap(
scheme =>
regressions(
baseline.schemes[scheme],
arm.schemes[scheme],
allowedFields,
replacedProbes,
insertionProbes,
).missing,
),
),
];
const contrast = uniqueObjects(
(['light', 'dark'] as const).flatMap(scheme =>
contrastFailures(
baseline.schemes[scheme],
arm.schemes[scheme],
4.5,
replacedProbes,
),
),
);
const variables = uniqueObjects(
(['light', 'dark'] as const).flatMap(scheme =>
variableCapture(baseline.schemes[scheme], arm.schemes[scheme]),
),
);
const layering = uniqueObjects(
(['light', 'dark'] as const).flatMap(scheme =>
nestedLayerFailures(arm.schemes[scheme]),
),
);
const runtimeErrors = (['light', 'dark'] as const).reduce(
(sum, scheme) =>
sum +
arm.schemes[scheme].consoleErrors.length +
arm.schemes[scheme].pageErrors.length,
0,
);
const failedRequests = (['light', 'dark'] as const).reduce(
(sum, scheme) => sum + arm.schemes[scheme].failedRequests.length,
0,
);
const taskFailures = taskContractFailures(baseline, arm);
const diffFailures = integrityFailures(arm);
return {
label: arm.label,
builds: arm.build.ok,
clean: runtimeErrors === 0 && failedRequests === 0,
validRun,
executionSucceeded,
consoleErrors: runtimeErrors,
failedRequests,
regressions: regressionDetails.length,
regressionDetails,
byCategory: countByCategory(regressionDetails),
baselineFailures,
missingProbes,
contrastFailures: contrast,
modeDependent: modeDependence(baseline, arm, replacedProbes),
variablesCaptured: variables,
measurementErrors: arm.measurementErrors ?? [],
layeringFailures: layering,
layerOrderFailures: layerOrderFailures(arm.layerOrder),
cascadeInverted: cascadeInverted(arm.layerOrder),
layerOrder: arm.layerOrder ?? [],
taskSuccess: taskFailures.length === 0,
taskFailures,
integrityFailures: diffFailures,
};
}
export type Verdict =
'broken-build' | 'noisy' | 'silent-damage' | 'cosmetic-drift' | 'clean';
export function verdict(score: SetupScore): Verdict {
if (!score.builds) {
return 'broken-build';
}
if (
score.measurementErrors.length > 0 ||
score.baselineFailures.length > 0 ||
score.contrastFailures.length > 0 ||
score.layeringFailures.length > 0 ||
score.layerOrderFailures.length > 0 ||
score.missingProbes.length > 0 ||
score.cascadeInverted ||
!score.taskSuccess ||
score.integrityFailures.length > 0
) {
return score.clean ? 'silent-damage' : 'noisy';
}
if (!score.clean) {
return 'noisy';
}
return score.regressions > 0 || score.modeDependent.length > 0
? 'cosmetic-drift'
: 'clean';
}
export function passesAcceptance(score: SetupScore): boolean {
return (
score.validRun &&
score.executionSucceeded &&
score.taskSuccess &&
score.integrityFailures.length === 0 &&
verdict(score) === 'clean'
);
}
/**
* The kinds of failure a run can have. They are different things and a report
* that adds them together says nothing useful.
*
* - `hostDamage` — the measurement found the host changed: computed styles,
* geometry, missing probes, contrast, layering, cascade order, mode
* dependence. This is the thing the whole stage exists to detect.
* - `runtime` — the built app logged errors or failed requests.
* - `integrity` — an escape hatch or a broken attestation in the source diff.
* The host may be pixel-perfect and this still fails.
* - `task` — the run did not do what it was asked to do.
* - `telemetry` — the run's own reporting is unusable: the executor was
* classified as failing, the measurement errored, or the baseline capture was
* incomplete. Nothing here says anything about the host; it says the cell has
* to be run again.
*
* `verdict` deliberately collapses several of these into `silent-damage`,
* because a run with any of them is not acceptable. That is right for a gate
* and wrong for a report: `silent-damage` from a comment misread as an
* `!important` and `silent-damage` from a repainted host are the same word for
* two unrelated problems. Report the breakdown next to the verdict.
*/
export type FailureKind =
'hostDamage' | 'runtime' | 'integrity' | 'task' | 'telemetry';
export type FailureBreakdown = Record<FailureKind, number>;
export function failureBreakdown(score: SetupScore): FailureBreakdown {
return {
hostDamage:
score.regressions +
score.missingProbes.length +
score.contrastFailures.length +
score.layeringFailures.length +
score.layerOrderFailures.length +
score.modeDependent.length +
(score.cascadeInverted ? 1 : 0),
runtime: score.consoleErrors + score.failedRequests,
integrity: score.integrityFailures.length,
task: score.taskFailures.length,
telemetry:
(score.executionSucceeded ? 0 : 1) +
score.measurementErrors.length +
score.baselineFailures.length,
};
}
/** The failure kinds actually present, in report order. */
export function failureCauses(score: SetupScore): FailureKind[] {
const breakdown = failureBreakdown(score);
return (Object.keys(breakdown) as FailureKind[]).filter(
kind => breakdown[kind] > 0,
);
}
/**
* Whether this run's measurement may be compared with another run's.
*
* A run the executor did not complete is still measured and still scored — the
* sandbox exists and the evaluator reads it — but its verdict describes an
* unfinished attempt. An operator run produced exactly that: a cell scored
* `clean` on a run whose executor stopped early, because nothing damaged the
* host once the work stopped. Counting that as a clean run makes the condition
* look better than the evidence supports, so comparison excludes it and says
* so.
*/
export function isComparableRun(score: SetupScore): boolean {
return score.validRun && score.executionSucceeded;
}
export type HardDimension =
| 'build'
| 'runtime'
| 'taskCompletion'
| 'color'
| 'font'
| 'radius'
| 'border'
| 'shadow'
| 'geometry'
| 'contrast'
| 'layering';
export type HardGateVector = Record<HardDimension, number>;
const regressionDimension = (property: string): HardDimension => {
if (property === 'boxShadow') {
return 'shadow';
}
if (/Radius$/.test(property)) {
return 'radius';
}
if (/^border/.test(property)) {
return 'border';
}
if (
property.startsWith('font') ||
property === 'lineHeight' ||
property === 'letterSpacing'
) {
return 'font';
}
if (property === 'color' || property === 'backgroundColor') {
return 'color';
}
return 'geometry';
};
export function hardGateVector(score: SetupScore): HardGateVector {
const vector: HardGateVector = {
build: score.builds ? 0 : 1,
runtime: score.consoleErrors + score.failedRequests,
taskCompletion:
score.taskFailures.length +
score.integrityFailures.length +
score.measurementErrors.length +
(score.executionSucceeded ? 0 : 1),
color: 0,
font: 0,
radius: 0,
border: 0,
shadow: 0,
geometry:
score.baselineFailures.length +
score.missingProbes.length +
score.modeDependent.length,
contrast: score.contrastFailures.length,
layering: score.layeringFailures.length + score.layerOrderFailures.length,
};
for (const regression of score.regressionDetails) {
vector[regressionDimension(regression.property)] += 1;
}
return vector;
}
export function compareCandidate(current: SetupScore, candidate: SetupScore) {
const before = hardGateVector(current);
const after = hardGateVector(candidate);
const deltas = Object.fromEntries(
(Object.keys(before) as HardDimension[]).map(dimension => [
dimension,
after[dimension] - before[dimension],
]),
) as HardGateVector;
const regressions = (Object.keys(deltas) as HardDimension[]).filter(
dimension => deltas[dimension] > 0,
);
const improvements = (Object.keys(deltas) as HardDimension[]).filter(
dimension => deltas[dimension] < 0,
);
return {
before,
after,
deltas,
regressions,
improvements,
};
}
export function strictAcceptanceSummary(
scores: SetupScore[],
expectedRuns: number,
coverageComplete: boolean,
) {
const valid = scores.filter(score => score.validRun);
const strict = valid.filter(passesAcceptance).length;
const damageFree = valid.filter(
score =>
score.builds &&
score.clean &&
score.baselineFailures.length === 0 &&
score.regressions === 0 &&
score.missingProbes.length === 0 &&
score.contrastFailures.length === 0 &&
score.layeringFailures.length === 0 &&
score.layerOrderFailures.length === 0 &&
score.measurementErrors.length === 0 &&
score.modeDependent.length === 0 &&
!score.cascadeInverted,
).length;
const complete =
coverageComplete &&
scores.length === expectedRuns &&
valid.length === expectedRuns;
return {
coverage: {
present: scores.length,
valid: valid.length,
expected: expectedRuns,
},
strictClean: {
numerator: strict,
denominator: valid.length,
rate: valid.length === 0 ? 0 : strict / valid.length,
},
damageFree: {
numerator: damageFree,
denominator: valid.length,
rate: valid.length === 0 ? 0 : damageFree / valid.length,
},
passes: complete && strict === expectedRuns,
};
}