An open source design system that's fully customizable and agent ready
TypeScript
feat(vibe-tests): test adding Astryx to existing Tailwind apps (#5268)
nynexman4464f0043d8f8cf7
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 31b25f0e9a..110456579e 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -562,6 +562,11 @@ jobs:- name: Install Playwrightrun: npx playwright install chromium+ - name: Verify canonical setup compositions+ env:+ ASTRYX_CANONICAL_SETUP_BROWSER: '1'+ run: pnpm vitest run internal/vibe-tests/setup-test/setup-canonical.test.ts --project node --maxWorkers=1+- name: Verify guest fixture contrastrun: pnpm -F @astryxdesign/vibe-tests fixtures:contrastdiff --git a/internal/vibe-tests/README.md b/internal/vibe-tests/README.mdindex b275d5a077..92dbd398ac 100644--- a/internal/vibe-tests/README.md+++ b/internal/vibe-tests/README.md@@ -150,6 +150,15 @@ internal/vibe-tests/├── fixture-recipes/ # Pinned provenance and SHA-256 manifests├── scripts/ # Fixture setup plus report helpers├── .baseline/ # Real shadcn/ui components for baseline tsc+├── setup-test/ # Setup evaluation over the canonical fixture matrix+│ ├── PLAN.md # controls, pilot stages, measures, decision rule+│ ├── matrix.json # fixtures × controls × prompts × bundles × reps+│ ├── run-setup.mjs # sandbox + task + provenance preparation only+│ ├── setup-interactions.mjs # marker-driven dialog and nested-overlay opener+│ ├── setup-measure.mjs # build + exact style, geometry, and overlay probes+│ ├── setup-integrity.mjs # read-only diff attestation and escape-hatch checks+│ ├── setup-eval.ts # strict deterministic deltas against each fixture+│ └── setup-aggregate.ts # matrix coverage, A/B deltas, and final acceptance├── results/ # Iteration results (gitignored)└── README.md # This file```@@ -158,7 +167,9 @@ internal/vibe-tests/The three apps under `fixtures/` represent a plain Tailwind v4 control, anestablished shadcn-style Tailwind v4 app, and an original enterprise-style app-with a marked guest design-system boundary. They are standalone packages with+with a marked guest design-system boundary. The two established-app fixtures+include controlled portal dialogs and nested tooltip/menu surfaces so setup can+measure clipping, occlusion, and layer order. They are standalone packages withexact dependencies and lockfiles; none has Astryx installed.Treat canonical fixture files as immutable inputs. Setup validates their pinned@@ -182,3 +193,9 @@ The root repository guard verifies provenance, exact manifests, deterministicsource, dependency isolation, workspace exclusion, and separation from everypublishable package root. This keeps future setup or migration tests independentof any unlanded harness branch.++Generated public artifacts never retain machine-local paths. Setup configs use+paths relative to their output directory; measurements identify the fixture and+redact private paths or hosts from build diagnostics; exported provenance uses a+generic usage-source label. Setup and universal aggregation fail closed before+emitting a report that still contains an absolute path or private host.diff --git a/internal/vibe-tests/fixture-recipes/tailwind-v4-control.json b/internal/vibe-tests/fixture-recipes/tailwind-v4-control.jsonindex 9402da06ef..209fcaaa21 100644--- a/internal/vibe-tests/fixture-recipes/tailwind-v4-control.json+++ b/internal/vibe-tests/fixture-recipes/tailwind-v4-control.json@@ -32,7 +32,7 @@"index.html": "d8daff351fd28efd42676b86e34c9312cf08410898a07f025082d619fe933240","package.json": "6388776258b624beb19538053dd7103c02f70024f3f4be79e4d7c489797db986","pnpm-lock.yaml": "5b50b51c8b1f5045813b3fbdcf9842527e684b34eb7aa1129a7c496b0be3158c",- "src/App.tsx": "fbd7fda6dc358d5be9c71484fe8afebf329079a46e737c375cee0c2b667cf6cb",+ "src/App.tsx": "aa4b438e70fcf7abebfda1e19f974ecc505d3ec0e225e9bb60fccb6360437644","src/index.css": "40c956dc7a7bfedf424aa3be1384390e728b912df21d3a0a55bfd0bbb01a0092","src/main.tsx": "47cc0b4b1cc9e21351c3a37b160d7f64277f59cd57937659022cdae97608a6a5","tsconfig.json": "39e5a743eb4be3f35a40bb28a216567a2df28de553728b5e9fb545ace0a2042c",diff --git a/internal/vibe-tests/fixtures/tailwind-v4-control/src/App.tsx b/internal/vibe-tests/fixtures/tailwind-v4-control/src/App.tsxindex be2ab9410a..9344a42b4b 100644--- a/internal/vibe-tests/fixtures/tailwind-v4-control/src/App.tsx+++ b/internal/vibe-tests/fixtures/tailwind-v4-control/src/App.tsx@@ -75,7 +75,14 @@ export default function App() {{row.item}</td><td className="px-5 py-3 text-zinc-600">{row.owner}</td>- <td className="px-5 py-3">{row.state}</td>+ <td className="px-5 py-3">+ <span+ data-vibe-probe={+ row.item === 'Quarterly plan' ? 'status' : undefined+ }>+ {row.state}+ </span>+ </td></tr>))}</tbody>diff --git a/internal/vibe-tests/package.json b/internal/vibe-tests/package.jsonindex 22139c8f29..58e7de00db 100644--- a/internal/vibe-tests/package.json+++ b/internal/vibe-tests/package.json@@ -29,6 +29,8 @@"correct": "tsx src/correct-previews.ts","correct:dry": "tsx src/correct-previews.ts --dry-run","preview:tsc-only": "tsx src/build-previews.ts --tsc-only",+ "setup:prepare": "node setup-test/run-setup.mjs",+ "setup:test": "vitest run --root ../.. internal/vibe-tests/setup-test --project node","fixtures:verify": "node scripts/fixture-suite.mjs verify","fixtures:prepare": "node scripts/fixture-suite.mjs prepare","fixtures:refresh": "node scripts/fixture-suite.mjs refresh",diff --git a/internal/vibe-tests/setup-test/PLAN.md b/internal/vibe-tests/setup-test/PLAN.mdnew file mode 100644index 0000000000..e3753aec10--- /dev/null+++ b/internal/vibe-tests/setup-test/PLAN.md@@ -0,0 +1,110 @@+# Setup test — zero-damage Astryx adoption in established apps++**Status:** integrated preparation and measurement harness. No executor runs or model findings are committed here.++## Goal and hard stop++The goal is zero unintended host damage. Installing Astryx alone must not change any existing host radius, color, typography or font metric, spacing, shadow, border, geometry, color-mode behavior, or layer order. A requested insertion or replacement may change only that explicitly named surface.++`cosmetic-drift` remains a useful diagnostic label, but it is a failed run. The iteration loop may use failures and per-dimension deltas to refine candidate instructions; it does not authorize shipping. Candidate instructions stop and ship only after a separate final confirmation has **100% valid runs classified `clean` with task-specific success** across every supported fixture × prompt × harness/model bundle × repetition. The final confirmation permits no broken, noisy, silent-damage, or cosmetic outcomes; missing probes; visible variable, contrast, layer, overlay, or geometry regressions; escape hatches; or post-run manual edits.++A classified stochastic infrastructure failure may be rerun only under the preregistered policy in `matrix.json`. The failed attempt remains recorded. A real agent failure is a valid failing run and may not be retried away. The configured Codex bundle is blocked by an upstream empty-turn issue, so final confirmation cannot start or claim all configured bundles until that blocker clears.++Damage in the directed `direct` control is a product gap. It blocks candidate-instruction shipping; it is not an accepted limitation or a documentation result.++## Canonical fixtures++Every run starts from the shared immutable fixtures in `../fixtures/`:++- `tailwind-v4-control` — a plain React, Vite, and Tailwind v4 app with only background/foreground semantics.+- `shadcn-tailwind-v4-established` — an established shadcn-style Tailwind v4 token vocabulary and card/form/table/status surface plus a portal dialog with independently controlled tooltip and menu portals.+- `enterprise-scoped-synthetic` — an original dense app with explicit light/dark control, non-shadcn tokens, a guest design-system subtree, and a guest-scoped portal dialog whose nested menu crosses the host boundary.++`run-setup.mjs` validates each fixture recipe and hash manifest, copies the fixture into a run sandbox, and never writes to committed fixture source. Tailwind v3 remains explicitly deferred until the v4 controls and measures separate.++## Tasks++The basic setup tasks remain: install-only proof (`s0`), button insertion (`s1`), status replacement (`s2`), and selector replacement (`s3`). `s0` deliberately measures the new proof component separately from every pre-existing host probe.++The two composition tasks run on different established fixtures:++- `s4`: an Astryx tooltip and selector popup inside the existing Tailwind/shadcn dialog;+- `s5`: the existing Tailwind menu inside a new Astryx dialog in the scoped enterprise fixture.++Each task declares stable task-owned `data-vibe-result` markers. They are measurement hooks, not implementation hints. The evaluator uses them to verify exact presence, task completion, keyboard reachability, visibility, clipping, center hit-testing, stack order, and—where host surfaces move into an Astryx dialog—baseline host-token styling. Fixture-owned `data-vibe-probe` markers remain the host baseline and may not be repurposed. Container text protection compares full normalized text from unmarked host descendants while excluding nested probe, result, and replacement subtrees. Each intentional replacement names exactly one fixture probe, requires the original probe to be absent afterward, and names a result that preserves its host text. Other allowances are fixture- and field-specific: host allowances may permit only text or exact position coordinates, while overlay allowances may permit only exact bounds. They never exempt host or overlay computed styles, or an entire neighboring probe.++## Controls++`conditions.json` defines four ordered controls. Astryx is preinstalled in none of them.++| Condition | Purpose |+| ----------- | ----------------------------------------------------------------------------------- |+| `floor` | Task only; no added setup pointer. |+| `current` | A pointer to the current public CLI documentation. |+| `candidate` | Current pointer plus proposed existing-app guidance. |+| `direct` | Exact installation edits; any residue identifies a product gap and blocks shipping. |++Guidance is committed into the sandbox baseline before execution, so the resulting diff contains only executor work.++## Measurement and integrity++All conditions use the same evaluator against a pristine copy of the same fixture in light and dark modes:++1. Typecheck and production build.+2. Console, page, and failed-request errors.+3. Exact computed-style comparisons for color/background; per-side border color, width, and style; all four radii; shadow; font family, size, weight, line height, and letter spacing; per-side padding and margin; gaps; dimensions; position; and transform.+4. Exact `getBoundingClientRect()` snapshots after Chromium values are serialized to 1/64 CSS px, the browser's subpixel grid. This is normalization, not a pass tolerance: normalized values must match exactly.+5. WCAG AA contrast regressions, color-mode behavior, emitted cascade order, and visible consequences of changed host variables. Variable changes alone remain diagnostic only when every marked host computed style and geometry snapshot is unchanged.+6. Existing host overlays and both cross-system composition directions: presence, keyboard reachability, visibility, viewport intersection, clipping ancestors, center hit target, top-layer state, and stack order above dialog and backdrop.+7. Task-specific output markers and semantics, kept separate from host baseline probes.+8. The executor's source diff. The evaluator rejects deleted or wholesale-replaced host files, blanket resets, hardcoded `!important`, disabled dark-mode behavior, missing Astryx use, missing requested composition, and changed/deleted host probes or overlays. Replacement is judged on content, not on raw diff lines: lines are compared with their indentation and interior whitespace normalized, so wrapping a component tree in a provider — which re-indents every line under it — deletes nothing, while minifying or rewriting a file still deletes all of it.+9. A runner-recorded agent diff digest. The evaluator only measures; it never patches. A missing or mismatched digest is post-run manual intervention and fails acceptance.+10. Public artifact safety. Measurements use stable fixture identifiers instead of local app paths, redact private paths and hosts from diagnostics, normalize provenance usage sources to a generic label, and fail closed before setup or universal aggregate reports can emit private path data.++Verdicts remain `broken-build`, `noisy`, `silent-damage`, `cosmetic-drift`, and `clean`. `passesAcceptance` is true only when the verdict is `clean`, the task contract succeeds, no escape hatch is present, and the measured diff exactly matches the runner's agent-output digest.++## Exploratory iteration rule++The guidance report compares current and candidate runs cell-for-cell and prints candidate-minus-current deltas for every hard dimension: build, runtime, task completion, color, font, radius, border, shadow, geometry, contrast, and layering. It never collapses those results into a weighted score or an automatic advance/reject decision.++The operator reviews the explicit tradeoffs while refining instructions. A large gain with a small regression may be a useful next iteration, while a balanced tradeoff may not be; the harness does not invent a threshold for that judgment. This flexibility applies only to exploratory A/B selection. Final confirmation has no tradeoff allowance: every valid run must pass strict acceptance.++## Matrix and exact counts++Four paired executor bundles remain configured: native/Claude-family, Claude Code/Claude-family, native/GPT-family, and Codex/GPT-family. Exact model versions come from execution provenance.++Prompt applicability produces 14 fixture-prompt pairs: four basic tasks across three fixtures (12), plus one cross-system task on each established fixture (2).++1. **Separation — 48 prepared runs:** `3 fixtures × 2 controls × s1 × 4 bundles × K=2`.+2. **Guidance iteration — 400 additional prepared runs:** the full four-condition matrix has 448 unique cells; reuse the 48 separation cells and prepare the other 400. No control evidence is rerun or double-counted.+3. **Final confirmation — 112 required valid runs:** `14 supported fixture-prompt pairs × candidate × 4 bundles × K=2`.++The repository prepares tasks and sandboxes but does not launch executors. Missing, blocked, invalid, or omitted cells make the confirmation incomplete; they do not shrink its denominator.++## Commands++```bash+# Validate matrix expansion, evaluators, integrity checks, and provenance.+pnpm vitest run internal/vibe-tests/setup-test --project node++# Prepare one small cell without launching an executor.+node internal/vibe-tests/setup-test/run-setup.mjs \+ --stage separation \+ --fixtures tailwind-v4-control \+ --conditions floor \+ --prompts s1 \+ --bundles native-claude \+ --reps 1 \+ --out /tmp/setup-test++# Immediately after the executor settles, record this command's diffSha256 as+# execution.agentDiffSha256 in the sidecar before measurement.+node internal/vibe-tests/setup-test/setup-integrity.mjs --app <sandbox>++# Measure without modifying source and carry the sidecar forward.+node internal/vibe-tests/setup-test/setup-measure.mjs \+ --app <sandbox> \+ --fixture tailwind-v4-control \+ --provenance <run.provenance.json> \+ --out <measurement.json>+```diff --git a/internal/vibe-tests/setup-test/apply-recipe.mjs b/internal/vibe-tests/setup-test/apply-recipe.mjsnew file mode 100644index 0000000000..ecbea21324--- /dev/null+++ b/internal/vibe-tests/setup-test/apply-recipe.mjs@@ -0,0 +1,150 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * Apply deterministic setup recipes to a copied canonical fixture.+ *+ * These recipes verify the measurement mechanism without invoking an executor.+ * Every edit asserts that the shared React + Vite + Tailwind fixture shape is+ * still present so drift fails loudly.+ */++import * as fs from 'node:fs';+import * as path from 'node:path';++const CSS = path.join('src', 'index.css');+const MAIN = path.join('src', 'main.tsx');++const read = (app, relative) =>+ fs.readFileSync(path.join(app, relative), 'utf8');+const write = (app, relative, source) =>+ fs.writeFileSync(path.join(app, relative), source);++function edit(app, relative, transform) {+ const before = read(app, relative);+ const after = transform(before);+ if (after === before) {+ throw new Error(`recipe edit did not apply: ${relative}`);+ }+ write(app, relative, after);+}++const ASTRYX_IMPORTS = [+ "@import '@astryxdesign/core/reset.css';",+ "@import '@astryxdesign/core/astryx.css';",+ "@import '@astryxdesign/theme-neutral/theme.css';",+];++function documentedTailwindBlock({bridge}) {+ return [+ '@layer reset, theme, base, astryx-base, astryx-theme, components, utilities;',+ '',+ "@import 'tailwindcss/theme.css' layer(theme);",+ "@import 'tailwindcss/preflight.css' layer(base);",+ ...ASTRYX_IMPORTS,+ ...(bridge ? ["@import '@astryxdesign/core/tailwind-theme.css';"] : []),+ "@import 'tailwindcss/utilities.css' layer(utilities);",+ ].join('\n');+}++function replaceTailwindEntry(app, block) {+ edit(app, CSS, css => {+ if (!/@import ['"]tailwindcss['"];/.test(css)) {+ throw new Error("fixture no longer starts from `@import 'tailwindcss';`");+ }+ return css.replace(/@import ['"]tailwindcss['"];/, block);+ });+}++function addThemeAndProbe(app, {mode}) {+ edit(app, MAIN, source =>+ source+ .replace(+ "import App from './App.tsx';",+ [+ "import {Button as AstryxButton} from '@astryxdesign/core/Button';",+ "import {Theme} from '@astryxdesign/core/theme';",+ "import {neutralTheme} from '@astryxdesign/theme-neutral/built';",+ "import App from './App.tsx';",+ ].join('\n'),+ )+ .replace(+ ' <App />',+ [+ ` <Theme theme={neutralTheme} mode="${mode}">`,+ ' <App />',+ ' <div data-setup-system-probe>',+ ' <AstryxButton label="Deploy" variant="primary" />',+ ' </div>',+ ' </Theme>',+ ].join('\n'),+ ),+ );+}++function appendAtEnd(app) {+ edit(app, CSS, css => {+ if (!/@import ['"]tailwindcss['"];/.test(css)) {+ throw new Error("fixture no longer starts from `@import 'tailwindcss';`");+ }+ return `${css}\n${[+ '/* Astryx */',+ '@layer reset, theme, base, astryx-base, astryx-theme, components, utilities;',+ ...ASTRYX_IMPORTS,+ '',+ ].join('\n')}`;+ });+}++export const RECIPES = {+ 'docs-verbatim': (app, options) => {+ replaceTailwindEntry(app, documentedTailwindBlock({bridge: true}));+ addThemeAndProbe(app, options);+ },+ 'docs-no-bridge': (app, options) => {+ replaceTailwindEntry(app, documentedTailwindBlock({bridge: false}));+ addThemeAndProbe(app, options);+ },+ 'appended-at-end': (app, options) => {+ appendAtEnd(app);+ addThemeAndProbe(app, options);+ },+ 'layered-in-place': (app, options) => {+ edit(app, CSS, css => {+ if (!/@import ['"]tailwindcss['"];/.test(css)) {+ throw new Error(+ "fixture no longer starts from `@import 'tailwindcss';`",+ );+ }+ return css.replace(+ /@import ['"]tailwindcss['"];/,+ [+ '@layer reset, theme, base, astryx-base, astryx-theme, components, utilities;',+ '',+ "@import 'tailwindcss';",+ ...ASTRYX_IMPORTS,+ ].join('\n'),+ );+ });+ addThemeAndProbe(app, options);+ },+};++if (import.meta.url === `file://${process.argv[1]}`) {+ const args = process.argv.slice(2);+ const get = name => {+ const index = args.indexOf(`--${name}`);+ return index === -1 ? undefined : args[index + 1];+ };+ const app = path.resolve(get('app') ?? '.');+ const recipeId = get('recipe');+ const recipe = RECIPES[recipeId];+ if (!recipe) {+ console.error(+ `Unknown recipe: ${recipeId}. Known: ${Object.keys(RECIPES).join(', ')}`,+ );+ process.exit(2);+ }+ recipe(app, {mode: get('mode') ?? 'light'});+ console.log(`applied ${recipeId} to ${app}`);+}diff --git a/internal/vibe-tests/setup-test/apply-recipe.test.mjs b/internal/vibe-tests/setup-test/apply-recipe.test.mjsnew file mode 100644index 0000000000..4e459cc820--- /dev/null+++ b/internal/vibe-tests/setup-test/apply-recipe.test.mjs@@ -0,0 +1,69 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {afterEach, describe, expect, it} from 'vitest';+import {copyFixture, FIXTURE_IDS} from '../src/fixture-suite.mjs';+import {RECIPES} from './apply-recipe.mjs';++const temporaryDirectories = [];++afterEach(() => {+ for (const directory of temporaryDirectories.splice(0)) {+ fs.rmSync(directory, {recursive: true, force: true});+ }+});++function copiedFixture(fixtureId) {+ const parent = fs.mkdtempSync(path.join(os.tmpdir(), 'setup-recipe-test-'));+ temporaryDirectories.push(parent);+ const fixture = path.join(parent, fixtureId);+ copyFixture(fixtureId, fixture);+ return fixture;+}++describe('deterministic setup recipes', () => {+ it.each(FIXTURE_IDS)(+ 'applies the candidate recipe to canonical fixture %s',+ fixtureId => {+ const fixture = copiedFixture(fixtureId);+ RECIPES['layered-in-place'](fixture, {mode: 'light'});++ expect(+ fs.readFileSync(path.join(fixture, 'src', 'index.css'), 'utf8'),+ ).toContain(+ '@layer reset, theme, base, astryx-base, astryx-theme, components, utilities;',+ );+ expect(+ fs.readFileSync(path.join(fixture, 'src', 'main.tsx'), 'utf8'),+ ).toContain('<Theme theme={neutralTheme} mode="light">');+ },+ );++ it.each(Object.keys(RECIPES))(+ 'keeps recipe %s compatible with the shared fixture entrypoints',+ recipeId => {+ const fixture = copiedFixture('tailwind-v4-control');+ RECIPES[recipeId](fixture, {mode: 'light'});+ expect(+ fs.readFileSync(path.join(fixture, 'src', 'main.tsx'), 'utf8'),+ ).toContain('data-setup-system-probe');+ },+ );++ it('rejects a fixture whose Tailwind entry has drifted', () => {+ const fixture = copiedFixture('tailwind-v4-control');+ const cssFile = path.join(fixture, 'src', 'index.css');+ fs.writeFileSync(+ cssFile,+ fs+ .readFileSync(cssFile, 'utf8')+ .replace("@import 'tailwindcss';", "@import 'other.css';"),+ );++ expect(() => RECIPES['layered-in-place'](fixture, {mode: 'light'})).toThrow(+ /fixture no longer starts/,+ );+ });+});diff --git a/internal/vibe-tests/setup-test/conditions.json b/internal/vibe-tests/setup-test/conditions.jsonnew file mode 100644index 0000000000..d77afc6e30--- /dev/null+++ b/internal/vibe-tests/setup-test/conditions.json@@ -0,0 +1,48 @@+{+ "name": "setup",+ "description": "Can an executor add Astryx to an established Tailwind app without damaging the host UI? Astryx is absent from every canonical fixture. The fixture, condition, prompt, repetition, and executor bundle are explicit matrix dimensions.",+ "base": {+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "notes": "Every run starts from a validated copy of a canonical fixture. The committed fixture is never modified."+ },+ "conditions": [+ {+ "id": "floor",+ "label": "Task only (floor)",+ "role": "floor",+ "patches": [],+ "hypothesis": "Without an added discovery cue, setup depends on what the executor finds from the package and project alone."+ },+ {+ "id": "current",+ "label": "Current public guidance",+ "role": "current-control",+ "patches": ["patch:pointer"],+ "hypothesis": "A pointer makes the current public setup guidance discoverable but does not change its instructions."+ },+ {+ "id": "candidate",+ "label": "Candidate existing-app guidance",+ "role": "candidate",+ "patches": ["patch:pointer", "patch:existing-app"],+ "hypothesis": "Existing-app guidance preserves the host Tailwind entry, cascade order, color mode, and app-owned tokens."+ },+ {+ "id": "direct",+ "label": "Directed setup control",+ "role": "direct-control",+ "patches": ["patch:pointer", "patch:existing-app", "patch:directed"],+ "hypothesis": "Precise setup instructions remove discovery ambiguity. Any remaining visible host damage is a product gap and blocks instruction shipping."+ }+ ],+ "measurement": {+ "reference": "A pristine copied fixture, measured by the same code before setup.",+ "schemes": ["light", "dark"],+ "notes": "Both preferred color schemes are measured even when the fixture has its own explicit mode control.",+ "emittedCss": "Cascade order is read from the built CSS rather than inferred from source order."+ }+}diff --git a/internal/vibe-tests/setup-test/guidance/directed.md b/internal/vibe-tests/setup-test/guidance/directed.mdnew file mode 100644index 0000000000..ce8ad4ecd5--- /dev/null+++ b/internal/vibe-tests/setup-test/guidance/directed.md@@ -0,0 +1,12 @@+## The exact edits++Do these in order:++1. Run `pnpm add @astryxdesign/core @astryxdesign/theme-neutral`.+2. In `src/index.css`, insert the layer declaration **above** the existing+ `@import 'tailwindcss';`, then add the three Astryx stylesheets immediately+ after the Tailwind import. Keep the rest of the host stylesheet intact.+3. In `src/main.tsx`, wrap `<App />` in+ `<Theme theme={neutralTheme} mode="light">`. Preserve the app's own mode+ control and change the explicit mode only when the host's initial mode differs.+4. Build the app and inspect the rendered page before finishing.diff --git a/internal/vibe-tests/setup-test/guidance/existing-app.md b/internal/vibe-tests/setup-test/guidance/existing-app.mdnew file mode 100644index 0000000000..6b7e82109b--- /dev/null+++ b/internal/vibe-tests/setup-test/guidance/existing-app.md@@ -0,0 +1,25 @@+## Adding Astryx to an established Tailwind app++The public greenfield recipe is not a replacement for an existing stylesheet.+Preserve the host application's CSS and add Astryx around it:++- Keep `@import 'tailwindcss';` and put the layer declaration **above it**:++ ```css+ @layer reset, theme, base, astryx-base, astryx-theme, components, utilities;++ @import 'tailwindcss';+ @import '@astryxdesign/core/reset.css';+ @import '@astryxdesign/core/astryx.css';+ @import '@astryxdesign/theme-neutral/theme.css';+ ```++- Do not import `@astryxdesign/core/tailwind-theme.css` unless the host wants+ Astryx to own its Tailwind vocabulary. It intentionally maps common color,+ radius, shadow, type, and spacing names.+- Pass an explicit `mode` to `<Theme>` that matches the host's initial mode.+ Preserve any app-controlled light/dark switch rather than replacing it.+- If both systems own the same custom-property name, reassert the host value at+ the theme boundary instead of deleting either vocabulary.+- Verify the rendered app as well as the build. Cascade and inherited-color+ changes can compile cleanly.diff --git a/internal/vibe-tests/setup-test/guidance/pointer.md b/internal/vibe-tests/setup-test/guidance/pointer.mdnew file mode 100644index 0000000000..f668f9f2a1--- /dev/null+++ b/internal/vibe-tests/setup-test/guidance/pointer.md@@ -0,0 +1,12 @@+## Design system++This app is starting to use the Astryx design system. Its docs are in the CLI,+not in this repo:++```bash+npx @astryxdesign/cli docs # list reference docs+npx @astryxdesign/cli component --list+npx @astryxdesign/cli component <Name>+```++Look a component up before writing it.diff --git a/internal/vibe-tests/setup-test/matrix.json b/internal/vibe-tests/setup-test/matrix.jsonnew file mode 100644index 0000000000..91b329c4cc--- /dev/null+++ b/internal/vibe-tests/setup-test/matrix.json@@ -0,0 +1,140 @@+{+ "schemaVersion": 1,+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "promptFixtures": {+ "s0": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "s1": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "s2": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "s3": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "s4": ["shadcn-tailwind-v4-established"],+ "s5": ["enterprise-scoped-synthetic"]+ },+ "bundles": [+ {+ "id": "native-claude",+ "harness": "native",+ "model": "claude-family",+ "effort": "high"+ },+ {+ "id": "claude-code-claude",+ "harness": "claude-code",+ "model": "claude-family",+ "effort": "high"+ },+ {+ "id": "native-gpt",+ "harness": "native",+ "model": "gpt-family",+ "effort": "high"+ },+ {+ "id": "codex-gpt",+ "harness": "codex",+ "model": "gpt-family",+ "effort": "high",+ "blocked": true,+ "blocker": "Upstream executor can return an empty successful turn."+ }+ ],+ "infraRetryPolicy": {+ "maxRetriesPerCell": 2,+ "retryableStatuses": [+ "infrastructure-failure",+ "executor-unavailable",+ "rate-limited"+ ],+ "agentFailureStatus": "agent-failure",+ "rule": "Retry only a classified infrastructure failure. A real agent failure is a completed failing run and is never discarded or retried into a pass."+ },+ "stages": [+ {+ "id": "separation",+ "description": "Confirm that the floor and direct controls separate before expanding the iteration loop.",+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "conditions": ["floor", "direct"],+ "prompts": ["s1"],+ "bundles": [+ "native-claude",+ "claude-code-claude",+ "native-gpt",+ "codex-gpt"+ ],+ "reps": 2+ },+ {+ "id": "guidance",+ "description": "Iterate on all guidance conditions. Failures and explicit per-dimension tradeoffs guide instruction refinement; this stage does not rank candidates or authorize shipping.",+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "conditions": ["floor", "current", "candidate", "direct"],+ "reuseFromStages": ["separation"],+ "prompts": ["s0", "s1", "s2", "s3", "s4", "s5"],+ "bundles": [+ "native-claude",+ "claude-code-claude",+ "native-gpt",+ "codex-gpt"+ ],+ "reps": 2+ },+ {+ "id": "confirmation",+ "description": "Final candidate-only confirmation. Every supported fixture and prompt combination, bundle, and repetition must be present and clean.",+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "conditions": ["candidate"],+ "prompts": ["s0", "s1", "s2", "s3", "s4", "s5"],+ "bundles": [+ "native-claude",+ "claude-code-claude",+ "native-gpt",+ "codex-gpt"+ ],+ "reps": 2,+ "acceptance": {+ "requiredVerdict": "clean",+ "requireTaskSuccess": true,+ "requireDiffAttestation": true,+ "requiredRate": 1,+ "requireCompleteMatrix": true+ }+ }+ ],+ "deferred": [+ {+ "fixtureFamily": "tailwind-v3",+ "reason": "Explicitly deferred until the Tailwind v4 experiment establishes that the controls and measures separate."+ }+ ]+}diff --git a/internal/vibe-tests/setup-test/probe-selectors.test.mjs b/internal/vibe-tests/setup-test/probe-selectors.test.mjsnew file mode 100644index 0000000000..a5700caa45--- /dev/null+++ b/internal/vibe-tests/setup-test/probe-selectors.test.mjs@@ -0,0 +1,189 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import fs from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {JSDOM} from 'jsdom';+import {describe, expect, it} from 'vitest';++const HERE = path.dirname(fileURLToPath(import.meta.url));+const VIBE_DIR = path.resolve(HERE, '..');+const probeConfig = JSON.parse(+ fs.readFileSync(path.join(HERE, 'probes.json'), 'utf8'),+);+const prompts = JSON.parse(+ fs.readFileSync(path.join(HERE, 'prompts.json'), 'utf8'),+).prompts;++function probe(fixtureId, name) {+ return probeConfig.fixtures[fixtureId].probes.find(+ entry => entry.name === name,+ );+}++function attributeFragments(source, attribute) {+ return (+ source.match(+ new RegExp(+ `${attribute}\\s*=\\s*(?:["'][^"']+["']|\\{[\\s\\S]*?\\})`,+ 'g',+ ),+ ) ?? []+ );+}++describe('setup host probe selectors', () => {+ it('captures every required paint, type, box, spacing, and position property', () => {+ expect(new Set(probeConfig.properties).size).toBe(+ probeConfig.properties.length,+ );+ for (const property of [+ 'color',+ 'backgroundColor',+ 'borderTopColor',+ 'borderRightColor',+ 'borderBottomColor',+ 'borderLeftColor',+ 'borderTopWidth',+ 'borderRightWidth',+ 'borderBottomWidth',+ 'borderLeftWidth',+ 'borderTopStyle',+ 'borderRightStyle',+ 'borderBottomStyle',+ 'borderLeftStyle',+ 'borderTopLeftRadius',+ 'borderTopRightRadius',+ 'borderBottomRightRadius',+ 'borderBottomLeftRadius',+ 'boxShadow',+ 'fontFamily',+ 'fontSize',+ 'fontWeight',+ 'lineHeight',+ 'letterSpacing',+ 'paddingTop',+ 'paddingRight',+ 'paddingBottom',+ 'paddingLeft',+ 'marginTop',+ 'marginRight',+ 'marginBottom',+ 'marginLeft',+ 'gap',+ 'rowGap',+ 'columnGap',+ 'width',+ 'height',+ 'position',+ 'top',+ 'right',+ 'bottom',+ 'left',+ ]) {+ expect(probeConfig.properties, property).toContain(property);+ }+ expect(probeConfig.geometryNormalization).toMatch(/1\/64 CSS pixel/);+ });++ it('uses only fixture-owned marker selectors', () => {+ for (const fixture of Object.values(probeConfig.fixtures)) {+ for (const entry of fixture.probes) {+ expect(entry.selector).toBe(`[data-vibe-probe="${entry.name}"]`);+ }+ }+ });++ it('matches exactly one real marker attribute in each canonical fixture source', () => {+ for (const [fixtureId, fixture] of Object.entries(probeConfig.fixtures)) {+ const source = fs.readFileSync(+ path.join(VIBE_DIR, 'fixtures', fixtureId, 'src', 'App.tsx'),+ 'utf8',+ );+ const attributes = attributeFragments(source, 'data-vibe-probe');+ for (const entry of fixture.probes) {+ expect(+ attributes.filter(attribute =>+ new RegExp(`['"]${entry.name}['"]`).test(attribute),+ ),+ `${fixtureId}:${entry.name}`,+ ).toHaveLength(1);+ }+ }+ });++ it('defines complete fixture-owned nested overlay interactions', () => {+ expect(+ probeConfig.fixtures['tailwind-v4-control'].interaction,+ ).toBeUndefined();+ for (const fixtureId of [+ 'shadcn-tailwind-v4-established',+ 'enterprise-scoped-synthetic',+ ]) {+ const interaction = probeConfig.fixtures[fixtureId].interaction;+ const source = fs.readFileSync(+ path.join(VIBE_DIR, 'fixtures', fixtureId, 'src', 'App.tsx'),+ 'utf8',+ );+ const attributes = attributeFragments(source, 'data-vibe-probe');+ const kinds = new Set(interaction.surfaces.map(surface => surface.kind));+ expect(kinds).toContain('backdrop');+ expect(kinds).toContain('dialog');+ expect(+ [...kinds].some(kind => kind === 'popover' || kind === 'tooltip'),+ ).toBe(true);+ for (const marker of [+ ...interaction.open,+ ...interaction.surfaces.map(surface => surface.name),+ ]) {+ expect(+ attributes.filter(attribute => attribute.includes(`"${marker}"`)),+ `${fixtureId}:${marker}`,+ ).toHaveLength(1);+ }+ }+ });++ it('declares exact task-owned result markers for both composition directions', () => {+ const promptById = new Map(prompts.map(prompt => [prompt.id, prompt]));+ const directions = new Set();+ for (const id of ['s4', 's5']) {+ const prompt = promptById.get(id);+ expect(prompt.kind).toBe('composition');+ for (const interaction of prompt.contract.interactions) {+ directions.add(interaction.direction);+ expect(+ interaction.surfaces.some(surface => surface.kind === 'dialog'),+ ).toBe(true);+ expect(+ interaction.surfaces.some(+ surface => surface.kind === 'tooltip' || surface.kind === 'popover',+ ),+ ).toBe(true);+ for (const marker of interaction.surfaces.filter(+ surface => surface.source === 'result',+ )) {+ expect(marker.name).toMatch(/^(astryx|host)-/);+ }+ }+ }+ expect(directions).toEqual(new Set(['astryx-in-host', 'host-in-astryx']));+ });++ it('keeps the original primary action when a new adjacent button is inserted', () => {+ const selector = probe(+ 'shadcn-tailwind-v4-established',+ 'primary-action',+ ).selector;+ const dom = new JSDOM(`+ <header>+ <button data-vibe-probe="primary-action">New request</button>+ <button data-vibe-result="astryx-button">Deploy</button>+ </header>+ `);++ expect(dom.window.document.querySelector(selector)?.textContent).toBe(+ 'New request',+ );+ });+});diff --git a/internal/vibe-tests/setup-test/probes.json b/internal/vibe-tests/setup-test/probes.jsonnew file mode 100644index 0000000000..8f5fb3d6fb--- /dev/null+++ b/internal/vibe-tests/setup-test/probes.json@@ -0,0 +1,238 @@+{+ "$comment": "Fixture-specific probes for existing host UI. Every selector uses a unique fixture-owned marker, never position or text. Missing probes are failures rather than skips.",+ "properties": [+ "color",+ "backgroundColor",+ "borderTopColor",+ "borderRightColor",+ "borderBottomColor",+ "borderLeftColor",+ "borderTopWidth",+ "borderRightWidth",+ "borderBottomWidth",+ "borderLeftWidth",+ "borderTopStyle",+ "borderRightStyle",+ "borderBottomStyle",+ "borderLeftStyle",+ "borderTopLeftRadius",+ "borderTopRightRadius",+ "borderBottomRightRadius",+ "borderBottomLeftRadius",+ "boxShadow",+ "fontFamily",+ "fontSize",+ "fontWeight",+ "lineHeight",+ "letterSpacing",+ "paddingTop",+ "paddingRight",+ "paddingBottom",+ "paddingLeft",+ "marginTop",+ "marginRight",+ "marginBottom",+ "marginLeft",+ "gap",+ "rowGap",+ "columnGap",+ "width",+ "height",+ "minWidth",+ "minHeight",+ "maxWidth",+ "maxHeight",+ "position",+ "top",+ "right",+ "bottom",+ "left",+ "transform"+ ],+ "fixtures": {+ "tailwind-v4-control": {+ "probes": [+ {+ "name": "page-title",+ "selector": "[data-vibe-probe=\"page-title\"]",+ "role": "host text"+ },+ {+ "name": "primary-action",+ "selector": "[data-vibe-probe=\"primary-action\"]",+ "role": "host control"+ },+ {+ "name": "table-header",+ "selector": "[data-vibe-probe=\"table-header\"]",+ "role": "host text"+ },+ {+ "name": "table-cell",+ "selector": "[data-vibe-probe=\"table-cell\"]",+ "role": "host text"+ },+ {+ "name": "status",+ "selector": "[data-vibe-probe=\"status\"]",+ "role": "host status"+ },+ {+ "name": "form-control",+ "selector": "[data-vibe-probe=\"form-control\"]",+ "role": "host control"+ },+ {+ "name": "host-shell",+ "selector": "[data-vibe-probe=\"host-shell\"]",+ "role": "host surface"+ }+ ],+ "rootVariables": [+ "--color-background",+ "--color-foreground",+ "--color-border",+ "--color-accent",+ "--spacing",+ "--text-sm",+ "--font-sans"+ ]+ },+ "shadcn-tailwind-v4-established": {+ "probes": [+ {+ "name": "page-title",+ "selector": "[data-vibe-probe=\"page-title\"]",+ "role": "host text"+ },+ {+ "name": "primary-action",+ "selector": "[data-vibe-probe=\"primary-action\"]",+ "role": "host control"+ },+ {+ "name": "table-header",+ "selector": "[data-vibe-probe=\"table-header\"]",+ "role": "host text"+ },+ {+ "name": "status",+ "selector": "[data-vibe-probe=\"status\"]",+ "role": "host status"+ },+ {+ "name": "form-control",+ "selector": "[data-vibe-probe=\"form-control\"]",+ "role": "host control"+ },+ {+ "name": "host-shell",+ "selector": "[data-vibe-probe=\"host-shell\"]",+ "role": "host surface"+ }+ ],+ "rootVariables": [+ "--color-card",+ "--color-muted",+ "--color-border",+ "--color-accent",+ "--color-background",+ "--color-foreground",+ "--radius-md",+ "--spacing",+ "--text-sm",+ "--font-sans"+ ],+ "interaction": {+ "open": ["primary-action", "tooltip-trigger", "popover-trigger"],+ "surfaces": [+ {+ "name": "dialog-backdrop",+ "kind": "backdrop"+ },+ {+ "name": "dialog-surface",+ "kind": "dialog"+ },+ {+ "name": "tooltip-surface",+ "kind": "tooltip"+ },+ {+ "name": "popover-surface",+ "kind": "popover"+ }+ ]+ }+ },+ "enterprise-scoped-synthetic": {+ "probes": [+ {+ "name": "page-title",+ "selector": "[data-vibe-probe=\"page-title\"]",+ "role": "host text"+ },+ {+ "name": "mode-control",+ "selector": "[data-vibe-probe=\"mode-control\"]",+ "role": "host control"+ },+ {+ "name": "table-header",+ "selector": "[data-vibe-probe=\"table-header\"]",+ "role": "host text"+ },+ {+ "name": "status",+ "selector": "[data-vibe-probe=\"status\"]",+ "role": "host status"+ },+ {+ "name": "settings-control",+ "selector": "[data-vibe-probe=\"settings-control\"]",+ "role": "host control"+ },+ {+ "name": "guest-boundary",+ "selector": "[data-vibe-probe=\"guest-boundary\"]",+ "role": "guest surface"+ },+ {+ "name": "host-shell",+ "selector": "[data-vibe-probe=\"host-shell\"]",+ "role": "host surface"+ }+ ],+ "rootVariables": [+ "--color-border",+ "--color-accent",+ "--color-background",+ "--color-foreground",+ "--color-success",+ "--color-warning",+ "--color-error",+ "--spacing",+ "--text-sm",+ "--font-sans"+ ],+ "interaction": {+ "open": ["dialog-trigger", "popover-trigger"],+ "surfaces": [+ {+ "name": "dialog-backdrop",+ "kind": "backdrop"+ },+ {+ "name": "dialog-surface",+ "kind": "dialog"+ },+ {+ "name": "popover-surface",+ "kind": "popover"+ }+ ]+ }+ }+ },+ "geometryNormalization": "DOMRect values are rounded to the Chromium 1/64 CSS pixel serialization grid, then compared exactly."+}diff --git a/internal/vibe-tests/setup-test/prompts.json b/internal/vibe-tests/setup-test/prompts.jsonnew file mode 100644index 0000000000..c082a49585--- /dev/null+++ b/internal/vibe-tests/setup-test/prompts.json@@ -0,0 +1,370 @@+{+ "name": "setup",+ "description": "Setup and cross-system composition tasks. Every request allows changes only to its named insertion or replacement surface; all unrelated host UI must remain unchanged.",+ "prompts": [+ {+ "id": "s0",+ "tier": "T1",+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "prompt": "Install and wire the Astryx design system without replacing any existing UI. Add one minimal proof control labelled \"Astryx ready\" beside the existing primary action. Do not change any pre-existing surface.",+ "measures": "Installation-only coexistence plus one separately marked proof component.",+ "expected": "The proof component renders while every pre-existing host probe remains identical.",+ "contract": {+ "results": [+ {+ "name": "astryx-proof",+ "description": "the new proof control",+ "exact": 1,+ "visible": true,+ "text": "Astryx ready"+ }+ ],+ "interactions": [],+ "allowedHostChanges": [+ {+ "fixture": "tailwind-v4-control",+ "probe": "primary-action",+ "fields": ["geometry.x", "geometry.left", "geometry.right"]+ },+ {+ "fixture": "shadcn-tailwind-v4-established",+ "probe": "primary-action",+ "fields": ["geometry.x", "geometry.left", "geometry.right"]+ },+ {+ "fixture": "enterprise-scoped-synthetic",+ "probe": "mode-control",+ "fields": ["geometry.x", "geometry.left", "geometry.right"]+ }+ ],+ "replacedHostProbes": [],+ "allowedOverlayChanges": []+ },+ "kind": "installation"+ },+ {+ "id": "s1",+ "tier": "T1",+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "prompt": "Start using the Astryx design system in this app. Add one of its buttons beside the existing primary action, labelled \"Deploy\", so we can see the installation working.",+ "measures": "The installation plus one minimal component insertion.",+ "expected": "The existing app remains intact and gains one correctly styled Astryx button.",+ "contract": {+ "results": [+ {+ "name": "astryx-button",+ "description": "the new Deploy button",+ "exact": 1,+ "visible": true,+ "text": "Deploy"+ }+ ],+ "interactions": [],+ "allowedHostChanges": [+ {+ "fixture": "tailwind-v4-control",+ "probe": "primary-action",+ "fields": ["geometry.x", "geometry.left", "geometry.right"]+ },+ {+ "fixture": "shadcn-tailwind-v4-established",+ "probe": "primary-action",+ "fields": ["geometry.x", "geometry.left", "geometry.right"]+ },+ {+ "fixture": "enterprise-scoped-synthetic",+ "probe": "mode-control",+ "fields": ["geometry.x", "geometry.left", "geometry.right"]+ }+ ],+ "replacedHostProbes": [],+ "allowedOverlayChanges": []+ },+ "kind": "addition"+ },+ {+ "id": "s2",+ "tier": "T2",+ "kind": "replacement",+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "prompt": "Start using the Astryx design system here, and replace the status treatment in the existing table with its equivalent while preserving every label and status meaning.",+ "measures": "Installation plus a substitution that must preserve fixed host data and semantics.",+ "expected": "Only the requested status surface changes; the table, labels, and unrelated host styling remain unchanged.",+ "contract": {+ "results": [+ {+ "name": "astryx-status",+ "description": "the replacement status surface",+ "exact": 1,+ "visible": true,+ "preserveTextFromHostProbe": "status"+ }+ ],+ "interactions": [],+ "allowedHostChanges": [],+ "replacedHostProbes": [+ {+ "fixture": "tailwind-v4-control",+ "probe": "status",+ "result": "astryx-status"+ },+ {+ "fixture": "shadcn-tailwind-v4-established",+ "probe": "status",+ "result": "astryx-status"+ },+ {+ "fixture": "enterprise-scoped-synthetic",+ "probe": "status",+ "result": "astryx-status"+ }+ ],+ "allowedOverlayChanges": []+ }+ },+ {+ "id": "s3",+ "tier": "T2",+ "kind": "replacement",+ "fixtures": [+ "tailwind-v4-control",+ "shadcn-tailwind-v4-established",+ "enterprise-scoped-synthetic"+ ],+ "prompt": "Adopt the Astryx design system in this app and rebuild its existing select control with the closest accessible Astryx equivalent, preserving the current options and layout.",+ "measures": "Installation plus an interactive substitution shared by all fixtures.",+ "expected": "Only the requested select changes; it remains keyboard reachable, keeps its options, and does not disturb the surrounding form or page.",+ "contract": {+ "results": [+ {+ "name": "astryx-selector",+ "description": "the replacement selector trigger",+ "exact": 1,+ "visible": true,+ "focusable": true+ }+ ],+ "interactions": [],+ "allowedHostChanges": [+ {+ "fixture": "tailwind-v4-control",+ "probe": "form-control",+ "fields": ["geometry.y", "geometry.top", "geometry.bottom"]+ },+ {+ "fixture": "enterprise-scoped-synthetic",+ "probe": "table-header",+ "fields": ["geometry.y", "geometry.top", "geometry.bottom"]+ },+ {+ "fixture": "enterprise-scoped-synthetic",+ "probe": "status",+ "fields": ["geometry.y", "geometry.top", "geometry.bottom"]+ },+ {+ "fixture": "enterprise-scoped-synthetic",+ "probe": "settings-control",+ "fields": ["geometry.y", "geometry.top", "geometry.bottom"]+ }+ ],+ "replacedHostProbes": [],+ "allowedOverlayChanges": []+ }+ },+ {+ "id": "s4",+ "tier": "T3",+ "kind": "composition",+ "fixtures": ["shadcn-tailwind-v4-established"],+ "prompt": "Inside the existing Tailwind/shadcn dialog, add an Astryx tooltip and an Astryx selector with its popup. Both controls must be keyboard reachable, and each opened surface must remain visible, unclipped, and above the existing dialog and backdrop. Preserve the existing dialog and all unrelated host styling.",+ "measures": "Astryx tooltip and selector surfaces composed inside an existing host dialog.",+ "expected": "Both Astryx surfaces open from the keyboard above the host dialog stack without changing host styling.",+ "contract": {+ "results": [+ {+ "name": "astryx-tooltip-trigger",+ "description": "the Astryx tooltip trigger",+ "exact": 1+ },+ {+ "name": "astryx-selector-trigger",+ "description": "the Astryx selector trigger",+ "exact": 1+ }+ ],+ "interactions": [+ {+ "id": "astryx-tooltip-in-host-dialog",+ "direction": "astryx-in-host",+ "open": [+ {+ "name": "primary-action",+ "source": "probe",+ "method": "click"+ },+ {+ "name": "astryx-tooltip-trigger",+ "source": "result",+ "method": "keyboard-focus"+ }+ ],+ "surfaces": [+ {+ "name": "dialog-backdrop",+ "source": "probe",+ "kind": "backdrop"+ },+ {+ "name": "dialog-surface",+ "source": "probe",+ "kind": "dialog"+ },+ {+ "name": "astryx-tooltip-surface",+ "source": "result",+ "kind": "tooltip"+ }+ ]+ },+ {+ "id": "astryx-selector-in-host-dialog",+ "direction": "astryx-in-host",+ "open": [+ {+ "name": "primary-action",+ "source": "probe",+ "method": "click"+ },+ {+ "name": "astryx-selector-trigger",+ "source": "result",+ "method": "keyboard-activate",+ "key": "Enter"+ }+ ],+ "surfaces": [+ {+ "name": "dialog-backdrop",+ "source": "probe",+ "kind": "backdrop"+ },+ {+ "name": "dialog-surface",+ "source": "probe",+ "kind": "dialog"+ },+ {+ "name": "astryx-selector-surface",+ "source": "result",+ "kind": "popover"+ }+ ]+ }+ ],+ "allowedHostChanges": [],+ "replacedHostProbes": [],+ "allowedOverlayChanges": [+ {+ "fixture": "shadcn-tailwind-v4-established",+ "surface": "dialog-surface",+ "fields": [+ "geometry.y",+ "geometry.top",+ "geometry.bottom",+ "geometry.height"+ ]+ }+ ]+ }+ },+ {+ "id": "s5",+ "tier": "T3",+ "kind": "composition",+ "fixtures": ["enterprise-scoped-synthetic"],+ "prompt": "Add an Astryx dialog that contains the app's existing Tailwind menu. The existing host menu must open from the keyboard above the Astryx dialog and backdrop, keep its host token styling, and remain visible and unclipped. Do not change unrelated host UI.",+ "measures": "An existing host menu surface composed inside an Astryx dialog.",+ "expected": "The host menu opens from the keyboard above the Astryx dialog stack and retains its baseline host styling.",+ "contract": {+ "results": [+ {+ "name": "astryx-dialog-trigger",+ "description": "the Astryx dialog trigger",+ "exact": 1,+ "visible": true+ }+ ],+ "interactions": [+ {+ "id": "host-menu-in-astryx-dialog",+ "direction": "host-in-astryx",+ "open": [+ {+ "name": "astryx-dialog-trigger",+ "source": "result",+ "method": "keyboard-activate",+ "key": "Enter"+ },+ {+ "name": "host-menu-trigger",+ "source": "result",+ "method": "keyboard-activate",+ "key": "Enter"+ }+ ],+ "surfaces": [+ {+ "name": "astryx-dialog-backdrop",+ "source": "result",+ "kind": "backdrop",+ "marker": "astryx-dialog-surface"+ },+ {+ "name": "astryx-dialog-surface",+ "source": "result",+ "kind": "dialog"+ },+ {+ "name": "host-menu-surface",+ "source": "result",+ "kind": "popover",+ "styleReference": "popover-surface"+ }+ ]+ }+ ],+ "allowedHostChanges": [],+ "replacedHostProbes": [],+ "allowedOverlayChanges": [+ {+ "fixture": "enterprise-scoped-synthetic",+ "surface": "dialog-surface",+ "fields": [+ "geometry.y",+ "geometry.top",+ "geometry.bottom",+ "geometry.height"+ ]+ }+ ]+ }+ }+ ],+ "notes": [+ "Prompt text is identical across conditions, repetitions, and executor bundles for each supported fixture.",+ "The data-vibe-result values are task-owned measurement hooks, not implementation hints. Existing data-vibe-probe markers must remain in place.",+ "No prompt tells the executor which files, imports, CSS layers, or component implementation details to use."+ ]+}diff --git a/internal/vibe-tests/setup-test/public-artifact-cli.test.mjs b/internal/vibe-tests/setup-test/public-artifact-cli.test.mjsnew file mode 100644index 0000000000..539ddc1119--- /dev/null+++ b/internal/vibe-tests/setup-test/public-artifact-cli.test.mjs@@ -0,0 +1,204 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {spawnSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {afterEach, describe, expect, it} from 'vitest';+import {assertPublicArtifactSafe} from '../src/public-artifact.mjs';++const HERE = path.dirname(fileURLToPath(import.meta.url));+const REPO_ROOT = path.resolve(HERE, '../../..');+const TSX = path.join(REPO_ROOT, 'node_modules', '.bin', 'tsx');+const SETUP_AGGREGATE = path.join(HERE, 'setup-aggregate.ts');+const UNIVERSAL_AGGREGATE = path.join(+ REPO_ROOT,+ 'internal/vibe-tests/src/universal-aggregate.ts',+);+const UNIVERSAL_RESULTS = path.join(REPO_ROOT, 'internal/vibe-tests/results');+const temporaryDirectories = [];++afterEach(() => {+ for (const directory of temporaryDirectories.splice(0)) {+ fs.rmSync(directory, {recursive: true, force: true});+ }+});++function run(script, args) {+ return spawnSync(TSX, [script, ...args], {+ cwd: REPO_ROOT,+ encoding: 'utf8',+ });+}++describe('public aggregate CLI output', () => {+ it('keeps setup aggregate stdout and stderr free of its absolute result path', () => {+ const directory = path.join(+ os.tmpdir(),+ `setup report ${process.pid}-${Date.now()}`,+ );+ fs.mkdirSync(directory, {recursive: true});+ temporaryDirectories.push(directory);+ const probe = {+ style: {color: 'rgb(0, 0, 0)'},+ geometry: {+ x: 0,+ y: 0,+ top: 0,+ right: 10,+ bottom: 10,+ left: 0,+ width: 10,+ height: 10,+ },+ text: 'Host',+ contrast: 21,+ };+ const scheme = {+ probes: {host: probe},+ variables: {},+ colorScheme: 'light dark',+ consoleErrors: [],+ pageErrors: [],+ failedRequests: [],+ taskResults: {},+ taskInteractions: {},+ };+ const baseline = {+ label: 'baseline',+ fixture: 'tailwind-v4-control',+ build: {ok: true, status: 0, ms: 1, stdout: '', stderr: ''},+ layerOrder: ['astryx-base', 'astryx-theme', 'utilities'],+ schemes: {light: scheme, dark: scheme},+ };+ const arm = {+ ...baseline,+ label: 'current',+ task: {+ id: 's0',+ kind: 'installation',+ contract: {+ allowedHostChanges: [],+ replacedHostProbes: [],+ allowedOverlayChanges: [],+ results: [{name: 'astryx-proof', exact: 1, visible: true}],+ interactions: [],+ },+ },+ executionStatus: 'succeeded',+ integrity: {+ diffSha256: 'a'.repeat(64),+ attestedDiffSha256: 'a'.repeat(64),+ diffMatchesAttestation: true,+ usesAstryx: true,+ changedFiles: ['src/App.tsx'],+ escapeHatches: [],+ },+ };+ fs.writeFileSync(+ path.join(directory, 'baseline__tailwind-v4-control.json'),+ JSON.stringify(baseline),+ );+ fs.writeFileSync(path.join(directory, 'current.json'), JSON.stringify(arm));+ fs.writeFileSync(+ path.join(directory, 'current.provenance.json'),+ JSON.stringify({+ schemaVersion: 1,+ task: {id: 's0', sha256: 'a'.repeat(64)},+ fixture: {id: 'tailwind-v4-control'},+ condition: 'current',+ rep: 1,+ executor: {harness: 'native', model: 'claude-family'},+ execution: {status: 'succeeded'},+ matrix: {stage: 'guidance', bundle: 'native-claude'},+ }),+ );++ const result = run(SETUP_AGGREGATE, [+ '--dir',+ directory,+ '--stage',+ 'guidance',+ ]);+ expect(result.status, result.stderr).toBe(0);+ expect(result.stdout).toContain('s0');+ expect(result.stdout).toContain('NO');+ expect(result.stdout).not.toContain(directory);+ expect(result.stderr).not.toContain(directory);+ expect(() =>+ assertPublicArtifactSafe([result.stdout, result.stderr], {+ label: 'setup aggregate process output',+ privateValues: [directory],+ }),+ ).not.toThrow();+ });++ it('sanitizes universal JSON and human output from recursive private data', () => {+ fs.mkdirSync(UNIVERSAL_RESULTS, {recursive: true});+ const iterationDirectory = fs.mkdtempSync(+ path.join(UNIVERSAL_RESULTS, 'public-artifact-cli-'),+ );+ temporaryDirectories.push(iterationDirectory);+ const iteration = path.basename(iterationDirectory);+ const results = path.join(iterationDirectory, 'results');+ fs.mkdirSync(results);+ fs.writeFileSync(+ path.join(iterationDirectory, 'manifest.json'),+ JSON.stringify({+ config: {target: 'astryx'},+ prompts: [{id: 'p1', category: 'basic'}],+ }),+ );+ fs.writeFileSync(+ path.join(results, 'p1.tsx'),+ 'export default function Example(){return <button>Save</button>}\n',+ );+ const privateRoot = '/home/example/private results';+ fs.writeFileSync(+ path.join(results, 'p1.json'),+ JSON.stringify({+ docsRead: [`${privateRoot}/guide.md`],+ completedAt: '2026-08-27T00:00:01.000Z',+ }),+ );+ fs.writeFileSync(+ path.join(results, 'p1.provenance.json'),+ JSON.stringify({+ schemaVersion: 1,+ task: {id: 'p1', sha256: 'a'.repeat(64)},+ fixture: {id: 'fixture'},+ executor: {harness: 'native', model: 'test'},+ execution: {durationMs: 1},+ usage: {+ inputTokens: 1,+ outputTokens: 1,+ source: `${privateRoot}/usage.json`,+ complete: true,+ },+ }),+ );++ const result = run(UNIVERSAL_AGGREGATE, ['--iteration', iteration]);+ expect(result.status, result.stderr).toBe(0);+ const aggregateFile = path.join(iterationDirectory, 'universal.json');+ const aggregateText = fs.readFileSync(aggregateFile, 'utf8');+ expect(result.stdout).toContain('Saved: universal.json');+ expect(result.stdout).not.toContain(iterationDirectory);+ expect(+ [aggregateText, result.stdout, result.stderr].join('\n'),+ ).not.toContain(privateRoot);+ expect(() =>+ assertPublicArtifactSafe(+ [JSON.parse(aggregateText), result.stdout, result.stderr],+ {+ label: 'universal aggregate process output',+ privateValues: [iterationDirectory, privateRoot, 'example'],+ },+ ),+ ).not.toThrow();+ expect(JSON.parse(aggregateText).execution.runs[0].usage.source).toBe(+ 'runner-reported',+ );+ });+});diff --git a/internal/vibe-tests/setup-test/run-setup.mjs b/internal/vibe-tests/setup-test/run-setup.mjsnew file mode 100644index 0000000000..3d32f51330--- /dev/null+++ b/internal/vibe-tests/setup-test/run-setup.mjs@@ -0,0 +1,338 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * Prepare setup-test sandboxes without launching an executor.+ *+ * Every run is a validated copy of a canonical fixture. The matrix keeps+ * fixture, condition, prompt, repetition, and paired harness/model bundle as+ * separate dimensions and emits one versioned provenance sidecar per task.+ */++import * as crypto from 'node:crypto';+import {execFileSync} from 'node:child_process';+import * as fs from 'node:fs';+import * as path from 'node:path';+import {fileURLToPath} from 'node:url';+import {copyFixture, readRecipe} from '../src/fixture-suite.mjs';+import {+ createSetupProvenance,+ expandSetupMatrix,+ fixtureManifestSha256,+ sha256Text,+ taskContractText,+ validatePromptContracts,+} from './setup-matrix.mjs';+import {assertPublicArtifactSafe} from '../src/public-artifact.mjs';++const EXP_DIR = path.dirname(fileURLToPath(import.meta.url));+const VIBE_DIR = path.resolve(EXP_DIR, '..');+const REPO_ROOT = path.resolve(VIBE_DIR, '../..');+const GUIDANCE = path.join(EXP_DIR, 'guidance');+const REAL_CLI_BIN = path.join(+ REPO_ROOT,+ 'packages',+ 'cli',+ 'clients',+ 'cli',+ 'bin',+ 'astryx.mjs',+);++const args = process.argv.slice(2);+const flag = name => args.includes(`--${name}`);+const value = (name, fallback) => {+ const index = args.indexOf(`--${name}`);+ return index === -1 ? fallback : args[index + 1];+};+const list = name => {+ const selected = value(name);+ return selected+ ? selected+ .split(',')+ .map(item => item.trim())+ .filter(Boolean)+ : undefined;+};++const outRoot = path.resolve(value('out', path.join(EXP_DIR, 'results')));+const artifactPath = absolutePath =>+ path.relative(outRoot, absolutePath).split(path.sep).join('/');+const matrixFile = path.resolve(+ value('matrix', path.join(EXP_DIR, 'matrix.json')),+);+const matrixConfig = JSON.parse(fs.readFileSync(matrixFile, 'utf8'));+const stage = value('stage', matrixConfig.stages[0]?.id);+const requestedReps = value('reps');+const copyDeps = flag('copy-deps') || stage === 'confirmation';++const read = file => fs.readFileSync(file, 'utf8');+const ensureDir = directory => fs.mkdirSync(directory, {recursive: true});+const write = (file, content) => {+ ensureDir(path.dirname(file));+ fs.writeFileSync(file, content);+};+const run = (command, commandArgs, cwd) =>+ execFileSync(command, commandArgs, {cwd, stdio: 'pipe', encoding: 'utf8'});++function copyDirectory(source, destination) {+ ensureDir(destination);+ for (const entry of fs.readdirSync(source, {withFileTypes: true})) {+ const from = path.join(source, entry.name);+ const to = path.join(destination, entry.name);+ if (entry.isDirectory()) copyDirectory(from, to);+ else fs.copyFileSync(from, to);+ }+}++function prepareDependencies(fixtureId) {+ const fixtureRoot = path.join(VIBE_DIR, 'fixtures', fixtureId);+ const depsDir = path.join(outRoot, '.deps', fixtureId);+ if (fs.existsSync(path.join(depsDir, 'node_modules'))) return depsDir;+ ensureDir(depsDir);+ for (const file of ['package.json', 'pnpm-lock.yaml']) {+ fs.copyFileSync(path.join(fixtureRoot, file), path.join(depsDir, file));+ }+ run('pnpm', ['install', '--frozen-lockfile', '--ignore-scripts'], depsDir);+ return depsDir;+}++function linkDependencies(depsDir, sandboxDir) {+ const source = path.join(depsDir, 'node_modules');+ const destination = path.join(sandboxDir, 'node_modules');+ if (copyDeps) copyDirectory(source, destination);+ else run('cp', ['-al', source, destination], sandboxDir);+}++function installLoggingShim(sandboxDir) {+ const binDir = path.join(sandboxDir, 'node_modules', '.bin');+ ensureDir(binDir);+ const logPath = path.join(sandboxDir, '.astryx-invocations.log');+ const shim = `#!/usr/bin/env node+// Records setup-documentation lookups, then delegates to the real CLI.+import {appendFileSync} from 'node:fs';+import {spawnSync} from 'node:child_process';+const LOG = ${JSON.stringify(logPath)};+const REAL = ${JSON.stringify(REAL_CLI_BIN)};+const argv = process.argv.slice(2);+const started = new Date().toISOString();+const result = spawnSync(process.execPath, [REAL, ...argv], {stdio: 'inherit'});+try {+ appendFileSync(LOG, JSON.stringify({ts: started, argv, status: result.status ?? 0}) + '\\n');+} catch {}+process.exit(result.status ?? 0);+`;+ const shimPath = path.join(binDir, 'astryx');+ fs.rmSync(shimPath, {force: true});+ fs.writeFileSync(shimPath, shim);+ fs.chmodSync(shimPath, 0o755);+}++function createSandbox(fixtureId, sandboxDir, depsDir) {+ copyFixture(fixtureId, sandboxDir);+ linkDependencies(depsDir, sandboxDir);+ installLoggingShim(sandboxDir);+ run('pnpm', ['build'], sandboxDir);+}++function appendGuidance(sandboxDir, guidanceFile) {+ const agentsFile = path.join(sandboxDir, 'AGENTS.md');+ const prefix = fs.existsSync(agentsFile) ? '\n' : '';+ fs.appendFileSync(+ agentsFile,+ prefix + read(path.join(GUIDANCE, guidanceFile)),+ );+}++const PATCHES = {+ 'patch:pointer': sandboxDir => appendGuidance(sandboxDir, 'pointer.md'),+ 'patch:existing-app': sandboxDir =>+ appendGuidance(sandboxDir, 'existing-app.md'),+ 'patch:directed': sandboxDir => appendGuidance(sandboxDir, 'directed.md'),+};++function taskPrompt(prompt, fixtureId) {+ return `You are working in an existing application.++You have no prior knowledge of this codebase or of any specific component+library or styling convention. Use only what the project and its public+documentation make available.++## Project++Treat the current working directory as the project and explore it before editing.++## Task++${prompt.prompt}++${taskContractText(prompt, fixtureId)}++## Working notes++Leave the working tree ready for review. Do not commit.++Write a short note to \`.run-notes.json\`:++{+ "summary": "<what you changed>",+ "approach": "<what you did, and why>",+ "consulted": ["<what you read or looked up before deciding>"],+ "uncertain": ["<anything you could not confirm>"],+ "workarounds": ["<anything you had to override or work around>"],+ "verified": ["<how you checked the result>" ]+}+`;+}++function gitBaseline(sandboxDir) {+ const git = commandArgs =>+ execFileSync('git', ['-C', sandboxDir, ...commandArgs], {+ stdio: 'pipe',+ env: {+ ...process.env,+ GIT_AUTHOR_NAME: 'vibe-tests',+ GIT_AUTHOR_EMAIL: '[email protected]',+ GIT_COMMITTER_NAME: 'vibe-tests',+ GIT_COMMITTER_EMAIL: '[email protected]',+ },+ });+ write(+ path.join(sandboxDir, '.gitignore'),+ 'node_modules/\ndist/\n.astryx-invocations.log\n.run-notes.json\n',+ );+ git(['init', '-q']);+ git(['add', '-A']);+ git(['commit', '-qm', 'baseline: app before design-system setup']);+}++const conditionsFile = JSON.parse(read(path.join(EXP_DIR, 'conditions.json')));+const promptsFile = JSON.parse(read(path.join(EXP_DIR, 'prompts.json')));+const probeConfig = JSON.parse(read(path.join(EXP_DIR, 'probes.json')));+validatePromptContracts(promptsFile.prompts, probeConfig, matrixConfig);+const conditionById = new Map(+ conditionsFile.conditions.map(condition => [condition.id, condition]),+);+const promptById = new Map(+ promptsFile.prompts.map(prompt => [prompt.id, prompt]),+);+if (promptById.size !== promptsFile.prompts.length) {+ throw new Error('prompt ids must not contain duplicates');+}+for (const prompt of promptsFile.prompts) {+ if (!prompt.kind || !prompt.contract || !Array.isArray(prompt.fixtures)) {+ throw new Error(`prompt ${prompt.id} is missing its structured contract`);+ }+}+const entries = expandSetupMatrix(matrixConfig, {+ stage,+ fixtures: list('fixtures'),+ conditions: list('conditions'),+ prompts: list('prompts'),+ bundles: list('bundles'),+ ...(requestedReps ? {reps: Number(requestedReps)} : {}),+});++for (const entry of entries) {+ if (!conditionById.has(entry.condition)) {+ throw new Error(`matrix references unknown condition ${entry.condition}`);+ }+ if (!promptById.has(entry.prompt)) {+ throw new Error(`matrix references unknown prompt ${entry.prompt}`);+ }+}++const expId = crypto.randomBytes(4).toString('hex');+const experimentRoot = path.join(outRoot, expId);+const fixtureIds = [...new Set(entries.map(entry => entry.fixture))];+const depsByFixture = new Map();+const baselines = {};++for (const fixtureId of fixtureIds) {+ const depsDir = prepareDependencies(fixtureId);+ depsByFixture.set(fixtureId, depsDir);+ const baselineDir = path.join(experimentRoot, 'baselines', fixtureId, 'app');+ createSandbox(fixtureId, baselineDir, depsDir);+ baselines[fixtureId] = artifactPath(baselineDir);+}++const runs = [];+for (const entry of entries) {+ const condition = conditionById.get(entry.condition);+ const prompt = promptById.get(entry.prompt);+ const runRoot = path.join(experimentRoot, entry.id);+ const sandboxDir = path.join(runRoot, 'app');+ createSandbox(entry.fixture, sandboxDir, depsByFixture.get(entry.fixture));+ for (const patch of condition.patches ?? []) {+ const apply = PATCHES[patch];+ if (!apply) throw new Error(`unknown patch: ${patch}`);+ apply(sandboxDir);+ }+ gitBaseline(sandboxDir);++ const task = taskPrompt(prompt, entry.fixture);+ const recipe = readRecipe(entry.fixture);+ const fixtureSha256 = fixtureManifestSha256(recipe);+ const guidanceHash = sha256Text(+ JSON.stringify({+ fixtureSha256,+ condition: entry.condition,+ patches: (condition.patches ?? []).map(patch => [+ patch,+ read(+ path.join(+ GUIDANCE,+ patch === 'patch:pointer'+ ? 'pointer.md'+ : patch === 'patch:existing-app'+ ? 'existing-app.md'+ : 'directed.md',+ ),+ ),+ ]),+ }),+ );+ const provenance = createSetupProvenance({+ entry,+ taskSha256: sha256Text(task),+ fixtureSha256,+ environmentHash: guidanceHash,+ });+ const taskFile = path.join(runRoot, 'task.md');+ const provenanceFile = path.join(runRoot, `${entry.id}.provenance.json`);+ write(taskFile, task);+ write(provenanceFile, `${JSON.stringify(provenance, null, 2)}\n`);+ runs.push({+ ...entry,+ sandboxDir: artifactPath(sandboxDir),+ taskFile: artifactPath(taskFile),+ provenanceFile: artifactPath(provenanceFile),+ });+ console.log(`prepared ${entry.id}`);+}++const configFile = path.join(outRoot, `setup-config-${expId}.json`);+const publicMatrixFile = `setup-matrix-${expId}.json`;+assertPublicArtifactSafe(matrixConfig, {label: 'setup matrix'});+write(+ path.join(outRoot, publicMatrixFile),+ `${JSON.stringify(matrixConfig, null, 2)}\n`,+);+const publicConfig = {+ schemaVersion: 1,+ expId,+ stage,+ createdAt: new Date().toISOString(),+ matrixFile: publicMatrixFile,+ baselines,+ runs,+};+assertPublicArtifactSafe(publicConfig, {label: 'setup config'});+write(configFile, `${JSON.stringify(publicConfig, null, 2)}\n`);+console.log(`\n${runs.length} run(s) — ${path.basename(configFile)}`);+console.log(+ 'No executor was launched. Supply each task and sidecar to a runner.',+);+console.log(+ 'Measure each pristine fixture baseline and run with setup-measure.mjs --fixture <id>.',+);diff --git a/internal/vibe-tests/setup-test/setup-aggregate.test.ts b/internal/vibe-tests/setup-test/setup-aggregate.test.tsnew file mode 100644index 0000000000..1d0aecf143--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-aggregate.test.ts@@ -0,0 +1,141 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ assertAggregateReportPublicSafe,+ compareGuidanceRows,+ matrixCoverage,+ type SetupAggregateRow,+} from './setup-aggregate.js';+import type {Regression, SetupScore} from './setup-eval.js';++function score(regressions: Regression[] = []): SetupScore {+ return {+ label: 'run',+ builds: true,+ clean: true,+ validRun: true,+ executionSucceeded: true,+ consoleErrors: 0,+ failedRequests: 0,+ regressions: regressions.length,+ regressionDetails: regressions,+ byCategory: {+ typography: regressions.filter(item => item.category === 'typography')+ .length,+ color: regressions.filter(item => item.category === 'color').length,+ geometry: regressions.filter(item => item.category === 'geometry').length,+ spacing: regressions.filter(item => item.category === 'spacing').length,+ },+ baselineFailures: [],+ missingProbes: [],+ contrastFailures: [],+ modeDependent: [],+ variablesCaptured: [],+ measurementErrors: [],+ layeringFailures: [],+ layerOrderFailures: [],+ cascadeInverted: false,+ layerOrder: [],+ taskSuccess: true,+ taskFailures: [],+ integrityFailures: [],+ };+}++function regression(+ property: string,+ category: Regression['category'],+): Regression {+ return {+ probe: 'host',+ property,+ category,+ before: 'before',+ after: 'after',+ };+}++function row(+ condition: 'current' | 'candidate',+ value: SetupScore,+ suffix = '',+): SetupAggregateRow {+ return {+ id: `guidance__${condition}__fixture__prompt__bundle__r1${suffix}`,+ file: `${condition}${suffix}.json`,+ stage: 'guidance',+ fixture: 'fixture',+ prompt: 'prompt',+ condition,+ bundle: 'bundle',+ executor: 'harness/model',+ rep: 1,+ score: value,+ };+}++describe('setup aggregation policy', () => {+ it('fails closed on missing, duplicate, and unexpected matrix cells', () => {+ const current = row('current', score());+ const duplicate = {...current, file: 'duplicate.json'};+ const unexpected = {...current, id: 'guidance__unexpected'};+ expect(+ matrixCoverage(+ [current, duplicate, unexpected],+ [{id: current.id}, {id: 'guidance__required'}],+ ),+ ).toEqual({+ missing: ['guidance__required'],+ duplicate: [current.id],+ unexpected: ['guidance__unexpected'],+ });+ });++ it('reports every per-dimension tradeoff without assigning an automatic rank', () => {+ const current = row('current', score([regression('color', 'color')]));+ const candidate = row(+ 'candidate',+ score([regression('borderTopLeftRadius', 'geometry')]),+ );+ const result = compareGuidanceRows([current, candidate]);+ expect(result.comparisons[0]).toMatchObject({+ complete: true,+ deltas: {color: -1, radius: 1},+ regressions: ['radius'],+ improvements: ['color'],+ });+ expect(result).not.toHaveProperty('advances');+ expect(result).not.toHaveProperty('score');+ });++ it('marks an incomplete pair without turning it into a ranking verdict', () => {+ const candidate = row('candidate', score());+ expect(compareGuidanceRows([candidate])).toMatchObject({+ complete: false,+ regressions: [],+ improvements: [],+ });+ });++ it('rejects aggregate report data containing absolute paths or private hosts', () => {+ const absoluteFile = {+ ...row('candidate', score()),+ file: '/home/example/results/candidate.json',+ };+ expect(() => assertAggregateReportPublicSafe([absoluteFile])).toThrow(+ /contains private path or host data/,+ );++ const privateHost = {+ ...row('candidate', score()),+ executor: 'runner.internal.example/model',+ };+ expect(() => assertAggregateReportPublicSafe([privateHost])).toThrow(+ /contains private path or host data/,+ );+ expect(() =>+ assertAggregateReportPublicSafe([row('candidate', score())]),+ ).not.toThrow();+ });+});diff --git a/internal/vibe-tests/setup-test/setup-aggregate.ts b/internal/vibe-tests/setup-test/setup-aggregate.tsnew file mode 100644index 0000000000..ba81cc554d--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-aggregate.ts@@ -0,0 +1,362 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/** Aggregate setup measurements with strict matrix and acceptance accounting. */++import * as fs from 'node:fs';+import * as path from 'node:path';+import {fileURLToPath} from 'node:url';+import {loadOptionalExecutionProvenance} from '../src/provenance-aggregation.js';+import {+ compareCandidate,+ passesAcceptance,+ scoreArm,+ strictAcceptanceSummary,+ verdict,+ type HardDimension,+ type Measurement,+ type SetupScore,+} from './setup-eval.js';+// This JavaScript module is covered by its focused runtime tests.+// @ts-expect-error -- setup-matrix.mjs intentionally has no declaration output.+import {expandSetupMatrix, setupCellKey} from './setup-matrix.mjs';+// This JavaScript module owns the public-boundary checks for emitted reports.+// @ts-expect-error -- public-artifact.mjs intentionally has no declarations.+import {+ assertPublicArtifactSafe,+ sanitizePublicArtifact,+} from '../src/public-artifact.mjs';++type MatrixEntry = {+ id: string;+ fixture: string;+ condition: string;+ prompt: string;+ bundle: string;+ rep: number;+};+type MatrixConfig = {+ stages: Array<{id: string}>;+};+const expandMatrix = expandSetupMatrix as (+ config: MatrixConfig,+ selection: {stage: string},+) => MatrixEntry[];+const cellKey = setupCellKey as (entry: {+ condition: string;+ fixture: string;+ prompt: string;+ bundle: string;+ rep: number;+}) => string;++const HERE = path.dirname(fileURLToPath(import.meta.url));++function argument(name: string, fallback: string): string {+ const index = process.argv.indexOf(`--${name}`);+ return index === -1 ? fallback : (process.argv[index + 1] ?? fallback);+}++export type SetupAggregateRow = {+ id: string;+ file: string;+ stage: string;+ fixture: string;+ prompt: string;+ condition: string;+ bundle: string;+ executor: string;+ rep: number;+ score: SetupScore;+};++export function assertAggregateReportPublicSafe(value: unknown) {+ assertPublicArtifactSafe(value, {label: 'setup aggregate report'});+}++export function matrixCoverage(+ rows: SetupAggregateRow[],+ expectedEntries: Array<{id: string}>,+) {+ const expected = new Set(expectedEntries.map(entry => entry.id));+ const counts = new Map<string, number>();+ for (const row of rows) {+ counts.set(row.id, (counts.get(row.id) ?? 0) + 1);+ }+ return {+ missing: [...expected].filter(id => !counts.has(id)).sort(),+ duplicate: [...counts]+ .filter(([, count]) => count > 1)+ .map(([id]) => id)+ .sort(),+ unexpected: [...counts.keys()].filter(id => !expected.has(id)).sort(),+ };+}++export function compareGuidanceRows(rows: SetupAggregateRow[]) {+ const byCell = new Map<+ string,+ Partial<Record<'current' | 'candidate', SetupAggregateRow>>+ >();+ for (const row of rows) {+ if (row.condition !== 'current' && row.condition !== 'candidate') {+ continue;+ }+ const key = [row.fixture, row.prompt, row.bundle, `r${row.rep}`].join('__');+ const pair = byCell.get(key) ?? {};+ pair[row.condition] = row;+ byCell.set(key, pair);+ }+ type GuidanceComparison =+ | {cell: string; complete: false}+ | ({cell: string; complete: true} & ReturnType<typeof compareCandidate>);+ const comparisons: GuidanceComparison[] = [...byCell]+ .map(([cell, pair]): GuidanceComparison => {+ if (!pair.current || !pair.candidate) {+ return {cell, complete: false};+ }+ return {+ cell,+ complete: true,+ ...compareCandidate(pair.current.score, pair.candidate.score),+ };+ })+ .sort((left, right) => left.cell.localeCompare(right.cell));+ const complete = comparisons.every(comparison => comparison.complete);+ const regressions: string[] = [];+ const improvements = new Set<HardDimension>();+ for (const comparison of comparisons) {+ if (!comparison.complete) {+ continue;+ }+ for (const dimension of comparison.regressions) {+ regressions.push(`${comparison.cell}:${dimension}`);+ }+ for (const dimension of comparison.improvements) {+ improvements.add(dimension);+ }+ }+ return {+ comparisons,+ complete,+ regressions,+ improvements: [...improvements].sort(),+ };+}++function renderTable(rows: SetupAggregateRow[]) {+ const header = [+ 'stage',+ 'fixture',+ 'prompt',+ 'condition',+ 'executor',+ 'rep',+ 'builds',+ 'runtime',+ 'regressions',+ 'unreadable',+ 'layers',+ 'task',+ 'integrity',+ 'verdict',+ 'accepts',+ ];+ const body = rows.map(row => [+ row.stage,+ row.fixture,+ row.prompt,+ row.condition,+ row.executor,+ String(row.rep || '-'),+ row.score.builds ? 'yes' : 'NO',+ String(row.score.consoleErrors + row.score.failedRequests),+ `${row.score.regressions} (${Object.entries(row.score.byCategory)+ .filter(([, count]) => count > 0)+ .map(([category, count]) => `${category[0]}${count}`)+ .join(' ')})`,+ String(row.score.contrastFailures.length),+ String(+ row.score.layeringFailures.length + row.score.layerOrderFailures.length,+ ),+ row.score.taskSuccess ? 'yes' : 'NO',+ String(row.score.integrityFailures.length),+ verdict(row.score),+ passesAcceptance(row.score) ? 'yes' : 'NO',+ ]);+ const widths = header.map((heading, index) =>+ Math.max(heading.length, ...body.map(row => row[index].length)),+ );+ const line = (cells: string[]) =>+ cells.map((cell, index) => cell.padEnd(widths[index])).join(' | ');+ console.log(line(header));+ console.log(widths.map(width => '-'.repeat(width)).join('-+-'));+ for (const row of body) {+ console.log(line(row));+ }+}++async function main() {+ const directory = path.resolve(argument('dir', 'results'));+ const legacyBaseline = argument('baseline', 'baseline');+ const matrixFile = path.resolve(+ argument('matrix', path.join(HERE, 'matrix.json')),+ );+ const stageId = argument('stage', 'confirmation');+ const matrix = JSON.parse(+ fs.readFileSync(matrixFile, 'utf8'),+ ) as MatrixConfig;+ const stage = matrix.stages.find(entry => entry.id === stageId);+ if (!stage) {+ throw new Error(`unknown stage: ${stageId}`);+ }+ const expectedEntries = expandMatrix(matrix, {stage: stageId});++ const load = (file: string): Measurement =>+ JSON.parse(+ fs.readFileSync(path.join(directory, file), 'utf8'),+ ) as Measurement;+ const measurementFiles = fs+ .readdirSync(directory)+ .filter(+ file => file.endsWith('.json') && !file.endsWith('.provenance.json'),+ );+ const baselineFiles = new Map(+ measurementFiles+ .filter(file => file.startsWith('baseline__'))+ .map(file => [file.slice('baseline__'.length, -'.json'.length), file]),+ );+ const armFiles = measurementFiles.filter(+ file => !file.startsWith('baseline__') && file !== `${legacyBaseline}.json`,+ );++ const rows = armFiles.flatMap(file => {+ const stem = path.basename(file, '.json');+ const provenance = loadOptionalExecutionProvenance(+ path.join(directory, `${stem}.provenance.json`),+ );+ if (!provenance) {+ return [];+ }+ const matrixMetadata = provenance.matrix as+ {stage?: string; bundle?: string} | undefined;+ const rowStage = matrixMetadata?.stage ?? 'unknown';+ if (rowStage !== stageId) {+ return [];+ }+ const arm = load(file);+ const fixture = provenance.fixture?.id ?? arm.fixture;+ const baselineFile =+ (fixture && baselineFiles.get(fixture)) ?? `${legacyBaseline}.json`;+ if (!fixture || !fs.existsSync(path.join(directory, baselineFile))) {+ throw new Error(+ `No baseline measurement found for fixture ${fixture ?? 'unknown'}`,+ );+ }+ const dimensions = {+ condition: provenance.condition ?? 'unknown',+ fixture,+ prompt: provenance.task?.id ?? 'unknown',+ bundle: matrixMetadata?.bundle ?? 'unknown',+ rep: provenance.rep ?? 0,+ };+ return [+ {+ id: `${rowStage}__${cellKey(dimensions)}`,+ file,+ stage: rowStage,+ fixture,+ prompt: dimensions.prompt,+ condition: dimensions.condition,+ bundle: dimensions.bundle,+ executor: provenance.executor+ ? `${provenance.executor.harness}/${provenance.executor.model}`+ : 'unknown/unknown',+ rep: dimensions.rep,+ score: scoreArm(load(baselineFile), arm),+ },+ ];+ });++ const coverage = matrixCoverage(rows, expectedEntries);+ const completeCoverage =+ coverage.missing.length === 0 &&+ coverage.duplicate.length === 0 &&+ coverage.unexpected.length === 0;+ const summary = strictAcceptanceSummary(+ rows.map(row => row.score),+ expectedEntries.length,+ completeCoverage,+ );+ assertAggregateReportPublicSafe({stageId, rows, coverage});++ console.log(`\nSetup run — stage: ${stageId}\n`);+ renderTable(rows);+ console.log(+ `\nCoverage: ${summary.coverage.valid}/${summary.coverage.expected} valid` ++ ` (${summary.coverage.present} present)`,+ );+ console.log(+ `Strict clean rate: ${summary.strictClean.numerator}/${summary.strictClean.denominator}`,+ );+ console.log(+ `Damage-free: ${summary.damageFree.numerator}/${summary.damageFree.denominator}`,+ );+ if (coverage.missing.length > 0) {+ console.log(`Missing cells: ${coverage.missing.join(', ')}`);+ }+ if (coverage.duplicate.length > 0) {+ console.log(`Duplicate cells: ${coverage.duplicate.join(', ')}`);+ }+ if (coverage.unexpected.length > 0) {+ console.log(`Unexpected cells: ${coverage.unexpected.join(', ')}`);+ }++ if (stageId === 'guidance') {+ const comparison = compareGuidanceRows(rows);+ console.log('\nCandidate minus current by cell (negative is better):');+ for (const result of comparison.comparisons) {+ if (!result.complete) {+ console.log(` ${result.cell}: INCOMPLETE`);+ continue;+ }+ const deltas = Object.entries(result.deltas)+ .map(+ ([dimension, delta]) =>+ `${dimension}=${delta >= 0 ? '+' : ''}${delta}`,+ )+ .join(' ');+ console.log(` ${result.cell}: ${deltas}`);+ }+ console.log(+ 'Exploratory comparison only: review the explicit tradeoffs; no automatic ranking is applied.',+ );+ }++ if (stageId === 'confirmation') {+ console.log(+ `Final confirmation: ${completeCoverage && summary.passes ? 'PASS' : 'FAIL'}`,+ );+ if (!completeCoverage || !summary.passes) {+ process.exitCode = 1;+ }+ }+}++const isMain =+ process.argv[1] != null &&+ path.resolve(process.argv[1]) ===+ path.resolve(fileURLToPath(import.meta.url));+if (isMain) {+ await main().catch(error => {+ const message = sanitizePublicArtifact(String(error), {+ privateValues: [+ process.cwd(),+ process.env.HOME,+ process.env.USER,+ process.env.HOSTNAME,+ ].filter((value): value is string => typeof value === 'string'),+ });+ console.error(message);+ process.exitCode = 1;+ });+}diff --git a/internal/vibe-tests/setup-test/setup-canonical.test.ts b/internal/vibe-tests/setup-test/setup-canonical.test.tsnew file mode 100644index 0000000000..5b66953a1a--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-canonical.test.ts@@ -0,0 +1,1066 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {execFileSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {afterAll, beforeAll, describe, expect, it} from 'vitest';+// The fixture suite is a runtime JavaScript module covered by its own tests.+// @ts-expect-error -- fixture-suite.mjs intentionally has no declaration output.+import {copyFixture} from '../src/fixture-suite.mjs';+// @ts-expect-error -- public-artifact.mjs intentionally has no declaration output.+import {assertPublicArtifactSafe} from '../src/public-artifact.mjs';+// @ts-expect-error -- setup-integrity.mjs intentionally has no declaration output.+import {analyzeSetupIntegrity} from './setup-integrity.mjs';+import {+ passesAcceptance,+ scoreArm,+ verdict,+ type Measurement,+} from './setup-eval.js';++const HERE = path.dirname(fileURLToPath(import.meta.url));+const REPO_ROOT = path.resolve(HERE, '../../..');+const MEASURE = path.join(HERE, 'setup-measure.mjs');+const RUN_SETUP = path.join(HERE, 'run-setup.mjs');+const EVIDENCE_DIR = process.env.ASTRYX_SETUP_EVIDENCE_DIR+ ? path.resolve(process.env.ASTRYX_SETUP_EVIDENCE_DIR)+ : null;+const RUN_CANONICAL = process.env.ASTRYX_CANONICAL_SETUP_BROWSER === '1';+const temporaryDirectories: string[] = [];+const dependencyRoots = new Map<string, string>();++function run(command: string, args: string[], cwd: string) {+ return execFileSync(command, args, {+ cwd,+ encoding: 'utf8',+ env: {+ ...process.env,+ GIT_AUTHOR_NAME: 'setup-canonical-tests',+ GIT_AUTHOR_EMAIL: '[email protected]',+ GIT_COMMITTER_NAME: 'setup-canonical-tests',+ GIT_COMMITTER_EMAIL: '[email protected]',+ },+ });+}++function edit(file: string, transform: (source: string) => string) {+ const source = fs.readFileSync(file, 'utf8');+ const next = transform(source);+ if (next === source) {+ throw new Error(`canonical edit did not apply: ${file}`);+ }+ fs.writeFileSync(file, next);+}++function ensurePackageBuilds() {+ if (+ !fs.existsSync(path.join(REPO_ROOT, 'packages/core/dist/Button/index.js'))+ ) {+ run('pnpm', ['-F', '@astryxdesign/core', 'build'], REPO_ROOT);+ }+ if (+ !fs.existsSync(+ path.join(REPO_ROOT, 'packages/themes/neutral/dist/theme.css'),+ )+ ) {+ run('pnpm', ['-F', '@astryxdesign/theme-neutral', 'build'], REPO_ROOT);+ }+}++function dependencyRoot(fixture: string) {+ const existing = dependencyRoots.get(fixture);+ if (existing) {+ return existing;+ }+ const parent = fs.mkdtempSync(+ path.join(os.tmpdir(), `setup-canonical-deps-${fixture}-`),+ );+ temporaryDirectories.push(parent);+ const root = path.join(parent, 'app');+ copyFixture(fixture, root);+ run('pnpm', ['install', '--frozen-lockfile', '--ignore-scripts'], root);+ dependencyRoots.set(fixture, root);+ return root;+}++function preparePair(fixture: string) {+ const root = fs.mkdtempSync(+ path.join(os.tmpdir(), `setup-canonical-${fixture}-`),+ );+ temporaryDirectories.push(root);+ const deps = dependencyRoot(fixture);+ const baseline = path.join(root, 'baseline');+ const arm = path.join(root, 'arm');+ for (const destination of [baseline, arm]) {+ copyFixture(fixture, destination);+ run(+ 'cp',+ [+ '-al',+ path.join(deps, 'node_modules'),+ path.join(destination, 'node_modules'),+ ],+ root,+ );+ }++ const scope = path.join(arm, 'node_modules', '@astryxdesign');+ fs.mkdirSync(scope, {recursive: true});+ fs.symlinkSync(+ path.join(REPO_ROOT, 'packages/core'),+ path.join(scope, 'core'),+ 'dir',+ );+ fs.symlinkSync(+ path.join(REPO_ROOT, 'packages/themes/neutral'),+ path.join(scope, 'theme-neutral'),+ 'dir',+ );+ edit(path.join(arm, 'vite.config.ts'), source =>+ source.includes(' resolve: {')+ ? source.replace(+ ' resolve: {',+ " resolve: {\n dedupe: ['react', 'react-dom'],",+ )+ : source.replace(+ 'export default defineConfig({\n plugins:',+ "export default defineConfig({\n resolve: {dedupe: ['react', 'react-dom']},\n plugins:",+ ),+ );+ fs.writeFileSync(path.join(arm, '.gitignore'), 'node_modules/\ndist/\n');+ run('git', ['init', '-q'], arm);+ run('git', ['add', '-A'], arm);+ run('git', ['commit', '-qm', 'baseline'], arm);+ return {root, baseline, arm};+}++function installAstryxCss(arm: string) {+ edit(path.join(arm, 'src', 'index.css'), source =>+ source.replace(+ "@import 'tailwindcss';",+ [+ '@layer reset, theme, base, astryx-base, astryx-theme, components, utilities;',+ '',+ "@import 'tailwindcss';",+ "@import '@astryxdesign/core/reset.css';",+ "@import '@astryxdesign/core/astryx.css';",+ "@import '@astryxdesign/theme-neutral/theme.css';",+ ].join('\n'),+ ),+ );+}++function applyButtonAddition(arm: string) {+ installAstryxCss(arm);+ edit(path.join(arm, 'src', 'App.tsx'), source =>+ source+ .replace(+ 'const activity = [',+ "import {Button} from '@astryxdesign/core/Button';\n\nconst activity = [",+ )+ .replace(+ ` <button+ className="rounded-md bg-zinc-900 px-3 py-2 text-sm font-medium text-white"+ data-vibe-probe="primary-action">+ Create item+ </button>`,+ ` <div className="flex gap-2">+ <button+ className="rounded-md bg-zinc-900 px-3 py-2 text-sm font-medium text-white"+ data-vibe-probe="primary-action">+ Create item+ </button>+ <Button+ data-vibe-result="astryx-button"+ label="Deploy"+ size="md"+ variant="secondary"+ />+ </div>`,+ ),+ );+}++function applyStatusReplacement(arm: string, fixture: string) {+ installAstryxCss(arm);+ edit(path.join(arm, 'src', 'App.tsx'), source => {+ if (fixture === 'tailwind-v4-control') {+ return source+ .replace(+ 'const activity = [',+ "import {Badge} from '@astryxdesign/core/Badge';\n\nconst activity = [",+ )+ .replace(+ ` <span+ data-vibe-probe={+ row.item === 'Quarterly plan' ? 'status' : undefined+ }>+ {row.state}+ </span>`,+ ` {row.item === 'Quarterly plan' ? (+ <span+ className="relative inline-block"+ data-vibe-replacement>+ <span className="invisible">{row.state}</span>+ <Badge+ className="absolute left-0 top-1/2 -translate-y-1/2 whitespace-nowrap"+ data-vibe-result="astryx-status"+ label={row.state}+ variant="neutral"+ />+ </span>+ ) : (+ row.state+ )}`,+ );+ }+ if (fixture === 'shadcn-tailwind-v4-established') {+ return source+ .replace(+ "import {createPortal} from 'react-dom';",+ "import {createPortal} from 'react-dom';\nimport {Badge} from '@astryxdesign/core/Badge';",+ )+ .replace(+ ` <span+ className={\`inline-flex rounded-full px-2 py-1 text-xs font-medium \${statusClasses[request.status as keyof typeof statusClasses]}\`}+ data-vibe-probe={+ request.id === 'REQ-104' ? 'status' : undefined+ }>+ {request.status}+ </span>`,+ ` {request.id === 'REQ-104' ? (+ <span+ className="relative inline-block"+ data-vibe-replacement>+ <span+ aria-hidden+ className={\`invisible inline-flex rounded-full px-2 py-1 text-xs font-medium \${statusClasses[request.status as keyof typeof statusClasses]}\`}>+ {request.status}+ </span>+ <Badge+ className="absolute left-0 top-1/2 -translate-y-1/2 whitespace-nowrap"+ data-vibe-result="astryx-status"+ label={request.status}+ variant="neutral"+ />+ </span>+ ) : (+ <span+ className={\`inline-flex rounded-full px-2 py-1 text-xs font-medium \${statusClasses[request.status as keyof typeof statusClasses]}\`}>+ {request.status}+ </span>+ )}`,+ );+ }+ if (fixture === 'enterprise-scoped-synthetic') {+ return source+ .replace(+ "import {createPortal} from 'react-dom';",+ "import {createPortal} from 'react-dom';\nimport {Badge} from '@astryxdesign/core/Badge';",+ )+ .replace(+ ` <span+ className={\`inline-flex rounded px-2 py-1 text-xs font-semibold \${healthClasses[service.health]}\`}+ data-vibe-probe={+ service.name === 'Aster' ? 'status' : undefined+ }>+ {service.health}+ </span>`,+ ` {service.name === 'Aster' ? (+ <span+ className="relative inline-block"+ data-vibe-replacement>+ <span+ aria-hidden+ className={\`invisible inline-flex rounded px-2 py-1 text-xs font-semibold \${healthClasses[service.health]}\`}>+ {service.health}+ </span>+ <Badge+ className="absolute left-0 top-1/2 -translate-y-1/2 whitespace-nowrap"+ data-vibe-result="astryx-status"+ label={service.health}+ variant="neutral"+ />+ </span>+ ) : (+ <span+ className={\`inline-flex rounded px-2 py-1 text-xs font-semibold \${healthClasses[service.health]}\`}>+ {service.health}+ </span>+ )}`,+ );+ }+ throw new Error(`unsupported status fixture: ${fixture}`);+ });+}++function applySelectorReplacement(arm: string) {+ installAstryxCss(arm);+ edit(path.join(arm, 'src', 'App.tsx'), source =>+ source+ .replace(+ 'const activity = [',+ "import {useState} from 'react';\nimport {Selector} from '@astryxdesign/core/Selector';\n\nconst activity = [",+ )+ .replace(+ 'export default function App() {',+ "export default function App() {\n const [reviewer, setReviewer] = useState('avery');",+ )+ .replace(+ ` <label className="block text-sm font-medium">+ Reviewer+ <select className="mt-1 block w-full rounded-md border border-zinc-300 bg-white px-3 py-2">+ <option>Avery</option>+ <option>Morgan</option>+ </select>+ </label>`,+ ` <Selector+ data-vibe-result="astryx-selector"+ label="Reviewer"+ onChange={setReviewer}+ options={[+ {value: 'avery', label: 'Avery'},+ {value: 'morgan', label: 'Morgan'},+ ]}+ size="md"+ value={reviewer}+ width="100%"+ />`,+ ),+ );+}++function applyShadcnComposition(arm: string) {+ installAstryxCss(arm);+ edit(path.join(arm, 'src', 'App.tsx'), source =>+ source+ .replace(+ "import {Button} from '@/components/ui/button';",+ [+ "import {Selector} from '@astryxdesign/core/Selector';",+ "import {Tooltip} from '@astryxdesign/core/Tooltip';",+ "import {Button} from '@/components/ui/button';",+ ].join('\n'),+ )+ .replace(+ ' const [popoverOpen, setPopoverOpen] = useState(false);',+ " const [popoverOpen, setPopoverOpen] = useState(false);\n const [route, setRoute] = useState('operations');",+ )+ .replace(+ ` <div className="mt-4 flex items-center justify-between gap-3 rounded-md border border-border bg-background p-3">`,+ ` <div className="mt-4 flex items-end gap-3">+ <Tooltip+ content={+ <span data-vibe-result="astryx-tooltip-surface">+ Approval routing details+ </span>+ }+ delay={0}>+ <button+ aria-label="Explain Astryx routing"+ className="grid size-9 place-items-center rounded-md border border-input bg-background text-sm font-semibold"+ data-vibe-result="astryx-tooltip-trigger"+ type="button">+ i+ </button>+ </Tooltip>+ <Selector+ data-vibe-result="astryx-selector-trigger"+ isLabelHidden+ label="Approval route"+ onChange={setRoute}+ options={[+ {value: 'operations', label: 'Operations'},+ {value: 'research', label: 'Research'},+ {value: 'facilities', label: 'Facilities'},+ ]}+ renderOption={option => (+ <span+ data-vibe-result={+ option.value === 'operations'+ ? 'astryx-selector-surface'+ : undefined+ }>+ {option.label}+ </span>+ )}+ value={route}+ width={180}+ />+ </div>++ <div className="mt-4 flex items-center justify-between gap-3 rounded-md border border-border bg-background p-3">`,+ ),+ );+}++function applyEnterpriseComposition(arm: string) {+ installAstryxCss(arm);+ edit(+ path.join(arm, 'src', 'index.css'),+ source =>+ `${source}\n[data-vibe-result='astryx-dialog-surface']::backdrop {\n z-index: 40;\n}\n`,+ );+ edit(path.join(arm, 'src', 'App.tsx'), source => {+ let next = source+ .replace(+ "import {useState} from 'react';\nimport {createPortal} from 'react-dom';",+ "import {useRef, useState} from 'react';\nimport {Dialog} from '@astryxdesign/core/Dialog';",+ )+ .replace(+ ' const [popoverOpen, setPopoverOpen] = useState(false);',+ ' const [popoverOpen, setPopoverOpen] = useState(false);\n const hostMenuRef = useRef<HTMLDivElement>(null);',+ )+ .replace(+ ` const closeDialog = () => {+ setDialogOpen(false);+ setPopoverOpen(false);+ };`,+ ` const closeDialog = () => {+ if (hostMenuRef.current?.matches(':popover-open')) {+ hostMenuRef.current.hidePopover();+ }+ setDialogOpen(false);+ setPopoverOpen(false);+ };++ const toggleHostMenu = () => {+ const menu = hostMenuRef.current;+ if (!menu) return;+ const isOpen = menu.matches(':popover-open');+ if (isOpen) {+ menu.hidePopover();+ } else {+ menu.showPopover();+ }+ setPopoverOpen(!isOpen);+ };`,+ )+ .replace(+ ` data-vibe-probe="dialog-trigger"+ onClick={() => setDialogOpen(true)}`,+ ` data-vibe-probe="dialog-trigger"+ data-vibe-result="astryx-dialog-trigger"+ onClick={() => setDialogOpen(true)}`,+ );+ const start = next.indexOf(' {dialogOpen\n ? createPortal(');+ const end = next.indexOf(' </div>\n );', start);+ if (start === -1 || end === -1) {+ throw new Error('canonical enterprise dialog edit did not apply');+ }+ next = `${next.slice(0, start)} {dialogOpen ? (+ <Dialog+ aria-labelledby="service-dialog-title"+ className="fixture-shell block z-[50] rounded-lg border border-border bg-panel p-6 text-foreground shadow-2xl"+ data-guest-design-system+ data-mode={mode}+ data-vibe-probe="dialog-surface"+ data-vibe-result="astryx-dialog-surface"+ isOpen+ onOpenChange={open => {+ if (!open) closeDialog();+ }}+ width="min(30rem, calc(100% - 2rem))">+ <div data-vibe-probe="dialog-backdrop">+ <p className="text-xs font-semibold uppercase tracking-[0.16em] opacity-75">+ Guest action+ </p>+ <h2 className="text-lg font-semibold" id="service-dialog-title">+ Service actions+ </h2>+ </div>+ <p+ className="mt-3 text-sm opacity-75"+ data-vibe-probe="dialog-body">+ This guest dialog is portaled across the host boundary.+ </p>+ <div+ className="mt-4 rounded-md border border-border bg-subtle p-3 text-sm"+ data-vibe-probe="dialog-callout">+ Restarting a service pauses its queued work.+ </div>+ <div className="mt-4 flex items-center justify-between gap-3 rounded-md border border-border bg-panel p-3">+ <div>+ <p className="text-sm font-semibold">Target service</p>+ <p className="text-xs opacity-75">Aster · North</p>+ </div>+ <button+ aria-controls="guest-service-menu"+ aria-expanded={popoverOpen}+ aria-haspopup="menu"+ className="rounded-md border border-border bg-panel px-3 py-2 text-sm font-medium"+ data-vibe-probe="popover-trigger"+ data-vibe-result="host-menu-trigger"+ onClick={toggleHostMenu}+ type="button">+ Select service+ </button>+ </div>+ <div className="mt-5 flex justify-end gap-2">+ <button+ className="rounded-md border border-border px-3 py-2 text-sm font-medium"+ onClick={closeDialog}+ type="button">+ Cancel+ </button>+ <button+ className="rounded-md bg-error px-3 py-2 text-sm font-semibold text-white"+ data-vibe-probe="destructive-action"+ type="button">+ Restart service+ </button>+ </div>+ <div+ ref={hostMenuRef}+ aria-label="Guest service options"+ className="fixture-shell fixed left-[calc(50%+1.5rem)] top-[calc(50%+2rem)] z-[70] w-52 rounded-md border border-border bg-panel p-1 text-foreground shadow-2xl"+ data-guest-design-system+ data-mode={mode}+ data-vibe-probe="popover-surface"+ data-vibe-result="host-menu-surface"+ id="guest-service-menu"+ popover="manual"+ role="menu">+ {['Aster', 'Briar', 'Cedar'].map(service => (+ <button+ className="block w-full rounded px-3 py-2 text-left text-sm hover:bg-subtle"+ data-vibe-probe={+ service === 'Aster' ? 'popover-menu-item' : undefined+ }+ key={service}+ role="menuitem"+ type="button">+ {service}+ </button>+ ))}+ </div>+ </Dialog>+ ) : null}+${next.slice(end)}`;+ return next;+ });+}++function measure(+ app: string,+ fixture: string,+ out: string,+ provenance?: string,+ screenshotDir?: string,+): Measurement {+ run(+ process.execPath,+ [+ MEASURE,+ '--app',+ app,+ '--fixture',+ fixture,+ '--out',+ out,+ ...(provenance ? ['--provenance', provenance] : []),+ ...(screenshotDir ? ['--screenshot-dir', screenshotDir] : []),+ ],+ REPO_ROOT,+ );+ const serialized = fs.readFileSync(out, 'utf8');+ expect(serialized).not.toContain(path.resolve(app));+ const measurement = JSON.parse(serialized) as Measurement & {app?: string};+ expect(measurement.app).toBe(fixture);+ if (provenance) {+ const exportedProvenance = JSON.parse(+ fs.readFileSync(out.replace(/\.json$/, '.provenance.json'), 'utf8'),+ );+ expect(exportedProvenance.usage?.source).toBe('runner-reported');+ expect(JSON.stringify(exportedProvenance)).not.toContain(path.resolve(app));+ }+ return measurement;+}++function provenanceFor(arm: string, fixture: string, promptId: string) {+ const digest = analyzeSetupIntegrity(arm).diffSha256;+ const file = path.join(path.dirname(arm), `${promptId}.provenance.json`);+ fs.writeFileSync(+ file,+ `${JSON.stringify(+ {+ schemaVersion: 1,+ task: {id: promptId, sha256: 'a'.repeat(64)},+ fixture: {id: fixture, sha256: 'b'.repeat(64)},+ condition: 'candidate',+ rep: 1,+ executor: {harness: 'fixture-test', model: 'deterministic'},+ matrix: {stage: 'confirmation', bundle: 'fixture-test'},+ execution: {+ status: 'succeeded',+ attempt: 1,+ retry: 0,+ agentDiffSha256: digest,+ },+ usage: {+ complete: true,+ inputTokens: 1,+ outputTokens: 1,+ source: arm,+ },+ environmentHash: 'c'.repeat(64),+ },+ null,+ 2,+ )}\n`,+ );+ return file;+}++function outputPaths(root: string, task: string) {+ const directory = EVIDENCE_DIR ? path.join(EVIDENCE_DIR, task) : root;+ fs.mkdirSync(directory, {recursive: true});+ return {+ directory,+ baseline: path.join(directory, `${task}-baseline.json`),+ arm: path.join(directory, `${task}-canonical.json`),+ };+}++function hostProbe(measurement: Measurement, probeName: string) {+ const probe = measurement.schemes.light.probes[probeName];+ if (!probe || 'missing' in probe) {+ throw new Error(`missing ${probeName}`);+ }+ return probe;+}++function writeFalsePositiveEvidence(+ paths: ReturnType<typeof outputPaths>,+ task: string,+ baseline: Measurement,+ arm: Measurement,+ flags: Array<{probe: string; field: string}>,+) {+ if (!EVIDENCE_DIR) {+ return;+ }+ const oldEvaluatorFlags = flags.map(({probe, field}) => {+ const before = hostProbe(baseline, probe);+ const after = hostProbe(arm, probe);+ const read = (reading: typeof before) => {+ if (field === 'textContent') {+ return reading.descendantText;+ }+ const geometryField = field.replace(+ 'geometry.',+ '',+ ) as keyof typeof reading.geometry;+ return reading.geometry[geometryField];+ };+ return {probe, field, before: read(before), after: read(after)};+ });+ fs.writeFileSync(+ path.join(paths.directory, 'false-positive-evidence.json'),+ `${JSON.stringify(+ {+ task,+ viewport: {width: 1280, height: 720},+ baselineScreenshot: path.join(+ paths.directory,+ `${task}-baseline-light.png`,+ ),+ canonicalScreenshot: path.join(+ paths.directory,+ `${task}-canonical-light.png`,+ ),+ oldEvaluatorFlags,+ strictAcceptance: passesAcceptance(scoreArm(baseline, arm)),+ },+ null,+ 2,+ )}\n`,+ );+}++function assertStyleMutationFails(+ baseline: Measurement,+ arm: Measurement,+ probeName: string,+) {+ const mutations = [+ ['color', 'rgb(255, 0, 0)', 'rgb(0, 255, 0)'],+ ['borderTopLeftRadius', '999px', '777px'],+ ['boxShadow', 'rgb(255, 0, 0) 0px 0px 1px 0px', 'none'],+ ['borderTopWidth', '9px', '7px'],+ ['fontFamily', 'Mutation Font', 'Other Font'],+ ];+ for (const [property, first, second] of mutations) {+ const mutated = structuredClone(arm);+ for (const scheme of ['light', 'dark'] as const) {+ const probe = mutated.schemes[scheme].probes[probeName];+ if (!probe || 'missing' in probe) {+ throw new Error(`missing ${probeName}`);+ }+ probe.style[property] = probe.style[property] === first ? second : first;+ }+ expect(+ passesAcceptance(scoreArm(baseline, mutated)),+ `${probeName}:${property}`,+ ).toBe(false);+ }+}++const describeCanonical = RUN_CANONICAL ? describe : describe.skip;++describeCanonical('canonical intended changes', () => {+ beforeAll(ensurePackageBuilds, 180_000);+ afterAll(() => {+ for (const directory of temporaryDirectories) {+ fs.rmSync(directory, {recursive: true, force: true});+ }+ });+ it('writes a portable setup config with a resolvable matrix reference', () => {+ const root = fs.mkdtempSync(+ path.join(os.tmpdir(), 'setup-canonical-public-config-'),+ );+ temporaryDirectories.push(root);+ run(+ process.execPath,+ [+ RUN_SETUP,+ '--stage',+ 'separation',+ '--fixtures',+ 'tailwind-v4-control',+ '--conditions',+ 'floor',+ '--prompts',+ 's1',+ '--bundles',+ 'native-claude',+ '--reps',+ '1',+ '--out',+ root,+ ],+ REPO_ROOT,+ );+ const configFile = fs+ .readdirSync(root)+ .find(file => file.startsWith('setup-config-'));+ if (!configFile) {+ throw new Error('missing generated setup config');+ }+ const config = JSON.parse(+ fs.readFileSync(path.join(root, configFile), 'utf8'),+ );+ expect(() =>+ assertPublicArtifactSafe(config, {+ label: 'generated setup config',+ privateValues: [+ root,+ REPO_ROOT,+ process.env.HOME,+ process.env.USER,+ process.env.HOSTNAME,+ ],+ }),+ ).not.toThrow();+ expect(path.isAbsolute(config.matrixFile)).toBe(false);+ expect(fs.existsSync(path.join(root, config.matrixFile))).toBe(true);+ const taskText = fs.readFileSync(+ path.join(root, config.runs[0].taskFile),+ 'utf8',+ );+ expect(taskText).toContain(+ 'Treat the current working directory as the project',+ );+ expect(taskText).not.toContain(root);+ expect(() =>+ assertPublicArtifactSafe(taskText, {+ label: 'generated task',+ privateValues: [root, REPO_ROOT],+ }),+ ).not.toThrow();+ for (const reference of [+ ...Object.values(config.baselines),+ ...config.runs.flatMap((entry: Record<string, string>) => [+ entry.sandboxDir,+ entry.taskFile,+ entry.provenanceFile,+ ]),+ ] as string[]) {+ expect(path.isAbsolute(reference)).toBe(false);+ expect(fs.existsSync(path.join(root, reference))).toBe(true);+ }+ }, 120_000);++ it('builds, measures, and accepts a correct s1 button insertion', () => {+ const {+ root,+ baseline: baselineApp,+ arm,+ } = preparePair('tailwind-v4-control');+ applyButtonAddition(arm);+ const paths = outputPaths(root, 's1');+ const baseline = measure(+ baselineApp,+ 'tailwind-v4-control',+ paths.baseline,+ undefined,+ EVIDENCE_DIR ? paths.directory : undefined,+ );+ const provenance = provenanceFor(arm, 'tailwind-v4-control', 's1');+ const result = measure(+ arm,+ 'tailwind-v4-control',+ paths.arm,+ provenance,+ EVIDENCE_DIR ? paths.directory : undefined,+ );+ writeFalsePositiveEvidence(paths, 's1', baseline, result, [+ {probe: 'host-shell', field: 'textContent'},+ {probe: 'primary-action', field: 'geometry.x'},+ ]);+ const score = scoreArm(baseline, result);+ expect(score.regressionDetails).toEqual([]);+ expect(verdict(score)).toBe('clean');+ expect(passesAcceptance(score)).toBe(true);+ assertStyleMutationFails(baseline, result, 'primary-action');+ const geometryMutation = structuredClone(result);+ for (const scheme of ['light', 'dark'] as const) {+ const probe = geometryMutation.schemes[scheme].probes['primary-action'];+ if ('missing' in probe) {+ throw new Error('missing primary-action');+ }+ probe.geometry.width += 1;+ }+ expect(passesAcceptance(scoreArm(baseline, geometryMutation))).toBe(false);+ }, 120_000);++ it.each([+ 'tailwind-v4-control',+ 'shadcn-tailwind-v4-established',+ 'enterprise-scoped-synthetic',+ ])(+ 'builds, measures, and accepts a correct s2 status replacement in %s',+ fixture => {+ const {root, baseline: baselineApp, arm} = preparePair(fixture);+ applyStatusReplacement(arm, fixture);+ const baseline = measure(+ baselineApp,+ fixture,+ path.join(root, 's2-baseline.json'),+ );+ const provenance = provenanceFor(arm, fixture, 's2');+ const result = measure(+ arm,+ fixture,+ path.join(root, 's2-canonical.json'),+ provenance,+ );+ const score = scoreArm(baseline, result);+ expect(+ result.build.ok,+ `${result.build.stdout}\n${result.build.stderr}`,+ ).toBe(true);+ expect(score.regressionDetails, JSON.stringify(score, null, 2)).toEqual(+ [],+ );+ expect(verdict(score)).toBe('clean');+ expect(passesAcceptance(score)).toBe(true);+ for (const scheme of ['light', 'dark'] as const) {+ expect(result.schemes[scheme].probes.status).toEqual({missing: true});+ expect(+ result.schemes[scheme].taskResults?.['astryx-status'],+ ).toMatchObject({count: 1, visible: true});+ }+ const neighbor =+ fixture === 'tailwind-v4-control' ? 'table-cell' : 'table-header';+ assertStyleMutationFails(baseline, result, neighbor);+ },+ 120_000,+ );++ it('builds, measures, and accepts a correct s3 selector replacement', () => {+ const {+ root,+ baseline: baselineApp,+ arm,+ } = preparePair('tailwind-v4-control');+ applySelectorReplacement(arm);+ const paths = outputPaths(root, 's3');+ const baseline = measure(+ baselineApp,+ 'tailwind-v4-control',+ paths.baseline,+ undefined,+ EVIDENCE_DIR ? paths.directory : undefined,+ );+ const provenance = provenanceFor(arm, 'tailwind-v4-control', 's3');+ const result = measure(+ arm,+ 'tailwind-v4-control',+ paths.arm,+ provenance,+ EVIDENCE_DIR ? paths.directory : undefined,+ );+ writeFalsePositiveEvidence(paths, 's3', baseline, result, [+ {probe: 'form-control', field: 'geometry.y'},+ {probe: 'form-control', field: 'geometry.bottom'},+ ]);+ const score = scoreArm(baseline, result);+ expect(score.regressionDetails).toEqual([]);+ expect(verdict(score)).toBe('clean');+ expect(passesAcceptance(score)).toBe(true);+ assertStyleMutationFails(baseline, result, 'form-control');+ const geometryMutation = structuredClone(result);+ for (const scheme of ['light', 'dark'] as const) {+ const probe = geometryMutation.schemes[scheme].probes['form-control'];+ if ('missing' in probe) {+ throw new Error('missing form-control');+ }+ probe.geometry.width += 1;+ }+ expect(passesAcceptance(scoreArm(baseline, geometryMutation))).toBe(false);+ }, 120_000);++ it('builds, interacts with, and accepts Astryx surfaces in a host dialog', () => {+ const {+ root,+ baseline: baselineApp,+ arm,+ } = preparePair('shadcn-tailwind-v4-established');+ applyShadcnComposition(arm);+ const baseline = measure(+ baselineApp,+ 'shadcn-tailwind-v4-established',+ path.join(root, 's4-baseline.json'),+ );+ const provenance = provenanceFor(+ arm,+ 'shadcn-tailwind-v4-established',+ 's4',+ );+ const result = measure(+ arm,+ 'shadcn-tailwind-v4-established',+ path.join(root, 's4-canonical.json'),+ provenance,+ );+ const score = scoreArm(baseline, result);+ expect(+ result.build.ok,+ `${result.build.stdout}\n${result.build.stderr}`,+ ).toBe(true);+ expect(score.regressionDetails, JSON.stringify(score, null, 2)).toEqual([]);+ expect(score.layeringFailures).toEqual([]);+ expect(score.taskFailures).toEqual([]);+ expect(verdict(score)).toBe('clean');+ expect(passesAcceptance(score)).toBe(true);++ const baselineDialog =+ baseline.schemes.light.interaction?.surfaces['dialog-surface'];+ const resultDialog =+ result.schemes.light.interaction?.surfaces['dialog-surface'];+ if (+ !baselineDialog ||+ 'missing' in baselineDialog ||+ !resultDialog ||+ 'missing' in resultDialog+ ) {+ throw new Error('missing host dialog surface');+ }+ expect(resultDialog.bounds.height).not.toBe(baselineDialog.bounds.height);+ const baselineHostText = hostProbe(baseline, 'host-shell').text;+ expect(hostProbe(result, 'host-shell').text).toBe(baselineHostText);+ expect(baselineHostText.indexOf('Save settings')).toBeGreaterThan(200);++ const widthDamage = structuredClone(result);+ const styleDamage = structuredClone(result);+ const backdropDamage = structuredClone(result);+ for (const scheme of ['light', 'dark'] as const) {+ const widthSurface =+ widthDamage.schemes[scheme].interaction?.surfaces['dialog-surface'];+ const styleSurface =+ styleDamage.schemes[scheme].interaction?.surfaces['dialog-surface'];+ const backdropSurface =+ backdropDamage.schemes[scheme].interaction?.surfaces['dialog-backdrop'];+ if (+ !widthSurface ||+ 'missing' in widthSurface ||+ !styleSurface ||+ 'missing' in styleSurface ||+ !backdropSurface ||+ 'missing' in backdropSurface+ ) {+ throw new Error('missing host overlay surface');+ }+ widthSurface.bounds.width += 1;+ styleSurface.style.boxShadow = 'none';+ backdropSurface.style.backgroundColor = 'rgb(255, 0, 0)';+ }+ expect(passesAcceptance(scoreArm(baseline, widthDamage))).toBe(false);+ expect(passesAcceptance(scoreArm(baseline, styleDamage))).toBe(false);+ expect(passesAcceptance(scoreArm(baseline, backdropDamage))).toBe(false);++ edit(path.join(arm, 'src', 'App.tsx'), source =>+ source.replace('Save settings', 'Publish settings'),+ );+ const textDamage = measure(+ arm,+ 'shadcn-tailwind-v4-established',+ path.join(root, 's4-host-text-damage.json'),+ provenanceFor(arm, 'shadcn-tailwind-v4-established', 's4'),+ );+ expect(scoreArm(baseline, textDamage).regressionDetails).toEqual(+ expect.arrayContaining([+ expect.objectContaining({probe: 'host-shell', property: 'text'}),+ ]),+ );+ expect(passesAcceptance(scoreArm(baseline, textDamage))).toBe(false);+ }, 180_000);++ it('builds, interacts with, and accepts a host menu in an Astryx dialog', () => {+ const {+ root,+ baseline: baselineApp,+ arm,+ } = preparePair('enterprise-scoped-synthetic');+ applyEnterpriseComposition(arm);+ const baseline = measure(+ baselineApp,+ 'enterprise-scoped-synthetic',+ path.join(root, 's5-baseline.json'),+ );+ const provenance = provenanceFor(arm, 'enterprise-scoped-synthetic', 's5');+ const result = measure(+ arm,+ 'enterprise-scoped-synthetic',+ path.join(root, 's5-canonical.json'),+ provenance,+ );+ const score = scoreArm(baseline, result);+ expect(+ result.build.ok,+ `${result.build.stdout}\n${result.build.stderr}`,+ ).toBe(true);+ expect(score.regressionDetails, JSON.stringify(score, null, 2)).toEqual([]);+ expect(score.layeringFailures).toEqual([]);+ expect(score.taskFailures).toEqual([]);+ expect(verdict(score)).toBe('clean');+ expect(passesAcceptance(score)).toBe(true);++ for (const scheme of ['light', 'dark'] as const) {+ const interaction =+ result.schemes[scheme].taskInteractions?.['host-menu-in-astryx-dialog'];+ expect(interaction?.keyboardReached).toEqual({+ 'astryx-dialog-trigger': true,+ 'host-menu-trigger': true,+ });+ const menu = interaction?.surfaces['host-menu-surface'];+ if (!menu || 'missing' in menu) {+ throw new Error('missing host menu surface');+ }+ expect(menu.topLayer.inTopLayer).toBe(true);+ }+ }, 180_000);+});diff --git a/internal/vibe-tests/setup-test/setup-eval.test.ts b/internal/vibe-tests/setup-test/setup-eval.test.tsnew file mode 100644index 0000000000..4a2af41e08--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-eval.test.ts@@ -0,0 +1,843 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file setup-eval.test.ts+ * @input Synthetic measurements+ * @output Failing-first coverage for every strict acceptance dimension+ * @position internal/vibe-tests/setup-test — the scorer's own tests+ */++import {describe, expect, it} from 'vitest';+import {+ cascadeInverted,+ compareCandidate,+ contrastFailures,+ hardGateVector,+ nestedLayerFailures,+ passesAcceptance,+ regressions,+ scoreArm,+ strictAcceptanceSummary,+ variableCapture,+ verdict,+ type InteractionReading,+ type Measurement,+ type ProbeReading,+ type SchemeReading,+ type SetupIntegrity,+ type TaskContract,+} from './setup-eval.js';++const GEOMETRY = {+ x: 10,+ y: 20,+ top: 20,+ right: 110,+ bottom: 60,+ left: 10,+ width: 100,+ height: 40,+};+const STYLE = {+ color: 'rgb(230, 230, 233)',+ backgroundColor: 'rgb(20, 20, 23)',+ borderTopColor: 'rgb(63, 63, 70)',+ borderTopWidth: '1px',+ borderTopStyle: 'solid',+ borderTopLeftRadius: '8px',+ boxShadow: 'rgba(0, 0, 0, 0.1) 0px 1px 2px 0px',+ fontFamily: 'ui-sans-serif',+ fontSize: '12px',+ fontWeight: '400',+ lineHeight: '16px',+ letterSpacing: 'normal',+ paddingLeft: '12px',+ marginLeft: '0px',+ gap: '8px',+ width: '100px',+ height: '40px',+ position: 'static',+};++const probe = (+ style: Record<string, string> = STYLE,+ contrast: number | null = 12,+ text = 'Host',+ geometry = GEOMETRY,+): ProbeReading => ({style, text, contrast, geometry});++const layerSurface = (+ kind: string,+ zIndex: string,+ overrides: Record<string, unknown> = {},+) => ({+ kind,+ visible: true,+ display: 'block',+ visibility: 'visible',+ opacity: '1',+ position: 'fixed',+ zIndex,+ style: STYLE,+ bounds: GEOMETRY,+ intersectsViewport: true,+ clippingAncestor: null,+ centerHitSelf: true,+ centerHitProbe: `${kind}-surface`,+ topLayer: {+ tagName: 'div',+ role: kind === 'dialog' ? 'dialog' : kind === 'popover' ? 'menu' : null,+ open: false,+ popover: null,+ ariaModal: kind === 'dialog' ? 'true' : null,+ inTopLayer: false,+ portalChild: true,+ },+ ...overrides,+});++function interaction(+ direction:+ 'host-baseline' | 'astryx-in-host' | 'host-in-astryx' = 'host-baseline',+ nestedOverrides: Record<string, unknown> = {},+): InteractionReading {+ return {+ id: `${direction}-overlay`,+ direction,+ opened: true,+ keyboardReached: {'nested-trigger': true},+ surfaces: {+ 'dialog-backdrop': layerSurface('backdrop', '40'),+ 'dialog-surface': layerSurface('dialog', '50'),+ 'popover-surface': layerSurface('popover', '70', nestedOverrides),+ },+ };+}++function scheme(overrides: Partial<SchemeReading> = {}): SchemeReading {+ return {+ probes: {+ 'page-title': probe(),+ status: probe({...STYLE, color: 'rgb(110, 231, 183)'}, 8.1, 'Ready'),+ },+ variables: {'--spacing': '.25rem', '--color-card': '#141417'},+ colorScheme: 'light dark',+ consoleErrors: [],+ pageErrors: [],+ failedRequests: [],+ interaction: interaction(),+ taskResults: {+ 'astryx-proof': {+ count: 1,+ visible: true,+ focusable: true,+ text: 'Astryx ready',+ style: STYLE,+ geometry: GEOMETRY,+ },+ },+ taskInteractions: {},+ ...overrides,+ };+}++const TASK: TaskContract = {+ allowedHostChanges: [],+ replacedHostProbes: [],+ allowedOverlayChanges: [],+ results: [+ {+ name: 'astryx-proof',+ exact: 1,+ visible: true,+ focusable: true,+ text: 'Astryx ready',+ },+ ],+ interactions: [],+};++const INTEGRITY: SetupIntegrity = {+ diffSha256: 'a'.repeat(64),+ attestedDiffSha256: 'a'.repeat(64),+ diffMatchesAttestation: true,+ usesAstryx: true,+ changedFiles: ['src/App.tsx'],+ escapeHatches: [],+};++const CORRECT_LAYERS = [+ 'reset',+ 'theme',+ 'base',+ 'astryx-base',+ 'astryx-theme',+ 'utilities',+];++function measurement({+ light = scheme(),+ dark = scheme(),+ label = 'arm',+ build = true,+ layerOrder = CORRECT_LAYERS,+ task = true,+ contract = TASK,+ integrity = INTEGRITY,+ executionStatus = 'succeeded',+ fixture,+}: {+ light?: SchemeReading;+ dark?: SchemeReading;+ label?: string;+ build?: boolean;+ layerOrder?: string[];+ task?: boolean;+ contract?: TaskContract;+ integrity?: SetupIntegrity;+ executionStatus?: string;+ fixture?: string;+} = {}): Measurement {+ return {+ label,+ ...(fixture ? {fixture} : {}),+ build: {+ ok: build,+ status: build ? 0 : 1,+ ms: 1000,+ stdout: '',+ stderr: build ? '' : 'build failed',+ },+ layerOrder,+ ...(task ? {task: {id: 's0', kind: 'installation', contract}} : {}),+ ...(task ? {integrity, executionStatus} : {}),+ schemes: {light, dark},+ };+}++const baseline = () => measurement({label: 'baseline', task: false});++describe('host style and geometry regressions', () => {+ it('compares the full style key union, text, and exact normalized geometry', () => {+ const after = scheme();+ after.probes['page-title'] = probe(+ {...STYLE, futureProperty: 'new'},+ 12,+ 'Changed host',+ {...GEOMETRY, x: 10.015625},+ );+ const changed = regressions(scheme(), after).changed;+ expect(changed.map(result => result.property)).toEqual([+ 'geometry.x',+ 'text',+ 'futureProperty',+ ]);+ });++ it.each([+ ['radius', 'borderTopLeftRadius', '12px'],+ ['shadow', 'boxShadow', 'none'],+ ['border', 'borderTopWidth', '0px'],+ ['font', 'fontFamily', 'Inter'],+ ['color', 'color', 'rgb(255, 0, 0)'],+ ])('makes a %s regression fail strict acceptance', (_, property, value) => {+ const after = scheme();+ const current = after.probes['page-title'];+ if ('missing' in current) {+ throw new Error('test probe missing');+ }+ after.probes['page-title'] = probe({...current.style, [property]: value});+ const score = scoreArm(+ baseline(),+ measurement({light: after, dark: after}),+ );+ expect(score.regressionDetails).toEqual(+ expect.arrayContaining([expect.objectContaining({property})]),+ );+ expect(verdict(score)).toBe('cosmetic-drift');+ expect(passesAcceptance(score)).toBe(false);+ });++ it('allows only declared position fields while protecting every host style', () => {+ const contract: TaskContract = {+ ...TASK,+ allowedHostChanges: [+ {+ fixture: 'tailwind-v4-control',+ probe: 'page-title',+ fields: ['geometry.x', 'geometry.left', 'geometry.right'],+ },+ ],+ };+ const moved = scheme();+ moved.probes['page-title'] = probe(STYLE, 12, 'Host', {+ ...GEOMETRY,+ x: 20,+ left: 20,+ right: 120,+ });+ const movedScore = scoreArm(+ baseline(),+ measurement({+ light: moved,+ dark: moved,+ contract,+ fixture: 'tailwind-v4-control',+ }),+ );+ expect(movedScore.regressionDetails).toEqual([]);+ expect(passesAcceptance(movedScore)).toBe(true);++ for (const [property, value] of [+ ['color', 'rgb(255, 0, 0)'],+ ['borderTopLeftRadius', '999px'],+ ['boxShadow', 'none'],+ ['borderTopWidth', '0px'],+ ['fontFamily', 'Inter'],+ ]) {+ const damaged = scheme();+ damaged.probes['page-title'] = probe(+ {...STYLE, [property]: value},+ 12,+ 'Host',+ {...GEOMETRY, x: 20, left: 20, right: 120},+ );+ const score = scoreArm(+ baseline(),+ measurement({+ light: damaged,+ dark: damaged,+ contract,+ fixture: 'tailwind-v4-control',+ }),+ );+ expect(score.regressionDetails, `unallowlisted ${property}`).toEqual(+ expect.arrayContaining([expect.objectContaining({property})]),+ );+ expect(passesAcceptance(score)).toBe(false);+ }+ });++ it('detects damage that exists only in dark mode', () => {+ const dark = scheme();+ const title = dark.probes['page-title'];+ if ('missing' in title) {+ throw new Error('test probe missing');+ }+ dark.probes['page-title'] = probe({...title.style, fontSize: '10px'});+ const score = scoreArm(baseline(), measurement({dark}));+ expect(score.regressionDetails).toContainEqual(+ expect.objectContaining({probe: 'page-title', property: 'fontSize'}),+ );+ expect(passesAcceptance(score)).toBe(false);+ });++ it('rejects an incomplete baseline instead of disabling host comparison', () => {+ const incomplete = scheme({+ probes: {'page-title': {missing: true}, status: {missing: true}},+ });+ const score = scoreArm(+ measurement({+ label: 'baseline',+ task: false,+ light: incomplete,+ dark: incomplete,+ }),+ measurement(),+ );+ expect(score.baselineFailures).toContain(+ 'light:page-title:baseline-missing',+ );+ expect(passesAcceptance(score)).toBe(false);+ });++ it('records a removed host probe rather than skipping it', () => {+ const after = scheme();+ after.probes.status = {missing: true};+ const score = scoreArm(+ baseline(),+ measurement({light: after, dark: after}),+ );+ expect(score.missingProbes).toEqual(['status']);+ expect(verdict(score)).toBe('silent-damage');+ });++ it('exempts only an explicitly replaced probe and keeps neighbors strict', () => {+ const contract: TaskContract = {+ allowedHostChanges: [],+ replacedHostProbes: [+ {+ fixture: 'tailwind-v4-control',+ probe: 'status',+ result: 'astryx-status',+ },+ ],+ allowedOverlayChanges: [],+ results: [+ {+ name: 'astryx-status',+ exact: 1,+ visible: true,+ preserveTextFromHostProbe: 'status',+ },+ ],+ interactions: [],+ };+ const replaced = scheme({+ probes: {...scheme().probes, status: {missing: true}},+ taskResults: {+ 'astryx-status': {+ count: 1,+ visible: true,+ focusable: false,+ text: 'Ready',+ style: STYLE,+ geometry: GEOMETRY,+ },+ },+ });+ const accepted = scoreArm(+ baseline(),+ measurement({+ light: replaced,+ dark: replaced,+ contract,+ fixture: 'tailwind-v4-control',+ }),+ );+ expect(accepted.missingProbes).toEqual([]);+ expect(passesAcceptance(accepted)).toBe(true);++ const addedInstead = scheme({taskResults: replaced.taskResults});+ const additionScore = scoreArm(+ baseline(),+ measurement({+ light: addedInstead,+ dark: addedInstead,+ contract,+ fixture: 'tailwind-v4-control',+ }),+ );+ expect(additionScore.taskFailures).toEqual([+ 'light:status:not-replaced',+ 'dark:status:not-replaced',+ ]);+ expect(passesAcceptance(additionScore)).toBe(false);++ const damaged = structuredClone(replaced);+ const title = damaged.probes['page-title'];+ if ('missing' in title) {+ throw new Error('test probe missing');+ }+ title.style.color = 'rgb(255, 0, 0)';+ const rejected = scoreArm(+ baseline(),+ measurement({+ light: damaged,+ dark: damaged,+ contract,+ fixture: 'tailwind-v4-control',+ }),+ );+ expect(rejected.regressionDetails).toEqual(+ expect.arrayContaining([+ expect.objectContaining({probe: 'page-title', property: 'color'}),+ ]),+ );+ expect(passesAcceptance(rejected)).toBe(false);+ });+});++describe('contrast, variables, modes, and cascade', () => {+ it('fails when readable text drops below the AA floor', () => {+ const after = scheme();+ after.probes['page-title'] = probe(STYLE, 1.1);+ expect(contrastFailures(scheme(), after)).toEqual([+ {probe: 'page-title', before: 12, after: 1.1},+ ]);+ expect(+ passesAcceptance(+ scoreArm(baseline(), measurement({light: after, dark: after})),+ ),+ ).toBe(false);+ });++ it('keeps variable capture diagnostic when visible host output is unchanged', () => {+ const after = scheme({+ variables: {+ '--spacing': '.25rem',+ '--color-card': 'light-dark(#fff,#000)',+ },+ });+ expect(variableCapture(scheme(), after)).toHaveLength(1);+ const score = scoreArm(+ baseline(),+ measurement({light: after, dark: after}),+ );+ expect(verdict(score)).toBe('clean');+ expect(passesAcceptance(score)).toBe(true);+ });++ it('rejects a measurement error instead of guessing across multiple CSS assets', () => {+ const arm = measurement();+ arm.measurementErrors = ['multiple-css-assets:a.css,b.css'];+ const score = scoreArm(baseline(), arm);+ expect(score.measurementErrors).toEqual([+ 'multiple-css-assets:a.css,b.css',+ ]);+ expect(passesAcceptance(score)).toBe(false);+ });++ it('rejects missing layer evidence instead of treating it as clean', () => {+ const score = scoreArm(baseline(), measurement({layerOrder: []}));+ expect(score.layerOrderFailures).toEqual([+ 'missing-astryx-base',+ 'missing-astryx-theme',+ 'missing-utilities',+ ]);+ expect(passesAcceptance(score)).toBe(false);+ });++ it('rejects an inverted cascade even when every probe still reads', () => {+ const score = scoreArm(+ baseline(),+ measurement({+ layerOrder: ['theme', 'base', 'utilities', 'reset', 'astryx-base'],+ }),+ );+ expect(cascadeInverted(score.layerOrder)).toBe(true);+ expect(verdict(score)).toBe('silent-damage');+ });+});++describe('task-specific success and run integrity', () => {+ it('rejects a broken build before browser evidence is available', () => {+ const score = scoreArm(baseline(), measurement({build: false}));+ expect(verdict(score)).toBe('broken-build');+ expect(score.taskFailures).toContain('build-failed');+ expect(passesAcceptance(score)).toBe(false);+ });++ it('counts a real agent failure as valid evidence that cannot pass', () => {+ const failed = scoreArm(+ baseline(),+ measurement({executionStatus: 'agent-failure'}),+ );+ expect(failed.validRun).toBe(true);+ expect(failed.executionSucceeded).toBe(false);+ expect(passesAcceptance(failed)).toBe(false);+ });++ it('rejects a no-op or component task with no result marker', () => {+ const noTask = scoreArm(baseline(), measurement({task: false}));+ expect(noTask.taskFailures).toContain('missing-task-contract');+ expect(passesAcceptance(noTask)).toBe(false);++ const missing = scheme({taskResults: {}});+ const missingResult = scoreArm(+ baseline(),+ measurement({light: missing, dark: missing}),+ );+ expect(missingResult.taskFailures).toContain('light:astryx-proof:count');+ expect(passesAcceptance(missingResult)).toBe(false);+ });++ it('accepts an unchanged host only with task output and exact diff attestation', () => {+ const score = scoreArm(baseline(), measurement());+ expect(score.taskSuccess).toBe(true);+ expect(verdict(score)).toBe('clean');+ expect(passesAcceptance(score)).toBe(true);+ });++ it.each([+ [+ 'manual patch',+ {...INTEGRITY, diffMatchesAttestation: false},+ 'post-run-manual-edit',+ ],+ [+ 'missing attestation',+ {...INTEGRITY, attestedDiffSha256: null, diffMatchesAttestation: false},+ 'missing-agent-diff-attestation',+ ],+ [+ 'escape hatch',+ {...INTEGRITY, escapeHatches: ['hardcoded-important']},+ 'hardcoded-important',+ ],+ ['avoided Astryx', {...INTEGRITY, usesAstryx: false}, 'missing-astryx-use'],+ ])('rejects %s evidence', (_, integrity, failure) => {+ const score = scoreArm(baseline(), measurement({integrity}));+ expect(score.integrityFailures).toContain(failure);+ expect(passesAcceptance(score)).toBe(false);+ });+});++describe('cross-system overlay directions', () => {+ const contractFor = (+ direction: 'astryx-in-host' | 'host-in-astryx',+ ): TaskContract => ({+ allowedHostChanges: [],+ replacedHostProbes: [],+ allowedOverlayChanges: [],+ results: TASK.results,+ interactions: [+ {+ id: `${direction}-overlay`,+ direction,+ open: [+ {+ name: 'nested-trigger',+ source: 'result',+ method: 'keyboard-activate',+ },+ ],+ surfaces: [+ {name: 'dialog-backdrop', source: 'result', kind: 'backdrop'},+ {name: 'dialog-surface', source: 'result', kind: 'dialog'},+ {name: 'popover-surface', source: 'result', kind: 'popover'},+ ],+ },+ ],+ });++ it.each(['astryx-in-host', 'host-in-astryx'] as const)(+ 'rejects center occlusion and below-stack ordering for %s',+ direction => {+ const failed = interaction(direction, {+ zIndex: '30',+ centerHitSelf: false,+ centerHitProbe: 'dialog-backdrop',+ });+ const after = scheme({+ taskInteractions: {[`${direction}-overlay`]: failed},+ });+ const score = scoreArm(+ baseline(),+ measurement({+ light: after,+ dark: after,+ contract: contractFor(direction),+ }),+ );+ expect(nestedLayerFailures(after)).toEqual([+ expect.objectContaining({+ direction,+ surface: 'popover-surface',+ problems: ['center-occluded', 'below-backdrop', 'below-dialog'],+ }),+ ]);+ expect(score.layeringFailures).toHaveLength(1);+ expect(passesAcceptance(score)).toBe(false);+ },+ );++ it('rejects host dialog visibility and geometry drift', () => {+ const changedInteraction = interaction();+ changedInteraction.surfaces['dialog-surface'] = layerSurface(+ 'dialog',+ '50',+ {+ visible: false,+ display: 'none',+ bounds: {...GEOMETRY, x: 390, left: 390, right: 490},+ },+ );+ const after = scheme({interaction: changedInteraction});+ const score = scoreArm(+ baseline(),+ measurement({light: after, dark: after}),+ );+ expect(score.regressionDetails).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ probe: 'overlay:dialog-surface',+ property: 'geometry.x',+ }),+ expect.objectContaining({+ probe: 'overlay:dialog-surface',+ property: 'visible',+ }),+ ]),+ );+ expect(passesAcceptance(score)).toBe(false);+ });++ it('allows declared dialog bounds but never dialog or backdrop styles', () => {+ const contract: TaskContract = {+ ...TASK,+ allowedOverlayChanges: [+ {+ fixture: 'shadcn-tailwind-v4-established',+ surface: 'dialog-surface',+ fields: [+ 'geometry.y',+ 'geometry.top',+ 'geometry.bottom',+ 'geometry.height',+ ],+ },+ ],+ };+ const changedInteraction = interaction();+ changedInteraction.surfaces['dialog-surface'] = layerSurface(+ 'dialog',+ '50',+ {+ bounds: {...GEOMETRY, y: 10, top: 10, bottom: 70, height: 60},+ },+ );+ const changed = scheme({interaction: changedInteraction});+ const accepted = scoreArm(+ baseline(),+ measurement({+ light: changed,+ dark: changed,+ contract,+ fixture: 'shadcn-tailwind-v4-established',+ }),+ );+ expect(accepted.regressionDetails).toEqual([]);+ expect(passesAcceptance(accepted)).toBe(true);++ for (const [surfaceName, property] of [+ ['dialog-surface', 'boxShadow'],+ ['dialog-backdrop', 'backgroundColor'],+ ] as const) {+ const damaged = structuredClone(changed);+ const surface = damaged.interaction?.surfaces[surfaceName];+ if (!surface || 'missing' in surface) {+ throw new Error(`missing ${surfaceName}`);+ }+ surface.style[property] = 'rgb(255, 0, 0)';+ const rejected = scoreArm(+ baseline(),+ measurement({+ light: damaged,+ dark: damaged,+ contract,+ fixture: 'shadcn-tailwind-v4-established',+ }),+ );+ expect(rejected.regressionDetails).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ probe: `overlay:${surfaceName}`,+ property,+ }),+ ]),+ );+ expect(passesAcceptance(rejected)).toBe(false);+ }+ });++ it('rejects a missing interaction result marker and unreachable keyboard trigger', () => {+ const failed = interaction('host-in-astryx');+ failed.keyboardReached = {'nested-trigger': false};+ failed.surfaces['popover-surface'] = {kind: 'popover', missing: true};+ const after = scheme({+ taskInteractions: {'host-in-astryx-overlay': failed},+ });+ const score = scoreArm(+ baseline(),+ measurement({+ light: after,+ dark: after,+ contract: contractFor('host-in-astryx'),+ }),+ );+ expect(score.taskFailures).toContain(+ 'light:host-in-astryx-overlay:nested-trigger:keyboard',+ );+ expect(score.layeringFailures).toContainEqual(+ expect.objectContaining({+ surface: 'popover-surface',+ problems: ['missing'],+ }),+ );+ });+});++describe('exploratory candidate comparison', () => {+ function changed(property: string, value: string) {+ const after = scheme();+ const title = after.probes['page-title'];+ if ('missing' in title) {+ throw new Error('test probe missing');+ }+ after.probes['page-title'] = probe({...title.style, [property]: value});+ return scoreArm(baseline(), measurement({light: after, dark: after}));+ }++ it('exposes an improvement and a regression without automatically ranking them', () => {+ const current = changed('color', 'rgb(255, 0, 0)');+ const candidate = changed('borderTopLeftRadius', '999px');+ const comparison = compareCandidate(current, candidate);+ expect(comparison.deltas.color).toBeLessThan(0);+ expect(comparison.deltas.radius).toBeGreaterThan(0);+ expect(comparison.regressions).toEqual(['radius']);+ expect(comparison.improvements).toEqual(['color']);+ expect(comparison).not.toHaveProperty('advances');+ expect(comparison).not.toHaveProperty('score');+ });++ it('shows a clean targeted improvement without assigning an automatic verdict', () => {+ const current = changed('color', 'rgb(255, 0, 0)');+ const candidate = scoreArm(baseline(), measurement());+ const comparison = compareCandidate(current, candidate);+ expect(comparison.regressions).toEqual([]);+ expect(comparison.improvements).toContain('color');+ expect(comparison).not.toHaveProperty('advances');+ });++ it('exposes every hard dimension rather than a compensating aggregate score', () => {+ const vector = hardGateVector(scoreArm(baseline(), measurement()));+ expect(Object.keys(vector)).toEqual([+ 'build',+ 'runtime',+ 'taskCompletion',+ 'color',+ 'font',+ 'radius',+ 'border',+ 'shadow',+ 'geometry',+ 'contrast',+ 'layering',+ ]);+ });+});++describe('strict confirmation summary', () => {+ it('requires complete coverage and 100% clean valid runs', () => {+ const clean = scoreArm(baseline(), measurement());+ expect(strictAcceptanceSummary([clean, clean], 2, true)).toMatchObject({+ strictClean: {numerator: 2, denominator: 2, rate: 1},+ damageFree: {numerator: 2, denominator: 2, rate: 1},+ passes: true,+ });+ expect(strictAcceptanceSummary([clean], 2, false).passes).toBe(false);+ });++ it('counts cosmetic drift as damage and a strict failure', () => {+ const after = scheme();+ const title = after.probes['page-title'];+ if ('missing' in title) {+ throw new Error('test probe missing');+ }+ after.probes['page-title'] = probe({...title.style, boxShadow: 'none'});+ const drift = scoreArm(+ baseline(),+ measurement({light: after, dark: after}),+ );+ const summary = strictAcceptanceSummary([drift], 1, true);+ expect(verdict(drift)).toBe('cosmetic-drift');+ expect(summary.strictClean).toEqual({+ numerator: 0,+ denominator: 1,+ rate: 0,+ });+ expect(summary.damageFree).toEqual({numerator: 0, denominator: 1, rate: 0});+ expect(summary.passes).toBe(false);+ });+});diff --git a/internal/vibe-tests/setup-test/setup-eval.ts b/internal/vibe-tests/setup-test/setup-eval.tsnew file mode 100644index 0000000000..2c6385b21b--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-eval.ts@@ -0,0 +1,1129 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file setup-eval.ts+ * @input Pristine and post-executor measurements from setup-measure.mjs+ * @output Deterministic damage, task-completion, integrity, and acceptance results+ * @position internal/vibe-tests/setup-test — the scoring half+ */++export type GeometrySnapshot = {+ x: number;+ y: number;+ top: number;+ right: number;+ bottom: number;+ left: number;+ width: number;+ height: number;+};++export type ProbeReading =+ | {missing: true}+ | {+ style: Record<string, string>;+ geometry: GeometrySnapshot;+ text: string;+ descendantText?: string;+ contrast: number | null;+ };++export type TaskResultReading = {+ count: number;+ visible: boolean;+ focusable: boolean;+ text: string;+ style: Record<string, string>;+ geometry: GeometrySnapshot | null;+};++export type LayerSurfaceReading =+ | {kind: string; missing: true; styleReference?: string}+ | {+ kind: string;+ styleReference?: string;+ visible: boolean;+ display: string;+ visibility: string;+ opacity: string;+ position: string;+ zIndex: string;+ style: Record<string, string>;+ bounds: GeometrySnapshot;+ intersectsViewport: boolean;+ clippingAncestor: string | null;+ centerHitSelf: boolean;+ centerHitProbe: string | null;+ topLayer: {+ tagName: string;+ role: string | null;+ open: boolean;+ popover: string | null;+ ariaModal: string | null;+ inTopLayer: boolean;+ portalChild: boolean;+ };+ };++export type InteractionReading = {+ id?: string;+ direction?: 'host-baseline' | 'astryx-in-host' | 'host-in-astryx';+ opened: boolean;+ keyboardReached?: Record<string, boolean>;+ error?: string | null;+ surfaces: Record<string, LayerSurfaceReading>;+};++export type SchemeReading = {+ probes: Record<string, ProbeReading>;+ variables: Record<string, string>;+ colorScheme: string;+ consoleErrors: string[];+ pageErrors: string[];+ failedRequests: string[];+ interaction?: InteractionReading;+ taskResults?: Record<string, TaskResultReading>;+ taskInteractions?: Record<string, InteractionReading>;+};++export type ResultContract = {+ name: string;+ exact: number;+ visible?: boolean;+ focusable?: boolean;+ text?: string;+ preserveTextFromHostProbe?: string;+};++export type InteractionContract = {+ id: string;+ direction: 'astryx-in-host' | 'host-in-astryx';+ open: Array<{+ name: string;+ source: 'probe' | 'result';+ method: 'click' | 'keyboard-focus' | 'keyboard-activate';+ key?: string;+ }>;+ surfaces: Array<{+ name: string;+ marker?: string;+ source: 'probe' | 'result';+ kind: string;+ styleReference?: string;+ }>;+};++export type AllowedHostChange = {+ fixture: string;+ probe: string;+ fields: string[];+};++export type ReplacedHostProbe = {+ fixture: string;+ probe: string;+ result: string;+};++export type AllowedOverlayChange = {+ fixture: string;+ surface: string;+ fields: string[];+};++export type TaskContract = {+ allowedHostChanges: AllowedHostChange[];+ replacedHostProbes: ReplacedHostProbe[];+ allowedOverlayChanges: AllowedOverlayChange[];+ results: ResultContract[];+ interactions: InteractionContract[];+};++export type SetupIntegrity = {+ diffSha256: string;+ attestedDiffSha256: string | null;+ diffMatchesAttestation: boolean;+ usesAstryx: boolean;+ changedFiles: string[];+ escapeHatches: string[];+};++export type Measurement = {+ label: string;+ fixture?: string;+ build: {+ ok: boolean;+ status: number;+ ms: number;+ stdout: string;+ stderr: string;+ };+ measurementErrors?: string[];+ layerOrder?: string[];+ task?: {id: string; kind: string; contract: TaskContract};+ executionStatus?: string;+ integrity?: SetupIntegrity;+ schemes: Record<'light' | 'dark', SchemeReading>;+};++export type Scheme = 'light' | 'dark';+export type Category = 'typography' | 'color' | 'geometry' | 'spacing';++const CATEGORY_OF: Record<string, Category> = {+ color: 'color',+ backgroundColor: 'color',+ boxShadow: 'color',+ fontFamily: 'typography',+ fontSize: 'typography',+ fontWeight: 'typography',+ lineHeight: 'typography',+ letterSpacing: 'typography',+ paddingTop: 'spacing',+ paddingRight: 'spacing',+ paddingBottom: 'spacing',+ paddingLeft: 'spacing',+ marginTop: 'spacing',+ marginRight: 'spacing',+ marginBottom: 'spacing',+ marginLeft: 'spacing',+ gap: 'spacing',+ rowGap: 'spacing',+ columnGap: 'spacing',+};++export const categoryOf = (property: string): Category => {+ if (/^border.*Color$/.test(property)) {+ return 'color';+ }+ if (property.startsWith('geometry.')) {+ return 'geometry';+ }+ return CATEGORY_OF[property] ?? 'geometry';+};++export type Regression = {+ scheme?: Scheme;+ probe: string;+ property: string;+ category: Category;+ before: string;+ after: string;+};++const stringValue = (value: unknown) =>+ value === undefined ? '(missing)' : String(value);++/** Compare the complete captured style and geometry key union exactly. */+export function regressions(+ before: SchemeReading,+ after: SchemeReading,+ allowedFields: ReadonlyMap<string, ReadonlySet<string>> = new Map(),+ replacedProbes: ReadonlySet<string> = new Set(),+): {changed: Regression[]; missing: string[]} {+ const changed: Regression[] = [];+ const missing: string[] = [];+ for (const [probe, baselineReading] of Object.entries(before.probes)) {+ if (replacedProbes.has(probe)) {+ continue;+ }+ const allowed = allowedFields.get(probe) ?? new Set<string>();+ const armReading = after.probes[probe];+ if (!armReading || 'missing' in armReading) {+ if (!('missing' in baselineReading)) {+ missing.push(probe);+ }+ continue;+ }+ if ('missing' in baselineReading) {+ continue;+ }++ const beforeValues: Record<string, unknown> = {+ ...baselineReading.style,+ ...Object.fromEntries(+ Object.entries(baselineReading.geometry).map(([key, value]) => [+ `geometry.${key}`,+ value,+ ]),+ ),+ text: baselineReading.text,+ };+ const afterValues: Record<string, unknown> = {+ ...armReading.style,+ ...Object.fromEntries(+ Object.entries(armReading.geometry).map(([key, value]) => [+ `geometry.${key}`,+ value,+ ]),+ ),+ text: armReading.text,+ };+ for (const property of new Set([+ ...Object.keys(beforeValues),+ ...Object.keys(afterValues),+ ])) {+ if (+ beforeValues[property] !== afterValues[property] &&+ !allowed.has(property)+ ) {+ changed.push({+ probe,+ property,+ category: categoryOf(property),+ before: stringValue(beforeValues[property]),+ after: stringValue(afterValues[property]),+ });+ }+ }+ }+ return {changed, missing};+}++export function countByCategory(+ changed: Regression[],+): Record<Category, number> {+ const counts: Record<Category, number> = {+ typography: 0,+ color: 0,+ geometry: 0,+ spacing: 0,+ };+ for (const regression of changed) {+ counts[regression.category] += 1;+ }+ return counts;+}++export type ContrastFinding = {probe: string; before: number; after: number};++export function contrastFailures(+ before: SchemeReading,+ after: SchemeReading,+ floor = 4.5,+ replacedProbes: ReadonlySet<string> = new Set(),+): ContrastFinding[] {+ const findings: ContrastFinding[] = [];+ for (const [probe, baselineReading] of Object.entries(before.probes)) {+ if (replacedProbes.has(probe)) {+ continue;+ }+ const armReading = after.probes[probe];+ if (+ 'missing' in baselineReading ||+ !armReading ||+ 'missing' in armReading+ ) {+ continue;+ }+ const beforeContrast = baselineReading.contrast;+ const afterContrast = armReading.contrast;+ if (+ beforeContrast != null &&+ afterContrast != null &&+ beforeContrast >= floor &&+ afterContrast < floor+ ) {+ findings.push({+ probe,+ before: beforeContrast,+ after: afterContrast,+ });+ }+ }+ return findings;+}++export function modeDependence(+ baseline: Measurement,+ arm: Measurement,+ replacedProbes: ReadonlySet<string> = new Set(),+): string[] {+ const probes = Object.keys(arm.schemes.light.probes);+ return probes.filter(+ probe =>+ !replacedProbes.has(probe) &&+ stableAcrossSchemes(baseline, probe) &&+ !stableAcrossSchemes(arm, probe),+ );+}++function stableAcrossSchemes(measurement: Measurement, probe: string): boolean {+ const light = measurement.schemes.light.probes[probe];+ const dark = measurement.schemes.dark.probes[probe];+ if (!light || !dark || 'missing' in light || 'missing' in dark) {+ return true;+ }+ return (+ JSON.stringify(light.style) === JSON.stringify(dark.style) &&+ JSON.stringify(light.geometry) === JSON.stringify(dark.geometry)+ );+}++export function cascadeInverted(+ layerOrder: string[] | undefined,+ {systemPrefix = 'astryx', utilityLayer = 'utilities'} = {},+): boolean {+ if (!layerOrder?.length) {+ return false;+ }+ const utilities = layerOrder.indexOf(utilityLayer);+ if (utilities === -1) {+ return false;+ }+ return layerOrder.some(+ (layer, index) => index > utilities && layer.startsWith(systemPrefix),+ );+}++export function layerOrderFailures(layerOrder: string[] | undefined): string[] {+ const order = layerOrder ?? [];+ const failures = [];+ for (const required of ['astryx-base', 'astryx-theme', 'utilities']) {+ if (!order.includes(required)) {+ failures.push(`missing-${required}`);+ }+ }+ if (cascadeInverted(order)) {+ failures.push('cascade-inverted');+ }+ return failures;+}++export type VariableFinding = {name: string; before: string; after: string};++export function variableCapture(+ before: SchemeReading,+ after: SchemeReading,+): VariableFinding[] {+ const findings: VariableFinding[] = [];+ for (const [name, value] of Object.entries(before.variables)) {+ const armValue = after.variables[name];+ if (armValue !== undefined && armValue !== value) {+ findings.push({name, before: value, after: armValue});+ }+ }+ return findings;+}++export type LayeringFailure = {+ interaction?: string;+ direction?: string;+ surface: string;+ problems: string[];+ centerHitProbe: string | null;+};++const numericZIndex = (surface: LayerSurfaceReading | undefined) => {+ if (+ !surface ||+ 'missing' in surface ||+ !/^-?\d+(?:\.\d+)?$/.test(surface.zIndex)+ ) {+ return null;+ }+ return Number(surface.zIndex);+};++function failuresForInteraction(+ interaction: InteractionReading,+): LayeringFailure[] {+ const context = {+ ...(interaction.id ? {interaction: interaction.id} : {}),+ ...(interaction.direction ? {direction: interaction.direction} : {}),+ };+ if (!interaction.opened) {+ return [+ {+ ...context,+ surface: 'interaction',+ problems: [interaction.error ? 'open-error' : 'not-opened'],+ centerHitProbe: null,+ },+ ];+ }++ const backdrop = Object.values(interaction.surfaces).find(+ surface => surface.kind === 'backdrop',+ );+ const dialog = Object.values(interaction.surfaces).find(+ surface => surface.kind === 'dialog',+ );+ const backdropZ = numericZIndex(backdrop);+ const dialogZ = numericZIndex(dialog);+ const failures: LayeringFailure[] = [];++ for (const [name, surface] of Object.entries(interaction.surfaces)) {+ if ('missing' in surface) {+ failures.push({+ ...context,+ surface: name,+ problems: ['missing'],+ centerHitProbe: null,+ });+ continue;+ }+ const isNested = surface.kind === 'tooltip' || surface.kind === 'popover';+ const problems: string[] = [];+ if (!surface.visible) {+ problems.push('hidden');+ }+ if (surface.bounds.width <= 0 || surface.bounds.height <= 0) {+ problems.push('empty-bounds');+ }+ if (!surface.intersectsViewport) {+ problems.push('outside-viewport');+ }+ if (surface.clippingAncestor) {+ problems.push('clipped');+ }+ if (isNested && !surface.centerHitSelf) {+ problems.push('center-occluded');+ }++ if (isNested && !surface.topLayer.inTopLayer) {+ const nestedZ = numericZIndex(surface);+ if (nestedZ == null) {+ problems.push('unresolved-stack-order');+ } else {+ if (backdropZ == null || nestedZ <= backdropZ) {+ problems.push('below-backdrop');+ }+ if (dialogZ == null || nestedZ <= dialogZ) {+ problems.push('below-dialog');+ }+ }+ }+ if (problems.length > 0) {+ failures.push({+ ...context,+ surface: name,+ problems,+ centerHitProbe: surface.centerHitProbe,+ });+ }+ }+ return failures;+}++export function nestedLayerFailures(reading: SchemeReading): LayeringFailure[] {+ return [+ ...(reading.interaction ? failuresForInteraction(reading.interaction) : []),+ ...Object.values(reading.taskInteractions ?? {}).flatMap(+ failuresForInteraction,+ ),+ ];+}++function overlayStyleRegressions(+ before: SchemeReading,+ after: SchemeReading,+ allowedFields: ReadonlyMap<string, ReadonlySet<string>> = new Map(),+): Regression[] {+ if (!before.interaction || !after.interaction) {+ return [];+ }+ const changed: Regression[] = [];+ for (const [name, baselineSurface] of Object.entries(+ before.interaction.surfaces,+ )) {+ const allowed = allowedFields.get(name) ?? new Set<string>();+ const armSurface = after.interaction.surfaces[name];+ if (+ !armSurface ||+ 'missing' in armSurface ||+ 'missing' in baselineSurface+ ) {+ continue;+ }+ const styleProperties = new Set([+ ...Object.keys(baselineSurface.style),+ ...Object.keys(armSurface.style),+ ]);+ if (baselineSurface.kind === 'backdrop') {+ styleProperties.delete('color');+ styleProperties.delete('fontFamily');+ styleProperties.delete('fontSize');+ styleProperties.delete('fontWeight');+ styleProperties.delete('lineHeight');+ styleProperties.delete('letterSpacing');+ for (const side of ['Top', 'Right', 'Bottom', 'Left']) {+ if (+ baselineSurface.style[`border${side}Width`] === '0px' &&+ armSurface.style[`border${side}Width`] === '0px'+ ) {+ styleProperties.delete(`border${side}Color`);+ }+ }+ }+ for (const property of styleProperties) {+ if (+ baselineSurface.style[property] !== armSurface.style[property] &&+ !allowed.has(property)+ ) {+ changed.push({+ probe: `overlay:${name}`,+ property,+ category: categoryOf(property),+ before: stringValue(baselineSurface.style[property]),+ after: stringValue(armSurface.style[property]),+ });+ }+ }+ const baselineValues: Record<string, unknown> = {+ visible: baselineSurface.visible,+ display: baselineSurface.display,+ visibility: baselineSurface.visibility,+ opacity: baselineSurface.opacity,+ position: baselineSurface.position,+ zIndex: baselineSurface.zIndex,+ ...Object.fromEntries(+ Object.entries(baselineSurface.bounds).map(([key, value]) => [+ `geometry.${key}`,+ value,+ ]),+ ),+ };+ const armValues: Record<string, unknown> = {+ visible: armSurface.visible,+ display: armSurface.display,+ visibility: armSurface.visibility,+ opacity: armSurface.opacity,+ position: armSurface.position,+ zIndex: armSurface.zIndex,+ ...Object.fromEntries(+ Object.entries(armSurface.bounds).map(([key, value]) => [+ `geometry.${key}`,+ value,+ ]),+ ),+ };+ for (const property of Object.keys(baselineValues)) {+ if (+ baselineValues[property] !== armValues[property] &&+ !allowed.has(property)+ ) {+ changed.push({+ probe: `overlay:${name}`,+ property,+ category: categoryOf(property),+ before: stringValue(baselineValues[property]),+ after: stringValue(armValues[property]),+ });+ }+ }+ }+ return changed;+}++function taskContractFailures(+ baseline: Measurement,+ arm: Measurement,+): string[] {+ if (!arm.task) {+ return ['missing-task-contract'];+ }+ const failures: string[] = [];+ for (const scheme of ['light', 'dark'] as const) {+ const reading = arm.schemes[scheme];+ for (const replaced of arm.task.contract.replacedHostProbes) {+ if (replaced.fixture !== arm.fixture) {+ continue;+ }+ const original = reading.probes[replaced.probe];+ if (original && !('missing' in original)) {+ failures.push(`${scheme}:${replaced.probe}:not-replaced`);+ }+ }+ for (const result of arm.task.contract.results) {+ const measured = reading.taskResults?.[result.name];+ if (!measured || measured.count !== result.exact) {+ failures.push(`${scheme}:${result.name}:count`);+ continue;+ }+ if (result.visible && !measured.visible) {+ failures.push(`${scheme}:${result.name}:hidden`);+ }+ if (result.focusable && !measured.focusable) {+ failures.push(`${scheme}:${result.name}:not-focusable`);+ }+ if (result.text !== undefined && measured.text !== result.text) {+ failures.push(`${scheme}:${result.name}:text`);+ }+ if (result.preserveTextFromHostProbe) {+ const reference =+ baseline.schemes[scheme].probes[result.preserveTextFromHostProbe];+ if (+ !reference ||+ 'missing' in reference ||+ measured.text !== reference.text+ ) {+ failures.push(`${scheme}:${result.name}:host-text`);+ }+ }+ }++ for (const interaction of arm.task.contract.interactions) {+ const measured = reading.taskInteractions?.[interaction.id];+ if (!measured) {+ failures.push(`${scheme}:${interaction.id}:missing-interaction`);+ continue;+ }+ for (const step of interaction.open.filter(step =>+ step.method.startsWith('keyboard'),+ )) {+ if (measured.keyboardReached?.[step.name] !== true) {+ failures.push(`${scheme}:${interaction.id}:${step.name}:keyboard`);+ }+ }+ for (const surfaceContract of interaction.surfaces) {+ if (!surfaceContract.styleReference) {+ continue;+ }+ const surface = measured.surfaces[surfaceContract.name];+ const reference =+ baseline.schemes[scheme].interaction?.surfaces[+ surfaceContract.styleReference+ ];+ if (+ !surface ||+ !reference ||+ 'missing' in surface ||+ 'missing' in reference ||+ JSON.stringify(surface.style) !== JSON.stringify(reference.style)+ ) {+ failures.push(+ `${scheme}:${interaction.id}:${surfaceContract.name}:host-style`,+ );+ }+ }+ }+ }+ return [...new Set(failures)];+}++function integrityFailures(arm: Measurement): string[] {+ if (!arm.integrity) {+ return ['missing-diff-integrity'];+ }+ const failures = [...arm.integrity.escapeHatches];+ if (!arm.integrity.usesAstryx) {+ failures.push('missing-astryx-use');+ }+ if (!arm.integrity.attestedDiffSha256) {+ failures.push('missing-agent-diff-attestation');+ } else if (!arm.integrity.diffMatchesAttestation) {+ failures.push('post-run-manual-edit');+ }+ return [...new Set(failures)];+}++const uniqueObjects = <T>(values: T[]): T[] => [+ ...new Map(values.map(value => [JSON.stringify(value), value])).values(),+];++export type SetupScore = {+ label: string;+ builds: boolean;+ clean: boolean;+ validRun: boolean;+ executionSucceeded: boolean;+ consoleErrors: number;+ failedRequests: number;+ regressions: number;+ regressionDetails: Regression[];+ byCategory: Record<Category, number>;+ baselineFailures: string[];+ missingProbes: string[];+ contrastFailures: ContrastFinding[];+ modeDependent: string[];+ variablesCaptured: VariableFinding[];+ measurementErrors: string[];+ layeringFailures: LayeringFailure[];+ layerOrderFailures: string[];+ cascadeInverted: boolean;+ layerOrder: string[];+ taskSuccess: boolean;+ taskFailures: string[];+ integrityFailures: string[];+};++function baselineValidationFailures(baseline: Measurement): string[] {+ const failures: string[] = [];+ for (const scheme of ['light', 'dark'] as const) {+ const reading = baseline.schemes[scheme];+ if (!reading || Object.keys(reading.probes).length === 0) {+ failures.push(`${scheme}:no-baseline-probes`);+ continue;+ }+ for (const [name, probe] of Object.entries(reading.probes)) {+ if ('missing' in probe) {+ failures.push(`${scheme}:${name}:baseline-missing`);+ }+ }+ if (reading.interaction) {+ if (!reading.interaction.opened) {+ failures.push(`${scheme}:baseline-interaction-not-opened`);+ }+ for (const [name, surface] of Object.entries(+ reading.interaction.surfaces,+ )) {+ if ('missing' in surface) {+ failures.push(`${scheme}:${name}:baseline-missing`);+ }+ }+ for (const [name, reached] of Object.entries(+ reading.interaction.keyboardReached ?? {},+ )) {+ if (!reached) {+ failures.push(`${scheme}:${name}:baseline-keyboard`);+ }+ }+ }+ }+ return failures;+}++function allowedHostFields(arm: Measurement) {+ const fields = new Map<string, Set<string>>();+ if (!arm.fixture || !arm.task) {+ return fields;+ }+ for (const allowed of arm.task.contract.allowedHostChanges) {+ if (allowed.fixture !== arm.fixture) {+ continue;+ }+ const allowedForProbe = fields.get(allowed.probe) ?? new Set<string>();+ for (const field of allowed.fields) {+ allowedForProbe.add(field);+ }+ fields.set(allowed.probe, allowedForProbe);+ }+ return fields;+}++function replacedHostProbes(arm: Measurement) {+ if (!arm.fixture || !arm.task) {+ return new Set<string>();+ }+ return new Set(+ arm.task.contract.replacedHostProbes+ .filter(replaced => replaced.fixture === arm.fixture)+ .map(replaced => replaced.probe),+ );+}++function allowedOverlayFields(arm: Measurement) {+ const fields = new Map<string, Set<string>>();+ if (!arm.fixture || !arm.task) {+ return fields;+ }+ for (const allowed of arm.task.contract.allowedOverlayChanges) {+ if (allowed.fixture !== arm.fixture) {+ continue;+ }+ const allowedForSurface = fields.get(allowed.surface) ?? new Set<string>();+ for (const field of allowed.fields) {+ allowedForSurface.add(field);+ }+ fields.set(allowed.surface, allowedForSurface);+ }+ return fields;+}++export function scoreArm(baseline: Measurement, arm: Measurement): SetupScore {+ const baselineFailures = baselineValidationFailures(baseline);+ const status = arm.executionStatus;+ const executionSucceeded = status === 'succeeded';+ const validRun = executionSucceeded || status === 'agent-failure';+ if (!arm.build.ok) {+ return {+ label: arm.label,+ builds: false,+ clean: false,+ validRun,+ executionSucceeded,+ consoleErrors: 0,+ failedRequests: 0,+ regressions: 0,+ regressionDetails: [],+ byCategory: {typography: 0, color: 0, geometry: 0, spacing: 0},+ baselineFailures,+ missingProbes: [],+ contrastFailures: [],+ modeDependent: [],+ variablesCaptured: [],+ measurementErrors: arm.measurementErrors ?? [],+ layeringFailures: [],+ layerOrderFailures: ['build-failed'],+ cascadeInverted: false,+ layerOrder: arm.layerOrder ?? [],+ taskSuccess: false,+ taskFailures: ['build-failed'],+ integrityFailures: integrityFailures(arm),+ };+ }+ const allowedFields = allowedHostFields(arm);+ const replacedProbes = replacedHostProbes(arm);+ const overlayFields = allowedOverlayFields(arm);+ const regressionDetails = uniqueObjects(+ (['light', 'dark'] as const).flatMap(scheme =>+ [+ ...regressions(+ baseline.schemes[scheme],+ arm.schemes[scheme],+ allowedFields,+ replacedProbes,+ ).changed,+ ...overlayStyleRegressions(+ baseline.schemes[scheme],+ arm.schemes[scheme],+ overlayFields,+ ),+ ].map(regression => ({...regression, scheme})),+ ),+ );+ const missingProbes = [+ ...new Set(+ (['light', 'dark'] as const).flatMap(+ scheme =>+ regressions(+ baseline.schemes[scheme],+ arm.schemes[scheme],+ allowedFields,+ replacedProbes,+ ).missing,+ ),+ ),+ ];+ const contrast = uniqueObjects(+ (['light', 'dark'] as const).flatMap(scheme =>+ contrastFailures(+ baseline.schemes[scheme],+ arm.schemes[scheme],+ 4.5,+ replacedProbes,+ ),+ ),+ );+ const variables = uniqueObjects(+ (['light', 'dark'] as const).flatMap(scheme =>+ variableCapture(baseline.schemes[scheme], arm.schemes[scheme]),+ ),+ );+ const layering = uniqueObjects(+ (['light', 'dark'] as const).flatMap(scheme =>+ nestedLayerFailures(arm.schemes[scheme]),+ ),+ );+ const runtimeErrors = (['light', 'dark'] as const).reduce(+ (sum, scheme) =>+ sum ++ arm.schemes[scheme].consoleErrors.length ++ arm.schemes[scheme].pageErrors.length,+ 0,+ );+ const failedRequests = (['light', 'dark'] as const).reduce(+ (sum, scheme) => sum + arm.schemes[scheme].failedRequests.length,+ 0,+ );+ const taskFailures = taskContractFailures(baseline, arm);+ const diffFailures = integrityFailures(arm);++ return {+ label: arm.label,+ builds: arm.build.ok,+ clean: runtimeErrors === 0 && failedRequests === 0,+ validRun,+ executionSucceeded,+ consoleErrors: runtimeErrors,+ failedRequests,+ regressions: regressionDetails.length,+ regressionDetails,+ byCategory: countByCategory(regressionDetails),+ baselineFailures,+ missingProbes,+ contrastFailures: contrast,+ modeDependent: modeDependence(baseline, arm, replacedProbes),+ variablesCaptured: variables,+ measurementErrors: arm.measurementErrors ?? [],+ layeringFailures: layering,+ layerOrderFailures: layerOrderFailures(arm.layerOrder),+ cascadeInverted: cascadeInverted(arm.layerOrder),+ layerOrder: arm.layerOrder ?? [],+ taskSuccess: taskFailures.length === 0,+ taskFailures,+ integrityFailures: diffFailures,+ };+}++export type Verdict =+ 'broken-build' | 'noisy' | 'silent-damage' | 'cosmetic-drift' | 'clean';++export function verdict(score: SetupScore): Verdict {+ if (!score.builds) {+ return 'broken-build';+ }+ if (+ score.measurementErrors.length > 0 ||+ score.baselineFailures.length > 0 ||+ score.contrastFailures.length > 0 ||+ score.layeringFailures.length > 0 ||+ score.layerOrderFailures.length > 0 ||+ score.missingProbes.length > 0 ||+ score.cascadeInverted ||+ !score.taskSuccess ||+ score.integrityFailures.length > 0+ ) {+ return score.clean ? 'silent-damage' : 'noisy';+ }+ if (!score.clean) {+ return 'noisy';+ }+ return score.regressions > 0 || score.modeDependent.length > 0+ ? 'cosmetic-drift'+ : 'clean';+}++export function passesAcceptance(score: SetupScore): boolean {+ return (+ score.validRun &&+ score.executionSucceeded &&+ score.taskSuccess &&+ score.integrityFailures.length === 0 &&+ verdict(score) === 'clean'+ );+}++export type HardDimension =+ | 'build'+ | 'runtime'+ | 'taskCompletion'+ | 'color'+ | 'font'+ | 'radius'+ | 'border'+ | 'shadow'+ | 'geometry'+ | 'contrast'+ | 'layering';++export type HardGateVector = Record<HardDimension, number>;++const regressionDimension = (property: string): HardDimension => {+ if (property === 'boxShadow') {+ return 'shadow';+ }+ if (/Radius$/.test(property)) {+ return 'radius';+ }+ if (/^border/.test(property)) {+ return 'border';+ }+ if (+ property.startsWith('font') ||+ property === 'lineHeight' ||+ property === 'letterSpacing'+ ) {+ return 'font';+ }+ if (property === 'color' || property === 'backgroundColor') {+ return 'color';+ }+ return 'geometry';+};++export function hardGateVector(score: SetupScore): HardGateVector {+ const vector: HardGateVector = {+ build: score.builds ? 0 : 1,+ runtime: score.consoleErrors + score.failedRequests,+ taskCompletion:+ score.taskFailures.length ++ score.integrityFailures.length ++ score.measurementErrors.length ++ (score.executionSucceeded ? 0 : 1),+ color: 0,+ font: 0,+ radius: 0,+ border: 0,+ shadow: 0,+ geometry:+ score.baselineFailures.length ++ score.missingProbes.length ++ score.modeDependent.length,+ contrast: score.contrastFailures.length,+ layering: score.layeringFailures.length + score.layerOrderFailures.length,+ };+ for (const regression of score.regressionDetails) {+ vector[regressionDimension(regression.property)] += 1;+ }+ return vector;+}++export function compareCandidate(current: SetupScore, candidate: SetupScore) {+ const before = hardGateVector(current);+ const after = hardGateVector(candidate);+ const deltas = Object.fromEntries(+ (Object.keys(before) as HardDimension[]).map(dimension => [+ dimension,+ after[dimension] - before[dimension],+ ]),+ ) as HardGateVector;+ const regressions = (Object.keys(deltas) as HardDimension[]).filter(+ dimension => deltas[dimension] > 0,+ );+ const improvements = (Object.keys(deltas) as HardDimension[]).filter(+ dimension => deltas[dimension] < 0,+ );+ return {+ before,+ after,+ deltas,+ regressions,+ improvements,+ };+}++export function strictAcceptanceSummary(+ scores: SetupScore[],+ expectedRuns: number,+ coverageComplete: boolean,+) {+ const valid = scores.filter(score => score.validRun);+ const strict = valid.filter(passesAcceptance).length;+ const damageFree = valid.filter(+ score =>+ score.builds &&+ score.clean &&+ score.baselineFailures.length === 0 &&+ score.regressions === 0 &&+ score.missingProbes.length === 0 &&+ score.contrastFailures.length === 0 &&+ score.layeringFailures.length === 0 &&+ score.layerOrderFailures.length === 0 &&+ score.measurementErrors.length === 0 &&+ score.modeDependent.length === 0 &&+ !score.cascadeInverted,+ ).length;+ const complete =+ coverageComplete &&+ scores.length === expectedRuns &&+ valid.length === expectedRuns;+ return {+ coverage: {+ present: scores.length,+ valid: valid.length,+ expected: expectedRuns,+ },+ strictClean: {+ numerator: strict,+ denominator: valid.length,+ rate: valid.length === 0 ? 0 : strict / valid.length,+ },+ damageFree: {+ numerator: damageFree,+ denominator: valid.length,+ rate: valid.length === 0 ? 0 : damageFree / valid.length,+ },+ passes: complete && strict === expectedRuns,+ };+}diff --git a/internal/vibe-tests/setup-test/setup-integrity.mjs b/internal/vibe-tests/setup-test/setup-integrity.mjsnew file mode 100644index 0000000000..fbcfeb0ff5--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-integrity.mjs@@ -0,0 +1,597 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * Read-only integrity analysis for a setup-test sandbox.+ *+ * The wholesale-replacement guard rejects a modified host source or stylesheet+ * when at least 20 baseline lines and 80% of that file's baseline lines were+ * deleted. Both conditions are required so a small, legitimate rewrite does not+ * look the same as replacing an established host file.+ *+ * Deletion is counted on *substantive* lines, not raw diff lines. Wrapping a+ * component tree in a provider re-indents every line inside it, which a raw+ * `git diff` reports as deleting and re-adding the whole body even though not+ * one line of content is gone. Each line is therefore normalized — leading and+ * trailing whitespace trimmed, interior whitespace runs collapsed to a single+ * space — and a baseline line counts as deleted only when no line of the+ * current file still carries its content.+ *+ * Normalization is deliberately per line and never collapses line boundaries or+ * removes whitespace between tokens, so it cannot be used to hide a rewrite:+ * minifying a file onto one line, or stripping the spaces inside its lines,+ * leaves nothing that matches the baseline and still trips the guard.+ */++import * as crypto from 'node:crypto';+import {execFileSync} from 'node:child_process';+import * as fs from 'node:fs';+import * as path from 'node:path';+import {fileURLToPath} from 'node:url';++export const WHOLESALE_REPLACEMENT_THRESHOLD = Object.freeze({+ minimumDeletedLines: 20,+ deletedFraction: 0.8,+});++const HOST_SOURCE_EXTENSIONS = new Set([+ '.cjs',+ '.css',+ '.html',+ '.js',+ '.jsx',+ '.less',+ '.mjs',+ '.pcss',+ '.sass',+ '.scss',+ '.svelte',+ '.ts',+ '.tsx',+ '.vue',+]);++const ESCAPE_HATCHES = [+ {+ kind: 'hardcoded-important',+ pattern: /!\s*important\b/i,+ message: 'added a hardcoded !important declaration',+ },+ {+ kind: 'blanket-reset',+ pattern: /\ball\s*:\s*(?:unset|initial|revert)\b/i,+ message: 'added a blanket all-property reset',+ },+ {+ kind: 'dark-mode-disabled',+ pattern:+ /\bcolor-scheme\s*:\s*(?:only\s+light|light\s+only|light(?=\s*[;}]))\b|\bforcedTheme\s*=\s*["']light["']|\bdarkMode\s*[:=]\s*(?:false|["'](?:disabled|off|light)["'])|<meta\b[^>]*\bname\s*=\s*["']color-scheme["'][^>]*\bcontent\s*=\s*["']light(?:\s+only)?["']/i,+ message: 'explicitly disabled dark-mode behavior',+ },+];++const compareText = (left, right) => (left < right ? -1 : left > right ? 1 : 0);++function git(appDir, args, options = {}) {+ return execFileSync('git', ['-C', appDir, ...args], {+ encoding: options.encoding ?? 'utf8',+ maxBuffer: 64 * 1024 * 1024,+ stdio: ['ignore', 'pipe', 'pipe'],+ });+}++function assertRepositoryRoot(appDir) {+ const root = path.resolve(+ git(appDir, ['rev-parse', '--show-toplevel']).trim(),+ );+ const requested = fs.realpathSync(appDir);+ const actual = fs.realpathSync(root);+ if (requested !== actual) {+ throw new Error(`--app must name the sandbox repository root: ${appDir}`);+ }+ git(appDir, ['rev-parse', '--verify', 'HEAD']);+}++function trackedPatch(appDir) {+ return git(+ appDir,+ [+ '-c',+ 'core.quotePath=false',+ 'diff',+ '--binary',+ '--full-index',+ '--no-color',+ '--no-ext-diff',+ '--no-textconv',+ '--no-renames',+ 'HEAD',+ '--',+ ],+ {encoding: 'buffer'},+ );+}++function untrackedPaths(appDir) {+ const output = git(appDir, [+ '-c',+ 'core.quotePath=false',+ 'ls-files',+ '--others',+ '--exclude-standard',+ '-z',+ ]);+ return output.split('\0').filter(Boolean).sort(compareText);+}++function untrackedEntry(appDir, relativePath) {+ const absolutePath = path.join(appDir, relativePath);+ const stat = fs.lstatSync(absolutePath);+ if (stat.isSymbolicLink()) {+ return {+ bytes: Buffer.from(fs.readlinkSync(absolutePath)),+ mode: '120000',+ binary: false,+ };+ }+ const bytes = fs.readFileSync(absolutePath);+ return {+ bytes,+ mode: stat.mode & 0o111 ? '100755' : '100644',+ binary: bytes.includes(0),+ };+}++function updateFramed(hash, label, bytes) {+ const content = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);+ hash.update(`${label}\0${content.length}\0`);+ hash.update(content);+ hash.update('\0');+}++function digestDiff(patch, untracked) {+ const hash = crypto.createHash('sha256');+ updateFramed(hash, 'format', 'setup-integrity-v1');+ updateFramed(hash, 'tracked-diff', patch);+ for (const file of untracked) {+ updateFramed(hash, 'untracked-path', file.path);+ updateFramed(hash, 'untracked-mode', file.mode);+ updateFramed(hash, 'untracked-content', file.bytes);+ }+ return hash.digest('hex');+}++function lineCount(bytes) {+ if (bytes.length === 0) return 0;+ let count = 0;+ for (const byte of bytes) {+ if (byte === 10) count += 1;+ }+ return count + (bytes.at(-1) === 10 ? 0 : 1);+}++function trackedNumstat(appDir) {+ const output = git(appDir, [+ 'diff',+ '--numstat',+ '-z',+ '--no-renames',+ 'HEAD',+ '--',+ ]);+ return output+ .split('\0')+ .filter(Boolean)+ .map(record => {+ const firstTab = record.indexOf('\t');+ const secondTab = record.indexOf('\t', firstTab + 1);+ const addedText = record.slice(0, firstTab);+ const deletedText = record.slice(firstTab + 1, secondTab);+ return {+ path: record.slice(secondTab + 1),+ added: addedText === '-' ? 0 : Number(addedText),+ deleted: deletedText === '-' ? 0 : Number(deletedText),+ binary: addedText === '-' || deletedText === '-',+ };+ });+}++function baselineBytes(appDir, relativePath) {+ return git(appDir, ['show', `HEAD:${relativePath}`], {encoding: 'buffer'});+}++/**+ * Collapse a line to the content a reader would compare by eye: no leading or+ * trailing whitespace, and every interior run of whitespace as a single space.+ *+ * Runs collapse to one space rather than to nothing, so `label="Deploy"` and+ * `label = "Deploy"` are the same line while `a b` and `ab` are not.+ */+function normalizeLine(text) {+ return text.replace(/\s+/g, ' ').trim();+}++/**+ * Count how many of a file's baseline lines no longer appear anywhere in its+ * current content, comparing lines after whitespace normalization and ignoring+ * blank lines. Re-indenting a file therefore deletes nothing, while replacing+ * its content deletes all of it.+ */+export function substantiveLineDelta(baselineText, currentText) {+ const remaining = new Map();+ for (const line of currentText.split('\n')) {+ const normalized = normalizeLine(line);+ if (normalized === '') continue;+ remaining.set(normalized, (remaining.get(normalized) ?? 0) + 1);+ }++ let baselineLines = 0;+ let deletedLines = 0;+ for (const line of baselineText.split('\n')) {+ const normalized = normalizeLine(line);+ if (normalized === '') continue;+ baselineLines += 1;+ const available = remaining.get(normalized) ?? 0;+ if (available > 0) remaining.set(normalized, available - 1);+ else deletedLines += 1;+ }++ return {+ baselineLines,+ deletedLines,+ deletedFraction: baselineLines === 0 ? 0 : deletedLines / baselineLines,+ };+}++function isHostSource(relativePath) {+ return HOST_SOURCE_EXTENSIONS.has(path.extname(relativePath).toLowerCase());+}++function addedLinesFromPatch(patch) {+ const lines = [];+ let newLine = 0;+ for (const line of patch.split('\n')) {+ const hunk = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/);+ if (hunk) {+ newLine = Number(hunk[1]);+ continue;+ }+ if (line.startsWith('+++')) continue;+ if (line.startsWith('+')) {+ lines.push({line: newLine, text: line.slice(1)});+ newLine += 1;+ } else if (!line.startsWith('-') && !line.startsWith('\\ No newline')) {+ newLine += 1;+ }+ }+ return lines;+}++function trackedAddedLines(appDir, relativePath) {+ const patch = git(appDir, [+ '-c',+ 'core.quotePath=false',+ 'diff',+ '--unified=0',+ '--no-color',+ '--no-ext-diff',+ '--no-textconv',+ '--no-renames',+ 'HEAD',+ '--',+ relativePath,+ ]);+ return addedLinesFromPatch(patch);+}++function removedLinesFromPatch(patch) {+ const lines = [];+ let oldLine = 0;+ for (const line of patch.split('\n')) {+ const hunk = line.match(/^@@ -(\d+)(?:,\d+)? \+\d+(?:,\d+)? @@/);+ if (hunk) {+ oldLine = Number(hunk[1]);+ continue;+ }+ if (line.startsWith('---')) continue;+ if (line.startsWith('-')) {+ lines.push({line: oldLine, text: line.slice(1)});+ oldLine += 1;+ } else if (!line.startsWith('+') && !line.startsWith('\\ No newline')) {+ oldLine += 1;+ }+ }+ return lines;+}++function trackedRemovedLines(appDir, relativePath) {+ const patch = git(appDir, [+ '-c',+ 'core.quotePath=false',+ 'diff',+ '--unified=0',+ '--no-color',+ '--no-ext-diff',+ '--no-textconv',+ '--no-renames',+ 'HEAD',+ '--',+ relativePath,+ ]);+ return removedLinesFromPatch(patch);+}++function untrackedAddedLines(file) {+ if (file.binary) return [];+ return file.bytes+ .toString('utf8')+ .split('\n')+ .map((text, index) => ({line: index + 1, text}));+}++function evidenceFor(relativePath, lines) {+ const evidence = [];+ for (const added of lines) {+ if (/@astryxdesign\//.test(added.text)) {+ evidence.push({+ path: relativePath,+ line: added.line,+ kind: /(?:^|\s)(?:import|require\s*\()/.test(added.text)+ ? 'code-import'+ : /(?:\.css|theme\.css)/.test(added.text)+ ? 'stylesheet-import'+ : 'package-reference',+ text: added.text.trim(),+ });+ }+ }+ return evidence;+}++function findingsFor(relativePath, lines) {+ const findings = [];+ for (const added of lines) {+ for (const escapeHatch of ESCAPE_HATCHES) {+ if (escapeHatch.pattern.test(added.text)) {+ findings.push({+ kind: escapeHatch.kind,+ path: relativePath,+ line: added.line,+ message: escapeHatch.message,+ });+ }+ }+ }+ return findings;+}++/**+ * Analyze the working tree relative to its committed pre-agent HEAD.+ * This function only invokes read-only Git commands and reads file bytes.+ */+export function analyzeSetupIntegrity(appDir, attestedDiffSha256) {+ const resolvedAppDir = path.resolve(appDir);+ assertRepositoryRoot(resolvedAppDir);++ const patch = trackedPatch(resolvedAppDir);+ const untracked = untrackedPaths(resolvedAppDir).map(relativePath => ({+ path: relativePath,+ ...untrackedEntry(resolvedAppDir, relativePath),+ }));+ const untrackedByPath = new Map(untracked.map(file => [file.path, file]));+ const tracked = trackedNumstat(resolvedAppDir);+ const changedFiles = [];+ const escapeHatches = [];+ const astryxEvidence = [];++ for (const file of tracked) {+ const absolutePath = path.join(resolvedAppDir, file.path);+ const existedAtBaseline = (() => {+ try {+ baselineBytes(resolvedAppDir, file.path);+ return true;+ } catch {+ return false;+ }+ })();+ const existsNow = fs.existsSync(absolutePath);+ const status = !existedAtBaseline+ ? 'added'+ : !existsNow+ ? 'deleted'+ : 'modified';+ changedFiles.push({+ path: file.path,+ status,+ tracked: true,+ added: file.added,+ deleted: file.deleted,+ binary: file.binary,+ });++ if (status === 'deleted' && isHostSource(file.path)) {+ escapeHatches.push({+ kind: 'deleted-host-source',+ path: file.path,+ line: null,+ message: 'deleted a host source or stylesheet file',+ });+ continue;+ }++ if (status === 'modified' && isHostSource(file.path) && !file.binary) {+ const currentSource = fs.readFileSync(absolutePath, 'utf8');+ const {baselineLines, deletedLines, deletedFraction} =+ substantiveLineDelta(+ baselineBytes(resolvedAppDir, file.path).toString('utf8'),+ currentSource,+ );+ if (+ deletedLines >= WHOLESALE_REPLACEMENT_THRESHOLD.minimumDeletedLines &&+ deletedFraction >= WHOLESALE_REPLACEMENT_THRESHOLD.deletedFraction+ ) {+ escapeHatches.push({+ kind: 'wholesale-replacement',+ path: file.path,+ line: null,+ message: 'replaced most of an established host source or stylesheet',+ baselineLines,+ deletedLines,+ deletedFraction,+ rawDeletedLines: file.deleted,+ });+ } else if (+ path.extname(file.path).toLowerCase() === '.css' &&+ deletedFraction >= WHOLESALE_REPLACEMENT_THRESHOLD.deletedFraction+ ) {+ escapeHatches.push({+ kind: 'neutralized-host-css',+ path: file.path,+ line: null,+ message: 'removed most of an established host stylesheet',+ baselineLines,+ deletedLines,+ deletedFraction,+ rawDeletedLines: file.deleted,+ });+ }++ const removedModeLines = trackedRemovedLines(+ resolvedAppDir,+ file.path,+ ).filter(removed =>+ /\b(?:dark|color-scheme|data-theme)\b/i.test(removed.text),+ );+ if (+ removedModeLines.length > 0 &&+ !/\b(?:dark|color-scheme|data-theme)\b/i.test(currentSource)+ ) {+ escapeHatches.push({+ kind: 'dark-mode-removed',+ path: file.path,+ line: removedModeLines[0].line,+ message: 'removed the host dark-mode behavior',+ });+ }+ }++ if (file.path === '.gitignore' && status !== 'added') {+ escapeHatches.push({+ kind: 'gitignore-modified',+ path: file.path,+ line: null,+ message: 'modified ignore rules after the execution baseline',+ });+ }+ if (status !== 'deleted' && !file.binary) {+ const addedLines = trackedAddedLines(resolvedAppDir, file.path);+ if (isHostSource(file.path)) {+ escapeHatches.push(...findingsFor(file.path, addedLines));+ }+ astryxEvidence.push(...evidenceFor(file.path, addedLines));+ }+ }++ for (const file of untracked) {+ const added = file.binary ? 0 : lineCount(file.bytes);+ changedFiles.push({+ path: file.path,+ status: 'untracked',+ tracked: false,+ added,+ deleted: 0,+ binary: file.binary,+ });+ const addedLines = untrackedAddedLines(untrackedByPath.get(file.path));+ if (isHostSource(file.path)) {+ escapeHatches.push(...findingsFor(file.path, addedLines));+ }+ astryxEvidence.push(...evidenceFor(file.path, addedLines));+ }++ changedFiles.sort((left, right) => compareText(left.path, right.path));+ escapeHatches.sort(+ (left, right) =>+ compareText(left.path, right.path) ||+ (left.line ?? -1) - (right.line ?? -1) ||+ compareText(left.kind, right.kind),+ );+ astryxEvidence.sort(+ (left, right) =>+ compareText(left.path, right.path) ||+ left.line - right.line ||+ compareText(left.kind, right.kind),+ );++ const diffSha256 = digestDiff(patch, untracked);+ const attestationProvided = typeof attestedDiffSha256 === 'string';+ const attestationMatches =+ attestationProvided && attestedDiffSha256 === diffSha256;+ const rejectionReasons = escapeHatches.map(finding => finding.kind);+ if (astryxEvidence.length === 0)+ rejectionReasons.push('missing-astryx-usage');+ if (!attestationProvided) rejectionReasons.push('missing-attestation');+ else if (!attestationMatches) rejectionReasons.push('attestation-mismatch');++ return {+ schemaVersion: 1,+ diffSha256,+ changedFiles,+ counts: {+ files: changedFiles.length,+ added: changedFiles.reduce((sum, file) => sum + file.added, 0),+ deleted: changedFiles.reduce((sum, file) => sum + file.deleted, 0),+ },+ astryxUsage: {+ found: astryxEvidence.length > 0,+ evidence: astryxEvidence,+ },+ escapeHatches,+ wholesaleReplacementThreshold: WHOLESALE_REPLACEMENT_THRESHOLD,+ attestation: {+ provided: attestationProvided,+ expectedSha256: attestedDiffSha256 ?? null,+ matches: attestationMatches,+ },+ accepted: rejectionReasons.length === 0,+ rejectionReasons: [...new Set(rejectionReasons)].sort(compareText),+ };+}++function parseCli(argv) {+ const parsed = {};+ for (let index = 0; index < argv.length; index += 1) {+ const argument = argv[index];+ if (argument !== '--app' && argument !== '--attested-diff-sha256') {+ throw new Error(+ 'usage: setup-integrity.mjs --app <sandbox> [--attested-diff-sha256 <sha256>]',+ );+ }+ const value = argv[index + 1];+ if (!value || value.startsWith('--')) {+ throw new Error(`missing value for ${argument}`);+ }+ parsed[argument.slice(2)] = value;+ index += 1;+ }+ if (!parsed.app) {+ throw new Error(+ 'usage: setup-integrity.mjs --app <sandbox> [--attested-diff-sha256 <sha256>]',+ );+ }+ return parsed;+}++const isMain =+ process.argv[1] != null &&+ path.resolve(process.argv[1]) ===+ path.resolve(fileURLToPath(import.meta.url));++if (isMain) {+ const args = parseCli(process.argv.slice(2));+ const result = analyzeSetupIntegrity(+ path.resolve(args.app),+ args['attested-diff-sha256'],+ );+ process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);+}diff --git a/internal/vibe-tests/setup-test/setup-integrity.test.mjs b/internal/vibe-tests/setup-test/setup-integrity.test.mjsnew file mode 100644index 0000000000..b46f3a79fe--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-integrity.test.mjs@@ -0,0 +1,713 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {execFileSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {afterEach, describe, expect, it} from 'vitest';+import {+ analyzeSetupIntegrity,+ substantiveLineDelta,+ WHOLESALE_REPLACEMENT_THRESHOLD,+} from './setup-integrity.mjs';++const temporaryDirectories = [];+const INTEGRITY_CLI = fileURLToPath(+ new URL('./setup-integrity.mjs', import.meta.url),+);++function git(directory, args) {+ return execFileSync('git', ['-C', directory, ...args], {+ encoding: 'utf8',+ env: {+ ...process.env,+ GIT_AUTHOR_NAME: 'setup-integrity-tests',+ GIT_AUTHOR_EMAIL: '[email protected]',+ GIT_COMMITTER_NAME: 'setup-integrity-tests',+ GIT_COMMITTER_EMAIL: '[email protected]',+ },+ });+}++function write(directory, relativePath, content) {+ const file = path.join(directory, relativePath);+ fs.mkdirSync(path.dirname(file), {recursive: true});+ fs.writeFileSync(file, content);+}++function repository() {+ const directory = fs.mkdtempSync(+ path.join(os.tmpdir(), 'setup-integrity-test-'),+ );+ temporaryDirectories.push(directory);+ write(directory, '.gitignore', 'ignored.txt\n');+ write(+ directory,+ 'src/App.tsx',+ [+ 'export function App() {',+ ' return <main className="host">Host application</main>;',+ '}',+ '',+ ].join('\n'),+ );+ write(+ directory,+ 'src/index.css',+ [+ ':root {',+ ' color-scheme: light dark;',+ '}',+ '.host {',+ ' color: CanvasText;',+ '}',+ '',+ ].join('\n'),+ );+ git(directory, ['init', '-q']);+ git(directory, ['add', '-A']);+ git(directory, ['commit', '-qm', 'baseline']);+ return directory;+}++function addValidAstryxChange(directory) {+ write(+ directory,+ 'src/App.tsx',+ [+ "import {Button} from '@astryxdesign/core/Button';",+ '',+ 'export function App() {',+ ' return (',+ ' <main className="host">',+ ' Host application',+ ' <Button label="Continue" />',+ ' </main>',+ ' );',+ '}',+ '',+ ].join('\n'),+ );+}++function attest(directory) {+ return analyzeSetupIntegrity(directory).diffSha256;+}++function kinds(result) {+ return result.escapeHatches.map(finding => finding.kind);+}++const HOST_RECORDS = [+ {name: 'Quarterly plan', owner: 'Avery', state: 'Ready'},+ {name: 'Partner review', owner: 'Morgan', state: 'In progress'},+ {name: 'Launch checklist', owner: 'Riley', state: 'Blocked'},+ {name: 'Vendor renewal', owner: 'Jordan', state: 'Ready'},+ {name: 'Budget transfer', owner: 'Casey', state: 'In progress'},+ {name: 'Access request', owner: 'Quinn', state: 'Ready'},+ {name: 'Hardware order', owner: 'Reese', state: 'Blocked'},+ {name: 'Contract amendment', owner: 'Sasha', state: 'Ready'},+];++/**+ * The JSX a provider wrapper re-indents: everything between `return (` and the+ * closing paren, at the indentation an unwrapped component tree has.+ */+function hostAppTree() {+ return [+ " <div className={dark ? 'dark' : ''}>",+ ' <main',+ ' className="min-h-screen bg-background px-6 py-8 text-foreground"',+ ' data-vibe-probe="host-shell">',+ ' <div className="mx-auto max-w-5xl space-y-6">',+ ' <header className="flex items-center justify-between gap-4">',+ ' <div>',+ ' <p className="text-sm text-muted-foreground">Review queue</p>',+ ' <h1',+ ' className="text-2xl font-semibold tracking-tight"',+ ' data-vibe-probe="page-title">',+ ' Purchase requests',+ ' </h1>',+ ' </div>',+ ' <div className="flex gap-2">',+ ' <Button',+ ' variant="outline"',+ ' onClick={() => setDark(value => !value)}>',+ ' Toggle mode',+ ' </Button>',+ ' </div>',+ ' </header>',+ ' <Card className="divide-y" data-vibe-probe="record-list">',+ ...HOST_RECORDS.flatMap(record => [+ ' <div className="flex items-center justify-between px-4 py-3">',+ ` <span className="font-medium">${record.name}</span>`,+ ` <span className="text-muted-foreground">${record.owner}</span>`,+ ` <span className="text-sm">${record.state}</span>`,+ ' </div>',+ ]),+ ' </Card>',+ ' </div>',+ ' </main>',+ ' </div>',+ ];+}++/** An established host `App.tsx`, most of whose lines sit inside the tree. */+function establishedHostApp() {+ return [+ "import {useState} from 'react';",+ "import {Button} from '@/components/ui/button';",+ "import {Card} from '@/components/ui/card';",+ '',+ 'export default function App() {',+ ' const [dark, setDark] = useState(false);',+ '',+ ' return (',+ ...hostAppTree(),+ ' );',+ '}',+ '',+ ].join('\n');+}++/**+ * The same app after adding Astryx: three imports, a `<Theme>` wrapper, one new+ * button — and, because the wrapper adds a nesting level, two more spaces of+ * indentation on every line of the tree. This is the shape that made a raw+ * line diff report the whole component tree as deleted.+ */+function wrappedHostApp() {+ return [+ "import {useState} from 'react';",+ "import {Button as AstryxButton} from '@astryxdesign/core/Button';",+ "import {Theme} from '@astryxdesign/core/theme';",+ "import {neutralTheme} from '@astryxdesign/theme-neutral/built';",+ "import {Button} from '@/components/ui/button';",+ "import {Card} from '@/components/ui/card';",+ '',+ 'export default function App() {',+ ' const [dark, setDark] = useState(false);',+ '',+ ' return (',+ " <Theme theme={neutralTheme} mode={dark ? 'dark' : 'light'}>",+ ...hostAppTree().map(line => (line === '' ? line : ` ${line}`)),+ ' </Theme>',+ ' );',+ '}',+ '',+ ].join('\n');+}++function numstat(directory, relativePath) {+ return parseNumstat(+ git(directory, [+ 'diff',+ '--numstat',+ '--no-renames',+ 'HEAD',+ '--',+ relativePath,+ ]),+ );+}++function numstatIgnoringWhitespace(directory, relativePath) {+ return parseNumstat(+ git(directory, [+ 'diff',+ '--numstat',+ '--no-renames',+ '-w',+ 'HEAD',+ '--',+ relativePath,+ ]),+ );+}++function parseNumstat(output) {+ const [record] = output.split('\n').filter(Boolean);+ if (!record) return {added: 0, deleted: 0};+ const [added, deleted] = record.split('\t');+ return {added: Number(added), deleted: Number(deleted)};+}++function commitHostApp(directory, source) {+ write(directory, 'src/App.tsx', source);+ git(directory, ['add', 'src/App.tsx']);+ git(directory, ['commit', '-qm', 'establish host app']);+}++afterEach(() => {+ for (const directory of temporaryDirectories.splice(0)) {+ fs.rmSync(directory, {recursive: true, force: true});+ }+});++describe('setup integrity', () => {+ it('accepts a clean Astryx diff with an exact runner attestation and stays read-only', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ const digest = attest(directory);+ const before = git(directory, [+ 'status',+ '--porcelain=v1',+ '-z',+ '--untracked-files=all',+ ]);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(result).toMatchObject({+ schemaVersion: 1,+ diffSha256: digest,+ astryxUsage: {found: true},+ escapeHatches: [],+ attestation: {provided: true, expectedSha256: digest, matches: true},+ accepted: true,+ rejectionReasons: [],+ });+ expect(result.astryxUsage.evidence).toEqual([+ {+ path: 'src/App.tsx',+ line: 1,+ kind: 'code-import',+ text: "import {Button} from '@astryxdesign/core/Button';",+ },+ ]);+ expect(+ git(directory, [+ 'status',+ '--porcelain=v1',+ '-z',+ '--untracked-files=all',+ ]),+ ).toBe(before);+ });++ it('reports deterministic changed-file inventory and line totals', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ write(directory, 'src/new.css', '.panel { color: CanvasText; }\n');+ write(directory, 'ignored.txt', 'not part of the agent diff\n');++ const first = analyzeSetupIntegrity(directory);+ const second = analyzeSetupIntegrity(directory);++ expect(second.diffSha256).toBe(first.diffSha256);+ expect(first.changedFiles).toEqual([+ {+ path: 'src/App.tsx',+ status: 'modified',+ tracked: true,+ added: 8,+ deleted: 1,+ binary: false,+ },+ {+ path: 'src/new.css',+ status: 'untracked',+ tracked: false,+ added: 1,+ deleted: 0,+ binary: false,+ },+ ]);+ expect(first.counts).toEqual({files: 2, added: 9, deleted: 1});+ expect(first.rejectionReasons).toContain('missing-attestation');+ });++ it.each([+ {+ name: 'hardcoded !important',+ source: '.host { color: red !important; }\n',+ expected: 'hardcoded-important',+ },+ {+ name: 'blanket all-property reset',+ source: '.host { all: revert; }\n',+ expected: 'blanket-reset',+ },+ {+ name: 'explicit dark-mode disabling',+ source: ':root { color-scheme: light; }\n',+ expected: 'dark-mode-disabled',+ },+ ])('rejects added $name', ({source, expected}) => {+ const directory = repository();+ addValidAstryxChange(directory);+ write(directory, 'src/escape.css', source);+ const digest = attest(directory);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(kinds(result)).toContain(expected);+ expect(result.accepted).toBe(false);+ });++ it('allows the standard light-and-dark color-scheme declaration', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ write(directory, 'src/theme.css', ':root { color-scheme: light dark; }\n');+ const digest = attest(directory);+ expect(kinds(analyzeSetupIntegrity(directory, digest))).not.toContain(+ 'dark-mode-disabled',+ );+ });++ it('does not treat prose or lockfile text as a CSS escape hatch', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ write(+ directory,+ 'README.md',+ 'Avoid using !important in application CSS.\n',+ );+ const digest = attest(directory);+ expect(kinds(analyzeSetupIntegrity(directory, digest))).not.toContain(+ 'hardcoded-important',+ );+ });++ it('rejects changing ignore rules after the execution baseline', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ write(directory, '.gitignore', 'ignored.txt\nhidden.css\n');+ const digest = attest(directory);+ expect(kinds(analyzeSetupIntegrity(directory, digest))).toContain(+ 'gitignore-modified',+ );+ });++ it('rejects deleting host source or CSS', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ fs.rmSync(path.join(directory, 'src', 'index.css'));+ const digest = attest(directory);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(result.escapeHatches).toContainEqual({+ kind: 'deleted-host-source',+ path: 'src/index.css',+ line: null,+ message: 'deleted a host source or stylesheet file',+ });+ expect(result.accepted).toBe(false);+ });++ it('rejects neutralizing a short host stylesheet', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ write(directory, 'src/index.css', ':root {}\n');+ const digest = attest(directory);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(kinds(result)).toContain('neutralized-host-css');+ expect(result.accepted).toBe(false);+ });++ it('rejects removing host dark-mode behavior', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ write(directory, 'src/index.css', '.host {\n color: CanvasText;\n}\n');+ const digest = attest(directory);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(kinds(result)).toContain('dark-mode-removed');+ expect(result.accepted).toBe(false);+ });++ it('rejects wholesale replacement at the documented threshold', () => {+ const directory = repository();+ const baselineLines = Array.from(+ {length: WHOLESALE_REPLACEMENT_THRESHOLD.minimumDeletedLines + 5},+ (_, index) => `export const value${index} = ${index};`,+ );+ write(directory, 'src/large.ts', `${baselineLines.join('\n')}\n`);+ git(directory, ['add', 'src/large.ts']);+ git(directory, ['commit', '-qm', 'add established source']);+ addValidAstryxChange(directory);+ write(directory, 'src/large.ts', "export const replacement = 'new';\n");+ const digest = attest(directory);++ const result = analyzeSetupIntegrity(directory, digest);+ const finding = result.escapeHatches.find(+ candidate => candidate.kind === 'wholesale-replacement',+ );++ expect(finding).toMatchObject({+ path: 'src/large.ts',+ baselineLines: baselineLines.length,+ deletedLines: baselineLines.length,+ deletedFraction: 1,+ });+ expect(result.wholesaleReplacementThreshold).toEqual({+ minimumDeletedLines: 20,+ deletedFraction: 0.8,+ });+ expect(result.accepted).toBe(false);+ });++ it('rejects a mismatched runner attestation', () => {+ const directory = repository();+ addValidAstryxChange(directory);++ const result = analyzeSetupIntegrity(directory, '0'.repeat(64));++ expect(result.attestation).toEqual({+ provided: true,+ expectedSha256: '0'.repeat(64),+ matches: false,+ });+ expect(result.rejectionReasons).toContain('attestation-mismatch');+ expect(result.accepted).toBe(false);+ });++ it('rejects a missing runner attestation while still returning the digest', () => {+ const directory = repository();+ addValidAstryxChange(directory);++ const result = analyzeSetupIntegrity(directory);++ expect(result.diffSha256).toMatch(/^[a-f0-9]{64}$/);+ expect(result.attestation).toEqual({+ provided: false,+ expectedSha256: null,+ matches: false,+ });+ expect(result.rejectionReasons).toContain('missing-attestation');+ expect(result.accepted).toBe(false);+ });++ it('rejects a diff with no Astryx usage evidence', () => {+ const directory = repository();+ write(+ directory,+ 'src/App.tsx',+ 'export function App() { return <main>Changed host</main>; }\n',+ );+ const digest = attest(directory);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(result.astryxUsage).toEqual({found: false, evidence: []});+ expect(result.rejectionReasons).toContain('missing-astryx-usage');+ expect(result.accepted).toBe(false);+ });++ it('prints the immediate post-agent digest as JSON from the CLI', () => {+ const directory = repository();+ addValidAstryxChange(directory);++ const result = JSON.parse(+ execFileSync(process.execPath, [INTEGRITY_CLI, '--app', directory], {+ encoding: 'utf8',+ }),+ );++ expect(result.diffSha256).toMatch(/^[a-f0-9]{64}$/);+ expect(result.attestation).toEqual({+ provided: false,+ expectedSha256: null,+ matches: false,+ });+ });++ it('changes the digest when a nonignored untracked file changes', () => {+ const directory = repository();+ addValidAstryxChange(directory);+ write(directory, 'notes.txt', 'first\n');+ const first = attest(directory);++ write(directory, 'notes.txt', 'second\n');+ const second = attest(directory);++ expect(second).not.toBe(first);+ });+});++describe('substantive line delta', () => {+ it('sees no deletion when a wrapper only re-indents the tree', () => {+ expect(substantiveLineDelta(establishedHostApp(), wrappedHostApp())).toEqual(+ {+ baselineLines: expect.any(Number),+ deletedLines: 0,+ deletedFraction: 0,+ },+ );+ });++ it('sees every line deleted when the same content is minified onto one line', () => {+ const baseline = establishedHostApp();+ const minified = `${baseline+ .split('\n')+ .map(line => line.trim())+ .filter(Boolean)+ .join(' ')}\n`;++ const delta = substantiveLineDelta(baseline, minified);++ expect(delta.deletedLines).toBe(delta.baselineLines);+ expect(delta.deletedFraction).toBe(1);+ });++ it('does not equate lines that differ by whitespace between tokens', () => {+ expect(+ substantiveLineDelta(+ 'const label = "Purchase requests";\n',+ 'constlabel="Purchaserequests";\n',+ ),+ ).toEqual({baselineLines: 1, deletedLines: 1, deletedFraction: 1});+ });++ it('treats a line as unchanged when only its indentation moved', () => {+ expect(+ substantiveLineDelta(' <Card className="divide-y">\n', ' <Card className="divide-y">\n'),+ ).toEqual({baselineLines: 1, deletedLines: 0, deletedFraction: 0});+ });+});++describe('setup integrity — reformatting versus replacement', () => {+ it('accepts a provider wrapper that re-indents an established component tree', () => {+ const directory = repository();+ commitHostApp(directory, establishedHostApp());+ write(directory, 'src/App.tsx', wrappedHostApp());+ const digest = attest(directory);++ // Non-vacuous: on the raw line diff this change clears both halves of the+ // documented threshold, so a guard reading `git diff --numstat` rejects it.+ const raw = numstat(directory, 'src/App.tsx');+ const baselineLines = establishedHostApp().split('\n').length - 1;+ expect(raw.deleted).toBeGreaterThanOrEqual(+ WHOLESALE_REPLACEMENT_THRESHOLD.minimumDeletedLines,+ );+ expect(raw.deleted / baselineLines).toBeGreaterThanOrEqual(+ WHOLESALE_REPLACEMENT_THRESHOLD.deletedFraction,+ );+ // ...and nothing was actually removed: the whole diff is added lines.+ expect(numstatIgnoringWhitespace(directory, 'src/App.tsx').deleted).toBe(0);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(kinds(result)).not.toContain('wholesale-replacement');+ expect(result.rejectionReasons).toEqual([]);+ expect(result.accepted).toBe(true);+ });++ it('still rejects gutting the component tree, re-indentation notwithstanding', () => {+ const directory = repository();+ commitHostApp(directory, establishedHostApp());+ write(+ directory,+ 'src/App.tsx',+ [+ "import {Button as AstryxButton} from '@astryxdesign/core/Button';",+ '',+ 'export default function App() {',+ ' return (',+ ' <AstryxButton label="Deploy" variant="primary" />',+ ' );',+ '}',+ '',+ ].join('\n'),+ );+ const digest = attest(directory);++ const result = analyzeSetupIntegrity(directory, digest);+ const finding = result.escapeHatches.find(+ candidate => candidate.kind === 'wholesale-replacement',+ );++ expect(finding).toMatchObject({path: 'src/App.tsx'});+ expect(finding.deletedFraction).toBeGreaterThanOrEqual(+ WHOLESALE_REPLACEMENT_THRESHOLD.deletedFraction,+ );+ expect(result.accepted).toBe(false);+ });++ it('rejects a rewrite that minifies the host source onto one line', () => {+ const directory = repository();+ commitHostApp(directory, establishedHostApp());+ write(+ directory,+ 'src/App.tsx',+ [+ "import {Button as AstryxButton} from '@astryxdesign/core/Button';",+ establishedHostApp()+ .split('\n')+ .map(line => line.trim())+ .filter(Boolean)+ .join(' '),+ '',+ ].join('\n'),+ );+ const digest = attest(directory);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(kinds(result)).toContain('wholesale-replacement');+ expect(result.accepted).toBe(false);+ });++ it('rejects a rewrite that only removes the whitespace inside each line', () => {+ const directory = repository();+ const baselineLines = Array.from(+ {length: WHOLESALE_REPLACEMENT_THRESHOLD.minimumDeletedLines + 10},+ (_, index) => `export const value${index} = ${index};`,+ );+ write(directory, 'src/large.ts', `${baselineLines.join('\n')}\n`);+ git(directory, ['add', 'src/large.ts']);+ git(directory, ['commit', '-qm', 'add established source']);+ addValidAstryxChange(directory);+ write(+ directory,+ 'src/large.ts',+ `${baselineLines.map(line => line.replaceAll(' ', '')).join('\n')}\n`,+ );+ const digest = attest(directory);++ // A guard that asked Git to ignore whitespace would see nothing at all+ // here, because `-w` equates `a b` with `ab`.+ expect(numstatIgnoringWhitespace(directory, 'src/large.ts').deleted).toBe(0);++ const result = analyzeSetupIntegrity(directory, digest);++ expect(kinds(result)).toContain('wholesale-replacement');+ expect(result.accepted).toBe(false);+ });++ it('does not treat wrapping a stylesheet in a layer as neutralizing it', () => {+ const directory = repository();+ const rules = Array.from(+ {length: WHOLESALE_REPLACEMENT_THRESHOLD.minimumDeletedLines + 4},+ (_, index) => `.host-row-${index} {\n color: CanvasText;\n}`,+ ).join('\n');+ write(directory, 'src/index.css', `:root {\n color-scheme: light dark;\n}\n${rules}\n`);+ git(directory, ['add', 'src/index.css']);+ git(directory, ['commit', '-qm', 'establish host stylesheet']);+ addValidAstryxChange(directory);+ write(+ directory,+ 'src/index.css',+ `@layer base {\n${`:root {\n color-scheme: light dark;\n}\n${rules}`+ .split('\n')+ .map(line => (line === '' ? line : ` ${line}`))+ .join('\n')}\n}\n`,+ );+ const digest = attest(directory);++ const raw = numstat(directory, 'src/index.css');+ expect(raw.deleted).toBeGreaterThanOrEqual(+ WHOLESALE_REPLACEMENT_THRESHOLD.minimumDeletedLines,+ );++ const result = analyzeSetupIntegrity(directory, digest);++ expect(kinds(result)).not.toContain('neutralized-host-css');+ expect(kinds(result)).not.toContain('wholesale-replacement');+ });+});diff --git a/internal/vibe-tests/setup-test/setup-interactions.mjs b/internal/vibe-tests/setup-test/setup-interactions.mjsnew file mode 100644index 0000000000..dccab9a544--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-interactions.mjs@@ -0,0 +1,85 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/** Open a declared interaction without relying on text, position, or pointer-only focus. */++export const markerSelector = marker =>+ `[data-vibe-${marker.source === 'result' ? 'result' : 'probe'}="${marker.marker ?? marker.name}"]`;+export const probeSelector = name => markerSelector({name, source: 'probe'});++async function requireUnique(page, marker) {+ const locator = page.locator(markerSelector(marker));+ const count = await locator.count();+ if (count !== 1) {+ throw new Error(+ `interaction marker ${marker.name} matched ${count} elements; expected exactly one`,+ );+ }+ return locator;+}++async function reachByKeyboard(page, marker, maxTabs = 50) {+ const locator = await requireUnique(page, marker);+ for (let index = 0; index <= maxTabs; index += 1) {+ const reached = await locator.evaluate(element => {+ const active = element.ownerDocument.activeElement;+ return active === element || (active != null && element.contains(active));+ });+ if (reached) return true;+ await page.keyboard.press('Tab');+ }+ return false;+}++export async function openInteractionState(page, interaction) {+ if (!interaction) {+ return {opened: false, keyboardReached: {}};+ }++ const keyboardReached = {};+ for (const rawStep of interaction.open) {+ const step =+ typeof rawStep === 'string'+ ? {name: rawStep, source: 'probe', method: 'click'}+ : rawStep;+ const locator = await requireUnique(page, step);+ if (step.method === 'click') {+ await locator.click();+ continue;+ }+ if (+ step.method !== 'keyboard-focus' &&+ step.method !== 'keyboard-activate'+ ) {+ throw new Error(`unknown interaction method ${step.method}`);+ }+ const reached = await reachByKeyboard(page, step);+ keyboardReached[step.name] = reached;+ if (!reached) {+ throw new Error(+ `interaction marker ${step.name} is not keyboard reachable`,+ );+ }+ if (step.method === 'keyboard-activate') {+ await page.keyboard.press(step.key ?? 'Enter');+ }+ }++ for (const surface of interaction.surfaces) {+ const locator = await requireUnique(page, surface);+ await locator.waitFor({state: 'visible', timeout: 3000});+ await locator.evaluate(async element => {+ let topLayer = element;+ try {+ topLayer = element.closest(':modal, :popover-open') ?? element;+ } catch {}+ const finiteAnimations = topLayer.getAnimations().filter(animation => {+ const endTime = animation.effect?.getComputedTiming().endTime;+ return typeof endTime === 'number' && Number.isFinite(endTime);+ });+ await Promise.allSettled(+ finiteAnimations.map(animation => animation.finished),+ );+ });+ }+ return {opened: true, keyboardReached};+}diff --git a/internal/vibe-tests/setup-test/setup-interactions.test.mjs b/internal/vibe-tests/setup-test/setup-interactions.test.mjsnew file mode 100644index 0000000000..17f046dfb7--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-interactions.test.mjs@@ -0,0 +1,124 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it, vi} from 'vitest';+import {+ markerSelector,+ openInteractionState,+ probeSelector,+} from './setup-interactions.mjs';++function fakePage({counts = {}, tabOrder = []} = {}) {+ const events = [];+ let active = null;+ let tabIndex = -1;+ return {+ events,+ page: {+ locator(selector) {+ return {+ count: async () => counts[selector] ?? 1,+ click: async () => events.push(`click:${selector}`),+ evaluate: async () => active === selector,+ waitFor: async ({state, timeout}) =>+ events.push(`wait:${selector}:${state}:${timeout}`),+ };+ },+ keyboard: {+ press: async key => {+ events.push(`key:${key}`);+ if (key === 'Tab') {+ tabIndex += 1;+ active = tabOrder[tabIndex] ?? null;+ }+ },+ },+ },+ };+}++describe('setup interaction runner', () => {+ it('opens legacy fixture controls before waiting for measured surfaces', async () => {+ const {page, events} = fakePage();+ await expect(+ openInteractionState(page, {+ open: ['dialog-trigger', 'popover-trigger'],+ surfaces: [+ {name: 'dialog-surface', kind: 'dialog'},+ {name: 'popover-surface', kind: 'popover'},+ ],+ }),+ ).resolves.toEqual({opened: true, keyboardReached: {}});++ expect(events).toEqual([+ `click:${probeSelector('dialog-trigger')}`,+ `click:${probeSelector('popover-trigger')}`,+ `wait:${probeSelector('dialog-surface')}:visible:3000`,+ `wait:${probeSelector('popover-surface')}:visible:3000`,+ ]);+ });++ it('proves task triggers are reached and activated from the keyboard', async () => {+ const dialog = markerSelector({name: 'dialog', source: 'result'});+ const menu = markerSelector({name: 'menu', source: 'result'});+ const {page, events} = fakePage({tabOrder: [dialog, menu]});+ const state = await openInteractionState(page, {+ open: [+ {+ name: 'dialog',+ source: 'result',+ method: 'keyboard-activate',+ key: 'Enter',+ },+ {name: 'menu', source: 'result', method: 'keyboard-focus'},+ ],+ surfaces: [],+ });++ expect(state).toEqual({+ opened: true,+ keyboardReached: {dialog: true, menu: true},+ });+ expect(events).toEqual(['key:Tab', 'key:Enter', 'key:Tab']);+ });++ it.each([0, 2])('rejects a marker that matches %i elements', async count => {+ const selector = probeSelector('dialog-trigger');+ const {page} = fakePage({counts: {[selector]: count}});+ await expect(+ openInteractionState(page, {+ open: ['dialog-trigger'],+ surfaces: [],+ }),+ ).rejects.toThrow(new RegExp(`matched ${count} elements`));+ });++ it('fails when a declared trigger is not keyboard reachable', async () => {+ const {page} = fakePage();+ await expect(+ openInteractionState(page, {+ open: [{name: 'menu', source: 'result', method: 'keyboard-activate'}],+ surfaces: [],+ }),+ ).rejects.toThrow(/not keyboard reachable/);+ });++ it('uses a declared marker alias for a logical surface name', () => {+ expect(+ markerSelector({+ name: 'astryx-dialog-backdrop',+ marker: 'astryx-dialog-surface',+ source: 'result',+ }),+ ).toBe('[data-vibe-result="astryx-dialog-surface"]');+ });++ it('leaves fixtures without interactions untouched', async () => {+ const {page} = fakePage();+ const locator = vi.spyOn(page, 'locator');+ await expect(openInteractionState(page, null)).resolves.toEqual({+ opened: false,+ keyboardReached: {},+ });+ expect(locator).not.toHaveBeenCalled();+ });+});diff --git a/internal/vibe-tests/setup-test/setup-matrix.mjs b/internal/vibe-tests/setup-test/setup-matrix.mjsnew file mode 100644index 0000000000..37875bc199--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-matrix.mjs@@ -0,0 +1,540 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {createHash} from 'node:crypto';++const SHA256 = /^[a-f0-9]{64}$/;+const ALLOWABLE_HOST_FIELDS = new Set([+ 'text',+ 'geometry.x',+ 'geometry.y',+ 'geometry.top',+ 'geometry.right',+ 'geometry.bottom',+ 'geometry.left',+]);+const ALLOWABLE_OVERLAY_FIELDS = new Set([+ 'geometry.x',+ 'geometry.y',+ 'geometry.top',+ 'geometry.right',+ 'geometry.bottom',+ 'geometry.left',+ 'geometry.width',+ 'geometry.height',+]);++function assertUnique(values, label) {+ if (new Set(values).size !== values.length) {+ throw new Error(`${label} must not contain duplicates`);+ }+}++function requireNonEmpty(values, label) {+ if (!Array.isArray(values) || values.length === 0) {+ throw new Error(`${label} must be a non-empty array`);+ }+}++export function taskContractText(prompt, fixtureId) {+ if (!prompt.fixtures.includes(fixtureId)) {+ throw new Error(+ `prompt ${prompt.id} does not support fixture ${fixtureId}`,+ );+ }+ const replacements = prompt.contract.replacedHostProbes.filter(+ replaced => replaced.fixture === fixtureId,+ );+ const replacementResults = new Set(+ replacements.map(replaced => replaced.result),+ );+ const resultByName = new Map(+ prompt.contract.results.map(result => [result.name, result]),+ );+ const markers = new Map();+ const marker = name => {+ const existing = markers.get(name);+ if (existing) return existing;+ const created = {description: null, surfaces: []};+ markers.set(name, created);+ return created;+ };++ for (const result of prompt.contract.results) {+ if (!replacementResults.has(result.name)) {+ marker(result.name).description = result.description;+ }+ }+ for (const interaction of prompt.contract.interactions) {+ for (const step of interaction.open) {+ if (step.source !== 'result') continue;+ const entry = marker(step.name);+ entry.description ??= `the ${step.name.replaceAll('-', ' ')}`;+ }+ for (const surface of interaction.surfaces) {+ if (surface.source !== 'result') continue;+ const name = surface.marker ?? surface.name;+ const entry = marker(name);+ if (!entry.surfaces.some(role => role.name === surface.name)) {+ entry.surfaces.push({name: surface.name, kind: surface.kind});+ }+ }+ }++ const markerLines = [...markers].map(([name, entry]) => {+ const surfaceRoles = [...entry.surfaces].sort(+ (left, right) => Number(right.name === name) - Number(left.name === name),+ );+ let description = entry.description;+ if (surfaceRoles.length === 1) {+ const role = surfaceRoles[0];+ description =+ role.kind === 'backdrop'+ ? `the ${role.name.replaceAll('-', ' ')} (the \`::backdrop\` of this element)`+ : `the ${role.name.replaceAll('-', ' ')} (${role.kind} surface)`;+ } else if (surfaceRoles.length > 1) {+ const roles = surfaceRoles.map(role =>+ role.kind === 'backdrop'+ ? `the ${role.name.replaceAll('-', ' ')} (the \`::backdrop\` of this element)`+ : `the ${role.name.replaceAll('-', ' ')} (${role.kind} surface)`,+ );+ description = `one element serving all measured roles: ${roles.join('; ')}`;+ }+ return `- \`data-vibe-result="${name}"\` on ${description}`;+ });++ const replacementLines = replacements.map(replaced => {+ const result = resultByName.get(replaced.result);+ const preserveText =+ result?.preserveTextFromHostProbe === replaced.probe+ ? ' Preserve the original probe text exactly.'+ : '';+ return `- Replace \`data-vibe-probe="${replaced.probe}"\`: remove that probe attribute from the original host treatment so it no longer exists as a host probe, and put \`data-vibe-result="${replaced.result}"\` on its replacement.${preserveText}`;+ });+ const markerSection =+ markerLines.length > 0+ ? `\nAdd exactly one of each other task-owned marker below to the element it describes:\n\n${markerLines.join('\n')}\n`+ : '';+ const replacementSection =+ replacementLines.length > 0+ ? `\nThe following host probes are intentional replacements for this fixture. Keep every other existing \`data-vibe-probe\` attribute:\n\n${replacementLines.join('\n')}\n`+ : '\nKeep every existing `data-vibe-probe` attribute on pre-existing host UI.\n';++ return `## Measurement contract+${replacementSection}${markerSection}+The markers are measurement hooks only. They do not choose the implementation.+Do not patch or replace unrelated host UI, delete or neutralize host CSS, disable+color modes, add blanket resets or \`!important\`, or avoid the requested+Astryx/Tailwind composition.`;+}++export function setupCellKey(entry) {+ return [+ entry.condition,+ entry.fixture,+ entry.prompt,+ entry.bundle,+ `r${entry.rep}`,+ ].join('__');+}++export function validatePromptContracts(prompts, probeConfig, matrixConfig) {+ assertUnique(+ prompts.map(prompt => prompt.id),+ 'prompt ids',+ );+ for (const prompt of prompts) {+ const contract = prompt.contract;+ if (!contract || !Array.isArray(contract.allowedHostChanges)) {+ throw new Error(`prompt ${prompt.id} is missing allowedHostChanges`);+ }+ if ('allowedHostProbeChanges' in contract || 'requiresAstryx' in contract) {+ throw new Error(`prompt ${prompt.id} uses a retired contract field`);+ }+ const configuredFixtures = matrixConfig.promptFixtures[prompt.id];+ if (+ !configuredFixtures ||+ JSON.stringify([...configuredFixtures].sort()) !==+ JSON.stringify([...prompt.fixtures].sort())+ ) {+ throw new Error(+ `prompt ${prompt.id} fixture applicability does not match matrix.json`,+ );+ }+ const seen = new Set();+ for (const allowed of contract.allowedHostChanges) {+ if (!prompt.fixtures.includes(allowed.fixture)) {+ throw new Error(+ `prompt ${prompt.id} allows a change in unsupported fixture ${allowed.fixture}`,+ );+ }+ const fixture = probeConfig.fixtures[allowed.fixture];+ if (!fixture?.probes.some(probe => probe.name === allowed.probe)) {+ throw new Error(+ `prompt ${prompt.id} allows unknown probe ${allowed.fixture}:${allowed.probe}`,+ );+ }+ requireNonEmpty(+ allowed.fields,+ `prompt ${prompt.id} ${allowed.fixture}:${allowed.probe} fields`,+ );+ assertUnique(+ allowed.fields,+ `prompt ${prompt.id} ${allowed.fixture}:${allowed.probe} fields`,+ );+ for (const field of allowed.fields) {+ if (!ALLOWABLE_HOST_FIELDS.has(field)) {+ throw new Error(+ `prompt ${prompt.id} cannot allow protected host field ${field}`,+ );+ }+ const key = `${allowed.fixture}:${allowed.probe}:${field}`;+ if (seen.has(key)) {+ throw new Error(+ `prompt ${prompt.id} duplicates allowed field ${key}`,+ );+ }+ seen.add(key);+ }+ }+ const resultsByName = new Map(+ (contract.results ?? []).map(result => [result.name, result]),+ );+ if (!Array.isArray(contract.replacedHostProbes)) {+ throw new Error(`prompt ${prompt.id} is missing replacedHostProbes`);+ }+ const replacedKeys = new Set();+ for (const replaced of contract.replacedHostProbes) {+ if (!prompt.fixtures.includes(replaced.fixture)) {+ throw new Error(+ `prompt ${prompt.id} replaces a probe in unsupported fixture ${replaced.fixture}`,+ );+ }+ const fixture = probeConfig.fixtures[replaced.fixture];+ if (!fixture?.probes.some(probe => probe.name === replaced.probe)) {+ throw new Error(+ `prompt ${prompt.id} replaces unknown probe ${replaced.fixture}:${replaced.probe}`,+ );+ }+ const replacementResult = resultsByName.get(replaced.result);+ if (!replacementResult) {+ throw new Error(+ `prompt ${prompt.id} replacement ${replaced.probe} references unknown result ${replaced.result}`,+ );+ }+ if (replacementResult.preserveTextFromHostProbe !== replaced.probe) {+ throw new Error(+ `prompt ${prompt.id} replacement ${replaced.probe} must preserve its host text in result ${replaced.result}`,+ );+ }+ const key = `${replaced.fixture}:${replaced.probe}`;+ if (replacedKeys.has(key)) {+ throw new Error(`prompt ${prompt.id} duplicates replacement ${key}`);+ }+ replacedKeys.add(key);+ }+ if (prompt.id === 's2') {+ for (const fixture of prompt.fixtures) {+ const replacementCount = contract.replacedHostProbes.filter(+ replaced => replaced.fixture === fixture,+ ).length;+ if (replacementCount !== 1) {+ throw new Error(+ `prompt s2 must replace exactly one host probe in ${fixture}`,+ );+ }+ }+ }++ if (!Array.isArray(contract.allowedOverlayChanges)) {+ throw new Error(`prompt ${prompt.id} is missing allowedOverlayChanges`);+ }+ const overlayKeys = new Set();+ for (const allowed of contract.allowedOverlayChanges) {+ if (!prompt.fixtures.includes(allowed.fixture)) {+ throw new Error(+ `prompt ${prompt.id} allows an overlay change in unsupported fixture ${allowed.fixture}`,+ );+ }+ const surfaces =+ probeConfig.fixtures[allowed.fixture]?.interaction?.surfaces;+ if (!surfaces?.some(surface => surface.name === allowed.surface)) {+ throw new Error(+ `prompt ${prompt.id} allows unknown overlay ${allowed.fixture}:${allowed.surface}`,+ );+ }+ requireNonEmpty(+ allowed.fields,+ `prompt ${prompt.id} ${allowed.fixture}:${allowed.surface} overlay fields`,+ );+ assertUnique(+ allowed.fields,+ `prompt ${prompt.id} ${allowed.fixture}:${allowed.surface} overlay fields`,+ );+ for (const field of allowed.fields) {+ if (!ALLOWABLE_OVERLAY_FIELDS.has(field)) {+ throw new Error(+ `prompt ${prompt.id} cannot allow protected overlay field ${field}`,+ );+ }+ const key = `${allowed.fixture}:${allowed.surface}:${field}`;+ if (overlayKeys.has(key)) {+ throw new Error(+ `prompt ${prompt.id} duplicates allowed overlay field ${key}`,+ );+ }+ overlayKeys.add(key);+ }+ }+ }+ return prompts;+}++export function validateSetupMatrixConfig(config) {+ if (config?.schemaVersion !== 1) {+ throw new Error('setup matrix schemaVersion must be 1');+ }+ requireNonEmpty(config.fixtures, 'fixtures');+ requireNonEmpty(config.bundles, 'bundles');+ requireNonEmpty(config.stages, 'stages');+ assertUnique(config.fixtures, 'fixtures');+ assertUnique(+ config.bundles.map(bundle => bundle.id),+ 'bundle ids',+ );+ assertUnique(+ config.stages.map(stage => stage.id),+ 'stage ids',+ );++ for (const bundle of config.bundles) {+ for (const field of ['id', 'harness', 'model', 'effort']) {+ if (typeof bundle[field] !== 'string' || bundle[field].length === 0) {+ throw new Error(`bundle ${field} must be a non-empty string`);+ }+ }+ if (bundle.blocked !== undefined && typeof bundle.blocked !== 'boolean') {+ throw new Error(`bundle ${bundle.id} blocked must be boolean`);+ }+ if (bundle.blocked && !bundle.blocker) {+ throw new Error(`blocked bundle ${bundle.id} must name its blocker`);+ }+ }++ if (!config.promptFixtures || typeof config.promptFixtures !== 'object') {+ throw new Error('promptFixtures must be an object');+ }+ const fixtureSet = new Set(config.fixtures);+ for (const [prompt, fixtures] of Object.entries(config.promptFixtures)) {+ requireNonEmpty(fixtures, `prompt ${prompt} fixtures`);+ assertUnique(fixtures, `prompt ${prompt} fixtures`);+ for (const fixture of fixtures) {+ if (!fixtureSet.has(fixture)) {+ throw new Error(+ `prompt ${prompt} references unknown fixture ${fixture}`,+ );+ }+ }+ }++ const bundleSet = new Set(config.bundles.map(bundle => bundle.id));+ const seenStages = new Set();+ for (const stage of config.stages) {+ for (const field of ['fixtures', 'conditions', 'prompts', 'bundles']) {+ requireNonEmpty(stage[field], `stage ${stage.id} ${field}`);+ assertUnique(stage[field], `stage ${stage.id} ${field}`);+ }+ if (!Number.isInteger(stage.reps) || stage.reps < 1) {+ throw new Error(`stage ${stage.id} reps must be a positive integer`);+ }+ for (const fixture of stage.fixtures) {+ if (!fixtureSet.has(fixture)) {+ throw new Error(+ `stage ${stage.id} references unknown fixture ${fixture}`,+ );+ }+ }+ for (const prompt of stage.prompts) {+ if (!config.promptFixtures[prompt]) {+ throw new Error(+ `stage ${stage.id} references unknown prompt ${prompt}`,+ );+ }+ }+ for (const bundle of stage.bundles) {+ if (!bundleSet.has(bundle)) {+ throw new Error(+ `stage ${stage.id} references unknown bundle ${bundle}`,+ );+ }+ }+ for (const reused of stage.reuseFromStages ?? []) {+ if (!seenStages.has(reused)) {+ throw new Error(+ `stage ${stage.id} can reuse only an earlier stage, not ${reused}`,+ );+ }+ }+ seenStages.add(stage.id);+ }++ const retryPolicy = config.infraRetryPolicy;+ if (+ !retryPolicy ||+ !Number.isInteger(retryPolicy.maxRetriesPerCell) ||+ retryPolicy.maxRetriesPerCell < 0+ ) {+ throw new Error('infraRetryPolicy.maxRetriesPerCell must be nonnegative');+ }+ requireNonEmpty(+ retryPolicy.retryableStatuses,+ 'infraRetryPolicy.retryableStatuses',+ );+ return config;+}++function select(available, requested, label) {+ if (!requested?.length) return available;+ const availableSet = new Set(available);+ for (const value of requested) {+ if (!availableSet.has(value)) {+ throw new Error(`unknown ${label}: ${value}`);+ }+ }+ return requested;+}++function rawEntries(config, stage, selections = {}) {+ const fixtures = select(stage.fixtures, selections.fixtures, 'fixture');+ const conditions = select(+ stage.conditions,+ selections.conditions,+ 'condition',+ );+ const prompts = select(stage.prompts, selections.prompts, 'prompt');+ const bundleIds = select(stage.bundles, selections.bundles, 'bundle');+ const reps = selections.reps ?? stage.reps;+ if (!Number.isInteger(reps) || reps < 1) {+ throw new Error('reps must be a positive integer');+ }+ const bundleById = new Map(config.bundles.map(bundle => [bundle.id, bundle]));+ const entries = [];++ for (const fixture of fixtures) {+ for (const condition of conditions) {+ for (const prompt of prompts) {+ if (!config.promptFixtures[prompt].includes(fixture)) continue;+ for (const bundleId of bundleIds) {+ const bundle = bundleById.get(bundleId);+ for (let rep = 1; rep <= reps; rep += 1) {+ const dimensions = {+ stage: stage.id,+ fixture,+ condition,+ prompt,+ bundle: bundleId,+ harness: bundle.harness,+ model: bundle.model,+ effort: bundle.effort,+ rep,+ blocked: bundle.blocked === true,+ blocker: bundle.blocker ?? null,+ };+ entries.push({+ id: `${stage.id}__${setupCellKey(dimensions)}`,+ ...dimensions,+ });+ }+ }+ }+ }+ }+ return entries;+}++export function expandSetupMatrix(+ config,+ {+ stage: stageId = config.stages[0]?.id,+ fixtures,+ conditions,+ prompts,+ bundles,+ reps,+ } = {},+) {+ validateSetupMatrixConfig(config);+ const stage = config.stages.find(entry => entry.id === stageId);+ if (!stage) throw new Error(`unknown stage: ${stageId}`);++ const entries = rawEntries(config, stage, {+ fixtures,+ conditions,+ prompts,+ bundles,+ reps,+ });+ const reusedKeys = new Set(+ (stage.reuseFromStages ?? []).flatMap(reusedStageId => {+ const reusedStage = config.stages.find(+ entry => entry.id === reusedStageId,+ );+ return rawEntries(config, reusedStage).map(setupCellKey);+ }),+ );+ const freshEntries = entries.filter(+ entry => !reusedKeys.has(setupCellKey(entry)),+ );+ if (freshEntries.length === 0) {+ throw new Error('selection contains no supported fixture-prompt cells');+ }+ return freshEntries;+}++export function sha256Text(value) {+ return createHash('sha256').update(value).digest('hex');+}++export function fixtureManifestSha256(recipe) {+ const files = Object.entries(recipe.manifest?.files ?? {}).sort(([a], [b]) =>+ a.localeCompare(b),+ );+ if (files.length === 0) throw new Error('fixture recipe manifest is empty');+ return sha256Text(JSON.stringify(files));+}++export function createSetupProvenance({+ entry,+ taskSha256,+ fixtureSha256,+ environmentHash,+}) {+ for (const [label, value] of [+ ['taskSha256', taskSha256],+ ['fixtureSha256', fixtureSha256],+ ['environmentHash', environmentHash],+ ]) {+ if (!SHA256.test(value)) {+ throw new Error(`${label} must be a SHA-256 digest`);+ }+ }+ return {+ schemaVersion: 1,+ task: {id: entry.prompt, sha256: taskSha256},+ fixture: {id: entry.fixture, sha256: fixtureSha256},+ condition: entry.condition,+ rep: entry.rep,+ executor: {+ harness: entry.harness,+ model: entry.model,+ effort: entry.effort,+ },+ matrix: {stage: entry.stage, bundle: entry.bundle},+ execution: {+ status: entry.blocked ? 'blocked' : 'prepared',+ attempt: 1,+ retry: 0,+ ...(entry.blocker ? {blocker: entry.blocker} : {}),+ },+ environmentHash,+ };+}diff --git a/internal/vibe-tests/setup-test/setup-matrix.test.mjs b/internal/vibe-tests/setup-test/setup-matrix.test.mjsnew file mode 100644index 0000000000..d5c4b92746--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-matrix.test.mjs@@ -0,0 +1,372 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import fs from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {describe, expect, it} from 'vitest';+import {parseExecutionProvenanceV1} from '../src/provenance';+import {+ createSetupProvenance,+ expandSetupMatrix,+ fixtureManifestSha256,+ setupCellKey,+ sha256Text,+ taskContractText,+ validatePromptContracts,+ validateSetupMatrixConfig,+} from './setup-matrix.mjs';++const HERE = path.dirname(fileURLToPath(import.meta.url));+const matrix = JSON.parse(+ fs.readFileSync(path.join(HERE, 'matrix.json'), 'utf8'),+);+const prompts = JSON.parse(+ fs.readFileSync(path.join(HERE, 'prompts.json'), 'utf8'),+).prompts;+const probes = JSON.parse(+ fs.readFileSync(path.join(HERE, 'probes.json'), 'utf8'),+);++function recipe(fixture) {+ return JSON.parse(+ fs.readFileSync(+ path.join(HERE, '..', 'fixture-recipes', `${fixture}.json`),+ 'utf8',+ ),+ );+}++describe('setup execution matrix', () => {+ it('uses three fixtures, four paired bundles, and K=2 in every stage', () => {+ expect(() => validateSetupMatrixConfig(matrix)).not.toThrow();+ expect(matrix.fixtures).toHaveLength(3);+ expect(matrix.bundles).toHaveLength(4);+ expect(matrix.bundles.map(bundle => bundle.effort)).toEqual([+ 'high',+ 'high',+ 'high',+ 'high',+ ]);+ expect(new Set(matrix.bundles.map(bundle => bundle.model))).toEqual(+ new Set(['claude-family', 'gpt-family']),+ );+ expect(matrix.stages.map(stage => stage.reps)).toEqual([2, 2, 2]);+ });++ it('keeps the Codex bundle configured and explicitly blocked', () => {+ expect(+ matrix.bundles.find(bundle => bundle.id === 'codex-gpt'),+ ).toMatchObject({harness: 'codex', blocked: true});+ const entries = expandSetupMatrix(matrix, {+ stage: 'confirmation',+ fixtures: ['tailwind-v4-control'],+ prompts: ['s0'],+ bundles: ['codex-gpt'],+ reps: 1,+ });+ expect(entries).toHaveLength(1);+ expect(entries[0]).toMatchObject({blocked: true});+ });++ it('validates every field-granular intended change against supported fixture probes', () => {+ expect(() =>+ validatePromptContracts(prompts, probes, matrix),+ ).not.toThrow();+ const allFields = prompts.flatMap(prompt =>+ prompt.contract.allowedHostChanges.flatMap(allowed => allowed.fields),+ );+ expect(allFields.every(field => field.startsWith('geometry.'))).toBe(true);+ expect(allFields).not.toContain('geometry.width');+ expect(allFields).not.toContain('color');++ const invalid = structuredClone(prompts);+ invalid+ .find(prompt => prompt.id === 's1')+ .contract.allowedHostChanges.push({+ fixture: 'tailwind-v4-control',+ probe: 'primary-action',+ fields: ['color'],+ });+ expect(() => validatePromptContracts(invalid, probes, matrix)).toThrow(+ /cannot allow protected host field color/,+ );+ });++ it('rejects an intended change for a nonexistent probe or unsupported fixture', () => {+ const missingProbe = structuredClone(prompts);+ missingProbe.find(+ prompt => prompt.id === 's1',+ ).contract.allowedHostChanges[0].probe = 'missing';+ expect(() => validatePromptContracts(missingProbe, probes, matrix)).toThrow(+ /allows unknown probe/,+ );++ const wrongFixture = structuredClone(prompts);+ wrongFixture+ .find(prompt => prompt.id === 's4')+ .contract.allowedHostChanges.push({+ fixture: 'tailwind-v4-control',+ probe: 'primary-action',+ fields: ['geometry.x'],+ });+ expect(() => validatePromptContracts(wrongFixture, probes, matrix)).toThrow(+ /unsupported fixture/,+ );+ });++ it('requires s2 to replace one validated host probe in every fixture', () => {+ const s2 = prompts.find(prompt => prompt.id === 's2');+ expect(+ new Set(+ s2.contract.replacedHostProbes.map(+ replaced =>+ `${replaced.fixture}:${replaced.probe}:${replaced.result}`,+ ),+ ),+ ).toEqual(+ new Set([+ 'tailwind-v4-control:status:astryx-status',+ 'shadcn-tailwind-v4-established:status:astryx-status',+ 'enterprise-scoped-synthetic:status:astryx-status',+ ]),+ );++ const incomplete = structuredClone(prompts);+ incomplete+ .find(prompt => prompt.id === 's2')+ .contract.replacedHostProbes.pop();+ expect(() => validatePromptContracts(incomplete, probes, matrix)).toThrow(+ /must replace exactly one host probe/,+ );++ const mismatchedText = structuredClone(prompts);+ mismatchedText+ .find(prompt => prompt.id === 's2')+ .contract.replacedHostProbes.push({+ fixture: 'tailwind-v4-control',+ probe: 'page-title',+ result: 'astryx-status',+ });+ expect(() =>+ validatePromptContracts(mismatchedText, probes, matrix),+ ).toThrow(/must preserve its host text/);++ const duplicateFixture = structuredClone(prompts);+ const duplicateS2 = duplicateFixture.find(prompt => prompt.id === 's2');+ duplicateS2.contract.results.push({+ name: 'astryx-title',+ exact: 1,+ preserveTextFromHostProbe: 'page-title',+ });+ duplicateS2.contract.replacedHostProbes.push({+ fixture: 'tailwind-v4-control',+ probe: 'page-title',+ result: 'astryx-title',+ });+ expect(() =>+ validatePromptContracts(duplicateFixture, probes, matrix),+ ).toThrow(/must replace exactly one host probe/);++ const unknownResult = structuredClone(prompts);+ unknownResult.find(+ prompt => prompt.id === 's2',+ ).contract.replacedHostProbes[0].result = 'missing-result';+ expect(() =>+ validatePromptContracts(unknownResult, probes, matrix),+ ).toThrow(/references unknown result/);+ });++ it('allows only validated overlay bounds and rejects style exemptions', () => {+ const overlayFields = prompts.flatMap(prompt =>+ prompt.contract.allowedOverlayChanges.flatMap(allowed => allowed.fields),+ );+ expect(overlayFields.length).toBeGreaterThan(0);+ expect(overlayFields.every(field => field.startsWith('geometry.'))).toBe(+ true,+ );++ const backdropStyle = structuredClone(prompts);+ backdropStyle+ .find(prompt => prompt.id === 's4')+ .contract.allowedOverlayChanges.push({+ fixture: 'shadcn-tailwind-v4-established',+ surface: 'dialog-backdrop',+ fields: ['backgroundColor'],+ });+ expect(() =>+ validatePromptContracts(backdropStyle, probes, matrix),+ ).toThrow(/cannot allow protected overlay field backgroundColor/);++ const missingSurface = structuredClone(prompts);+ missingSurface.find(+ prompt => prompt.id === 's5',+ ).contract.allowedOverlayChanges[0].surface = 'missing';+ expect(() =>+ validatePromptContracts(missingSurface, probes, matrix),+ ).toThrow(/allows unknown overlay/);+ });++ it.each([+ 'tailwind-v4-control',+ 'shadcn-tailwind-v4-established',+ 'enterprise-scoped-synthetic',+ ])('renders an explicit s2 replacement contract for %s', fixture => {+ const text = taskContractText(+ prompts.find(prompt => prompt.id === 's2'),+ fixture,+ );+ expect(text).toContain(+ 'The following host probes are intentional replacements for this fixture. Keep every other existing `data-vibe-probe` attribute:',+ );+ expect(text).toContain(+ '- Replace `data-vibe-probe="status"`: remove that probe attribute from the original host treatment so it no longer exists as a host probe, and put `data-vibe-result="astryx-status"` on its replacement. Preserve the original probe text exactly.',+ );+ expect(text).not.toContain(+ 'Keep every existing `data-vibe-probe` attribute on pre-existing host UI.',+ );+ expect(text.match(/data-vibe-result="astryx-status"/g)).toHaveLength(1);+ });++ it('renders aliased surface roles on one unambiguous marker instruction', () => {+ const text = taskContractText(+ prompts.find(prompt => prompt.id === 's5'),+ 'enterprise-scoped-synthetic',+ );+ const markerLines = text+ .split('\n')+ .filter(line => line.includes('data-vibe-result'));+ const dialogLines = markerLines.filter(line =>+ line.includes('data-vibe-result="astryx-dialog-surface"'),+ );+ expect(dialogLines).toHaveLength(1);+ expect(dialogLines[0]).toContain(+ 'the astryx dialog surface (dialog surface)',+ );+ expect(dialogLines[0]).toContain(+ 'the astryx dialog backdrop (the `::backdrop` of this element)',+ );+ expect(dialogLines[0].indexOf('dialog surface')).toBeLessThan(+ dialogLines[0].indexOf('dialog backdrop'),+ );+ expect(dialogLines[0]).not.toMatch(/on the astryx dialog backdrop/);+ });++ it('prepares 48 separation cells, then 400 new guidance cells', () => {+ const separation = expandSetupMatrix(matrix, {stage: 'separation'});+ const guidance = expandSetupMatrix(matrix, {stage: 'guidance'});+ expect(separation).toHaveLength(48);+ expect(guidance).toHaveLength(400);++ const reused = new Set(separation.map(setupCellKey));+ expect(guidance.some(entry => reused.has(setupCellKey(entry)))).toBe(false);+ expect(+ new Set([...separation, ...guidance].map(setupCellKey)),+ ).toHaveLength(448);+ });++ it('requires 112 fresh candidate confirmation cells', () => {+ const confirmation = expandSetupMatrix(matrix, {stage: 'confirmation'});+ expect(confirmation).toHaveLength(112);+ expect(new Set(confirmation.map(entry => entry.fixture))).toEqual(+ new Set(matrix.fixtures),+ );+ expect(new Set(confirmation.map(entry => entry.prompt))).toEqual(+ new Set(['s0', 's1', 's2', 's3', 's4', 's5']),+ );+ expect(new Set(confirmation.map(entry => entry.condition))).toEqual(+ new Set(['candidate']),+ );+ expect(+ confirmation.every(entry => entry.id.startsWith('confirmation__')),+ ).toBe(true);+ });++ it('applies each cross-system prompt only to its supported established fixture', () => {+ const confirmation = expandSetupMatrix(matrix, {+ stage: 'confirmation',+ prompts: ['s4', 's5'],+ reps: 1,+ });+ expect(+ new Set(confirmation.map(entry => `${entry.prompt}:${entry.fixture}`)),+ ).toEqual(+ new Set([+ 's4:shadcn-tailwind-v4-established',+ 's5:enterprise-scoped-synthetic',+ ]),+ );+ expect(confirmation).toHaveLength(2 * 4);+ });++ it('keeps harness/model bundles paired instead of taking their cross-product', () => {+ const entries = expandSetupMatrix(matrix, {stage: 'separation'});+ expect(new Set(entries.map(entry => entry.bundle))).toEqual(+ new Set([+ 'native-claude',+ 'claude-code-claude',+ 'native-gpt',+ 'codex-gpt',+ ]),+ );+ });++ it('emits a sidecar accepted by the shared provenance parser', () => {+ const entry = expandSetupMatrix(matrix, {+ stage: 'separation',+ fixtures: [matrix.fixtures[0]],+ conditions: ['floor'],+ prompts: ['s1'],+ bundles: [matrix.bundles[0].id],+ reps: 1,+ })[0];+ const fixtureSha256 = fixtureManifestSha256(recipe(entry.fixture));+ const sidecar = createSetupProvenance({+ entry,+ taskSha256: sha256Text('stable task'),+ fixtureSha256,+ environmentHash: sha256Text(`${fixtureSha256}:${entry.condition}`),+ });+ expect(parseExecutionProvenanceV1(sidecar)).toMatchObject({+ task: {id: 's1'},+ fixture: {id: entry.fixture},+ condition: 'floor',+ executor: {harness: entry.harness, model: entry.model},+ execution: {status: 'prepared'},+ matrix: {stage: 'separation', bundle: entry.bundle},+ rep: 1,+ });+ });++ it('rejects invalid or empty selections', () => {+ expect(() =>+ expandSetupMatrix(matrix, {stage: 'separation', reps: 0}),+ ).toThrow(/positive integer/);+ expect(() =>+ expandSetupMatrix(matrix, {+ stage: 'separation',+ fixtures: ['missing-fixture'],+ }),+ ).toThrow(/unknown fixture/);+ expect(() =>+ expandSetupMatrix(matrix, {+ stage: 'confirmation',+ fixtures: ['tailwind-v4-control'],+ prompts: ['s4'],+ }),+ ).toThrow(/no supported fixture-prompt cells/);+ });++ it('rejects duplicate bundle labels and invalid reuse references', () => {+ const duplicate = structuredClone(matrix);+ duplicate.bundles[1].id = duplicate.bundles[0].id;+ expect(() => validateSetupMatrixConfig(duplicate)).toThrow(+ /bundle ids must not contain duplicates/,+ );++ const forwardReuse = structuredClone(matrix);+ forwardReuse.stages[0].reuseFromStages = ['confirmation'];+ expect(() => validateSetupMatrixConfig(forwardReuse)).toThrow(+ /only an earlier stage/,+ );+ });+});diff --git a/internal/vibe-tests/setup-test/setup-measure.mjs b/internal/vibe-tests/setup-test/setup-measure.mjsnew file mode 100644index 0000000000..b64e3150bc--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-measure.mjs@@ -0,0 +1,721 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/** Build, serve, and measure one copied canonical fixture in two color schemes. */++import {spawnSync} from 'node:child_process';+import * as fs from 'node:fs';+import * as http from 'node:http';+import * as path from 'node:path';+import {fileURLToPath} from 'node:url';+import {analyzeSetupIntegrity} from './setup-integrity.mjs';+import {openInteractionState} from './setup-interactions.mjs';+import {validatePromptContracts} from './setup-matrix.mjs';+import {+ assertPublicArtifactSafe,+ publicProvenance,+ sanitizePublicArtifact,+} from '../src/public-artifact.mjs';++const HERE = path.dirname(fileURLToPath(import.meta.url));+const REPO_ROOT = path.resolve(HERE, '../../..');++function parseArgs(argv) {+ const parsed = {};+ for (let index = 0; index < argv.length; index += 1) {+ const argument = argv[index];+ if (!argument.startsWith('--')) continue;+ const key = argument.slice(2);+ const next = argv[index + 1];+ if (next && !next.startsWith('--')) {+ parsed[key] = next;+ index += 1;+ } else {+ parsed[key] = true;+ }+ }+ return parsed;+}++export const normalizeSubpixel = value => Math.round(value * 64) / 64;++export function parseLayerOrder(css) {+ const order = [];+ for (const match of css.matchAll(/@layer\s+([^;{]+)\s*[;{]/g)) {+ for (const name of match[1].split(',').map(part => part.trim())) {+ if (/^[a-zA-Z0-9_-]+$/.test(name) && !order.includes(name)) {+ order.push(name);+ }+ }+ }+ return order;+}++function build(appDir) {+ const started = Date.now();+ const result = spawnSync('pnpm', ['build'], {cwd: appDir, encoding: 'utf8'});+ return {+ ok: result.status === 0,+ status: result.status ?? -1,+ ms: Date.now() - started,+ stdout: (result.stdout ?? '').slice(-4000),+ stderr: (result.stderr ?? '').slice(-4000),+ };+}++export function classifyCssAssets(files) {+ if (files.length === 0) return ['missing-css-asset'];+ if (files.length > 1) return [`multiple-css-assets:${files.join(',')}`];+ return [];+}++export function measurementPrivateValues({+ appDir,+ outFile,+ environment = process.env,+ cwd = process.cwd(),+ repoRoot = REPO_ROOT,+}) {+ return [+ appDir,+ repoRoot,+ cwd,+ path.dirname(outFile),+ environment.HOME,+ environment.USER,+ environment.HOSTNAME,+ ].filter(value => typeof value === 'string');+}++export function publicMeasurement(+ measurement,+ {fixtureId, privateValues = []},+) {+ const sanitized = sanitizePublicArtifact(+ {...measurement, app: fixtureId},+ {privateValues},+ );+ assertPublicArtifactSafe(sanitized, {+ label: 'measurement',+ privateValues,+ });+ return sanitized;+}++function emittedLayerOrder(distDir) {+ const assets = path.join(distDir, 'assets');+ if (!fs.existsSync(assets)) {+ return {order: [], errors: ['missing-css-assets-directory']};+ }+ const cssFiles = fs+ .readdirSync(assets)+ .filter(name => name.endsWith('.css'))+ .sort();+ const errors = classifyCssAssets(cssFiles);+ if (errors.length > 0) return {order: [], errors};+ const order = [];+ for (const file of cssFiles) {+ const css = fs.readFileSync(path.join(assets, file), 'utf8');+ for (const layer of parseLayerOrder(css)) {+ if (!order.includes(layer)) order.push(layer);+ }+ }+ return {order, errors: []};+}++const MIME = {+ '.html': 'text/html',+ '.js': 'text/javascript',+ '.mjs': 'text/javascript',+ '.css': 'text/css',+ '.json': 'application/json',+ '.svg': 'image/svg+xml',+ '.woff2': 'font/woff2',+};++function serve(distDir) {+ const server = http.createServer((request, response) => {+ const url = new URL(request.url ?? '/', 'http://localhost');+ const relative = url.pathname === '/' ? '/index.html' : url.pathname;+ const file = path.join(distDir, relative);+ if (+ !file.startsWith(distDir) ||+ !fs.existsSync(file) ||+ fs.statSync(file).isDirectory()+ ) {+ response.statusCode = 404;+ response.end();+ return;+ }+ response.setHeader(+ 'content-type',+ MIME[path.extname(file)] ?? 'application/octet-stream',+ );+ fs.createReadStream(file).pipe(response);+ });+ return new Promise(resolve =>+ server.listen(0, () => resolve({server, port: server.address().port})),+ );+}++/* c8 ignore start -- executed in the page rather than in Node. */+function readPage(spec) {+ const normalizedRect = rect => {+ const normalize = value => Math.round(value * 64) / 64;+ return {+ x: normalize(rect.x),+ y: normalize(rect.y),+ top: normalize(rect.top),+ right: normalize(rect.right),+ bottom: normalize(rect.bottom),+ left: normalize(rect.left),+ width: normalize(rect.width),+ height: normalize(rect.height),+ };+ };+ const markerSelector = marker =>+ `[data-vibe-${marker.source === 'result' ? 'result' : 'probe'}="${marker.marker ?? marker.name}"]`;+ const styleRecord = (computed, properties) =>+ Object.fromEntries(+ properties.map(property => [property, computed[property]]),+ );+ const luminance = rgb => {+ const [red, green, blue] = rgb.map(value => {+ const channel = value / 255;+ return channel <= 0.03928+ ? channel / 12.92+ : ((channel + 0.055) / 1.055) ** 2.4;+ });+ return 0.2126 * red + 0.7152 * green + 0.0722 * blue;+ };+ const parseColor = color => {+ const match = color.match(/rgba?\(([^)]+)\)/);+ if (!match) return null;+ const parts = match[1]+ .split(/[\s,/]+/)+ .filter(Boolean)+ .map(Number);+ return {rgb: parts.slice(0, 3), alpha: parts.length > 3 ? parts[3] : 1};+ };+ const effectiveBackground = element => {+ let current = element;+ while (current) {+ const color = parseColor(getComputedStyle(current).backgroundColor);+ if (color && color.alpha > 0.5) return color.rgb;+ current = current.parentElement;+ }+ return [255, 255, 255];+ };+ const contrast = (foreground, background) => {+ const light = luminance(foreground) + 0.05;+ const dark = luminance(background) + 0.05;+ return (+ Math.round((Math.max(light, dark) / Math.min(light, dark)) * 100) / 100+ );+ };+ const protectedText = element => {+ const parts = [];+ const visit = node => {+ for (const child of node.childNodes) {+ if (child.nodeType === Node.TEXT_NODE) {+ parts.push(child.textContent ?? '');+ continue;+ }+ if (!(child instanceof Element)) continue;+ if (+ child.matches(+ '[data-vibe-probe],[data-vibe-result],[data-vibe-replacement]',+ )+ ) {+ continue;+ }+ visit(child);+ }+ };+ visit(element);+ return parts.join(' ').trim().replace(/\s+/g, ' ');+ };+ const focusableSelector =+ 'button:not([disabled]),a[href],input:not([disabled]),select:not([disabled]),textarea:not([disabled]),[tabindex]:not([tabindex="-1"]):not([disabled])';+ const isVisible = (computed, rect) =>+ computed.display !== 'none' &&+ computed.visibility !== 'hidden' &&+ Number(computed.opacity) > 0 &&+ rect.width > 0 &&+ rect.height > 0;++ const readLayerSurface = surface => {+ const element = document.querySelector(markerSelector(surface));+ if (!(element instanceof HTMLElement)) {+ return {+ kind: surface.kind,+ missing: true,+ ...(surface.styleReference+ ? {styleReference: surface.styleReference}+ : {}),+ };+ }+ let topLayerElement = element;+ try {+ let candidate = element;+ while (candidate) {+ if (candidate.matches(':modal') || candidate.matches(':popover-open')) {+ topLayerElement = candidate;+ break;+ }+ candidate = candidate.parentElement;+ }+ } catch {}+ let inTopLayer = false;+ try {+ inTopLayer =+ topLayerElement.matches(':modal') ||+ topLayerElement.matches(':popover-open');+ } catch {}+ const readsDialogBackdrop =+ surface.kind === 'backdrop' &&+ topLayerElement instanceof HTMLDialogElement &&+ inTopLayer;+ const computed = readsDialogBackdrop+ ? getComputedStyle(topLayerElement, '::backdrop')+ : getComputedStyle(element);+ const rect = readsDialogBackdrop+ ? {+ x: 0,+ y: 0,+ top: 0,+ right: innerWidth,+ bottom: innerHeight,+ left: 0,+ width: innerWidth,+ height: innerHeight,+ }+ : element.getBoundingClientRect();+ const center = {+ x: rect.left + rect.width / 2,+ y: rect.top + rect.height / 2,+ };+ const centerHit =+ center.x >= 0 &&+ center.x <= innerWidth &&+ center.y >= 0 &&+ center.y <= innerHeight+ ? document.elementFromPoint(center.x, center.y)+ : null;+ let clippingAncestor = null;+ let ancestor =+ readsDialogBackdrop || (inTopLayer && topLayerElement === element)+ ? null+ : element.parentElement;+ while (ancestor && ancestor !== document.documentElement) {+ const ancestorStyle = getComputedStyle(ancestor);+ if (+ /hidden|clip/.test(+ `${ancestorStyle.overflowX} ${ancestorStyle.overflowY}`,+ )+ ) {+ const ancestorRect = ancestor.getBoundingClientRect();+ if (+ rect.left < ancestorRect.left ||+ rect.right > ancestorRect.right ||+ rect.top < ancestorRect.top ||+ rect.bottom > ancestorRect.bottom+ ) {+ clippingAncestor =+ ancestor.getAttribute('data-vibe-probe') ||+ ancestor.getAttribute('data-vibe-result') ||+ ancestor.tagName.toLowerCase();+ break;+ }+ }+ ancestor = ancestor.parentElement;+ }+ return {+ kind: surface.kind,+ ...(surface.styleReference+ ? {styleReference: surface.styleReference}+ : {}),+ visible: isVisible(computed, rect),+ display: computed.display,+ visibility: computed.visibility,+ opacity: computed.opacity,+ position: computed.position,+ zIndex: computed.zIndex,+ style: styleRecord(computed, spec.surfaceProperties),+ bounds: normalizedRect(rect),+ intersectsViewport:+ rect.right > 0 &&+ rect.bottom > 0 &&+ rect.left < innerWidth &&+ rect.top < innerHeight,+ clippingAncestor,+ centerHitSelf:+ centerHit === element ||+ (centerHit instanceof Node && element.contains(centerHit)),+ centerHitProbe:+ centerHit instanceof Element+ ? (centerHit+ .closest('[data-vibe-probe],[data-vibe-result]')+ ?.getAttribute('data-vibe-probe') ??+ centerHit+ .closest('[data-vibe-result]')+ ?.getAttribute('data-vibe-result') ??+ null)+ : null,+ topLayer: {+ tagName: topLayerElement.tagName.toLowerCase(),+ role: topLayerElement.getAttribute('role'),+ open: topLayerElement.hasAttribute('open'),+ popover: topLayerElement.getAttribute('popover'),+ ariaModal: topLayerElement.getAttribute('aria-modal'),+ inTopLayer,+ portalChild: topLayerElement.parentElement === document.body,+ },+ };+ };++ const probes = {};+ for (const probe of spec.probes) {+ const element = document.querySelector(probe.selector);+ if (!(element instanceof HTMLElement)) {+ probes[probe.name] = {missing: true};+ continue;+ }+ const computed = getComputedStyle(element);+ const foreground = parseColor(computed.color);+ probes[probe.name] = {+ style: styleRecord(computed, spec.properties),+ geometry: normalizedRect(element.getBoundingClientRect()),+ text: protectedText(element),+ descendantText: (element.textContent ?? '')+ .trim()+ .replace(/\s+/g, ' ')+ .slice(0, 200),+ contrast: foreground+ ? contrast(foreground.rgb, effectiveBackground(element))+ : null,+ };+ }++ const taskResults = {};+ for (const result of spec.results ?? []) {+ const elements = document.querySelectorAll(+ `[data-vibe-result="${result.name}"]`,+ );+ const element = elements[0];+ if (!(element instanceof HTMLElement)) {+ taskResults[result.name] = {+ count: elements.length,+ visible: false,+ focusable: false,+ text: '',+ style: {},+ geometry: null,+ };+ continue;+ }+ const computed = getComputedStyle(element);+ const rect = element.getBoundingClientRect();+ taskResults[result.name] = {+ count: elements.length,+ visible: isVisible(computed, rect),+ focusable:+ (element.matches(focusableSelector) ||+ element.querySelector(focusableSelector) != null) &&+ !element.hasAttribute('disabled') &&+ element.getAttribute('aria-disabled') !== 'true',+ text: (element.textContent ?? '')+ .trim()+ .replace(/\s+/g, ' ')+ .slice(0, 200),+ style: styleRecord(computed, spec.properties),+ geometry: normalizedRect(rect),+ };+ }++ const root = getComputedStyle(document.documentElement);+ const variables = {};+ for (const variable of spec.rootVariables) {+ variables[variable] = root.getPropertyValue(variable).trim();+ }+ const interaction = spec.interaction+ ? {+ id: spec.interaction.id,+ direction: spec.interaction.direction ?? 'host-baseline',+ opened: true,+ keyboardReached: {},+ error: null,+ surfaces: Object.fromEntries(+ spec.interaction.surfaces.map(surface => [+ surface.name,+ readLayerSurface(surface),+ ]),+ ),+ }+ : undefined;+ return {+ probes,+ variables,+ colorScheme: root.colorScheme,+ taskResults,+ ...(interaction ? {interaction} : {}),+ };+}+/* c8 ignore stop */++async function readScheme({+ browser,+ port,+ colorScheme,+ probeSpec,+ interaction,+}) {+ const page = await browser.newPage({+ viewport: {width: 1280, height: 720},+ colorScheme,+ });+ const consoleErrors = [];+ const pageErrors = [];+ const failedRequests = [];+ page.on(+ 'console',+ message => message.type() === 'error' && consoleErrors.push(message.text()),+ );+ page.on('pageerror', error => pageErrors.push(String(error)));+ page.on('requestfailed', request => failedRequests.push(request.url()));+ await page.goto(`http://127.0.0.1:${port}/`);+ await page.waitForLoadState('networkidle');+ await page.evaluate(() => document.fonts.ready);+ let state = {opened: false, keyboardReached: {}};+ let interactionError = null;+ if (interaction) {+ try {+ state = await openInteractionState(page, interaction);+ } catch (error) {+ interactionError = String(error);+ }+ }+ const reading = await page.evaluate(readPage, {...probeSpec, interaction});+ if (reading.interaction) {+ reading.interaction.opened = state.opened && interactionError === null;+ reading.interaction.keyboardReached = state.keyboardReached;+ reading.interaction.error = interactionError;+ }+ await page.close();+ return {reading, consoleErrors, pageErrors, failedRequests};+}++async function main() {+ const args = parseArgs(process.argv.slice(2));+ if (typeof args.app !== 'string' || typeof args.fixture !== 'string') {+ throw new Error(+ 'usage: setup-measure.mjs --app <dir> --fixture <id> --out <file>',+ );+ }+ const appDir = path.resolve(args.app);+ const fixtureId = args.fixture;+ const outFile = path.resolve(+ typeof args.out === 'string'+ ? args.out+ : path.join(HERE, 'results', 'measurement.json'),+ );+ const screenshotDir =+ typeof args['screenshot-dir'] === 'string'+ ? path.resolve(args['screenshot-dir'])+ : null;+ const label =+ typeof args.label === 'string'+ ? args.label+ : path.basename(outFile, '.json');++ const probeFile = JSON.parse(+ fs.readFileSync(path.join(HERE, 'probes.json'), 'utf8'),+ );+ const fixtureProbes = probeFile.fixtures[fixtureId];+ if (!fixtureProbes) throw new Error(`unknown fixture probes: ${fixtureId}`);++ const provenance =+ typeof args.provenance === 'string'+ ? JSON.parse(fs.readFileSync(args.provenance, 'utf8'))+ : null;+ if (+ provenance &&+ (provenance.schemaVersion !== 1 || provenance.fixture?.id !== fixtureId)+ ) {+ throw new Error('provenance sidecar does not match the measured fixture');+ }+ const prompts = JSON.parse(+ fs.readFileSync(path.join(HERE, 'prompts.json'), 'utf8'),+ ).prompts;+ const matrixConfig = JSON.parse(+ fs.readFileSync(path.join(HERE, 'matrix.json'), 'utf8'),+ );+ validatePromptContracts(prompts, probeFile, matrixConfig);+ const taskDefinition = provenance?.task?.id+ ? prompts.find(prompt => prompt.id === provenance.task.id)+ : null;+ if (provenance?.task?.id && !taskDefinition) {+ throw new Error(`unknown task contract: ${provenance.task.id}`);+ }+ if (taskDefinition && !taskDefinition.fixtures.includes(fixtureId)) {+ throw new Error(+ `task ${taskDefinition.id} does not support fixture ${fixtureId}`,+ );+ }++ const surfaceProperties = probeFile.properties.filter(+ property =>+ !property.startsWith('margin') &&+ ![+ 'position',+ 'top',+ 'right',+ 'bottom',+ 'left',+ 'transform',+ 'width',+ 'height',+ 'minWidth',+ 'minHeight',+ 'maxWidth',+ 'maxHeight',+ ].includes(property),+ );+ const probeSpec = {+ properties: probeFile.properties,+ surfaceProperties,+ probes: fixtureProbes.probes,+ rootVariables: fixtureProbes.rootVariables,+ interaction: fixtureProbes.interaction ?? null,+ results: taskDefinition?.contract.results ?? [],+ };++ let integrity;+ if (taskDefinition) {+ const analyzed = analyzeSetupIntegrity(+ appDir,+ provenance?.execution?.agentDiffSha256,+ );+ integrity = {+ diffSha256: analyzed.diffSha256,+ attestedDiffSha256: analyzed.attestation.expectedSha256,+ diffMatchesAttestation: analyzed.attestation.matches,+ usesAstryx: analyzed.astryxUsage.found,+ changedFiles: analyzed.changedFiles.map(file => file.path),+ escapeHatches: analyzed.escapeHatches.map(finding => finding.kind),+ };+ }++ const measurement = {+ label,+ fixture: fixtureId,+ app: appDir,+ measuredAt: new Date().toISOString(),+ build: build(appDir),+ layerOrder: [],+ measurementErrors: [],+ ...(taskDefinition+ ? {+ task: {+ id: taskDefinition.id,+ kind: taskDefinition.kind,+ contract: taskDefinition.contract,+ },+ executionStatus: provenance?.execution?.status ?? 'missing',+ integrity,+ }+ : {}),+ schemes: {},+ };++ if (measurement.build.ok) {+ const distDir = path.join(appDir, 'dist');+ const layers = emittedLayerOrder(distDir);+ measurement.layerOrder = layers.order;+ measurement.measurementErrors = layers.errors;+ const {chromium} = await import('playwright');+ const {server, port} = await serve(distDir);+ const browser = await chromium.launch();++ for (const colorScheme of ['light', 'dark']) {+ const mainReading = await readScheme({+ browser,+ port,+ colorScheme,+ probeSpec,+ interaction: probeSpec.interaction,+ });+ const taskInteractions = {};+ for (const interaction of taskDefinition?.contract.interactions ?? []) {+ const taskReading = await readScheme({+ browser,+ port,+ colorScheme,+ probeSpec: {...probeSpec, probes: [], rootVariables: [], results: []},+ interaction,+ });+ taskInteractions[interaction.id] = taskReading.reading.interaction;+ mainReading.consoleErrors.push(...taskReading.consoleErrors);+ mainReading.pageErrors.push(...taskReading.pageErrors);+ mainReading.failedRequests.push(...taskReading.failedRequests);+ }+ if (screenshotDir) {+ fs.mkdirSync(screenshotDir, {recursive: true});+ const page = await browser.newPage({+ viewport: {width: 1280, height: 720},+ colorScheme,+ });+ await page.goto(`http://127.0.0.1:${port}/`);+ await page.screenshot({+ path: path.join(screenshotDir, `${label}-${colorScheme}.png`),+ fullPage: true,+ });+ await page.close();+ }+ measurement.schemes[colorScheme] = {+ ...mainReading.reading,+ taskInteractions,+ consoleErrors: mainReading.consoleErrors,+ pageErrors: mainReading.pageErrors,+ failedRequests: mainReading.failedRequests,+ };+ }++ await browser.close();+ server.close();+ }++ const privateValues = measurementPrivateValues({appDir, outFile});+ const exportedMeasurement = publicMeasurement(measurement, {+ fixtureId,+ privateValues,+ });+ fs.mkdirSync(path.dirname(outFile), {recursive: true});+ fs.writeFileSync(+ outFile,+ `${JSON.stringify(exportedMeasurement, null, 2)}\n`,+ );++ if (provenance) {+ const sidecarOut = outFile.replace(/\.json$/, '.provenance.json');+ const exportedProvenance = publicProvenance(provenance, {privateValues});+ fs.writeFileSync(+ sidecarOut,+ `${JSON.stringify(exportedProvenance, null, 2)}\n`,+ );+ }++ console.log(+ `${label}: build ${measurement.build.ok ? 'ok' : `FAILED (${measurement.build.status})`}` ++ (measurement.build.ok+ ? `, ${measurement.schemes.light.consoleErrors.length} console errors` ++ `, ${Object.values(measurement.schemes.light.probes).filter(probe => probe.missing).length} probes missing`+ : ''),+ );+}++const isMain =+ process.argv[1] != null &&+ path.resolve(process.argv[1]) ===+ path.resolve(fileURLToPath(import.meta.url));+if (isMain) await main();diff --git a/internal/vibe-tests/setup-test/setup-measure.test.mjs b/internal/vibe-tests/setup-test/setup-measure.test.mjsnew file mode 100644index 0000000000..01f091e691--- /dev/null+++ b/internal/vibe-tests/setup-test/setup-measure.test.mjs@@ -0,0 +1,201 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {describe, expect, it} from 'vitest';+import {+ classifyCssAssets,+ measurementPrivateValues,+ normalizeSubpixel,+ parseLayerOrder,+ publicMeasurement,+} from './setup-measure.mjs';+import {+ assertPublicArtifactSafe,+ publicProvenance,+} from '../src/public-artifact.mjs';++describe('setup browser measurement helpers', () => {+ it('normalizes Chromium geometry to its 1/64 CSS pixel grid without tolerance', () => {+ expect(normalizeSubpixel(10.015625)).toBe(10.015625);+ expect(normalizeSubpixel(10.01)).toBe(10.015625);+ expect(normalizeSubpixel(10.02)).toBe(10.015625);+ expect(normalizeSubpixel(10.03125)).toBe(10.03125);+ });++ it('classifies missing and multiple emitted stylesheets as measurement errors', () => {+ expect(classifyCssAssets([])).toEqual(['missing-css-asset']);+ expect(classifyCssAssets(['a.css', 'b.css'])).toEqual([+ 'multiple-css-assets:a.css,b.css',+ ]);+ expect(classifyCssAssets(['app.css'])).toEqual([]);+ });++ it('reads both ordering statements and block declarations from emitted CSS', () => {+ expect(+ parseLayerOrder(`+ @layer reset, theme, base, astryx-base, astryx-theme, components, utilities;+ @layer components { .button { display: flex } }+ `),+ ).toEqual([+ 'reset',+ 'theme',+ 'base',+ 'astryx-base',+ 'astryx-theme',+ 'components',+ 'utilities',+ ]);+ });++ it('preserves first registration order instead of letting repeats reorder layers', () => {+ expect(+ parseLayerOrder(+ '@layer utilities {} @layer reset, utilities; @layer theme {}',+ ),+ ).toEqual(['utilities', 'reset', 'theme']);+ });++ it('emits a stable fixture id and redacts private paths and hosts', () => {+ const privateRoot = '/home/example/checkouts/secret project';+ const exported = publicMeasurement(+ {+ app: privateRoot,+ build: {+ stdout: `built ${privateRoot}/dist/app.js`,+ stderr: [+ 'fetch failed at https://builder.internal.example/private/token runner-reported rebuild',+ 'email mailto:[email protected]',+ 'mirror ftp://private-host/files/package',+ 'checkout ssh://private-host/repository',+ 'fetch git://private-host/repository',+ 'mount smb://private-host/share',+ 'load file://private-host/share/file.json',+ 'spaced file file:///Users/alice/Secret Project/src/App.tsx runner-reported rebuild',+ 'quoted "file:///Users/alice/Secret Project/src/App.tsx?mode=test#result" after',+ 'punctuation (file:///home/example/secret.log?mode=test#result), after',+ 'probe http://10.20.30.40:8080/status',+ 'path error:/home/other/private project/src/App.tsx',+ 'dash path error:/-secret/file',+ String.raw`windows error:C:\Users\person\private project\App.tsx`,+ String.raw`unc error:\\build-host\share\artifact`,+ 'host=::1 host=fd00::1',+ ].join('\n'),+ },+ schemes: {+ light: {+ consoleErrors: [`file://${privateRoot}/src/App.tsx`],+ },+ },+ extra: {+ [`${privateRoot}/key`]: 'value',+ 'worker9257.region.facebook.com': 'host-key',+ 'service.internal': 'private-key',+ },+ },+ {+ fixtureId: 'tailwind-v4-control',+ privateValues: [privateRoot, 'example'],+ },+ );+ const serialized = JSON.stringify(exported);+ expect(exported.app).toBe('tailwind-v4-control');+ expect(serialized).not.toContain(privateRoot);+ expect(serialized).not.toContain('secret project');+ expect(serialized).not.toContain('example');+ expect(serialized).not.toContain('10.20.30.40');+ expect(serialized).not.toContain('::1');+ expect(serialized).not.toContain('fd00::1');+ expect(serialized).not.toContain('/-secret/file');+ expect(serialized).not.toContain('service.internal');+ expect(serialized).not.toContain('builder.internal.example');+ expect(exported.build.stderr).toContain(+ 'spaced file <external-uri> runner-reported rebuild',+ );+ expect(exported.build.stderr).toContain('quoted "<external-uri>" after');+ expect(exported.build.stderr).toContain(+ 'punctuation (<external-uri>), after',+ );+ expect(serialized).toContain('runner-reported rebuild');+ expect(serialized).not.toMatch(/(?:https?|ftp|ssh|git|smb|file|mailto):/);+ expect(serialized).not.toContain('/home/other');+ expect(serialized).not.toContain('C:\\\\Users');+ expect(serialized).not.toContain('person');+ expect(serialized).not.toContain('build-host');+ expect(serialized).not.toContain('worker9257');+ expect(serialized).not.toMatch(/file:\/\/\/[a-z]/i);+ expect(serialized).not.toMatch(/\binternal\b/i);+ expect(() => assertPublicArtifactSafe(exported)).not.toThrow();+ });++ it('exports a generic provenance usage source without private paths', () => {+ const privateRoot = '/Users/example/work/setup-fixture';+ const exported = publicProvenance(+ {+ schemaVersion: 1,+ fixture: {id: 'tailwind-v4-control'},+ usage: {+ complete: true,+ inputTokens: 10,+ outputTokens: 5,+ source: `${privateRoot}/usage.json`,+ },+ },+ {privateValues: [privateRoot, 'example']},+ );+ expect(exported.usage.source).toBe('runner-reported');+ expect(JSON.stringify(exported)).not.toContain(privateRoot);+ expect(() => assertPublicArtifactSafe(exported)).not.toThrow();+ });++ it.each(['runner', 'build', 'missing', undefined])(+ 'does not corrupt normal words when USER is %s',+ user => {+ const privateValues = measurementPrivateValues({+ appDir: '/tmp/setup-fixture',+ outFile: '/tmp/results/measurement.json',+ cwd: '/workspace/project',+ repoRoot: '/workspace/project',+ environment: {+ ...(user === undefined ? {} : {USER: user}),+ HOSTNAME: user,+ },+ });+ const exported = publicMeasurement(+ {+ message:+ 'runner-reported rebuild buildable <private-host> <private-path> <private-value>',+ },+ {fixtureId: 'fixture', privateValues},+ );+ expect(exported.message).toBe(+ 'runner-reported rebuild buildable <private-host> <private-path> <private-value>',+ );+ expect(() => assertPublicArtifactSafe(exported)).not.toThrow();+ },+ );++ it('preserves evaluator diagnostics and CSS selectors verbatim', () => {+ const labels = [+ 'light:status:not-replaced',+ 'multiple-css-assets:app.css,theme.css',+ 'overlay:dialog-surface',+ 'a:hover',+ ];+ const exported = publicMeasurement(+ {labels},+ {+ fixtureId: 'fixture',+ privateValues: ['runner', 'build', 'missing'],+ },+ );+ expect(exported.labels).toEqual(labels);+ });++ it('redacts a specific bare username atomically without changing larger words', () => {+ const exported = publicMeasurement(+ {owner: 'owner-42', message: 'owner-420 can rebuild'},+ {fixtureId: 'fixture', privateValues: ['owner-42']},+ );+ expect(exported.owner).toBe('<private-value>');+ expect(exported.message).toBe('owner-420 can rebuild');+ });+});diff --git a/internal/vibe-tests/src/fixture-suite.mjs b/internal/vibe-tests/src/fixture-suite.mjsindex a5181f2fdc..1bd584dcdf 100644--- a/internal/vibe-tests/src/fixture-suite.mjs+++ b/internal/vibe-tests/src/fixture-suite.mjs@@ -172,6 +172,7 @@ function validateFixtureShape(fixtureId, fixtureRoot) {'primary-action','table-header','table-cell',+ 'status','form-control',],'shadcn-tailwind-v4-established': [diff --git a/internal/vibe-tests/src/provenance-aggregation.test.ts b/internal/vibe-tests/src/provenance-aggregation.test.tsindex d8cbd24414..81fa1283d4 100644--- a/internal/vibe-tests/src/provenance-aggregation.test.ts+++ b/internal/vibe-tests/src/provenance-aggregation.test.ts@@ -88,7 +88,7 @@ describe('provenance aggregation', () => {usage: {inputTokens: 7,outputTokens: 3,- source: 'runner',+ source: '/home/example/private-usage.json',complete: true,},};@@ -108,7 +108,7 @@ describe('provenance aggregation', () => {).toEqual({inputTokens: 7,outputTokens: 3,- source: 'runner',+ source: 'runner-reported',quality: 'complete',complete: true,});diff --git a/internal/vibe-tests/src/provenance-aggregation.ts b/internal/vibe-tests/src/provenance-aggregation.tsindex cd4f776175..fa74730966 100644--- a/internal/vibe-tests/src/provenance-aggregation.ts+++ b/internal/vibe-tests/src/provenance-aggregation.ts@@ -21,6 +21,8 @@ import {parseExecutionProvenanceV1,type ExecutionProvenanceV1,} from './provenance.js';+// @ts-expect-error -- public-artifact.mjs intentionally has no declaration output.+import {publicSourceLabel} from './public-artifact.mjs';export function loadOptionalExecutionProvenance(filePath: string,@@ -120,7 +122,7 @@ export function resolveUsage(options: {return {inputTokens: usage.inputTokens ?? null,outputTokens: usage.outputTokens ?? null,- source: usage.source ?? 'provenance',+ source: publicSourceLabel(usage.source),quality: usage.complete ? 'complete' : 'incomplete',complete: usage.complete,};diff --git a/internal/vibe-tests/src/public-artifact.mjs b/internal/vibe-tests/src/public-artifact.mjsnew file mode 100644index 0000000000..5727b57886--- /dev/null+++ b/internal/vibe-tests/src/public-artifact.mjs@@ -0,0 +1,190 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++const QUOTED_URI = /(["'`])(?:[a-z][a-z0-9+.-]*:\/\/|mailto:)[^\r\n]*?\1/giu;+const FILE_URI = /\bfile:\/\/[^\s"'`)<]+(?:\s+(?=[^\s"'`]*[\\/])[^\s"'`]*)*/giu;+const NETWORK_URI = /\b[a-z][a-z0-9+.-]*:\/\/[^\s"'`)<]+/giu;+const MAILTO_URI = /\bmailto:[^\s"'`)<]+/giu;+const UNIX_ABSOLUTE_PATH =+ /(^|[^A-Za-z0-9_])\/(?!\/)[-A-Za-z0-9._~][^\r\n"'`]*/gmu;+const WINDOWS_ABSOLUTE_PATH = /(^|[^A-Za-z0-9_])[A-Za-z]:[\\/][^\r\n"'`]*/gmu;+const UNC_PATH = /(^|[^A-Za-z0-9_])\\\\[^\\\r\n"'`]+\\[^\r\n"'`]*/gmu;+const NON_PUBLIC_HOST =+ /\b(?:localhost|(?:[a-z0-9-]+\.)+(?:internal|local|lan|corp)|[a-z0-9.-]*internal[a-z0-9.-]*\.[a-z]{2,})\b/giu;+const CORPORATE_HOST = /\b(?!www\.)(?:[a-z0-9-]+\.)+facebook\.com\b/giu;+const PRIVATE_ADDRESS =+ /\b(?:127(?:\.\d{1,3}){3}|10(?:\.\d{1,3}){3}|192\.168(?:\.\d{1,3}){2}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|169\.254(?:\.\d{1,3}){2})\b/giu;+const PRIVATE_IPV6 =+ /(^|[^A-Za-z0-9_])(?:\[?::1\]?|\[?f[cd][0-9a-f]{2}:[0-9a-f:]+\]?|\[?fe[89ab][0-9a-f]:[0-9a-f:]+\]?)/gimu;+const BARE_HOST_WITH_PORT = /\b[a-z][a-z0-9-]{1,62}:\d{2,5}\b/giu;+const GENERIC_IDENTITIES = new Set([+ 'build',+ 'ci',+ 'missing',+ 'root',+ 'runner',+ 'unknown',+ 'user',+]);++function escaped(value) {+ return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+}++function usablePrivateValues(privateValues) {+ return [...privateValues]+ .filter(+ value =>+ typeof value === 'string' &&+ value.length > 2 &&+ !GENERIC_IDENTITIES.has(value.toLowerCase()),+ )+ .sort((left, right) => right.length - left.length);+}++function replacePrivateValue(value, privateValue) {+ const pathLike = /[\\/]/.test(privateValue);+ const pattern = pathLike+ ? escaped(privateValue)+ : `(?<![A-Za-z0-9_])${escaped(privateValue)}(?![A-Za-z0-9_])`;+ return value.replace(new RegExp(pattern, 'gu'), '<private-value>');+}++function redactUriToken(token) {+ const punctuation = token.match(/[),.;!?]+$/u)?.[0] ?? '';+ return `<external-uri>${punctuation}`;+}++function redactQuotedUri(token) {+ const quote = token[0];+ return `${quote}<external-uri>${quote}`;+}++function sanitizeString(value, privateValues) {+ let sanitized = value+ .replace(QUOTED_URI, redactQuotedUri)+ .replace(FILE_URI, redactUriToken)+ .replace(NETWORK_URI, redactUriToken)+ .replace(MAILTO_URI, redactUriToken);+ for (const privateValue of usablePrivateValues(privateValues)) {+ sanitized = replacePrivateValue(sanitized, privateValue);+ }+ return sanitized+ .replace(UNC_PATH, '$1<private-path>')+ .replace(UNIX_ABSOLUTE_PATH, '$1<private-path>')+ .replace(WINDOWS_ABSOLUTE_PATH, '$1<private-path>')+ .replace(PRIVATE_ADDRESS, '<private-host>')+ .replace(PRIVATE_IPV6, '$1<private-host>')+ .replace(BARE_HOST_WITH_PORT, '<private-host>')+ .replace(NON_PUBLIC_HOST, '<private-host>')+ .replace(CORPORATE_HOST, '<private-host>');+}++export function sanitizePublicArtifact(value, {privateValues = []} = {}) {+ if (typeof value === 'string') {+ return sanitizeString(value, privateValues);+ }+ if (Array.isArray(value)) {+ return value.map(item => sanitizePublicArtifact(item, {privateValues}));+ }+ if (value && typeof value === 'object') {+ return Object.fromEntries(+ Object.entries(value).map(([key, item]) => [+ sanitizeString(key, privateValues),+ sanitizePublicArtifact(item, {privateValues}),+ ]),+ );+ }+ return value;+}++function matches(pattern, value) {+ pattern.lastIndex = 0;+ return pattern.test(value);+}++function containsPrivateValue(value, privateValue) {+ if (/[\\/]/.test(privateValue)) {+ return value.includes(privateValue);+ }+ return matches(+ new RegExp(+ `(?<![A-Za-z0-9_])${escaped(privateValue)}(?![A-Za-z0-9_])`,+ 'gu',+ ),+ value,+ );+}++function stringViolation(value, privateValues) {+ if (+ matches(QUOTED_URI, value) ||+ matches(FILE_URI, value) ||+ matches(NETWORK_URI, value) ||+ matches(MAILTO_URI, value) ||+ matches(UNC_PATH, value) ||+ matches(UNIX_ABSOLUTE_PATH, value) ||+ matches(WINDOWS_ABSOLUTE_PATH, value) ||+ matches(PRIVATE_ADDRESS, value) ||+ matches(PRIVATE_IPV6, value) ||+ matches(BARE_HOST_WITH_PORT, value) ||+ matches(NON_PUBLIC_HOST, value) ||+ matches(CORPORATE_HOST, value)+ ) {+ return 'private-path-or-host';+ }+ if (+ usablePrivateValues(privateValues).some(privateValue =>+ containsPrivateValue(value, privateValue),+ )+ ) {+ return 'private-value';+ }+ return null;+}++export function publicArtifactViolation(value, {privateValues = []} = {}) {+ if (typeof value === 'string') {+ return stringViolation(value, privateValues);+ }+ if (Array.isArray(value)) {+ for (const item of value) {+ const violation = publicArtifactViolation(item, {privateValues});+ if (violation) return violation;+ }+ return null;+ }+ if (value && typeof value === 'object') {+ for (const [key, item] of Object.entries(value)) {+ const keyViolation = stringViolation(key, privateValues);+ if (keyViolation) return keyViolation;+ const valueViolation = publicArtifactViolation(item, {privateValues});+ if (valueViolation) return valueViolation;+ }+ }+ return null;+}++export function assertPublicArtifactSafe(+ value,+ {label = 'public artifact', privateValues = []} = {},+) {+ if (publicArtifactViolation(value, {privateValues})) {+ throw new Error(`${label} contains private path or host data`);+ }+}++export function publicSourceLabel(source) {+ return source == null ? 'provenance' : 'runner-reported';+}++export function publicProvenance(provenance, options = {}) {+ const sanitized = sanitizePublicArtifact(provenance, options);+ if (sanitized?.usage?.source !== undefined) {+ sanitized.usage.source = publicSourceLabel(sanitized.usage.source);+ }+ assertPublicArtifactSafe(sanitized, {+ ...options,+ label: 'exported provenance',+ });+ return sanitized;+}diff --git a/internal/vibe-tests/src/universal-aggregate.ts b/internal/vibe-tests/src/universal-aggregate.tsindex 4bdd43331f..74efcee647 100644--- a/internal/vibe-tests/src/universal-aggregate.ts+++ b/internal/vibe-tests/src/universal-aggregate.ts@@ -24,6 +24,11 @@ import type {import {getResultsDir, writeJson, ensureTsxFiles} from './utils.js';import {evaluate, getDimensionNames} from './universal-eval.js';import {provenanceFilename} from './provenance.js';+// @ts-expect-error -- public-artifact.mjs intentionally has no declaration output.+import {+ assertPublicArtifactSafe,+ sanitizePublicArtifact,+} from './public-artifact.mjs';import {buildExecutionBreakdown,loadOptionalExecutionProvenance,@@ -41,6 +46,19 @@ const DIMENSION_LABELS: Partial<Record<UniversalDimension, string>> = {design: 'Design',};+function runtimePrivateValues(): string[] {+ return [+ process.cwd(),+ process.env.HOME,+ process.env.USER,+ process.env.HOSTNAME,+ ].filter((value): value is string => typeof value === 'string');+}++function publicMessage(value: unknown, privateValues = runtimePrivateValues()) {+ return sanitizePublicArtifact(String(value), {privateValues}) as string;+}+function parseArgs(): {iteration: string;json: boolean;@@ -84,14 +102,15 @@ async function main() {const iterDir = path.join(resultsDir, iteration);const codeDir = path.join(iterDir, 'results');const manifestPath = path.join(iterDir, 'manifest.json');+ const privateValues = [iterDir, resultsDir, ...runtimePrivateValues()];if (!fs.existsSync(manifestPath)) {- console.error(`No manifest.json found at ${manifestPath}`);+ console.error('No manifest.json found for the requested iteration');process.exit(1);}if (!fs.existsSync(codeDir)) {- console.error(`No results directory found at ${codeDir}`);+ console.error('No results directory found for the requested iteration');process.exit(1);}@@ -296,7 +315,9 @@ async function main() {}}} catch (err) {- console.error(`Warning: Failed to load design scores: ${err}`);+ console.error(+ `Warning: Failed to load design scores: ${publicMessage(err, privateValues)}`,+ );}}@@ -378,7 +399,7 @@ async function main() {(durationSources[entry.durationSource] ?? 0) + 1;}- const aggregate: UniversalAggregate = {+ let aggregate: UniversalAggregate = {averages,overall,byPrompt,@@ -403,8 +424,24 @@ async function main() {execution: buildExecutionBreakdown(runs),};+ aggregate = sanitizePublicArtifact(aggregate, {+ privateValues,+ }) as UniversalAggregate;+ assertPublicArtifactSafe(aggregate, {+ label: 'universal aggregate report',+ privateValues,+ });+// Saveconst outputPath = path.join(iterDir, 'universal.json');+ const reportContext = sanitizePublicArtifact(+ {iteration, target, output: path.basename(outputPath)},+ {privateValues},+ ) as {iteration: string; target: string; output: string};+ assertPublicArtifactSafe(reportContext, {+ label: 'universal aggregate stdout',+ privateValues,+ });writeJson(outputPath, aggregate);if (json) {@@ -413,8 +450,10 @@ async function main() {}// Print formatted table- console.log(`\n📊 Universal Evaluation — Iteration ${iteration}`);- console.log(` ${promptCount} prompts, target: ${target}\n`);+ console.log(+ `\n📊 Universal Evaluation — Iteration ${reportContext.iteration}`,+ );+ console.log(` ${promptCount} prompts, target: ${reportContext.target}\n`);console.log('┌─────────────────────┬───────┐');console.log('│ Dimension │ Score │');@@ -522,22 +561,24 @@ async function main() {}// Category breakdown- const categories = Object.keys(byCategory).sort();+ const categories = Object.keys(aggregate.byCategory).sort();if (categories.length > 1) {console.log('\n📂 By Category:');for (const cat of categories) {const catOverall = Math.round(- dimensions.reduce((s, d) => s + byCategory[cat][d], 0) /- dimensions.length,+ dimensions.reduce(+ (sum, dimension) => sum + aggregate.byCategory[cat][dimension],+ 0,+ ) / dimensions.length,);console.log(` ${cat.padEnd(25)} ${catOverall}`);}}- console.log(`\nSaved: ${outputPath}\n`);+ console.log(`\nSaved: ${reportContext.output}\n`);}main().catch(err => {- console.error(err);+ console.error(publicMessage(err));process.exit(1);});