Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

feat(review): add content-based visual signal to the design review gate (#3748)

Jiun Shin5a722f68d592
diff --git a/.github/scripts/classify-visual-pr.js b/.github/scripts/classify-visual-pr.js
new file mode 100644
index 0000000000..b900502fdd
--- /dev/null
+++ b/.github/scripts/classify-visual-pr.js
@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+
+/**
+ * @description Local calibration/debug CLI for the static visual-diff
+ * classifier (#3667). CI consumes the classifier through review-signal.yml;
+ * this runs the same scoring against any local ref range so a threshold or
+ * signal change can be validated against real diffs before shipping
+ * (e.g. `gh pr checkout <n> && node .github/scripts/classify-visual-pr.js`).
+ * @input --base <ref> --head <ref> --output <file>
+ * @output JSON file with the classification ({ score, bucket, signals, … })
+ */
+
+const fs = require('node:fs');
+const { execSync } = require('node:child_process');
+const { classifyVisualDiff } = require('./lib/classify-visual');
+
+const args = process.argv.slice(2);
+const getArg = (name) => {
+ const idx = args.indexOf(`--${name}`);
+ return idx !== -1 ? args[idx + 1] : null;
+};
+
+const baseRef = getArg('base') || 'origin/main';
+const headRef = getArg('head') || 'HEAD';
+const outputFile = getArg('output') || 'visual-classification.json';
+
+// A missing merge base means the clone is too shallow — the three-dot diff
+// would be empty and every PR would classify as a confident "non-visual".
+// Fail loudly instead of reporting a bogus result.
+try {
+ execSync(`git merge-base ${baseRef} ${headRef}`, { encoding: 'utf8', stdio: 'pipe' });
+} catch {
+ console.error(
+ `No merge base between ${baseRef} and ${headRef} — the checkout is too shallow to diff. ` +
+ 'Fetch full history (fetch-depth: 0) before running the classifier.'
+ );
+ process.exit(1);
+}
+
+const diff = execSync(
+ `git diff ${baseRef}...${headRef}`,
+ { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }
+);
+
+const result = classifyVisualDiff(diff);
+console.log(`Visual classification: ${result.bucket} (score ${result.score})`);
+
+fs.writeFileSync(outputFile, JSON.stringify(result, null, 2));
+console.log(`Classification written to ${outputFile}`);
diff --git a/.github/scripts/lib/classify-visual.js b/.github/scripts/lib/classify-visual.js
new file mode 100644
index 0000000000..d25769c1b8
--- /dev/null
+++ b/.github/scripts/lib/classify-visual.js
@@ -0,0 +1,338 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+
+/**
+ * @description Static diff classifier that scores how likely a PR changes rendered
+ * appearance (issue #3667). Pure, dependency-free function over unified diff
+ * text — no git access, no rendering. Consumed by review-signal.yml as the
+ * content-based design signal (loaded from the BASE ref, run against the PR's
+ * API diff — complements the path-based signals, which cannot see an inline
+ * `stylex.create` edit in a component .tsx). Also runnable locally via
+ * classify-visual-pr.js for calibration.
+ * @input Unified diff text (git diff <base>...<head>)
+ * @output classifyVisualDiff: { score, bucket, signals, styleSurfacePaths, evidence }
+ *
+ * SYNC: review-signal.yml loads this file with new Function(module, exports,
+ * require) — keep it dependency-free CJS with a plain module.exports object.
+ *
+ * Signals and weights:
+ * 1. CSS-property edit (×3) — added/removed line starting with a known CSS
+ * property + `:`. In this codebase such lines essentially only occur inside
+ * `stylex.create({...})` blocks, so one means a pixel moved.
+ * 2. Design-token reference (×2) — `colorVars.accent`, `radiusVars.lg`, … —
+ * matched generically as `<something>Vars.<key>` so every token group
+ * (shadowVars, borderVars, durationVars, …) counts without a hardcoded list.
+ * 3. Style-surface file (×3, per file) — any non-comment edit to `*.stylex.ts(x)`,
+ * `*.css`, or `packages/themes/<name>/src/**`. `*.markers.stylex.ts` files are
+ * excluded: they define stylex.when scoping markers, not appearance, so marker
+ * renames/refactors move no pixels (their content still counts via signals 1–2).
+ * 4. JSX structural change (×1, capped at 10) — add/remove of a rendered element
+ * in a non-test `packages/**` .tsx file. Noisy (aria/VisuallyHidden wrappers),
+ * hence the low weight and cap.
+ *
+ * Deliberately NOT a signal: "touches a component .tsx". Many PRs edit component
+ * files but change only logic/state/a11y — the classifier looks for
+ * appearance-affecting content, not file paths.
+ *
+ * Scope: only product surfaces (packages/, apps/) are scanned. Tooling —
+ * .github/, scripts/, root configs — can mention CSS-ish content (mocks,
+ * fixtures, CSS generators) without moving a pixel.
+ */
+
+// CSS properties (camelCase, as written in stylex.create) that mark a line as a
+// style edit. Kebab-case properties in .css files are normalized before lookup.
+const CSS_PROPERTIES = new Set([
+ // Layout & box model
+ 'display', 'position', 'top', 'right', 'bottom', 'left', 'inset',
+ 'insetBlock', 'insetBlockStart', 'insetBlockEnd',
+ 'insetInline', 'insetInlineStart', 'insetInlineEnd',
+ 'width', 'height', 'minWidth', 'minHeight', 'maxWidth', 'maxHeight',
+ 'blockSize', 'inlineSize', 'minBlockSize', 'minInlineSize',
+ 'maxBlockSize', 'maxInlineSize',
+ 'margin', 'marginTop', 'marginRight', 'marginBottom', 'marginLeft',
+ 'marginBlock', 'marginBlockStart', 'marginBlockEnd',
+ 'marginInline', 'marginInlineStart', 'marginInlineEnd',
+ 'padding', 'paddingTop', 'paddingRight', 'paddingBottom', 'paddingLeft',
+ 'paddingBlock', 'paddingBlockStart', 'paddingBlockEnd',
+ 'paddingInline', 'paddingInlineStart', 'paddingInlineEnd',
+ 'boxSizing', 'aspectRatio', 'zIndex', 'overflow', 'overflowX', 'overflowY',
+ 'overflowWrap', 'overscrollBehavior', 'visibility', 'contain', 'containerType',
+ 'containerName', 'float', 'clear', 'objectFit', 'objectPosition',
+ // Flex & grid
+ 'flex', 'flexDirection', 'flexWrap', 'flexFlow', 'flexGrow', 'flexShrink',
+ 'flexBasis', 'justifyContent', 'justifyItems', 'justifySelf',
+ 'alignContent', 'alignItems', 'alignSelf', 'placeContent', 'placeItems',
+ 'placeSelf', 'gap', 'rowGap', 'columnGap', 'order',
+ 'grid', 'gridArea', 'gridAutoColumns', 'gridAutoFlow', 'gridAutoRows',
+ 'gridColumn', 'gridColumnStart', 'gridColumnEnd',
+ 'gridRow', 'gridRowStart', 'gridRowEnd',
+ 'gridTemplate', 'gridTemplateAreas', 'gridTemplateColumns', 'gridTemplateRows',
+ // Color & background
+ 'color', 'background', 'backgroundColor', 'backgroundImage',
+ 'backgroundPosition', 'backgroundRepeat', 'backgroundSize', 'backgroundClip',
+ 'backgroundAttachment', 'backgroundBlendMode', 'accentColor', 'caretColor',
+ 'colorScheme', 'mixBlendMode', 'backdropFilter', 'filter',
+ // Border & outline
+ 'border', 'borderTop', 'borderRight', 'borderBottom', 'borderLeft',
+ 'borderWidth', 'borderTopWidth', 'borderRightWidth', 'borderBottomWidth',
+ 'borderLeftWidth', 'borderStyle', 'borderColor', 'borderTopColor',
+ 'borderRightColor', 'borderBottomColor', 'borderLeftColor',
+ 'borderBlock', 'borderBlockStart', 'borderBlockEnd',
+ 'borderInline', 'borderInlineStart', 'borderInlineEnd',
+ 'borderInlineStartWidth', 'borderInlineEndWidth',
+ 'borderRadius', 'borderTopLeftRadius', 'borderTopRightRadius',
+ 'borderBottomLeftRadius', 'borderBottomRightRadius',
+ 'borderStartStartRadius', 'borderStartEndRadius',
+ 'borderEndStartRadius', 'borderEndEndRadius',
+ 'borderCollapse', 'borderSpacing', 'outline', 'outlineWidth', 'outlineStyle',
+ 'outlineColor', 'outlineOffset', 'boxShadow',
+ // Typography
+ 'font', 'fontFamily', 'fontSize', 'fontStyle', 'fontWeight', 'fontVariant',
+ 'fontVariantNumeric', 'fontFeatureSettings', 'fontStretch',
+ 'lineHeight', 'letterSpacing', 'textAlign', 'textDecoration',
+ 'textDecorationLine', 'textDecorationColor', 'textDecorationStyle',
+ 'textDecorationThickness', 'textUnderlineOffset', 'textTransform',
+ 'textOverflow', 'textWrap', 'textShadow', 'textIndent', 'verticalAlign',
+ 'whiteSpace', 'wordBreak', 'wordSpacing', 'hyphens', 'lineClamp',
+ 'webkitLineClamp', 'webkitBoxOrient', 'tabSize', 'userSelect',
+ // Transform, transition & animation
+ 'transform', 'transformOrigin', 'transformStyle', 'translate', 'rotate',
+ 'scale', 'perspective', 'perspectiveOrigin', 'transition',
+ 'transitionProperty', 'transitionDuration', 'transitionTimingFunction',
+ 'transitionDelay', 'transitionBehavior', 'animation', 'animationName',
+ 'animationDuration', 'animationTimingFunction', 'animationDelay',
+ 'animationIterationCount', 'animationDirection', 'animationFillMode',
+ 'animationPlayState', 'willChange',
+ // Misc visual
+ 'opacity', 'cursor', 'pointerEvents', 'resize', 'appearance', 'content',
+ 'listStyle', 'listStyleType', 'listStylePosition', 'clipPath', 'mask',
+ 'scrollbarWidth', 'scrollbarGutter', 'scrollBehavior', 'scrollMargin',
+ 'scrollPadding', 'scrollSnapType', 'scrollSnapAlign', 'touchAction',
+ 'strokeWidth', 'stroke', 'fill', 'viewTransitionName',
+]);
+
+// Values that mark a `prop: value` line as a TypeScript type member rather than
+// a style declaration (e.g. `width: number;` in a Props interface).
+const TYPE_ANNOTATION_VALUE = /^(?:string|number|boolean|void|never|unknown|any)(?:\[\])?[;,]?\s*(?:\/\/.*)?$/;
+
+// `<something>Vars.<key>` or `<something>Vars['--key']` — matches every token
+// group exported from packages/core/src/theme (colorVars, spacingVars,
+// shadowVars, durationVars, …) in both dot and bracket access.
+const TOKEN_REFERENCE = /\b[a-zA-Z][a-zA-Z0-9]*Vars\s*(?:\.[a-zA-Z$_]|\[)/;
+
+// Opening or closing tag of a rendered element at line start. Fragments (`<>`,
+// `</>`) render nothing and are excluded.
+const JSX_TAG = /^<\/?[a-zA-Z][\w.]*(?:[\s/>]|$)/;
+
+const WEIGHTS = {cssProperties: 3, tokenReferences: 2, styleSurfaceFiles: 3, jsxStructuralChanges: 1};
+const JSX_CAP = 10;
+const MAX_EVIDENCE_PER_SIGNAL = 5;
+
+// Only product surfaces can change rendered appearance.
+function isProductFile(file) {
+ return /^(?:packages|apps)\//.test(file);
+}
+
+// Files whose diff content is never scanned (tests, docs, snapshots, generated).
+// `.test[.-]` covers colocated tests and test fixtures alike (Button.test.tsx,
+// Badge.test-violations.tsx — the ESLint-plugin fixture full of intentional
+// style violations).
+function isIgnoredFile(file) {
+ return (
+ /\.test[.-]/.test(file) ||
+ /\.stories\.[jt]sx?$/.test(file) ||
+ /\.doc\.mjs$/.test(file) ||
+ /\.md$/.test(file) ||
+ /\.snap$/.test(file) ||
+ /(^|\/)__snapshots__\//.test(file) ||
+ /(^|\/)__tests__\//.test(file)
+ );
+}
+
+// Only these file types get line-level scanning — keeps YAML/JSON keys like
+// `width:` in configs from matching the CSS-property signal.
+function isScannableFile(file) {
+ return /\.(?:tsx?|jsx?|mjs|css)$/.test(file);
+}
+
+function isStyleSurfaceFile(file) {
+ if (/\.markers\.stylex\.tsx?$/.test(file)) return false;
+ return (
+ /\.stylex\.tsx?$/.test(file) ||
+ /\.css$/.test(file) ||
+ /^packages\/themes\/[^/]+\/src\/.+\.tsx?$/.test(file)
+ );
+}
+
+// kebab-case → camelCase for properties in plain .css files
+function toCamelCase(prop) {
+ return prop.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
+}
+
+function isCssPropertyLine(content, file) {
+ if (/\.css$/.test(file)) {
+ // Custom property definitions (`--brand-accent: …`) are visual by definition.
+ if (/^--[\w-]+\s*:/.test(content)) return true;
+ const match = content.match(/^([a-z][a-z-]*)\s*:/);
+ return match !== null && CSS_PROPERTIES.has(toCamelCase(match[1]));
+ }
+ const match = content.match(/^([a-zA-Z][a-zA-Z0-9]*)\s*:\s*(.*)$/);
+ if (!match || !CSS_PROPERTIES.has(match[1])) return false;
+ // `width: number;` in a Props type is not a style edit.
+ return !TYPE_ANNOTATION_VALUE.test(match[2]);
+}
+
+function isCommentLine(content) {
+ return (
+ content.startsWith('//') ||
+ content.startsWith('/*') ||
+ content.startsWith('*')
+ );
+}
+
+// Advance the block-comment state across one line. Returns [wasInside,
+// nowInside]: whether the line began inside a `/* … */` comment, and the state
+// after consuming it. Best-effort — does not understand string literals.
+function advanceBlockComment(content, inside) {
+ const wasInside = inside;
+ // A line comment consumes the rest of the line — `/*` after `//` is inert.
+ if (!inside && content.startsWith('//')) return [false, false];
+ let i = 0;
+ while (i < content.length) {
+ if (inside) {
+ const close = content.indexOf('*/', i);
+ if (close === -1) return [wasInside, true];
+ inside = false;
+ i = close + 2;
+ } else {
+ const open = content.indexOf('/*', i);
+ if (open === -1) return [wasInside, false];
+ inside = true;
+ i = open + 2;
+ }
+ }
+ return [wasInside, inside];
+}
+
+/**
+ * Classify a unified diff as visual / likely-visual / maybe-visual / non-visual.
+ *
+ * @param {string} diffText - output of `git diff <base>...<head>`
+ * @param {{ignorePath?: (file: string) => boolean}} [opts] - `ignorePath`
+ * excludes additional paths beyond the built-in rules (e.g. review-signal
+ * passes its safe-space predicate so lab/sandbox/storybook never count)
+ * @returns {{
+ * score: number,
+ * bucket: 'visual'|'likely-visual'|'maybe-visual'|'non-visual',
+ * signals: {cssProperties: number, tokenReferences: number, styleSurfaceFiles: number, jsxStructuralChanges: number},
+ * styleSurfacePaths: string[],
+ * evidence: {cssProperties: Array<{file: string, line: string}>, tokenReferences: Array<{file: string, line: string}>, jsxStructuralChanges: Array<{file: string, line: string}>},
+ * }}
+ */
+function classifyVisualDiff(diffText, opts = {}) {
+ const ignorePath = typeof opts.ignorePath === 'function' ? opts.ignorePath : null;
+ const signals = {
+ cssProperties: 0,
+ tokenReferences: 0,
+ styleSurfaceFiles: 0,
+ jsxStructuralChanges: 0,
+ };
+ const evidence = {
+ cssProperties: [],
+ tokenReferences: [],
+ jsxStructuralChanges: [],
+ };
+ const styleSurfacePaths = [];
+ const touchedStyleSurfaces = new Set();
+
+ let currentFile = null;
+ // Block-comment state per diff side: a `-` line opening a comment must not
+ // silence subsequent `+` lines. Context lines advance both sides. State
+ // resets at file and hunk boundaries — a hunk can start mid-comment we never
+ // saw, which the tracker accepts rather than guessing (cf. the cross-hunk
+ // brace-tracking failure noted in #3667).
+ const inComment = {added: false, removed: false};
+
+ const addEvidence = (signal, content) => {
+ if (evidence[signal].length < MAX_EVIDENCE_PER_SIGNAL) {
+ evidence[signal].push({file: currentFile, line: content.slice(0, 120)});
+ }
+ };
+
+ for (const line of (diffText || '').split('\n')) {
+ // File boundary — `diff --git a/<old> b/<new>`; the b-side names the file
+ // for adds/renames, and still identifies deleted style files.
+ const fileMatch = line.match(/^diff --git a\/.* b\/(.*)$/);
+ if (fileMatch) {
+ currentFile = fileMatch[1];
+ inComment.added = false;
+ inComment.removed = false;
+ continue;
+ }
+ if (currentFile === null || !isProductFile(currentFile)) continue;
+ if (ignorePath && ignorePath(currentFile)) continue;
+ if (isIgnoredFile(currentFile) || !isScannableFile(currentFile)) continue;
+
+ if (line.startsWith('@@')) {
+ inComment.added = false;
+ inComment.removed = false;
+ continue;
+ }
+ if (/^\+\+\+|^---/.test(line)) continue;
+
+ // Context lines advance the comment state on both sides.
+ if (!/^[+-]/.test(line)) {
+ const context = line.slice(1).trim();
+ [, inComment.added] = advanceBlockComment(context, inComment.added);
+ [, inComment.removed] = advanceBlockComment(context, inComment.removed);
+ continue;
+ }
+
+ const side = line[0] === '+' ? 'added' : 'removed';
+ const content = line.slice(1).trim();
+ const [wasInsideComment, nowInsideComment] = advanceBlockComment(content, inComment[side]);
+ inComment[side] = nowInsideComment;
+ if (content === '' || wasInsideComment || isCommentLine(content)) continue;
+
+ if (isStyleSurfaceFile(currentFile) && !touchedStyleSurfaces.has(currentFile)) {
+ touchedStyleSurfaces.add(currentFile);
+ styleSurfacePaths.push(currentFile);
+ signals.styleSurfaceFiles++;
+ }
+
+ if (isCssPropertyLine(content, currentFile)) {
+ signals.cssProperties++;
+ addEvidence('cssProperties', content);
+ }
+
+ if (TOKEN_REFERENCE.test(content)) {
+ signals.tokenReferences++;
+ addEvidence('tokenReferences', content);
+ }
+
+ if (
+ /^packages\//.test(currentFile) &&
+ /\.tsx$/.test(currentFile) &&
+ JSX_TAG.test(content)
+ ) {
+ signals.jsxStructuralChanges++;
+ addEvidence('jsxStructuralChanges', content);
+ }
+ }
+
+ const score =
+ signals.cssProperties * WEIGHTS.cssProperties +
+ signals.tokenReferences * WEIGHTS.tokenReferences +
+ signals.styleSurfaceFiles * WEIGHTS.styleSurfaceFiles +
+ Math.min(signals.jsxStructuralChanges, JSX_CAP) * WEIGHTS.jsxStructuralChanges;
+
+ let bucket;
+ if (score >= 12) bucket = 'visual';
+ else if (score >= 5) bucket = 'likely-visual';
+ else if (score >= 2) bucket = 'maybe-visual';
+ else bucket = 'non-visual';
+
+ return {score, bucket, signals, styleSurfacePaths, evidence};
+}
+
+module.exports = {classifyVisualDiff, WEIGHTS, JSX_CAP};
diff --git a/.github/scripts/lib/classify-visual.test.mjs b/.github/scripts/lib/classify-visual.test.mjs
new file mode 100644
index 0000000000..c3a652051a
--- /dev/null
+++ b/.github/scripts/lib/classify-visual.test.mjs
@@ -0,0 +1,296 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file classify-visual.test.mjs
+ * Tests for the static visual-diff classifier (#3667): signal detection,
+ * file filtering, scoring buckets, and the review-signal loading contract.
+ */
+
+import {readFileSync} from 'node:fs';
+import {fileURLToPath} from 'node:url';
+import {describe, it, expect} from 'vitest';
+import {classifyVisualDiff, JSX_CAP} from './classify-visual.js';
+
+// Build a minimal unified diff for a single file. Lines are raw content;
+// they get prefixed with +/- here.
+function makeDiff(file, added = [], removed = []) {
+ return [
+ `diff --git a/${file} b/${file}`,
+ 'index 1111111..2222222 100644',
+ `--- a/${file}`,
+ `+++ b/${file}`,
+ '@@ -1,9 +1,9 @@',
+ ...removed.map(l => `-${l}`),
+ ...added.map(l => `+${l}`),
+ ].join('\n');
+}
+
+describe('classifyVisualDiff', () => {
+ it('returns non-visual for an empty diff', () => {
+ const result = classifyVisualDiff('');
+ expect(result.score).toBe(0);
+ expect(result.bucket).toBe('non-visual');
+ });
+
+ it('counts CSS property edits inside a component file', () => {
+ const diff = makeDiff('packages/core/src/Button/Button.tsx', [
+ 'paddingBlock: spacingVars.sm,',
+ "minHeight: '2.5rem',",
+ ]);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.cssProperties).toBe(2);
+ // 2×3 (props) + 1×2 (spacingVars token) = 8
+ expect(result.score).toBe(8);
+ expect(result.bucket).toBe('likely-visual');
+ });
+
+ it('counts removed lines as well as added lines', () => {
+ const diff = makeDiff('packages/core/src/Card/Card.tsx', [], [
+ 'boxShadow: shadowVars.raised,',
+ ]);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.cssProperties).toBe(1);
+ expect(result.signals.tokenReferences).toBe(1);
+ });
+
+ it('does not count TypeScript type members as CSS properties', () => {
+ const diff = makeDiff('packages/core/src/Button/Button.tsx', [
+ 'width: number;',
+ 'height: string;',
+ 'color: string,',
+ ]);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.cssProperties).toBe(0);
+ });
+
+ it('matches every token group via the generic *Vars. pattern', () => {
+ const diff = makeDiff('packages/core/src/Card/Card.tsx', [
+ 'boxShadow: shadowVars.overlay,',
+ 'borderColor: borderVars.subtle,',
+ 'transitionDuration: durationVars.fast,',
+ "fontSize: textSizeVars['--font-size-sm'],",
+ ]);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.tokenReferences).toBe(4);
+ });
+
+ it('flags style-surface files by path', () => {
+ const diff = [
+ makeDiff('packages/core/src/Field/inputStyles.stylex.ts', ['const x = 1;']),
+ makeDiff('apps/docsite/src/app/globals.css', ['--brand-accent: #f00;']),
+ makeDiff('packages/themes/matcha/src/theme.ts', ['const y = 2;']),
+ ].join('\n');
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.styleSurfaceFiles).toBe(3);
+ expect(result.styleSurfacePaths).toContain('packages/core/src/Field/inputStyles.stylex.ts');
+ });
+
+ it('does not flag marker stylex files as style surfaces', () => {
+ const diff = makeDiff('packages/core/src/CheckboxInput/checkbox.markers.stylex.ts', [
+ 'export const hoverMarker = stylex.defineMarker();',
+ ]);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.styleSurfaceFiles).toBe(0);
+ });
+
+ it('counts kebab-case properties and custom properties in .css files', () => {
+ const diff = makeDiff('apps/docsite/src/app/globals.css', [
+ 'background-color: #fff;',
+ '--sidebar-width: 240px;',
+ ]);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.cssProperties).toBe(2);
+ expect(result.signals.styleSurfaceFiles).toBe(1);
+ });
+
+ it('still flags deleted style-surface files', () => {
+ const diff = [
+ 'diff --git a/packages/core/src/Layout/padding.stylex.ts b/packages/core/src/Layout/padding.stylex.ts',
+ 'deleted file mode 100644',
+ '--- a/packages/core/src/Layout/padding.stylex.ts',
+ '+++ /dev/null',
+ '@@ -1,2 +0,0 @@',
+ '-export const padding = stylex.create({',
+ '- root: {padding: spacingVars.md},',
+ ].join('\n');
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.styleSurfaceFiles).toBe(1);
+ expect(result.signals.tokenReferences).toBe(1);
+ });
+
+ it('ignores test, story, doc, and markdown files entirely', () => {
+ const styleLine = 'padding: spacingVars.md,';
+ const diff = [
+ makeDiff('packages/core/src/Button/Button.test.tsx', [styleLine]),
+ makeDiff('apps/storybook/stories/Button.stories.tsx', [styleLine]),
+ makeDiff('packages/core/src/Button/Button.doc.mjs', [styleLine]),
+ makeDiff('packages/core/src/__tests__/TestIcon.tsx', ['<svg viewBox="0 0 24 24">']),
+ // ESLint-plugin fixture of intentional style violations
+ makeDiff('packages/core/src/Badge/Badge.test-violations.tsx', [
+ "fontSize: '14px',",
+ "color: '#ff0000',",
+ ]),
+ makeDiff('CONTRIBUTING.md', [styleLine]),
+ ].join('\n');
+ const result = classifyVisualDiff(diff);
+ expect(result.score).toBe(0);
+ expect(result.bucket).toBe('non-visual');
+ });
+
+ it('does not scan YAML/JSON files for property-like keys', () => {
+ const diff = makeDiff('apps/docsite/config.yml', ['width: 100']);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.cssProperties).toBe(0);
+ });
+
+ it('scans only product surfaces — tooling files never count', () => {
+ const diff = [
+ makeDiff('.github/scripts/test-pr-enrichment.js', [
+ "{ file: 'packages/core/src/Button/Button.tsx', line: 'paddingBlock: spacingVars.sm,' },",
+ ]),
+ makeDiff('vitest.config.ts', ["fontSize: '14px',"]),
+ makeDiff('scripts/build-css.mjs', ['padding: spacingVars.md,']),
+ ].join('\n');
+ const result = classifyVisualDiff(diff);
+ expect(result.score).toBe(0);
+ expect(result.bucket).toBe('non-visual');
+ });
+
+ it('skips comment lines', () => {
+ const diff = makeDiff('packages/core/src/Button/Button.tsx', [
+ '// padding: spacingVars.md,',
+ '* color: colorVars.accent',
+ '/* opacity: 0.5 */',
+ ]);
+ const result = classifyVisualDiff(diff);
+ expect(result.score).toBe(0);
+ });
+
+ it('skips lines inside multi-line block comments', () => {
+ const diff = makeDiff('packages/core/src/Button/Button.tsx', [
+ '/*',
+ 'padding: spacingVars.md,',
+ 'opacity: 0.5,',
+ '*/',
+ 'gap: spacingVars.sm,',
+ ]);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.cssProperties).toBe(1);
+ expect(result.signals.tokenReferences).toBe(1);
+ expect(result.evidence.cssProperties[0].line).toBe('gap: spacingVars.sm,');
+ });
+
+ it('tracks block-comment state per diff side', () => {
+ // A removed `/*` must not silence added lines.
+ const diff = [
+ 'diff --git a/packages/core/src/Button/Button.tsx b/packages/core/src/Button/Button.tsx',
+ 'index 1111111..2222222 100644',
+ '--- a/packages/core/src/Button/Button.tsx',
+ '+++ b/packages/core/src/Button/Button.tsx',
+ '@@ -1,3 +1,3 @@',
+ '-/*',
+ '+opacity: 0.5,',
+ '-*/',
+ ].join('\n');
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.cssProperties).toBe(1);
+ });
+
+ it('counts JSX structural changes only in packages/**/*.tsx', () => {
+ const jsxLines = ['<VisuallyHidden>', '</VisuallyHidden>'];
+ const inPackages = classifyVisualDiff(
+ makeDiff('packages/core/src/Banner/Banner.tsx', jsxLines)
+ );
+ expect(inPackages.signals.jsxStructuralChanges).toBe(2);
+
+ const inApps = classifyVisualDiff(
+ makeDiff('apps/docsite/src/app/page.tsx', jsxLines)
+ );
+ expect(inApps.signals.jsxStructuralChanges).toBe(0);
+ });
+
+ it('excludes JSX fragments from the structural signal', () => {
+ const diff = makeDiff('packages/core/src/Banner/Banner.tsx', ['<>', '</>']);
+ const result = classifyVisualDiff(diff);
+ expect(result.signals.jsxStructuralChanges).toBe(0);
+ });
+
+ it('caps the JSX contribution to the score', () => {
+ const lines = Array.from({length: JSX_CAP + 5}, (_, i) => `<div data-i={${i}}>`);
+ const result = classifyVisualDiff(makeDiff('packages/core/src/Table/Table.tsx', lines));
+ expect(result.signals.jsxStructuralChanges).toBe(JSX_CAP + 5);
+ expect(result.score).toBe(JSX_CAP);
+ });
+
+ it('buckets scores per the #3667 thresholds', () => {
+ // 4 props + 4 token refs + 1 style file in a stylex file: 12+8+3 → visual
+ const visual = classifyVisualDiff(
+ makeDiff('packages/core/src/Field/inputStyles.stylex.ts', [
+ 'padding: spacingVars.md,',
+ 'color: colorVars.accent,',
+ 'borderRadius: radiusVars.lg,',
+ 'gap: spacingVars.xs,',
+ ])
+ );
+ expect(visual.bucket).toBe('visual');
+
+ // A lone token-reference swap: 2 → maybe
+ const maybe = classifyVisualDiff(
+ makeDiff('packages/core/src/Button/Button.tsx', ['const tone = colorVars.accent;'])
+ );
+ expect(maybe.score).toBe(2);
+ expect(maybe.bucket).toBe('maybe-visual');
+
+ // Pure logic change → non-visual
+ const logic = classifyVisualDiff(
+ makeDiff('packages/core/src/Table/useSort.ts', [
+ 'const sorted = useMemo(() => rows.sort(compare), [rows, compare]);',
+ ])
+ );
+ expect(logic.bucket).toBe('non-visual');
+ });
+
+ it('collects capped evidence samples with file attribution', () => {
+ const result = classifyVisualDiff(
+ makeDiff('packages/core/src/Button/Button.tsx', ['opacity: 0.5,'])
+ );
+ expect(result.evidence.cssProperties).toEqual([
+ {file: 'packages/core/src/Button/Button.tsx', line: 'opacity: 0.5,'},
+ ]);
+ });
+
+ it('honors opts.ignorePath for caller-defined exclusions (review-signal safe spaces)', () => {
+ const isSafeSpace = (p) =>
+ /^packages\/lab\//.test(p) || /^apps\/(sandbox|storybook)\//.test(p);
+ const diff = [
+ makeDiff('packages/lab/src/Rating/Rating.tsx', ['padding: spacingVars.md,']),
+ makeDiff('apps/sandbox/src/page.tsx', ['opacity: 0.5,']),
+ makeDiff('packages/core/src/Button/Button.tsx', ['gap: spacingVars.sm,']),
+ ].join('\n');
+ const result = classifyVisualDiff(diff, {ignorePath: isSafeSpace});
+ // Only the core Button line counts: 1 prop + 1 token ref
+ expect(result.signals.cssProperties).toBe(1);
+ expect(result.signals.tokenReferences).toBe(1);
+ });
+});
+
+describe('review-signal loading contract', () => {
+ // review-signal.yml has no checkout: it fetches this file from the base ref
+ // via the API and instantiates it with new Function(module, exports, require).
+ // Pin that contract so a refactor (ESM syntax, a require of another module)
+ // fails here instead of silently breaking the workflow's content signal.
+ it('instantiates via new Function without a real require', () => {
+ const src = readFileSync(
+ fileURLToPath(new URL('./classify-visual.js', import.meta.url)),
+ 'utf8'
+ );
+ const mod = {exports: {}};
+ new Function('module', 'exports', 'require', src)(mod, mod.exports, () => {
+ throw new Error('classify-visual.js must stay dependency-free');
+ });
+ const result = mod.exports.classifyVisualDiff(
+ makeDiff('packages/core/src/Button/Button.tsx', ['opacity: 0.5,'])
+ );
+ expect(result.signals.cssProperties).toBe(1);
+ });
+});
diff --git a/.github/workflows/review-signal.yml b/.github/workflows/review-signal.yml
index f4ac2e49f7..f3664e66db 100644
--- a/.github/workflows/review-signal.yml
+++ b/.github/workflows/review-signal.yml
@@ -21,11 +21,17 @@
# area-gated — a theme/template edit is where design
# lives. Requests review from DESIGNOWNERS. Skipped when
# the author is a DESIGNOWNER (design self-serves design).
+# Besides the path signals, the DIFF CONTENT is scored
+# by .github/scripts/lib/classify-visual.js (#3667) —
+# most component styling here is an inline stylex.create
+# edit in a .tsx, which no path pattern can see.
#
# Either label DISABLES auto-merge so the PR can't merge unattended, and is
# cleared when an entitled owner approves (code → CODEOWNER; design → DESIGNOWNER
-# or CODEOWNER). Detection is path-based and deterministic; the Copilot reviewer
-# reads these labels to focus its review (see .github/copilot-instructions.md).
+# or CODEOWNER). Code detection is path-based and deterministic; design adds a
+# content-based diff score (deterministic too — same diff, same score). The
+# Copilot reviewer reads these labels to focus its review (see
+# .github/copilot-instructions.md).
#
# HARD GATE (code review only): this emits a "review-required" COMMIT STATUS.
# When code review is needed it's set to `pending`, which blocks a required
@@ -41,10 +47,11 @@
# self-serve code.
#
# Uses pull_request_target so it can label / request reviewers / disable
-# auto-merge on fork PRs. It reads the PR file LIST via the API and the eng/
-# design owner lists from repo VARIABLES (vars.ENGOWNERS / vars.DESIGNOWNERS) —
-# no checkout, no owner-file content API. It never checks out or runs
-# PR-controlled code. Do not add checkout or build steps here.
+# auto-merge on fork PRs. It reads the PR file LIST, the PR DIFF TEXT, and the
+# eng/design owner lists from repo VARIABLES (vars.ENGOWNERS / vars.DESIGNOWNERS)
+# via the API — no owner-file content API, no checkout. It never checks out or
+# runs PR-controlled code (the diff is scanned as data; the classifier it feeds
+# is loaded from the BASE ref, not the PR). Do not add checkout or build steps here.
name: Review signal
@@ -311,6 +318,44 @@ jobs:
if (paths.some((p) => /^packages\/themes\/.+\/src\/.+\.tsx?$/.test(p))) designReasons.push('theme/token change');
if (paths.some((p) => /^packages\/cli\/templates\/.+\.tsx$/.test(p))) designReasons.push('template change');
if (paths.some(isDocsiteVisual)) designReasons.push('docsite visual change');
+
+ // Content-based visual signal (#3667). The path signals above
+ // cannot see the most common visual change in this codebase: an
+ // inline `stylex.create` edit inside a component .tsx (no
+ // .stylex.ts touched). Score the diff CONTENT with the shared
+ // classifier. Safety: both the classifier source and the diff come
+ // from the API — the source from the BASE ref (trusted), the diff
+ // as inert text. Best-effort: any failure (huge diff 406, missing
+ // file on old base) logs a warning and falls back to path signals.
+ if (designReasons.length === 0) {
+ try {
+ const { data: libFile } = await github.rest.repos.getContent({
+ owner, repo, ref: pr.base.ref,
+ path: '.github/scripts/lib/classify-visual.js',
+ });
+ const libSrc = Buffer.from(libFile.content, 'base64').toString('utf8');
+ const mod = { exports: {} };
+ new Function('module', 'exports', 'require', libSrc)(
+ mod, mod.exports,
+ () => { throw new Error('classify-visual.js must stay dependency-free'); },
+ );
+ const { data: diff } = await github.request(
+ 'GET /repos/{owner}/{repo}/pulls/{pull_number}',
+ { owner, repo, pull_number: pr.number,
+ headers: { accept: 'application/vnd.github.diff' } },
+ );
+ const vc = mod.exports.classifyVisualDiff(String(diff), {
+ ignorePath: isSafeSpace,
+ });
+ core.info(`visual content score: ${vc.score} (${vc.bucket})`);
+ if (vc.bucket === 'visual' || vc.bucket === 'likely-visual') {
+ designReasons.push(`visual diff content (score ${vc.score})`);
+ }
+ } catch (e) {
+ core.warning(`Content-based visual signal skipped: ${e.message}`);
+ }
+ }
+
// Design owners self-serve design — no design tag for them.
const designAffecting = designReasons.length > 0 && !authorIsDesign;
diff --git a/vitest.config.ts b/vitest.config.ts
index 31acd414b3..729744b116 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -119,6 +119,7 @@ export default defineConfig({
'packages/**/src/**/*.test.{ts,tsx,mjs}',
'internal/**/*.test.{ts,tsx,mjs}',
'scripts/**/*.test.{ts,tsx,mjs}',
+ '.github/scripts/**/*.test.{ts,tsx,mjs}',
],
exclude: [
...configDefaults.exclude,