An open source design system that's fully customizable and agent ready
TypeScript
fix(cli): codemod output validation + CI verification + fix #910 (#912)
Cindy Zhangee0bc71ec5ab
diff --git a/.github/scripts/codemod-verify.mjs b/.github/scripts/codemod-verify.mjsnew file mode 100644index 0000000000..c1f52c4bda--- /dev/null+++ b/.github/scripts/codemod-verify.mjs@@ -0,0 +1,229 @@+/**+ * @file Codemod verification script+ *+ * Verifies that codemods in a PR can reproduce the callsite changes.+ * This enforces the process: "never hand-edit callsites for breaking changes,+ * always use the codemod."+ *+ * Algorithm:+ * 1. Detect which codemod version(s) were added/modified in this PR+ * 2. Find consumer directories that were also changed (apps/, e2e/)+ * 3. For each consumer dir: revert to base, run codemods, diff against PR+ * 4. Report mismatches+ *+ * Exit 0 if codemods reproduce changes (or no consumer files changed).+ * Exit 1 if there are mismatches β callsites were hand-edited.+ */++import {execSync} from 'node:child_process';+import * as fs from 'node:fs';+import * as path from 'node:path';+import {getTransformsBetween} from '../../packages/cli/src/codemods/registry.mjs';+import {runCodemods} from '../../packages/cli/src/codemods/runner.mjs';++// Consumer directories whose changes should be reproducible by codemods+const CONSUMER_DIRS = ['apps/storybook', 'apps/sandbox', 'e2e'];++function run(cmd) {+ return execSync(cmd, {encoding: 'utf-8'}).trim();+}++function getChangedFiles() {+ const base = run('git merge-base HEAD origin/main');+ return {+ base,+ files: run(`git diff --name-only ${base} HEAD`).split('\n').filter(Boolean),+ };+}++function getCodemodVersions(changedFiles) {+ const versionPattern = /^packages\/cli\/src\/codemods\/transforms\/v([\d.]+)\//;+ const versions = new Set();+ for (const file of changedFiles) {+ const match = file.match(versionPattern);+ if (match) versions.add(match[1]);+ }+ return [...versions].sort();+}++function getChangedConsumerFiles(changedFiles) {+ return changedFiles.filter((f) =>+ CONSUMER_DIRS.some(+ (dir) =>+ f.startsWith(dir + '/') &&+ /\.(tsx?|jsx?)$/.test(f),+ ),+ );+}++async function main() {+ console.log('π Codemod Verification\n');++ const {base, files: changedFiles} = getChangedFiles();+ const codemodVersions = getCodemodVersions(changedFiles);+ const consumerFiles = getChangedConsumerFiles(changedFiles);++ console.log(`Codemod versions modified: ${codemodVersions.join(', ') || 'none'}`);+ console.log(`Consumer files changed: ${consumerFiles.length}`);++ if (codemodVersions.length === 0) {+ console.log('\nβ No codemod changes β nothing to verify.');+ process.exit(0);+ }++ if (consumerFiles.length === 0) {+ console.log('\nβ No consumer file changes β codemod-only PR.');+ console.log(+ 'β οΈ Note: If this codemod has breaking changes, consumer files should also be updated via the codemod.',+ );+ process.exit(0);+ }++ console.log(`\nConsumer files to verify:\n${consumerFiles.map((f) => ` ${f}`).join('\n')}\n`);++ // Save the PR version of each consumer file+ const prVersions = {};+ for (const file of consumerFiles) {+ prVersions[file] = fs.readFileSync(file, 'utf-8');+ }++ // Revert consumer files to base branch state+ let revertedFiles = [];+ for (const file of consumerFiles) {+ try {+ const baseContent = run(`git show ${base}:${file}`);+ fs.writeFileSync(file, baseContent);+ revertedFiles.push(file);+ } catch {+ console.log(` β ${file} β new in this PR, skipping`);+ }+ }+ console.log(`Reverted ${revertedFiles.length} files to base (${base.slice(0, 8)})\n`);++ if (revertedFiles.length === 0) {+ console.log('β All consumer files are new β nothing to verify.');+ for (const file of consumerFiles) {+ fs.writeFileSync(file, prVersions[file]);+ }+ process.exit(0);+ }++ // Determine the version range for codemods+ // Run ALL codemods up to the latest modified version to be thorough+ const latestVersion = codemodVersions[codemodVersions.length - 1];++ // Run codemods on each affected consumer directory+ const affectedDirs = [+ ...new Set(+ revertedFiles.map((f) => CONSUMER_DIRS.find((dir) => f.startsWith(dir + '/'))).filter(Boolean),+ ),+ ];++ for (const dir of affectedDirs) {+ console.log(`Running codemods (0.0.0 β ${latestVersion}) on ${dir}/...`);+ const manifests = await getTransformsBetween('0.0.0', latestVersion);+ if (manifests.length === 0) {+ console.log(' No applicable codemods found.');+ continue;+ }++ // Suppress clack output during verification+ const origLog = console.log;+ const origErr = console.error;+ await runCodemods(manifests, {+ apply: true,+ path: dir,+ codemod: undefined,+ });+ }++ // Format codemod output with prettier to normalize style differences+ // (jscodeshift's toSource() may change quotes, spacing, etc.)+ console.log('Formatting codemod output with prettier...');+ try {+ run(`npx prettier --write ${revertedFiles.map((f) => `"${f}"`).join(' ')}`);+ } catch (e) {+ console.log(' β οΈ Prettier formatting failed, comparing raw output');+ }++ // Also format the expected (PR) versions for a fair comparison+ const formattedExpected = {};+ for (const file of revertedFiles) {+ const tmpFile = file + '.expected.tmp';+ fs.writeFileSync(tmpFile, prVersions[file]);+ try {+ run(`npx prettier --write "${tmpFile}"`);+ formattedExpected[file] = fs.readFileSync(tmpFile, 'utf-8');+ } catch {+ formattedExpected[file] = prVersions[file];+ }+ fs.unlinkSync(tmpFile);+ }++ // Compare formatted codemod output against formatted PR versions+ let mismatches = 0;+ let matches = 0;+ const mismatchDetails = [];++ for (const file of revertedFiles) {+ const expected = formattedExpected[file] || prVersions[file];+ let actual;+ try {+ actual = fs.readFileSync(file, 'utf-8');+ } catch {+ continue;+ }++ if (actual === expected) {+ matches++;+ console.log(` β ${file}`);+ } else {+ mismatches++;+ const actualLines = actual.split('\n');+ const expectedLines = expected.split('\n');+ let diffCount = 0;+ const maxLen = Math.max(actualLines.length, expectedLines.length);+ for (let i = 0; i < maxLen; i++) {+ if (actualLines[i] !== expectedLines[i]) diffCount++;+ }+ mismatchDetails.push({file, diffLines: diffCount});+ console.log(` β ${file} (${diffCount} lines differ)`);+ }+ }++ // Restore PR versions+ for (const file of consumerFiles) {+ if (prVersions[file]) {+ fs.writeFileSync(file, prVersions[file]);+ }+ }++ // Report+ console.log(`\n${'='.repeat(60)}`);+ console.log('Codemod Verification Results');+ console.log('='.repeat(60));+ console.log(` Files verified: ${matches + mismatches}`);+ console.log(` β Reproducible: ${matches}`);+ console.log(` β Mismatches: ${mismatches}`);++ if (mismatches > 0) {+ console.log('\nMismatched files (codemod output β committed changes):');+ for (const {file, diffLines} of mismatchDetails) {+ console.log(` β ${file} (${diffLines} line${diffLines === 1 ? '' : 's'} differ)`);+ }+ console.log('\nβ οΈ The codemod did not reproduce the committed consumer changes.');+ console.log('This means either:');+ console.log(' 1. Callsites were hand-edited instead of using the codemod');+ console.log(' 2. The codemod is incomplete (add missing transforms)');+ console.log(' 3. Changes are intentionally beyond codemod scope (add allow-manual-edit label)');+ process.exit(1);+ }++ console.log('\nβ All consumer changes are reproducible by codemods.');+ process.exit(0);+}++main().catch((err) => {+ console.error('Codemod verification failed:', err);+ process.exit(1);+});diff --git a/.github/workflows/codemod-verify.yml b/.github/workflows/codemod-verify.ymlnew file mode 100644index 0000000000..2296c76112--- /dev/null+++ b/.github/workflows/codemod-verify.yml@@ -0,0 +1,46 @@+# Codemod verification β ensures codemods can reproduce callsite changes+#+# When a PR adds/modifies codemods AND touches consumer code (apps/, stories),+# this workflow:+# 1. Reverts the consumer code changes to the base branch state+# 2. Runs the codemods from source against that reverted code+# 3. Verifies the codemod output matches the PR's committed changes+#+# This enforces the rule: "never manually update callsites β always use the codemod"+# If the codemod can't reproduce the changes, either the codemod is incomplete+# or callsites were hand-edited.+name: Codemod Verify++on:+ pull_request:+ branches: ["main"]+ paths:+ - "packages/cli/src/codemods/**"++permissions:+ contents: read+ pull-requests: write++concurrency:+ group: "codemod-verify-${{ github.head_ref }}"+ cancel-in-progress: true++jobs:+ verify-codemods:+ runs-on: ubuntu-latest+ steps:+ - name: Checkout PR+ uses: actions/checkout@v4+ with:+ fetch-depth: 0++ - uses: actions/setup-node@v4+ with:+ node-version: 20+ cache: "yarn"++ - run: yarn install --frozen-lockfile++ - name: Verify codemod reproduces callsite changes+ id: verify+ run: node .github/scripts/codemod-verify.mjsdiff --git a/packages/cli/src/codemods/__tests__/validation.test.mjs b/packages/cli/src/codemods/__tests__/validation.test.mjsnew file mode 100644index 0000000000..7992e1d1d2--- /dev/null+++ b/packages/cli/src/codemods/__tests__/validation.test.mjs@@ -0,0 +1,69 @@+import {describe, it, expect} from 'vitest';+import {validateOutput} from '../runner.mjs';++async function getJscodeshift(parser = 'tsx') {+ const jscodeshift = (await import('jscodeshift')).default;+ return jscodeshift.withParser(parser);+}++describe('validateOutput', () => {+ it('accepts valid transformed output', async () => {+ const j = await getJscodeshift();+ const source = '<XDSButton variant="primary" />';+ const result = '<XDSButton container="primary" />';+ expect(validateOutput(result, source, j)).toEqual({valid: true});+ });++ it('rejects output with syntax errors', async () => {+ const j = await getJscodeshift();+ const source = 'const x = 1;';+ const result = 'const x = {{{;';+ const validation = validateOutput(result, source, j);+ expect(validation.valid).toBe(false);+ expect(validation.reason).toMatch(/unparseable/);+ });++ it('rejects [native code] corruption in parseable output', async () => {+ const j = await getJscodeshift();+ // This is syntactically valid JS but contains corruption+ const source = 'const name = "toString";';+ const result = 'const name = "[native code] toString";';+ const validation = validateOutput(result, source, j);+ expect(validation.valid).toBe(false);+ expect(validation.reason).toMatch(/corruption/);+ });++ it('allows pre-existing [native code] strings', async () => {+ const j = await getJscodeshift();+ const source = 'const msg = "[native code] is a thing";';+ const result = 'const msg = "[native code] is a thing";\nconst x = 1;';+ expect(validateOutput(result, source, j)).toEqual({valid: true});+ });++ it('catches prototype pollution that produces unparseable output', async () => {+ const j = await getJscodeshift();+ const source = `+import {lineHeightVars} from '@xds/core/tokens';+const x = someValue.toString();+`;+ // Buggy codemod replaces toString identifier with native function string+ const result = `+import {typeScaleVars} from '@xds/core/tokens';+const x = someValue.function toString() { [native code] }();+`;+ const validation = validateOutput(result, source, j);+ expect(validation.valid).toBe(false);+ // Could fail on parse OR corruption pattern β either is correct+ expect(validation.reason).toMatch(/unparseable|corruption|native/i);+ });++ it('rejects when new [native code] appears even if parseable', async () => {+ const j = await getJscodeshift();+ const source = 'const x = "hello";';+ // Someone somehow produced valid JS with [native code] in a string+ const result = 'const x = "function toString() { [native code] }";';+ const validation = validateOutput(result, source, j);+ expect(validation.valid).toBe(false);+ expect(validation.reason).toMatch(/corruption/);+ });+});diff --git a/packages/cli/src/codemods/runner.mjs b/packages/cli/src/codemods/runner.mjsindex 46137515cf..160f3d4471 100644--- a/packages/cli/src/codemods/runner.mjs+++ b/packages/cli/src/codemods/runner.mjs@@ -2,13 +2,21 @@* @file Codemod runner** Orchestrates running jscodeshift transforms against source files.- * Handles dry-run previews, file writing, and summary reporting.+ * Handles dry-run previews, file writing, summary reporting, and+ * output validation to prevent corrupted transforms from reaching disk.*/import * as fs from 'node:fs';import * as path from 'node:path';import * as p from '@clack/prompts';+// Known corruption patterns that indicate a broken transform.+// Each entry: [regex, human-readable description]+const CORRUPTION_PATTERNS = [+ [/\[native code\]/g, '[native code] injection (prototype pollution in identifier map)'],+ [/function \w+\(\) \{ \[native code\] \}/g, 'native function toString() leak'],+];+/*** Recursively find all source files in a directory.* @param {string} dir@@ -40,6 +48,74 @@ function findSourceFiles(dir) {return results.sort();}+/**+ * Detect the project's formatter and run it on changed files.+ * Tries prettier, then biome. Silently skips if none found.+ *+ * @param {string[]} files - Absolute paths to files that were modified+ */+async function formatChangedFiles(files) {+ if (files.length === 0) return;++ const {execSync} = await import('node:child_process');+ const fileArgs = files.map((f) => `"${f}"`).join(' ');++ const formatters = [+ {name: 'prettier', cmd: `npx prettier --write ${fileArgs}`},+ {name: 'biome', cmd: `npx biome format --write ${fileArgs}`},+ ];++ for (const {name, cmd} of formatters) {+ try {+ execSync(cmd, {stdio: 'pipe', timeout: 30000});+ p.log.info(`Formatted ${files.length} file${files.length === 1 ? '' : 's'} with ${name}`);+ return;+ } catch {+ // Formatter not available or failed β try next+ }+ }+}++/**+ * Validate transform output before writing to disk.+ *+ * Checks:+ * 1. The output can be re-parsed by jscodeshift (no syntax corruption)+ * 2. No known corruption patterns are present that weren't in the original+ *+ * @param {string} result - The transformed source code+ * @param {string} source - The original source code+ * @param {Function} j - jscodeshift instance (with parser configured)+ * @returns {{ valid: boolean, reason?: string }}+ */+export function validateOutput(result, source, j) {+ // Check 1: Re-parse the output β catches syntax-breaking corruption+ try {+ j(result);+ } catch (parseError) {+ return {+ valid: false,+ reason: `transform produced unparseable output: ${parseError.message}`,+ };+ }++ // Check 2: Known corruption patterns (only flag new ones, not pre-existing)+ for (const [pattern, description] of CORRUPTION_PATTERNS) {+ const resultMatches = result.match(pattern);+ const sourceMatches = source.match(pattern);+ const resultCount = resultMatches ? resultMatches.length : 0;+ const sourceCount = sourceMatches ? sourceMatches.length : 0;+ if (resultCount > sourceCount) {+ return {+ valid: false,+ reason: `detected corruption: ${description} (${resultCount - sourceCount} new occurrence${resultCount - sourceCount > 1 ? 's' : ''})`,+ };+ }+ }++ return {valid: true};+}+/*** Run codemods against source files.*@@ -72,7 +148,9 @@ export async function runCodemods(versionManifests, {apply, path: srcPath, codemlet totalFilesChanged = 0;let totalTransformsApplied = 0;+ let totalValidationBlocked = 0;const errors = [];+ const writtenFiles = [];for (const {version, transforms} of versionManifests) {p.log.step(`Applying v${version} codemods...`);@@ -92,13 +170,11 @@ export async function runCodemods(versionManifests, {apply, path: srcPath, codemtry {const source = fs.readFileSync(filePath, 'utf-8');// Configure parser based on file extension- const parser = filePath.endsWith('.tsx')- ? 'tsx'- : filePath.endsWith('.jsx')- ? 'babel'- : 'babel';+ const ext = path.extname(filePath);+ const parser = ext === '.tsx' ? 'tsx' : ext === '.jsx' ? 'babel' : 'babel';+ const j = jscodeshift.withParser(parser);const api = {- jscodeshift: jscodeshift.withParser(parser),+ jscodeshift: j,stats: () => {},report: () => {},};@@ -107,12 +183,22 @@ export async function runCodemods(versionManifests, {apply, path: srcPath, codemconst result = transform(file, api);if (result != null && result !== source) {+ // Validate output before writing+ const validation = validateOutput(result, source, j);+ if (!validation.valid) {+ totalValidationBlocked++;+ p.log.error(` β ${relativePath} β ${validation.reason}`);+ errors.push({file: relativePath, codemod: name, error: validation.reason});+ continue;+ }+filesChanged++;totalFilesChanged++;totalTransformsApplied++;if (apply) {fs.writeFileSync(filePath, result, 'utf-8');+ writtenFiles.push(filePath);p.log.success(` β ${relativePath}`);} else {p.log.warn(` ~ ${relativePath} (would change)`);@@ -143,6 +229,21 @@ export async function runCodemods(versionManifests, {apply, path: srcPath, codem}}+ // Post-codemod formatting: run the project's formatter on changed files+ // so codemods don't introduce style drift (jscodeshift may change quotes, etc.)+ if (apply && writtenFiles.length > 0) {+ await formatChangedFiles(writtenFiles);+ }++ if (totalValidationBlocked > 0) {+ p.log.warn(+ `${totalValidationBlocked} file${totalValidationBlocked === 1 ? ' was' : 's were'} blocked by validation β no changes written to ${totalValidationBlocked === 1 ? 'that file' : 'those files'}.`,+ );+ p.log.info(+ 'This means a codemod produced invalid output. Please report this as a bug.',+ );+ }+if (totalFilesChanged === 0 && errors.length === 0) {p.log.success('No changes needed β your code is already up to date!');} else if (apply) {@@ -159,4 +260,6 @@ export async function runCodemods(versionManifests, {apply, path: srcPath, codem);p.log.info('Run with --apply to write changes to disk.');}++ return {totalFilesChanged, totalTransformsApplied, totalValidationBlocked, errors};}diff --git a/packages/cli/src/codemods/transforms/v0.0.8/__tests__/migrate-token-renames.test.mjs b/packages/cli/src/codemods/transforms/v0.0.8/__tests__/migrate-token-renames.test.mjsindex 53ca74ca41..6895480bcc 100644--- a/packages/cli/src/codemods/transforms/v0.0.8/__tests__/migrate-token-renames.test.mjs+++ b/packages/cli/src/codemods/transforms/v0.0.8/__tests__/migrate-token-renames.test.mjs@@ -248,4 +248,72 @@ const tokens = { ...lineHeightDefaults };`;const result = transform({source: `const x = '--color-accent';`, path: 'test.tsx'}, api);expect(result).toBeUndefined();});++ // === Regression: #910 β prototype pollution corrupts .toString() calls ===++ it('does not corrupt .toString() calls', async () => {+ const input = `const qs = params.toString();\nconst x = '--color-wash';`;+ const output = await applyTransform(input);+ expect(output).toContain('params.toString()');+ expect(output).not.toContain('[native code]');+ expect(output).toContain('--color-background-body');+ });++ it('does not corrupt .toLocaleString() calls', async () => {+ const input = `const n = count.toLocaleString();\nconst x = '--shadow-base';`;+ const output = await applyTransform(input);+ expect(output).toContain('count.toLocaleString()');+ expect(output).not.toContain('[native code]');+ expect(output).toContain('--shadow-low');+ });++ it('does not corrupt .hasOwnProperty() calls', async () => {+ const input = `const ok = obj.hasOwnProperty('key');\nconst x = '--radius-0';`;+ const output = await applyTransform(input);+ expect(output).toContain("obj.hasOwnProperty('key')");+ expect(output).not.toContain('[native code]');+ expect(output).toContain('--radius-none');+ });++ it('does not corrupt .valueOf() calls', async () => {+ const input = `const v = date.valueOf();\nconst x = '--font-body';`;+ const output = await applyTransform(input);+ expect(output).toContain('date.valueOf()');+ expect(output).not.toContain('[native code]');+ expect(output).toContain('--font-family-body');+ });++ // === Regression: duplicate import when target already exists ===++ it('removes old import specifier when target is already imported', async () => {+ const input = `import { lineHeightDefaults, typeScaleDefaults } from '@xds/core/theme';+const x = { ...lineHeightDefaults, ...typeScaleDefaults };`;+ const output = await applyTransform(input);+ // lineHeightDefaults import should be removed (not renamed to duplicate)+ expect(output).not.toContain('lineHeightDefaults');+ // typeScaleDefaults should appear once in import, twice in usage+ const importMatch = output.match(/import.*typeScaleDefaults/);+ expect(importMatch).toBeTruthy();+ // Usage should rename lineHeightDefaults β typeScaleDefaults+ expect(output).toContain('...typeScaleDefaults');+ });++ it('renames import specifier when target is NOT already imported', async () => {+ const input = `import { lineHeightDefaults } from '@xds/core/theme';+const x = lineHeightDefaults;`;+ const output = await applyTransform(input);+ expect(output).toContain('typeScaleDefaults');+ expect(output).not.toContain('lineHeightDefaults');+ });++ it('handles lineHeightVars + typeScaleVars dual import', async () => {+ const input = `import { lineHeightVars, typeScaleVars } from '@xds/core/theme';+const a = lineHeightVars;+const b = typeScaleVars;`;+ const output = await applyTransform(input);+ // lineHeightVars import should be removed+ expect(output).not.toContain('lineHeightVars');+ // All usages should be typeScaleVars+ expect((output.match(/typeScaleVars/g) || []).length).toBeGreaterThanOrEqual(2);+ });});diff --git a/packages/cli/src/codemods/transforms/v0.0.8/migrate-token-renames.mjs b/packages/cli/src/codemods/transforms/v0.0.8/migrate-token-renames.mjsindex 961b4fb082..b2e5af79fd 100644--- a/packages/cli/src/codemods/transforms/v0.0.8/migrate-token-renames.mjs+++ b/packages/cli/src/codemods/transforms/v0.0.8/migrate-token-renames.mjs@@ -256,11 +256,48 @@ export default function transformer(file, api) {});// --- Pass 3: Rename JS identifiers (lineHeightVars β typeScaleVars, etc.) ---- // This handles imports, member expressions, and any other reference in one pass.+ // Uses Object.hasOwn to avoid prototype pollution (toString, hasOwnProperty, etc.)+ //+ // Special handling for imports: if the target name already exists as an import+ // specifier in the same declaration, remove the old specifier instead of renaming+ // it (avoids duplicate identifier declarations).+ root.find(j.ImportSpecifier).forEach((importPath) => {+ const oldName = importPath.node.imported.name;+ if (!Object.hasOwn(IDENTIFIER_MAP, oldName)) return;++ const newName = IDENTIFIER_MAP[oldName];+ const declaration = importPath.parent.node;+ const siblings = declaration.specifiers || [];+ const targetExists = siblings.some(+ (s) =>+ s !== importPath.node &&+ s.type === 'ImportSpecifier' &&+ s.imported.name === newName,+ );++ if (targetExists) {+ // Target already imported β remove this specifier entirely+ const idx = siblings.indexOf(importPath.node);+ if (idx !== -1) {+ siblings.splice(idx, 1);+ hasChanges = true;+ }+ } else {+ // Safe to rename+ importPath.node.imported.name = newName;+ if (importPath.node.local.name === oldName) {+ importPath.node.local.name = newName;+ }+ hasChanges = true;+ }+ });++ // Rename remaining non-import identifier referencesroot.find(j.Identifier).forEach((path) => {- const newName = IDENTIFIER_MAP[path.node.name];- if (!newName) return;- path.node.name = newName;+ if (!Object.hasOwn(IDENTIFIER_MAP, path.node.name)) return;+ // Skip import specifiers (already handled above)+ if (path.parent.node.type === 'ImportSpecifier') return;+ path.node.name = IDENTIFIER_MAP[path.node.name];hasChanges = true;});