Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(ci): fall back to two-dot diff in analyze-pr when merge base is missing (#4914)

Gonzalo Blasco152e956f3298
diff --git a/.github/scripts/analyze-pr.js b/.github/scripts/analyze-pr.js
index 826b458365..f45e061b84 100644
--- a/.github/scripts/analyze-pr.js
+++ b/.github/scripts/analyze-pr.js
@@ -88,24 +88,89 @@ function componentIndexRef(pkg, componentName) {
: `${pkgSrc(pkg)}/${componentName}/index.ts`;
}
-// Get changed files between base and head
+// Get changed files between base and head.
+//
+// Prefers a three-dot diff (base...head), which only reports files changed on
+// the PR branch and excludes commits already merged to base. But a three-dot
+// diff needs a merge base, which a shallow clone (fetch-depth: 50) may not
+// have once the PR has been open long enough for base to advance past that
+// window — the diff then fails with "no merge base" and the whole job dies.
+//
+// On that failure we first try to deepen the shallow clone and retry the
+// three-dot diff — the real fix for "base advanced past the fetch window" is
+// to fetch more history, not to degrade. Only if that still cannot find a
+// merge base do we fall back to a two-dot diff (base..head), which needs no
+// merge base and never hard-fails the CI.
+//
+// The two-dot fallback is inherently lossy: it compares two trees, so a file
+// whose change also exists on base (a cherry-picked/duplicated fix, or the
+// same codegen output landing from another PR) vanishes from the diff
+// entirely. That means two-dot can UNDER-report a PR's own changes, not just
+// over-report base churn. Because downstream gates (pr-a11y) derive their
+// component list from this file list, under-reporting silently skips the
+// audit for a component the PR did change. The caller records the diff mode
+// in the analysis output so consumers know when the file list is approximate.
+// Returns { files, diffMode: 'three-dot' | 'two-dot' }.
function getChangedFiles() {
+ const threeDot = `${baseBranch}...${headRef}`;
+ const tryThreeDot = () => {
+ const output = execSync(`git diff --name-only ${threeDot}`, { encoding: 'utf8' });
+ return output.trim().split('\n').filter(Boolean);
+ };
try {
- const output = execSync(
- `git diff --name-only ${baseBranch}...${headRef}`,
- { encoding: 'utf8' }
+ return { files: tryThreeDot(), diffMode: 'three-dot' };
+ } catch (e) {
+ console.warn(
+ `::warning::three-dot diff ${threeDot} failed - attempting to deepen the clone and retry`
);
- return output.trim().split('\n').filter(Boolean);
+ console.warn(diffErrorDetail(e));
+ }
+
+ // The three-dot diff failed, almost certainly because the shallow clone is
+ // too shallow to contain the merge base. Deepen before giving up on the
+ // accurate path; a deepened clone makes three-dot correct again. The base
+ // arg arrives as "origin/main" (see ci.yml) but the fetch refspec needs the
+ // bare branch name and must update the origin/<base> ref explicitly —
+ // `--deepen` alone only updates FETCH_HEAD, which the three-dot ref
+ // (origin/main...HEAD) needs to actually exist.
+ try {
+ const baseName = baseBranch.replace(/^origin\//, '');
+ execSync(
+ `git fetch --deepen=200 origin ${baseName}:refs/remotes/origin/${baseName}`,
+ { encoding: 'utf8', stdio: 'pipe' },
+ );
+ return { files: tryThreeDot(), diffMode: 'three-dot' };
+ } catch (e) {
+ console.warn(
+ `::warning::deepen failed (${diffErrorDetail(e)}) - falling back to two-dot diff`
+ );
+ }
+
+ // Last resort: a two-dot diff needs no merge base. Its file list is
+ // approximate (may over-report base churn and, worse, under-report the PR's
+ // own changes when they also exist on base), so the caller marks the output
+ // with diffMode: 'two-dot' for any consumer to caveat.
+ const twoDot = `${baseBranch}..${headRef}`;
+ try {
+ const output = execSync(`git diff --name-only ${twoDot}`, { encoding: 'utf8' });
+ return { files: output.trim().split('\n').filter(Boolean), diffMode: 'two-dot' };
} catch (e) {
- // A failed diff (e.g. no merge base on a too-shallow clone) is not the
+ // A failed diff (e.g. base ref missing on a too-shallow clone) is not the
// same as "no changes" — fail loudly instead of publishing an empty
// analysis report that looks legitimate.
- console.error(`::error::git diff ${baseBranch}...${headRef} failed: ${e.message}`);
- console.error('The clone is likely too shallow to contain the merge base (see fetch-depth in ci.yml).');
+ console.error(`::error::git diff ${twoDot} failed: ${e.message}`);
+ console.error('The clone is likely too shallow to contain the base ref (see fetch-depth in ci.yml).');
process.exit(1);
}
}
+// Human-readable reason for a failed diff/deepen, preferring the fatal stderr
+// line (e.g. "fatal: origin/main...HEAD: no merge base") over execSync's
+// boilerplate "Command failed: ..." first line.
+function diffErrorDetail(e) {
+ return (e.stderr && e.stderr.toString().trim()) || e.message.split('\n')[0];
+}
+
// Check if a component exists in the base branch.
function componentExistsInBase(pkg, componentName) {
try {
@@ -454,8 +519,13 @@ function getPackageBundleStats(pkg) {
// Main analysis
function analyze() {
console.log(`Analyzing changes from ${baseBranch} to ${headRef}...`);
- const changedFiles = getChangedFiles();
- console.log(`Found ${changedFiles.length} changed files`);
+ const { files: changedFiles, diffMode } = getChangedFiles();
+ console.log(`Found ${changedFiles.length} changed files (diff mode: ${diffMode})`);
+ if (diffMode === 'two-dot') {
+ console.warn(
+ '::warning::using approximate two-dot diff - the file list may include base-only churn and may miss the PR\'s own changes when they also exist on base'
+ );
+ }
const newComponents = [];
const modifiedComponents = [];
@@ -521,6 +591,10 @@ function analyze() {
newExports,
componentStats,
changedPackages: [...changedPackages],
+ // Records whether the file list is exact (three-dot) or approximate
+ // (two-dot fallback). Consumers that gate on the component list (pr-a11y)
+ // or render it in the report should caveat when this is 'two-dot'.
+ diffMode,
bundlePackages,
// Back-compat: keep totalBundle pointed at core when core changed, so any
// older consumer of this field still resolves.
diff --git a/.github/scripts/analyze-pr.test.mjs b/.github/scripts/analyze-pr.test.mjs
new file mode 100644
index 0000000000..74a61f4812
--- /dev/null
+++ b/.github/scripts/analyze-pr.test.mjs
@@ -0,0 +1,121 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file analyze-pr.test.mjs
+ * Pins the shallow-clone recovery contract of analyze-pr.js. The pr-analysis
+ * job runs analyze-pr.js with a `--depth=50` fetch; once a PR has been open
+ * long enough for base to advance past that window, the three-dot diff
+ * (base...head) fails with "no merge base" and — without this fix — the whole
+ * job dies. The script must recover by deepening the clone and retrying the
+ * three-dot diff, and it must record which diff mode it used so consumers
+ * know whether the component list is exact or approximate.
+ *
+ * We build a synthetic repo: a branch point, 60 churn commits on main, and a
+ * feature branch off the branch point that touches exactly one component.
+ * A shallow single-branch clone of the feature (depth 5) has no merge base
+ * with origin/main, reproducing the CI failure; running analyze-pr.js against
+ * it must exit 0 and report exactly the component the branch touched.
+ */
+
+import {execFileSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {describe, it, expect} from 'vitest';
+
+const SCRIPTS_DIR = path.dirname(fileURLToPath(import.meta.url));
+const SCRIPT = path.join(SCRIPTS_DIR, 'analyze-pr.js');
+
+/** Run a git command in a repo dir; return trimmed stdout. */
+function git(dir, args) {
+ return execFileSync('git', args, {cwd: dir, encoding: 'utf8'}).trim();
+}
+
+/** Run a git command that is allowed to fail; return {status, stdout, stderr}. */
+function gitTry(dir, args) {
+ try {
+ const stdout = execFileSync('git', args, {cwd: dir, encoding: 'utf8'});
+ return {status: 0, stdout, stderr: ''};
+ } catch (e) {
+ return {status: e.status, stdout: e.stdout || '', stderr: e.stderr || ''};
+ }
+}
+
+/**
+ * Build the synthetic repo and return the shallow clone dir:
+ * upstream/: branch point, 60 churn commits on main, feature off the point
+ * touching only packages/core/src/Card/.
+ * clone/ : --depth=5 --single-branch of feature (no merge base with main).
+ */
+function buildFixture() {
+ const base = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-analyze-'));
+ const upstream = path.join(base, 'upstream');
+ const clone = path.join(base, 'clone');
+
+ fs.mkdirSync(path.join(upstream, 'packages/core/src/Card'), {recursive: true});
+ fs.mkdirSync(path.join(upstream, 'packages/core/src/Button'), {recursive: true});
+ fs.mkdirSync(path.join(upstream, 'packages/core/src/Line'), {recursive: true});
+
+ git(upstream, ['init', '-q', '-b', 'main']);
+ git(upstream, ['config', 'user.email', '[email protected]']);
+ git(upstream, ['config', 'user.name', 'Test']);
+ fs.writeFileSync(path.join(upstream, 'package.json'), '{}');
+ fs.writeFileSync(path.join(upstream, 'packages/core/src/Card/index.ts'), 'export {}\n');
+ fs.writeFileSync(path.join(upstream, 'packages/core/src/Button/index.ts'), 'export {}\n');
+ fs.writeFileSync(path.join(upstream, 'packages/core/src/Line/index.ts'), 'export {}\n');
+ git(upstream, ['add', '-A']);
+ git(upstream, ['commit', '-qm', 'branch point']);
+ const branchPoint = git(upstream, ['rev-parse', 'HEAD']);
+
+ // 60 churn commits on main so the branch point is beyond any small depth.
+ for (let i = 1; i <= 60; i++) {
+ fs.appendFileSync(path.join(upstream, 'packages/core/src/Button/index.ts'), `b${i}\n`);
+ fs.appendFileSync(path.join(upstream, 'packages/core/src/Line/index.ts'), `l${i}\n`);
+ git(upstream, ['add', '-A']);
+ git(upstream, ['commit', '-qm', `churn ${i}`]);
+ }
+
+ // Feature branch off the branch point, touching only Card.
+ git(upstream, ['branch', 'feature', branchPoint]);
+ git(upstream, ['checkout', '-q', 'feature']);
+ fs.appendFileSync(path.join(upstream, 'packages/core/src/Card/index.ts'), 'export const Card = {}\n');
+ git(upstream, ['add', '-A']);
+ git(upstream, ['commit', '-qm', 'touch Card only']);
+
+ // Shallow single-branch clone of the feature branch.
+ git(null, [
+ 'clone', '-q', '--depth=5', '--single-branch', '--branch=feature',
+ `file://${upstream}`, clone,
+ ]);
+
+ return {base, clone, branchPoint};
+}
+
+describe('analyze-pr shallow-clone recovery', () => {
+ it('recovers the three-dot diff by deepening and reports the exact component', () => {
+ const {base, clone} = buildFixture();
+ try {
+ // Precondition: the clone really has no merge base (the CI failure).
+ const mb = gitTry(clone, ['merge-base', 'HEAD', 'origin/main']);
+ expect(mb.status).not.toBe(0);
+
+ const stdout = execFileSync(
+ process.execPath,
+ [SCRIPT, '--base', 'origin/main', '--head', 'HEAD', '--output', 'analysis.json'],
+ {cwd: clone, encoding: 'utf8'},
+ );
+ const analysis = JSON.parse(
+ fs.readFileSync(path.join(clone, 'analysis.json'), 'utf8'),
+ );
+
+ // Reaching the assertions proves exit 0 (execFileSync throws otherwise).
+ expect(stdout).toContain('diff mode: three-dot');
+ expect(analysis.diffMode).toBe('three-dot');
+ expect(analysis.modifiedComponents).toEqual(['Card']);
+ expect(analysis.changedPackages).toEqual(['@astryxdesign/core']);
+ } finally {
+ fs.rmSync(base, {recursive: true, force: true});
+ }
+ });
+});
diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.js
index d9fd820e7d..c82d11c77a 100644
--- a/.github/scripts/generate-pr-comment.js
+++ b/.github/scripts/generate-pr-comment.js
@@ -172,10 +172,19 @@ if (sandboxUrl) footerLinks.push(extLink('Sandbox', sandboxUrl));
if (runUrl) footerLinks.push(extLink('View full report', runUrl));
const footerLinksStr = footerLinks.length > 0 ? ` | ${footerLinks.join(' | ')}` : '';
+// A one-line caveat shown when the analysis fell back to the approximate
+// two-dot diff. The file list may include base-only churn and, worse, may
+// miss a component the PR actually changed (if that change also exists on
+// base), so consumers should not treat the component list as exact.
+const diffModeCaveat =
+ analysis.diffMode === 'two-dot'
+ ? '> ⚠️ **Approximate analysis** - the diff could not be resolved to a merge base, so a two-dot fallback was used. The component list may include base-only changes or miss a component whose change also landed on base.\n\n'
+ : '';
+
// Build the full comment
const body = `## PR Analysis Report
-${storybookSection}${sandboxSection}${componentSection || '_No new or modified components detected._\n\n'}
+${diffModeCaveat}${storybookSection}${sandboxSection}${componentSection || '_No new or modified components detected._\n\n'}
${bundleSection}
${a11ySection}
---
diff --git a/.github/scripts/generate-pr-comment.test.mjs b/.github/scripts/generate-pr-comment.test.mjs
new file mode 100644
index 0000000000..2ec670512c
--- /dev/null
+++ b/.github/scripts/generate-pr-comment.test.mjs
@@ -0,0 +1,63 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file generate-pr-comment.test.mjs
+ * Pins the diffMode caveat contract of generate-pr-comment.js. analyze-pr.js
+ * writes `diffMode` ('three-dot' | 'two-dot') into analysis.json; when the
+ * analysis fell back to the approximate two-dot diff, the PR comment must tell
+ * readers the component list is not exact, so they don't treat an approximate
+ * report as authoritative.
+ */
+
+import {execFileSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {describe, it, expect} from 'vitest';
+
+const SCRIPTS_DIR = path.dirname(fileURLToPath(import.meta.url));
+const SCRIPT = path.join(SCRIPTS_DIR, 'generate-pr-comment.js');
+
+/** Run the comment generator with the given analysis.json; return stdout. */
+function runComment(analysis) {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-comment-'));
+ try {
+ const analysisFile = path.join(dir, 'analysis.json');
+ fs.writeFileSync(analysisFile, JSON.stringify(analysis));
+ return execFileSync(process.execPath, [SCRIPT, '--analysis', analysisFile], {
+ cwd: dir,
+ encoding: 'utf8',
+ });
+ } finally {
+ fs.rmSync(dir, {recursive: true, force: true});
+ }
+}
+
+const baseAnalysis = {
+ newComponents: [],
+ modifiedComponents: ['Card'],
+ componentStats: {},
+ changedPackages: ['@astryxdesign/core'],
+ bundlePackages: [],
+ totalBundle: null,
+};
+
+describe('generate-pr-comment diffMode caveat', () => {
+ it('flags an approximate two-dot analysis in the report', () => {
+ const stdout = runComment({...baseAnalysis, diffMode: 'two-dot'});
+ expect(stdout).toContain('Approximate analysis');
+ expect(stdout).toContain('two-dot fallback');
+ });
+
+ it('does not caveat an exact three-dot analysis', () => {
+ const stdout = runComment({...baseAnalysis, diffMode: 'three-dot'});
+ expect(stdout).not.toContain('Approximate analysis');
+ });
+
+ it('treats a missing diffMode (older analysis.json) as exact', () => {
+ // Back-compat: analyses written before diffMode existed must render clean.
+ const stdout = runComment(baseAnalysis);
+ expect(stdout).not.toContain('Approximate analysis');
+ });
+});