Skip to content

3D engine that AI agents drive

16.3 KB
Raw Blame
// Runs the pull request gates of the `Test` workflow on this checkout, picked by the files the branch changed, and ends with
// one summary line per check and how to fix each failure. Made for people and coding agents alike: every check runs even
// after one fails, full logs go to artifacts/preflight/, and only the tail of a failing log is printed.
// Usage: npm run preflight                 every check that applies to the changed files
//        npm run preflight -- --skip-tests  run only the sub-second unit tests (config/fast-tests.json) and skip the rest and the coverage gate
//        npm run preflight -- --json        print the summary as JSON after the logs
//        npm run preflight -- --base <ref>  compare with another ref than origin/main
import { spawn, spawnSync } from 'node:child_process';
import { createWriteStream, mkdirSync, readFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { isScannedPath } from './check-names.mjs';
import { RELEASE_FIX, readReleaseProblem } from './check-release-changelog.mjs';
import { isFragmentName } from './changelog-fragments.mjs';

const root = fileURLToPath(new URL('..', import.meta.url));
const logFolder = join(root, 'artifacts', 'preflight');
const failingLogTailLines = 30;

// Files a pull request must never edit. Each one is reset by a maintainer at release time.
export const PROTECTED_FILES = Object.freeze({
  'CHANGELOG.md': 'Add a changes/<issue>.md note instead (changes/README.md). Two pull requests that edit CHANGELOG.md conflict.',
  'bench/size/budgets.json': 'Accept a size increase with a changes/<issue>.size.md line instead (bench/size/README.md).',
  'bench/size/baseline.json': 'Accept a size increase with a changes/<issue>.size.md line instead (bench/size/README.md).',
  'bench/tool-cost-budgets.json': 'Accept a tool cost increase with a changes/<issue>.size.md line instead (bench/tool-cost/README.md).',
});

const userFacingPattern = /^(src|bin|packages|skills)\//;
const shippedCodePattern = /^(src|bin|packages)\/|^package(-lock)?\.json$|^(config\/)?vite\.lib\.config\.js$|^scripts\/(build-mcp|vite-cook-optional)\.mjs$/;
const toolCodePattern = /^(src\/agent-|src\/mcp|bin\/)|^scripts\/(build-mcp|measure-tool-cost|check-tool-cost|mcp-server|mcp-(input|union)-errors|capabilities-topics|tool-examples)/;
const linkedTextPattern = /\.(md|html)$/;
const blenderAddonPattern = /^packages\/blender-addon\//;

// Checks a GPU or a person's eyes must run. CONTRIBUTING.md lists them; these patterns only point out the ones that may apply.
const MANUAL_CHECKS = Object.freeze([
  { pattern: /shader|light|material|environment|shadow|tone|fog|sky/, command: 'npm run reference', reason: 'shaders, lighting or materials' },
  { pattern: /skin|gltf|pose|retarget|skeleton|(^|[-/])ik[-.]/, command: 'npm run smoke:character', reason: 'skinning, glTF, poses or IK' },
  { pattern: /^src\/(agent-|mcp)|^bin\//, command: 'npm run smoke:mcp', reason: 'MCP tools or the preview' },
  { pattern: /^src\/(renderer|frame|render)/, command: 'npm run benchmark:runtime', reason: 'the frame path' },
]);

// The same rule the fold uses, so a note this gate accepts is a note changes:fold writes into CHANGELOG.md.
// A note in a subfolder of changes/ or named readme.md passes a looser check and is then skipped by the fold.
function isNewChangeNote(path) {
  const [folder, name, ...deeper] = path.split('/');
  return folder === 'changes' && name !== undefined && deeper.length === 0 && isFragmentName(name);
}

const RELEASE_SKIP_REASON = 'a release: the version changed, so the notes are folded and the budgets reset (CONTRIBUTING.md#release)';

// True when the package version differs between two package.json texts. A release branch is the only change that bumps it.
export function isVersionBump(basePackageText, headPackageText) {
  if (!basePackageText || !headPackageText) return false;
  return JSON.parse(basePackageText).version !== JSON.parse(headPackageText).version;
}

// Decides which checks apply to a list of changed paths. Pure, so tests can call it without git.
// `baseRef` is the ref size and tool cost compare with; `releaseProblem` is what check-release-changelog.mjs found on a release.
// `addedPaths` are the changed paths this branch adds. Only an added note is this pull request's note: a deleted note, or an
// edit of a note another pull request added, is a change to something else. Without it every changed path counts as added.
export function planChecks(changedPaths, { skipTests = false, release = false, baseRef = null, releaseProblem = null, addedPaths = null } = {}) {
  const baseArguments = baseRef ? ['--', '--base', baseRef] : [];
  const paths = changedPaths.map(path => path.replace(/\\/g, '/'));
  const addedNotePaths = (addedPaths ?? changedPaths).map(path => path.replace(/\\/g, '/'));
  const changesShippedCode = paths.some(path => shippedCodePattern.test(path));
  const changesToolCode = paths.some(path => toolCodePattern.test(path));
  const checks = [];

  const protectedEdits = paths.filter(path => path in PROTECTED_FILES);
  checks.push({
    name: 'protected files',
    kind: 'inline',
    skipped: release ? RELEASE_SKIP_REASON : null,
    failure: !release && protectedEdits.length ? protectedEdits.map(path => `${path}: ${PROTECTED_FILES[path]}`).join('\n') : null,
    fix: 'Revert the edits named above: git checkout origin/main -- <file>',
  });

  checks.push({
    name: 'release changelog',
    kind: 'inline',
    skipped: release ? null : 'the version did not change',
    failure: release ? releaseProblem : null,
    fix: RELEASE_FIX,
  });

  const needsNote = !release && paths.some(path => userFacingPattern.test(path));
  const hasNote = addedNotePaths.some(path => isNewChangeNote(path) && !path.endsWith('.size.md'));
  let noteSkipReason = needsNote ? null : 'no change under src/, bin/, packages/ or skills/';
  if (release) noteSkipReason = RELEASE_SKIP_REASON;
  checks.push({
    name: 'change note',
    kind: 'inline',
    skipped: noteSkipReason,
    failure: needsNote && !hasNote ? 'The branch changes shipped code but adds no changes/<issue>.md.' : null,
    fix: 'Add changes/<issue>.md with one bullet, such as "- `bar` takes a `foo` option (#123)" (changes/README.md).',
  });

  checks.push(linkedTextCheck(paths));
  checks.push({
    name: 'names',
    command: ['node', 'scripts/check-names.mjs'],
    skipped: paths.some(path => isScannedPath(path)) ? null : 'no scanned source file changed',
    fix: 'Rename each declaration the report names to what it holds; only i j k n, x y z u v, t, w h, r g b a and _ may stay short (scripts/check-names.mjs).',
  });
  checks.push({
    name: 'tool schemas',
    command: ['npm', 'run', 'check:tool-schemas'],
    skipped: changesToolCode ? null : 'no MCP tool code changed',
    fix: 'Fix the schema the report names; every client profile must accept it (scripts/check-tool-schemas.mjs).',
  });
  checks.push({
    name: 'size',
    command: ['npm', 'run', 'size', ...baseArguments],
    skipped: changesShippedCode ? null : 'no shipped code changed',
    fix: 'Shrink the change, or add "- entry:<name>: <why it is worth the bytes>" to changes/<issue>.size.md (bench/size/README.md).',
  });
  checks.push({
    name: 'tool cost',
    command: ['npm', 'run', 'tool-cost', ...baseArguments],
    skipped: changesToolCode ? null : 'no MCP tool code changed',
    fix: 'Move rare detail into scene_capabilities or a shared schema, or add "- tool:<name>: <why>" to changes/<issue>.size.md (bench/tool-cost/README.md).',
  });

  checks.push({
    name: 'site build',
    command: ['npm', 'run', 'check:site-build'],
    skipped: changesSitePages(paths) ? null : 'no shipped code or page changed',
    fix: 'Fix the module or import error in the log; npm run deploy runs the same build.',
  });

  checks.push({
    name: 'add-on python tests',
    command: ['uv', 'run', '--no-project', '--with', 'numpy', '--with', 'pytest', 'python', '-m', 'pytest', 'packages/blender-addon/tests'],
    skipped: paths.some(path => blenderAddonPattern.test(path)) ? null : 'no file under packages/blender-addon/ changed',
    fix: 'Run the command above and fix the failing test; the add-on tests need no Blender.',
  });

  const testSkip = skipTests ? 'skipped with --skip-tests' : null;
  checks.push({
    name: 'fast tests',
    command: ['node', 'scripts/fast-tests.mjs'],
    skipped: skipTests ? null : 'the full unit test pass below covers them',
    fix: 'Run the failing file alone with node --test tests/<file>.test.js. Never weaken a test to make it pass.',
  });
  checks.push({
    name: 'build library',
    command: ['npm', 'run', 'build:lib'],
    skipped: testSkip,
    fix: 'Fix the build error in the log; the MCP tests need build/.',
  });
  checks.push({
    name: 'unit tests and coverage',
    command: ['npm', 'run', 'test:coverage', '--', '--test-reporter=dot', '--test-reporter-destination=stdout', '--test-reporter=lcov', '--test-reporter-destination=coverage/lcov.info'],
    skipped: testSkip,
    fix: 'Run the failing file alone with node --test tests/<file>.test.js. Never weaken a test to make it pass.',
  });
  checks.push({
    name: 'coverage gate',
    command: ['npm', 'run', 'coverage:gate', '--', 'coverage'],
    skipped: testSkip,
    dependsOn: 'unit tests and coverage',
    fix: 'Add tests for the new code. config/coverage-baseline.json may only go up.',
  });
  return checks;
}

function changesSitePages(paths) {
  return paths.some(path => shippedCodePattern.test(path) || /\.html$|^public\/|^vite\.config\.js$|^scripts\/vite-site-pages\.mjs$/.test(path));
}

function linkedTextCheck(paths) {
  const changesLinkedText = paths.some(path => linkedTextPattern.test(path));
  return {
    name: 'links',
    command: ['npm', 'run', 'check:links'],
    skipped: changesLinkedText ? null : 'no Markdown or HTML changed',
    fix: 'Fix or remove each link the report names.',
  };
}

export function manualChecksFor(changedPaths) {
  const paths = changedPaths.map(path => path.replace(/\\/g, '/'));
  return MANUAL_CHECKS.filter(check => paths.some(path => check.pattern.test(path)));
}

function git(args, folder = root) {
  const result = spawnSync('git', args, { cwd: folder, encoding: 'utf8' });
  return result.status === 0 ? result.stdout.trim() : null;
}

// --no-renames lists both sides of a rename, so a moved protected file still counts as a change to it.
export function changedPathsSince(baseRef, folder = root) {
  const mergeBase = git(['merge-base', baseRef, 'HEAD'], folder) ?? baseRef;
  const committedAndStaged = git(['diff', '--name-only', '--no-renames', mergeBase], folder) ?? '';
  const untracked = git(['ls-files', '--others', '--exclude-standard'], folder) ?? '';
  const lines = `${committedAndStaged}\n${untracked}`.split('\n').filter(Boolean);
  return [...new Set(lines)].sort();
}

// The paths the branch adds (committed, staged or untracked), for the change note check.
export function addedPathsSince(baseRef, folder = root) {
  const mergeBase = git(['merge-base', baseRef, 'HEAD'], folder) ?? baseRef;
  const committedAndStaged = git(['diff', '--name-only', '--no-renames', '--diff-filter=A', mergeBase], folder) ?? '';
  const untracked = git(['ls-files', '--others', '--exclude-standard'], folder) ?? '';
  const lines = `${committedAndStaged}\n${untracked}`.split('\n').filter(Boolean);
  return [...new Set(lines)].sort();
}

function logPathFor(check) {
  return join(logFolder, `${check.name.replace(/\W+/g, '-')}.log`);
}

function runCommand(check) {
  return new Promise(resolvePromise => {
    const logPath = logPathFor(check);
    const log = createWriteStream(logPath);
    const [program, ...args] = check.command;
    const child = spawn(program, args, { cwd: root, shell: process.platform === 'win32', env: process.env });
    child.stdout.pipe(log, { end: false });
    child.stderr.pipe(log, { end: false });
    child.on('close', code => {
      log.end(() => resolvePromise({ code, logPath }));
    });
  });
}

function tailOf(path, lineCount) {
  const lines = readFileSync(path, 'utf8').trimEnd().split(/\r?\n/);
  return lines.slice(-lineCount).join('\n');
}

async function runChecks(checks) {
  const results = [];
  for (const check of checks) {
    const dependency = check.dependsOn && results.find(result => result.name === check.dependsOn);
    if (check.skipped) {
      results.push({ name: check.name, status: 'skip', detail: check.skipped });
    } else if (dependency && dependency.status !== 'ok') {
      results.push({ name: check.name, status: 'skip', detail: `${check.dependsOn} did not pass` });
    } else if (check.kind === 'inline') {
      results.push(check.failure
        ? { name: check.name, status: 'fail', detail: check.failure, fix: check.fix }
        : { name: check.name, status: 'ok' });
    } else {
      process.stdout.write(`… ${check.name}: ${check.command.join(' ')}\n`);
      const startedAt = Date.now();
      const { code, logPath } = await runCommand(check);
      const seconds = Math.round((Date.now() - startedAt) / 1000);
      const log = logPath.slice(root.length).replace(/\\/g, '/');
      if (code === 0) {
        results.push({ name: check.name, status: 'ok', detail: `${seconds} s`, log });
      } else {
        process.stdout.write(`\n--- last ${failingLogTailLines} lines of ${log} ---\n${tailOf(logPath, failingLogTailLines)}\n---\n\n`);
        results.push({ name: check.name, status: 'fail', detail: `exit ${code} after ${seconds} s`, log, fix: check.fix });
      }
    }
  }
  return results;
}

function printSummary(results, manualChecks) {
  const marks = { ok: 'ok  ', fail: 'FAIL', skip: 'skip' };
  console.log('\nPreflight summary');
  for (const result of results) {
    const detail = result.detail ? ` (${result.detail.split('\n')[0]})` : '';
    console.log(`${marks[result.status]} ${result.name}${detail}`);
    if (result.status === 'fail') {
      for (const line of result.detail.split('\n').slice(1)) console.log(`     ${line}`);
      if (result.log) console.log(`     log: ${result.log}`);
      console.log(`     fix: ${result.fix}`);
    }
  }
  if (manualChecks.length) {
    console.log('\nNot run here, may apply (they need a GPU or a person, see CONTRIBUTING.md):');
    for (const check of manualChecks) console.log(`     ${check.command}  for ${check.reason}`);
  }
}

function valueAfter(args, flag) {
  const index = args.indexOf(flag);
  return index < 0 ? null : args[index + 1];
}

// Builds the checks for one run: what changed since `baseRef`, whether it is a release, and the plan with `baseRef` handed to size and tool cost.
export async function planFromArguments(args, baseRef, folder = root) {
  const changedPaths = changedPathsSince(baseRef, folder);
  const release = isVersionBump(git(['show', `${baseRef}:package.json`], folder), readFileSync(join(folder, 'package.json'), 'utf8'));
  const releaseProblem = release ? await readReleaseProblem(folder) : null;
  const checks = planChecks(changedPaths, { skipTests: args.includes('--skip-tests'), release, baseRef, releaseProblem, addedPaths: addedPathsSince(baseRef, folder) });
  return { changedPaths, checks };
}

async function main(args) {
  const baseRef = valueAfter(args, '--base') ?? 'origin/main';
  if (baseRef === 'origin/main' && git(['fetch', '--quiet', 'origin', 'main']) === null) {
    console.warn('Could not fetch origin/main; comparing with the local copy, which may be stale.');
  }
  if (git(['rev-parse', '--verify', '--quiet', `${baseRef}^{commit}`]) === null) {
    console.error(`The base ref ${baseRef} does not exist. Fetch it, or pass --base <ref>.`);
    process.exit(2);
  }
  mkdirSync(logFolder, { recursive: true });
  mkdirSync(join(root, 'coverage'), { recursive: true });

  const { changedPaths, checks } = await planFromArguments(args, baseRef);
  console.log(`${changedPaths.length} files changed since ${baseRef}.`);
  const results = await runChecks(checks);
  const manualChecks = manualChecksFor(changedPaths);
  printSummary(results, manualChecks);
  if (args.includes('--json')) {
    console.log(JSON.stringify({ base: baseRef, results, manualChecks: manualChecks.map(check => check.command) }, null, 2));
  }
  const failed = results.some(result => result.status === 'fail');
  console.log(failed ? '\nPreflight failed. The Test workflow would fail the same way.' : '\nPreflight passed.');
  process.exit(failed ? 1 : 0);
}

if (import.meta.url === pathToFileURL(resolve(process.argv[1] ?? '')).href) {
  await main(process.argv.slice(2));
}