Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

37.8 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
 * @file  verify-golden.mjs — carry each golden project through package → launch → drive → compare.
 *
 * The registry says which projects a tier certifies and for which targets; this
 * runs that matrix. Web, playable and the mini-game targets are driven here
 * because all three launch in the Electron already on hand; desktop/android/ios
 * packages go to verify-desktop-render and verify-native-boot, which own their
 * toolchains.
 *
 * Reports every pair rather than stopping at the first failure — a release
 * argument wants the whole matrix, not the first thing that broke.
 *
 *   node tools/verify-golden.mjs --tier pr
 *   node tools/verify-golden.mjs --tier nightly --only platformer,spine-demo
 *   node tools/verify-golden.mjs --tier pr --shots <dir>
 *   node tools/verify-golden.mjs --tier pr --shard 2/4
 *
 * A shard is one machine's share of the tier, launched one project at a time;
 * `ESTELLA_GOLDEN_SHARD` names it where the command line is not the runner's to
 * write, as in the release gate. `--jobs` is for a machine running the whole list.
 */
import { spawn, spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { mkdirSync, rmSync, readFileSync, existsSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { atTier, sharesOf, projectDir, parityFor, interactFor, audioFor, suspendFor, safeAreaFor, atlasFor, subpackageFor, firstScreenFor, packagePixels, launchTimeoutFor, ROOT } from './goldenProjects.mjs';
import { frameDistance, frameCellMax, readPNG } from './frameCompare.mjs';
import { suspendDrive } from './lib/suspendDrive.mjs';
import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict, failureLines } from './lib/deadGpu.mjs';
import { NEEDS_XVFB, runWithElectronDisplay, runElectron, ensureElectronBinary } from './lib/electronRun.mjs';
import { requireCurrentEngine } from './lib/engineBuild.mjs';

const argv = process.argv.slice(2);
const flag = (n, d) => {
  const i = argv.indexOf(`--${n}`);
  return i >= 0 && argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : d;
};

const TIER = flag('tier', 'pr');
const ONLY = flag('only', '');
const shotsArg = flag('shots', '');
const SHOTS = shotsArg ? path.resolve(shotsArg) : '';
const WORK = path.resolve(flag('work', path.join(ROOT, '.golden')));
const NO_PARITY = argv.includes('--no-parity');
/** Targets this runner owns; the rest belong to the native/desktop verifiers.
 *  A mini-game needs its vendor's globals, so it goes through its own launcher. */
const OWNED = new Set(['web', 'playable', 'wechat', 'douyin']);
const MINIGAME = new Set(['wechat', 'douyin']);
const LAUNCHER = (target) => path.join(ROOT, 'tools', 'launchers',
  MINIGAME.has(target) ? 'launch-minigame.mjs' : 'launch-export.mjs');
/** Targets whose surface this runner can size to the editor's, which is what
 *  makes a frame comparable at all. Measured on one project: web 0.0009,
 *  playable 0.0027, wechat 0.0027 — the packaging wrapper is not what differs. */
const COMPARABLE = OWNED;

const DESKTOP = path.join(ROOT, 'desktop');

/** A throttled boot is the whole package over a phone's link — minutes, not the
 *  30s an unthrottled launch gets. */
const FIRST_SCREEN_TIMEOUT_MS = 180_000;

/**
 * Where the engine binary a target packages comes from, and which build variant
 * it is. Checked PER TARGET: the variants are separate builds, and a stale one
 * does not announce itself — the package boots, fails its ABI handshake and
 * reads as the game being broken.
 */
const ENGINE_OF = (target) => (target === 'wechat'
  ? { dir: path.join(ROOT, 'build', 'wasm', 'wechat'), variant: 'wechat' }
  : { dir: path.join(DESKTOP, 'public', 'wasm'), variant: 'web' });
if (SHOTS) mkdirSync(SHOTS, { recursive: true });

/**
 * The summary took the LAST line of a recorded reason, which for the exporter's
 * JSON report is `}`. So a run whose only red was a missing wechat runtime said
 * `✗ input-actions wechat (package) — }` and a reader learned nothing.
 */
function lastSpeakingLine(why) {
  const said = why.split('\n').map((l) => l.trim()).filter((l) => /[A-Za-z]/.test(l));
  return (said.pop() ?? why.trim()).slice(0, 300);
}

/** The exporter states its reasons in `errors`; anything else is its shape. */
function exporterReason(out) {
  const at = out.indexOf('{');
  if (at >= 0) {
    try {
      const errors = JSON.parse(out.slice(at))?.errors;
      if (Array.isArray(errors) && errors.length) return errors.join('; ').slice(0, 300);
    } catch { /* not a report — the trailing text is the best there is */ }
  }
  return out.slice(-300);
}

const only = ONLY ? new Set(ONLY.split(',').map((s) => s.trim())) : null;
const tiered = atTier(TIER).filter((g) => !only || only.has(g.id));
const shardArg = flag('shard', argv.includes('--worker') ? '' : (process.env.ESTELLA_GOLDEN_SHARD ?? ''));
const shard = /^\d+\/\d+$/.test(shardArg)
  ? { index: Number(shardArg.split('/')[0]) - 1, of: Number(shardArg.split('/')[1]) }
  : null;
if (shardArg && (!shard || shard.index < 0 || shard.index >= shard.of)) {
  console.error(`golden: a shard is i/n with 1 <= i <= n, e.g. 2/4 — got "${shardArg}"`);
  process.exit(2);
}
if (NEEDS_XVFB) {
  const run = runWithElectronDisplay([fileURLToPath(import.meta.url), ...argv], { cwd: ROOT, stdio: 'inherit' });
  if (run.error) throw run.error;
  process.exit(run.status ?? 1);
}
const projects = shard ? sharesOf(tiered, shard.of, OWNED)[shard.index] : tiered;
if (shard) {
  console.log(`golden ${TIER}: shard ${shard.index + 1}/${shard.of} — ${projects.map((g) => g.id).join(', ') || 'nothing'}`
    + ` of ${tiered.length} project(s)`);
}

const pairs = projects.flatMap((g) => g.targets.filter((t) => OWNED.has(t)).map((t) => ({ id: g.id, target: t })));
const deferred = projects.flatMap((g) => g.targets.filter((t) => !OWNED.has(t)).map((t) => `${g.id}:${t}`));

console.log(`golden ${TIER}: ${projects.length} project(s), ${pairs.length} pair(s) here`
  + (deferred.length ? `, ${deferred.length} left to the platform verifiers` : ''));

/**
 * `--jobs N`: the projects, N at a time, each worker this script over its share.
 * The release tier's seventeen took 33 minutes in a row with three cores idle.
 */
const JOBS = shard ? 1 : Math.max(1, Number(flag('jobs', '1')) || 1);
/** How many launches share this runner's rasterizer: a worker's, or one. */
const SHARE = argv.includes('--worker') ? JOBS : 1;
if (JOBS > 1 && !argv.includes('--worker') && projects.length > 1) {
  ensureElectronBinary();
  const bins = sharesOf(projects, Math.min(JOBS, projects.length), OWNED).map((share) => ({ ids: share.map((g) => g.id) }));
  // `--jobs` goes through: a worker does not fork again (`--worker`), and it
  // needs the count to know what it shares the rasterizer with — see SHARE.
  const own = new Set(['--only']);
  const passthrough = argv.filter((a, i) => !own.has(a) && !own.has(argv[i - 1]));
  const outcomes = await Promise.all(bins.map((bin, i) => new Promise((resolve) => {
    const tag = `[${i + 1}/${bins.length}]`;
    const child = spawn(process.execPath,
      [fileURLToPath(import.meta.url), ...passthrough, '--worker', '--only', bin.ids.join(',')],
      { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'] });
    // Relayed as they come, tagged: a worker's line is only readable if it says
    // whose it is, and a runner staring at nothing for ten minutes kills the job.
    let rest = '';
    const relay = (chunk) => {
      rest += chunk;
      const lines = rest.split('\n');
      rest = lines.pop();
      for (const l of lines) console.log(`${tag} ${l}`);
    };
    child.stdout.on('data', relay);
    child.stderr.on('data', relay);
    child.on('close', (code) => {
      if (rest) console.log(`${tag} ${rest}`);
      resolve({ tag, code, ids: bin.ids });
    });
  })));
  const failed = outcomes.filter((o) => o.code !== 0);
  console.log(`\ngolden ${TIER}: ${bins.length} worker(s) over ${projects.length} project(s) — `
    + (failed.length ? `${failed.length} did not pass` : 'all packaged and launched'));
  for (const f of failed) console.log(`  ✗ ${f.tag} exit ${f.code}: ${f.ids.join(', ')}`);
  process.exit(failed.length ? 1 : 0);
}

/** What the project declares: the shape it is authored for, and the scene a
 *  package will ship as its entry. */
function manifestOf(id) {
  try {
    return JSON.parse(readFileSync(path.join(projectDir(id), 'project.esproject'), 'utf8'));
  } catch {
    return {};
  }
}

/**
 * The surface a project is drawn on, in the aspect it is authored for. A package
 * derives its orientation gate from that aspect, and a point is a fraction of the
 * surface — so the editor's frame and the package's take this one size, whether or
 * not the editor is opened at all.
 */
function surfaceOf(id) {
  const r = manifestOf(id).designResolution;
  const a = r?.width > 0 && r?.height > 0 ? { w: r.width, h: r.height } : { w: 16, h: 9 };
  const major = 820;
  return a.h >= a.w
    ? { width: Math.round((major * a.w) / a.h), height: major }
    : { width: major, height: Math.round((major * a.h) / a.w) };
}

/**
 * A failed attempt's whole output, beside the frames: CI uploads that directory when a
 * run fails, and the lines printed inline are a summary of what only this file holds.
 */
function keepAttempt(label, n, output) {
  try {
    writeFileSync(path.join(WORK, `${label}-attempt${n}.log`), String(output ?? ''));
  } catch { /* the log is evidence, not a verdict */ }
}

/**
 * The editor's frame of the running game, through the screenshot hook the UI
 * shots use. The play panel takes the project's aspect first: a portrait game on
 * a landscape surface draws the package's rotate gate against the editor's
 * letterbox — a difference of surface, not of game.
 */
function captureEditorFrame(id, out, timeoutMs) {
  const panel = surfaceOf(id);
  const attempt = () => {
    // A partial file from the attempt before would be read as this attempt's frame.
    rmSync(out, { force: true });
    const r = runElectron(['.'], {
      via: 'npx',
      encoding: 'utf8',
      cwd: DESKTOP,
      env: {
        ESTELLA_SHOT: out,
        ESTELLA_SHOT_PROJECT: projectDir(id),
        // The scene the PACKAGE ships, named rather than inherited: opening a
        // project reopens whatever was last open, which is untracked local state —
        // so the two sides would differ by a developer's workspace file.
        ...(manifestOf(id).defaultScene ? { ESTELLA_SHOT_SCENE: manifestOf(id).defaultScene } : {}),
        ESTELLA_SHOT_PLAY: '1',
        ESTELLA_SHOT_CROP: 'iframe[title="Game"]',
        ESTELLA_SHOT_EVAL: `window.__estellaEditor.setPanelSize('viewport', ${JSON.stringify(panel)})`,
        // The settle window the PACKAGE gets, so one policy covers both sides:
        // a project that needs longer needs it wherever its frames are drawn.
        ...(timeoutMs ? { ESTELLA_SHOT_PLAY_TIMEOUT: String(timeoutMs) } : {}),
        ESTELLA_WIN_W: '1500',
        ESTELLA_WIN_H: '1040',
      },
    });
    const output = `${r.stdout ?? ''}${r.stderr ?? ''}`;
    // The frame on disk is the verdict here: without one the editor never got
    // far enough to judge anything, whatever the log says about the GPU.
    if (!existsSync(out)) return { ok: false, output, measured: false };
    // A frame the editor was killed halfway through writing is not a frame, and
    // decoding it throws where a verdict belongs: one project's dead GPU took
    // the whole matrix's report with it.
    try {
      const png = readPNG(readFileSync(out));
      return { ok: true, output, w: png.w, h: png.h };
    } catch (e) {
      // Half a PNG is a run that was killed mid-write, not a frame to judge.
      return { ok: false, output: `${output}\nthe capture is not a whole PNG: ${e.message}`, measured: false };
    }
  };
  let tries = 0;
  const run = retryOnDeadGpu(attempt, (noVerdict, last) => {
    keepAttempt(`${id}-editor`, ++tries, last.output);
    console.log(`↻ ${id} — ${noVerdict ? 'no editor frame' : 'a blank editor frame after an outage'}; capturing again`);
    for (const l of failureLines(last.output)) console.log(`    ${l}`);
  });
  if (!run.ok) {
    return {
      ok: false,
      why: run.launchFailed
        ? `${launchNeverHappenedVerdict('the editor\'s frame')} (${run.output.trim().slice(-200)})`
        : run.gpuDied
          ? `${deadGpuVerdict('the editor\'s frame')} (${failureLines(run.output).join(' | ')})`
          : `no frame on any attempt: ${failureLines(run.output).join(' | ')}`,
    };
  }
  return { ok: true, w: run.w, h: run.h };
}

/** Points the package's own frame failed to draw, as messages. Empty = all held. */
function probePackagePixels(png, points) {
  let frame;
  try {
    frame = readPNG(readFileSync(png));
  } catch (e) {
    return [`the package capture is not a whole PNG: ${e.message}`];
  }
  const missed = [];
  for (const pt of points) {
    const x = Math.min(frame.w - 1, Math.max(0, Math.round(pt.x * (frame.w - 1))));
    const y = Math.min(frame.h - 1, Math.max(0, Math.round(pt.y * (frame.h - 1))));
    const got = frame.px(x, y);
    const tol = pt.tol ?? 30;
    if (!got.every((c, k) => Math.abs(c - pt.rgb[k]) <= tol)) {
      missed.push(`${pt.what ?? `${pt.x}x${pt.y}`}: want [${pt.rgb}] ±${tol}, got [${got}]`);
    }
  }
  return missed;
}

/** Where a packaged run says the named entities are, or null if it never said. */
function probePositions(target, dir, w, h, names, extra = []) {
  const r = launchPackage('probe', target, [
    '--dir', dir, '--w', String(w), '--h', String(h),
    '--probe', names.join(','), ...extra,
  ]);
  const line = (r.stdout || '').split('\n').find((l) => l.includes('probe:'));
  if (!line) return null;
  try {
    return JSON.parse(line.slice(line.indexOf('{'))).at ?? null;
  } catch { return null; }
}

/**
 * Launch a package, once more if the GPU never came up. Every launch here goes
 * through this: the runner kills the first electron of a job often enough that a
 * per-call-site answer means some launches retry and others report a broken game.
 */
function launchPackage(id, target, args) {
  // Three launches on one rasterizer each get the wait three would: alone,
  // model-import settles 30 frames inside its 30 s; beside two workers it made 22.
  const at = args.indexOf('--timeout');
  const timeoutMs = (at >= 0 ? Number(args[at + 1]) : 180_000) * SHARE;
  const patient = at >= 0
    ? [...args.slice(0, at + 1), String(timeoutMs), ...args.slice(at + 2)]
    : [...args, '--timeout', String(timeoutMs)];
  const run = retryOnDeadGpu(
    () => {
      // Opening a PACKAGE needs no editor — the launchers are engine-side, so a
      // checkout without one can still judge what it shipped.
      const launchArgs = [LAUNCHER(target), ...patient];
      // Present CI's browser packages on the owned virtual display so Chromium
      // does not throttle their requestAnimationFrame callbacks as hidden pages.
      if (process.env.CI && !MINIGAME.has(target) && !launchArgs.includes('--present')) launchArgs.push('--present');
      const r = runElectron(launchArgs, { encoding: 'utf8', cwd: ROOT });
      // The launcher prints a ✓/✗ line once it has looked at the frame; that
      // line existing is what says a measurement happened.
      const verdictLine = (r.stdout || '').split('\n').find((l) => l.startsWith('✓') || l.startsWith('✗'));
      return {
        ok: r.status === 0,
        output: `${r.stdout ?? ''}${r.stderr ?? ''}`,
        measured: Boolean(verdictLine),
        drew: verdictLine ? verdictLine.startsWith('✓') : undefined,
        r,
      };
    },
    (noVerdict, last) => {
      console.log(`↻ ${id} ${target} — ${noVerdict ? 'no verdict from the package' : 'a blank frame after an outage'}; launching again`);
      for (const l of failureLines(last.output)) console.log(`    ${l}`);
    },
  );
  // gpuDied travels with the result: without it a runner whose GPU never came up
  // is reported as a game that draws nothing, which is the one confusion this
  // whole retry exists to prevent.
  return { ...run.r, gpuDied: Boolean(run.gpuDied), launchFailed: Boolean(run.launchFailed) };
}

const fmtMB = (bytes) => `${(bytes / 1024 / 1024).toFixed(2)}MB`;

/**
 * The limits the export judged itself against, from the report it already
 * writes. Read rather than recomputed: the pipeline owns what a package weighs
 * and which limits are in force, and a second opinion here is the one that
 * drifts. No verdicts is not the same as being over one.
 */
function readSizeVerdicts(reportPath) {
  try {
    return JSON.parse(readFileSync(reportPath, 'utf8')).size?.verdicts ?? [];
  } catch {
    return [];
  }
}

// Every engine build this tier's pairs will package, before any of them runs:
// finding out at pair fourteen costs the thirteen packages before it.
for (const key of new Set(pairs.map((p) => JSON.stringify(ENGINE_OF(p.target))))) {
  const { dir, variant } = JSON.parse(key);
  requireCurrentEngine(ROOT, dir, process.argv, variant);
  // A package picks the lite engine from the directory beside the WeChat one when its content allows, so that binary is
  // judged too — a stale one boots, fails its ABI handshake and reads as the game being broken.
  const lite = `${dir}-lite`;
  if (variant === 'wechat' && existsSync(lite)) requireCurrentEngine(ROOT, lite, process.argv, 'wechat-lite');
}

const results = [];
for (const { id, target } of pairs) {
  const out = path.join(WORK, `${id}-${target}`);
  rmSync(out, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });

  const sizeReport = path.join(WORK, `${id}-${target}.size.json`);
  const exported = spawnSync(process.execPath, [
    path.join(ROOT, 'pipeline', 'bin', 'estella.mjs'), 'export', projectDir(id),
    '--platform', target, '--out', out, '--json', sizeReport,
  ], { encoding: 'utf8', cwd: ROOT });

  if (exported.status !== 0) {
    // The reason was recorded and never printed, so the whole account of a
    // failed export was two words. It is the only place the exporter speaks.
    const why = (exported.stderr || exported.stdout || '').trim();
    results.push({ id, target, stage: 'package', ok: false, why: exporterReason(why) });
    console.log(`✗ ${id} ${target} — package failed`);
    for (const l of why.split('\n').slice(-8)) if (l.trim()) console.log(`    ${l}`);
    continue;
  }

  // A package that does not fit is one nobody can ship. A stage rather than an
  // export flag, so a build that is over still launches and reports everything
  // else it can — the way an over-tolerance parity does.
  for (const verdict of readSizeVerdicts(sizeReport)) {
    const ok = verdict.status !== 'over';
    results.push({
      id, target, stage: 'size', ok,
      why: ok ? '' : `${fmtMB(verdict.measuredBytes)} against a ${fmtMB(verdict.budget.maxBytes)} limit`
        + ` (${verdict.ratio.toFixed(2)}x) — ${verdict.budget.note ?? verdict.budget.scope}`,
    });
    if (!ok) console.log(`✗ ${id} ${target} — over the ${fmtMB(verdict.budget.maxBytes)} limit at ${fmtMB(verdict.measuredBytes)}`);
  }

  // Parity compares like for like, so the package is opened at exactly the size
  // the editor's play surface came out — never a guessed one.
  const golden = atTier(TIER).find((g) => g.id === id);
  // Declared up here because four launches below need it and three were taking
  // the 30 s default: a project's settle window is the project's, whichever
  // question this is asking of its package.
  const timeoutMs = launchTimeoutFor(golden);

  // Did the cook actually pack? Parity cannot answer this: an atlas that stopped
  // working still draws the same frame from nine standalone textures.
  const atlas = atlasFor(golden);
  if (atlas) {
    let packed = 0;
    try {
      const manifest = JSON.parse(readFileSync(path.join(out, 'asset-manifest.json'), 'utf8'));
      for (const group of Object.values(manifest.groups ?? {})) {
        for (const asset of Object.values(group.assets ?? {})) {
          if (asset.metadata?.atlasPage !== undefined) packed++;
        }
      }
    } catch (e) {
      packed = -1;
    }
    const ok = packed === atlas.packed;
    results.push({
      id, target, stage: 'atlas', ok,
      why: ok ? '' : `${packed} texture(s) came out packed, the project claims ${atlas.packed}`,
    });
    console.log(`${ok ? '✓' : '✗'} ${id} ${target} — atlas: ${packed} texture(s) packed into a page`);
  }
  // Did anything reach the audio output? A frame cannot answer it: the control
  // that starts the sound redraws itself either way. So drive the toggle and read
  // the height the game wrote from an analyser bin — silence leaves it on its floor.
  const audio = audioFor(golden);
  if (golden.settings && target === 'web') {
    const settingsLog = path.join(WORK, `${id}-${target}-settings.log`);
    const run = spawnSync(process.execPath, [path.join(ROOT, 'tools/verify-settings.mjs'),
      '--dir', out, '--spec', JSON.stringify(golden.settings), '--log', settingsLog,
      ...(process.env.CI ? ['--present'] : [])], { cwd: ROOT, encoding: 'utf8' });
    const ok = run.status === 0;
    results.push({ id, target, stage: 'settings', ok, why: ok ? '' : String(run.stderr || run.stdout || 'settings verifier did not complete')
      + `\nsettings verifier exit=${run.status ?? 'none'}, signal=${run.signal ?? 'none'}; full launch trace: ${settingsLog}` });
    console.log(`${ok ? '✓' : '✗'} ${id} ${target} — settings process restart`);
    for (const line of (run.stdout || '').trim().split('\n')) if (line) console.log(`    ${line}`);
    if (!ok) for (const line of (run.stderr || '').trim().split('\n').slice(-8)) if (line) console.log(`    ${line}`);
  }
  const authoredAudio = golden.authoredAudio;
  if (authoredAudio && target === 'web') {
    // This gesture only unlocks browser audio. It avoids every pad and the
    // music toggle, so only the scene's authored source can drive the spectrum.
    const probed = launchPackage(id, target, [
      '--dir', out,
      ...(timeoutMs ? ['--timeout', String(timeoutMs)] : []),
      '--input', JSON.stringify({ pointer: authoredAudio.unlock, frames: authoredAudio.frames ?? 60 }),
      '--probe', authoredAudio.bars.join(','),
    ]);
    const found = /probe: (\{.*\})/.exec(probed.stdout || '');
    let heights = [];
    try {
      const at = found ? JSON.parse(found[1]).at : null;
      heights = authoredAudio.bars.map(name => at?.[name]?.h);
    } catch { heights = []; }
    const complete = heights.length === authoredAudio.bars.length && heights.length > 0
      && heights.every(height => typeof height === 'number' && Number.isFinite(height));
    const peak = complete ? Math.max(...heights) : null;
    const ok = probed.status === 0 && peak !== null && peak > authoredAudio.floor;
    results.push({ id, target, stage: 'authored-audio', ok,
      why: ok ? '' : probed.status !== 0 ? `authored audio probe exited with ${probed.status ?? 'no status'}`
        : peak === null ? 'authored source spectrum bars were not readable'
        : `authored source spectrum stayed at ${peak} (floor ${authoredAudio.floor})` });
    console.log(`${ok ? '✓' : '✗'} ${id} ${target} — authored audio: peak ${peak ?? 'unreadable'} (silent floor ${authoredAudio.floor})`);
  }
  if (audio && target === 'web') {
    const probed = launchPackage(id, target, [
      '--dir', out,
      ...(timeoutMs ? ['--timeout', String(timeoutMs)] : []),
      '--input', JSON.stringify({ pointer: { x: audio.toggle.x, y: audio.toggle.y }, frames: audio.frames ?? 60 }),
      '--probe', audio.bar,
    ]);
    const found = /probe: (\{.*\})/.exec(probed.stdout || '');
    let height = null;
    try { height = found ? JSON.parse(found[1]).at?.[audio.bar]?.h ?? null : null; } catch { height = null; }
    const ok = typeof height === 'number' && height > audio.floor;
    results.push({
      id, target, stage: 'audio', ok,
      why: ok ? '' : height === null
        ? `${audio.bar} did not report a height — the probe saw no such UI node`
        : `${audio.bar} stayed at ${height} (floor ${audio.floor}); nothing reached the output`,
    });
    console.log(`${ok ? '✓' : '✗'} ${id} ${target} — audio: ${audio.bar} at ${height ?? 'nothing'} (silent floor ${audio.floor})`);
  }

  // When the start screen appears, over a link a phone actually has. A machine
  // with a fast one cannot ask this: the whole boot lands inside the fade.
  const firstScreen = target === 'web' ? firstScreenFor(golden) : null;
  if (firstScreen) {
    const run = launchPackage(id, target, [
      '--dir', out, '--boot', '--throttle', firstScreen.link,
      '--timeout', String(FIRST_SCREEN_TIMEOUT_MS), '--settle', '10',
    ]);
    const found = /boot: (\{.*\})/.exec(run.stdout || '');
    let boot = null;
    try { boot = found ? JSON.parse(found[1]) : null; } catch { boot = null; }
    const at = boot?.firstScreenMs ?? null;
    const still = boot?.stillMs ?? Infinity;
    const ok = typeof at === 'number' && at <= firstScreen.maxMs && boot?.monotonic === true
      && still <= firstScreen.maxStillMs;
    results.push({
      id, target, stage: 'first-screen', ok,
      why: ok ? '' : at === null
        ? 'the page reported no first paint at all — nothing was on screen to time'
        : at > firstScreen.maxMs
          ? `the start screen appeared at ${at}ms over ${firstScreen.link} (limit ${firstScreen.maxMs}ms)`
          : boot?.monotonic === false ? 'the start screen\'s progress went backwards'
            : `the bar stood still for ${still}ms (limit ${firstScreen.maxStillMs}ms) — a stage is`
              + ' reporting nothing for its whole length',
    });
    console.log(`${ok ? '✓' : '✗'} ${id} ${target} — start screen at ${at ?? 'never'}ms over ${firstScreen.link}`
      + `, first frame at ${boot?.firstFrameMs ?? 'never'}ms, still at most ${still}ms`);
  }

  // The 分包 the host REFUSES. The ordinary launch above already proved the happy
  // half (this package's engine rides a 分包, so it booted through one); what has
  // never run is the half a player on a bad network gets.
  const subpackage = MINIGAME.has(target) ? subpackageFor(golden) : null;
  if (subpackage) {
    const refused = launchPackage(id, target, [
      '--dir', out,
      ...(timeoutMs ? ['--timeout', String(timeoutMs)] : []),
      '--fail-subpackage', subpackage.name,
    ]);
    const ok = refused.status === 0;
    results.push({
      id, target, stage: 'subpackage-refused', ok,
      why: ok ? '' : `the package said nothing when ${subpackage.name} was refused`
        + ' — a black frame is all a player would get',
    });
    console.log(`${ok ? '✓' : '✗'} ${id} ${target} — ${subpackage.name} refused:`
      + ` the package ${ok ? 'said why' : 'said nothing'}`);
  }

  const tolerance = COMPARABLE.has(target) && !NO_PARITY ? parityFor(golden) : null;
  const editorPng = path.join(WORK, `${id}-editor.png`);
  // ...times SHARE, as launchPackage already does for the other side: both frames
  // are taken on ONE rasterizer and only the package's half knew it was sharing.
  const editor = tolerance != null
    ? captureEditorFrame(id, editorPng, timeoutMs ? timeoutMs * SHARE : timeoutMs)
    : null;
  if (editor && !editor.ok) {
    results.push({ id, target, stage: 'editor-frame', ok: false, why: editor.why });
    console.log(`✗ ${id} ${target} — the editor never produced a play frame`);
    continue;
  }

  const packagePng = SHOTS ? path.join(SHOTS, `${id}-${target}.png`) : path.join(WORK, `${id}-${target}.png`);
  const surface = editor ? { width: editor.w, height: editor.h } : surfaceOf(id);
  const launch = launchPackage(id, target, [
    '--dir', out, '--out', packagePng,
    '--w', String(surface.width), '--h', String(surface.height),
    ...(timeoutMs ? ['--timeout', String(timeoutMs)] : []),
  ]);

  const line = (launch.stdout || '').split('\n').find((l) => l.startsWith('✓') || l.startsWith('✗')) ?? '';
  if (launch.status !== 0) {
    const why = launch.launchFailed
      ? `${launchNeverHappenedVerdict('the game')} (${(launch.stderr || '').trim().slice(-200)})`
      : launch.gpuDied
        ? `${deadGpuVerdict('the game')} (${line || 'no frame'})`
        : line || 'launch failed';
    results.push({ id, target, stage: 'launch', ok: false, why });
    console.log(`✗ ${id} ${target} — ${why}`);
    // Deep enough to carry the launcher's own diagnosis of a package that never
    // drew — six lines cut it off at the headline.
    for (const l of (launch.stdout || '').split('\n').slice(-20)) if (l.trim()) console.log(`    ${l}`);
    continue;
  }

  // What the PACKAGE drew, before comparing it to anything. Parity is an A/B, so
  // a capability the package lost and the editor never had passes it; the launch
  // check passes any frame that is not one flat colour.
  const wanted = COMPARABLE.has(target) ? packagePixels(golden) : null;
  if (wanted) {
    const missed = probePackagePixels(packagePng, wanted);
    if (missed.length > 0) {
      results.push({ id, target, stage: 'pixels', ok: false, why: missed.join('; ') });
      console.log(`✗ ${id} ${target} — the package did not draw what it claims`);
      for (const m of missed) console.log(`    ${m}`);
      continue;
    }
  }

  if (tolerance == null) {
    results.push({ id, target, stage: 'launch', ok: true });
    console.log(`✓ ${id} ${target}${wanted ? ` — ${wanted.length} point(s) drawn` : ''}`);
    continue;
  }

  let distance;
  try {
    distance = frameDistance(readFileSync(editorPng), readFileSync(packagePng));
  } catch (e) {
    results.push({ id, target, stage: 'parity', ok: false, why: e.message });
    console.log(`✗ ${id} ${target} — parity: ${e.message}`);
    continue;
  }
  if (distance > tolerance) {
    results.push({ id, target, stage: 'parity', ok: false, why: `distance ${distance.toFixed(4)} > ${tolerance}` });
    console.log(`✗ ${id} ${target} — parity ${distance.toFixed(4)} (limit ${tolerance})`);
    console.log(`    the package is not showing what the editor showed; compare ${editorPng} and ${packagePng}`);
    continue;
  }

  // Does the package answer a keyboard? An A/B against the undriven capture, not
  // a before/after: a game animates on its own, so only "it differs BECAUSE of
  // the key" is a claim. The worst-cell reducer is what sees a sprite move.
  const input = interactFor(golden, target);
  if (!input) {
    results.push({ id, target, stage: 'parity', ok: true });
    console.log(`✓ ${id} ${target} — parity ${distance.toFixed(4)}`
      + (wanted ? `, ${wanted.length} point(s) drawn` : ''));
    continue;
  }

  // Each input the project claims is driven ALONE. Sent together, a package
  // whose touch controls do nothing still answers the keyboard, and the gate
  // reads that as both of them working.
  const gestures = [];
  if (input.keys.length || input.pointer) {
    gestures.push({
      what: [input.keys.join('+'), input.pointer && `tap ${input.pointer.x}×${input.pointer.y}`]
        .filter(Boolean).join(' + '),
      spec: { keys: input.keys, pointer: input.pointer, frames: input.frames },
      touch: false,
    });
  }
  if (input.touches) {
    gestures.push({
      what: `${input.touches.length} touch(es)`,
      spec: { touches: input.touches, frames: input.frames },
      touch: true,
    });
  }
  if (input.pad) {
    gestures.push({
      what: 'a gamepad',
      spec: { pad: input.pad, frames: input.frames },
      touch: false,
    });
  }

  let allAnswered = true;
  for (const gesture of gestures) {
    const drivenPng = path.join(WORK, `${id}-${target}-driven-${gesture.what.replace(/\W+/g, '-')}.png`);
    // The same window the undriven launch got. Driving a package does not make
    // its frames cheaper, and a project that needs minutes to settle needs them
    // twice — this asked for 30 s and read the shortfall as a dead gesture.
    const drive = launchPackage(id, target, [
      '--dir', out, '--out', drivenPng,
      '--w', String(editor.w), '--h', String(editor.h),
      ...(timeoutMs ? ['--timeout', String(timeoutMs)] : []),
      ...(gesture.touch ? ['--touch'] : []),
      '--input', JSON.stringify(gesture.spec),
    ]);
    if (drive.status !== 0) {
      results.push({ id, target, stage: 'interact', ok: false, why: `the driven launch failed (${gesture.what})` });
      console.log(`✗ ${id} ${target} — the driven launch failed for ${gesture.what}`);
      // The tail, as the undriven launch already prints: "it failed" with no
      // reason is a red nobody can act on without re-running it by hand.
      for (const l of `${drive.stdout ?? ''}${drive.stderr ?? ''}`.split('\n').slice(-8)) {
        if (l.trim()) console.log(`    ${l}`);
      }
      allAnswered = false;
      continue;
    }

    let response;
    try {
      response = frameCellMax(readFileSync(packagePng), readFileSync(drivenPng));
    } catch (e) {
      results.push({ id, target, stage: 'interact', ok: false, why: e.message });
      console.log(`✗ ${id} ${target} — interact: ${e.message}`);
      allAnswered = false;
      continue;
    }
    const answered = response >= input.responds;
    allAnswered = allAnswered && answered;
    results.push({
      id, target, stage: 'interact', ok: answered,
      why: answered ? '' : `${gesture.what}: response ${response.toFixed(4)} < ${input.responds}`,
    });
    console.log(`${answered ? '✓' : '✗'} ${id} ${target} — parity ${distance.toFixed(4)}, `
      + `responds ${response.toFixed(4)} to ${gesture.what}`);
    if (!answered) console.log(`    the package did not visibly answer it; compare ${packagePng} and ${drivenPng}`);
  }
  void allAnswered;

  // Backgrounded, the world stops; brought back, it carries on. Three runs of
  // the same drive, with a new foreground key press after input cancellation.
  const suspend = suspendFor(golden);
  if (suspend) {
    const where = (hidden) => probePositions(target, out, editor.w, editor.h, [suspend.entity], [
      ...(timeoutMs ? ['--timeout', String(timeoutMs)] : []),
      '--input', JSON.stringify(suspendDrive(suspend, hidden[0] ?? null)),
    ])?.[suspend.entity]?.x ?? null;

    const never = where([]);
    const stayed = where([{ from: suspend.hideFrom, to: suspend.frames - 1 }]);
    const back = where([{ from: suspend.hideFrom, to: suspend.hideTo }]);

    if (never === null || stayed === null || back === null) {
      results.push({ id, target, stage: 'suspend', ok: false, why: `could not read ${suspend.entity}` });
      console.log(`✗ ${id} ${target} — suspend: could not read ${suspend.entity} in all three runs`);
    } else {
      // Ordered, not merely different: a run left in the background must sit
      // between nothing and a run that never stopped, or "resume" only means
      // the frame kept being drawn.
      const paused = Math.abs(never - stayed) >= suspend.moves;
      const resumed = Math.abs(back - stayed) >= suspend.moves && Math.abs(never - back) >= suspend.moves;
      const between = (back - stayed) * (never - back) > 0;
      const ok = paused && resumed && between;
      results.push({
        id, target, stage: 'suspend', ok,
        why: ok ? '' : `never ${never.toFixed(0)}, stayed hidden ${stayed.toFixed(0)}, came back ${back.toFixed(0)}`,
      });
      console.log(`${ok ? '✓' : '✗'} ${id} ${target} — suspend: ${suspend.entity} at `
        + `${stayed.toFixed(0)} hidden, ${back.toFixed(0)} back, ${never.toFixed(0)} never stopped`);
    }
  }

  // A notch takes screen away from one edge, and the HUD has to come out from
  // under it. Read as an offset from a node that rides the camera: a live game
  // is somewhere slightly different each run, and that cancels.
  const safe = safeAreaFor(golden);
  if (safe) {
    const names = [safe.entity, safe.reference];
    const offset = (insets) => {
      const at = probePositions(target, out, editor.w, editor.h, names, [
        ...(timeoutMs ? ['--timeout', String(timeoutMs)] : []),
        ...(insets ? ['--safe-area', insets] : []),
      ]);
      const node = at?.[safe.entity];
      const ref = at?.[safe.reference];
      return node && ref ? { x: node.x - ref.x, y: node.y - ref.y } : null;
    };

    const flat = offset(null);
    const notched = offset(`${safe.top},0,0,0`);
    const sided = offset(`0,0,0,${safe.left}`);

    if (!flat || !notched || !sided) {
      results.push({ id, target, stage: 'safe-area', ok: false, why: `could not read ${safe.entity} against ${safe.reference}` });
      console.log(`✗ ${id} ${target} — safe-area: could not read ${safe.entity} against ${safe.reference} in all three runs`);
    } else {
      // World y is up, so a notch at the top pushes the HUD DOWN. The axis the
      // inset did not come from must not move at all — swapped edges are the
      // failure this catches, and they move the right distance the wrong way.
      const down = flat.y - notched.y;
      const right = sided.x - flat.x;
      const QUIET = 1;
      const fellUnderNotch = down >= safe.moves && Math.abs(notched.x - flat.x) <= QUIET;
      const clearedSide = right >= safe.moves && Math.abs(sided.y - flat.y) <= QUIET;
      // The two edges carry different insets, so their moves must carry the same
      // ratio. One hardcoded nudge satisfies everything above and fails here.
      const scaled = down > 0 && Math.abs(right / down - safe.left / safe.top) <= 0.05;
      const ok = fellUnderNotch && clearedSide && scaled;
      results.push({
        id, target, stage: 'safe-area', ok,
        why: ok ? '' : `top ${safe.top} moved it (${(notched.x - flat.x).toFixed(1)}, ${(-down).toFixed(1)}), `
          + `left ${safe.left} moved it (${right.toFixed(1)}, ${(sided.y - flat.y).toFixed(1)})`,
      });
      console.log(`${ok ? '✓' : '✗'} ${id} ${target} — safe-area: ${safe.entity} drops ${down.toFixed(1)} under a `
        + `${safe.top} notch, clears ${right.toFixed(1)} past a ${safe.left} edge`);
    }
  }
}

const bad = results.filter((r) => !r.ok);
console.log(`\ngolden ${TIER}: ${results.length - bad.length}/${results.length} pair(s) packaged and launched`);
for (const d of deferred) console.log(`  deferred: ${d}`);
if (bad.length) {
  for (const b of bad) {
    console.log(`  ✗ ${b.id} ${b.target} (${b.stage})${b.why ? ` — ${lastSpeakingLine(b.why)}` : ''}`);
  }
  process.exit(1);
}