Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

12.6 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
 * @file  verify-render.mjs — run the pixel gates a tier pays for.
 *
 * One runner over {@link renderScenes.mjs}, so CI and a developer run the same
 * list by the same names. Reports the whole matrix rather than stopping at the
 * first failure: which scenes broke is the useful answer, not which broke first.
 *
 *   node tools/verify-render.mjs --tier pr
 *   node tools/verify-render.mjs --tier pr --backend webgpu --budget 45
 *   node tools/verify-render.mjs --tier nightly --only spine,video
 */
import path from 'node:path';
import { existsSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { TIERS, SCENES, scenesAtTier, sceneWatchdogMs } from './renderScenes.mjs';
import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict, resultMeasured } from './lib/deadGpu.mjs';
import { runTool } from './lib/runTool.mjs';
import { runElectron } from './lib/electronRun.mjs';
import { requireCurrentEngine } from './lib/engineBuild.mjs';

const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const DESKTOP = path.join(ROOT, 'desktop');

const argv = process.argv.slice(2);
const flag = (n, d) => {
  const i = argv.indexOf(`--${n}`);
  return i >= 0 && argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : d;
};

const TIER = flag('tier', 'pr');
// Which host's scenes to run. The engine's gates stand on their own, so a
// checkout without the editor submodule runs `--host engine` and is not quietly
// short a hundred checks — it says which list it took.
const HOST_FILTER = flag('host', '');
const ONLY = flag('only', '');
const BACKEND = flag('backend', 'webgl2');
const BUDGET = flag('budget', '');
const NO_BUILD = argv.includes('--no-build');
if (argv.includes('--no-xvfb')) process.env.ESTELLA_NO_XVFB = '1';

const only = ONLY ? new Set(ONLY.split(',').map((s) => s.trim())) : null;
if (only) {
  const known = new Set(SCENES.map((s) => s.id));
  const bogus = [...only].filter((id) => !known.has(id));
  if (bogus.length) {
    console.error(`verify-render: no such scene(s): ${bogus.join(', ')}`);
    process.exit(2);
  }
}
if (!TIERS.includes(TIER)) {
  console.error(`verify-render: unknown tier "${TIER}" (have: ${TIERS.join(', ')})`);
  process.exit(2);
}

if (HOST_FILTER && !['engine', 'editor'].includes(HOST_FILTER)) {
  console.error(`verify-render: unknown host "${HOST_FILTER}" (have: engine, editor)`);
  process.exit(2);
}
if (!['webgl2', 'webgpu'].includes(BACKEND)) {
  console.error(`verify-render: unknown backend "${BACKEND}" (have: webgl2, webgpu)`);
  process.exit(2);
}
if (BUDGET && !(Number(BUDGET) > 0)) {
  console.error(`verify-render: --budget takes minutes, not "${BUDGET}"`);
  process.exit(2);
}
// Minutes the whole run may spend before it stops and says what it never
// measured. A tier that outgrows its runner is a report either way; without one
// the report is a killed process, which names no scene at all.
const BUDGET_MS = Number(BUDGET || 0) * 60_000;
const watchdogFor = (host) => sceneWatchdogMs(BACKEND, host);
// The scene's own watchdog produces a verdict, so this kill is for the launch
// that never reaches it — a whole electron start-up beyond.
const LAUNCH_MARGIN_MS = 30_000;
// A scene runs on the second backend only where it says so: the two backends
// agree on most frames and deliberately differ on a couple of tolerances, and
// guessing that per scene is how the WebGPU list became a third copy.
// A tier run on the second backend takes the scenes that DECLARE it; naming
// scenes explicitly is a decision already made, so `--only` is not filtered —
// that is how a scene earns the declaration in the first place.
// A Linux runner has no GPU, so WebGL2 there is a software rasteriser, and a scene that says `softwareGpu: false` takes its GPU process
// down before it draws, every launch. That says nothing about the scene and spends a budget the other scenes need. Named below, and
// still run wherever there is a GPU, the second backend's Metal runner included.
const softwareGpu = process.platform === 'linux' && Boolean(process.env.CI);
const tierScenes = scenesAtTier(TIER);
const needsGpu = BACKEND === 'webgl2' && softwareGpu ? tierScenes.filter((s) => s.softwareGpu === false) : [];
const selected = only
  ? SCENES.filter((s) => only.has(s.id))
  : tierScenes.filter((s) => (BACKEND === 'webgpu' ? Boolean(s.webgpu) : !needsGpu.includes(s)));
if (!only && needsGpu.length > 0) {
  console.log(`not run on this machine's software rasteriser: ${needsGpu.map((s) => s.id).join(', ')}`);
}
const scenes = selected.map((s) => ({
  id: s.id,
  // Which host answers this scene's driving surface. The engine's is the
  // default; a scene names the editor's only when its SUBJECT is an editor door.
  host: s.host ?? 'engine',
  env: BACKEND === 'webgpu'
    ? {
      ...s.env,
      ...(typeof s.webgpu === 'object' ? s.webgpu : {}),
      ESTELLA_VERIFY_BACKEND: 'webgpu',
      // The adapter Dawn picks for itself: a real driver is the only kind that
      // answers the portability question. `swiftshader` names the bundled software
      // one, for a machine with no GPU. The boot log says which it got.
      ESTELLA_VERIFY_WEBGPU_ADAPTER: process.env.ESTELLA_VERIFY_WEBGPU_ADAPTER ?? '',
    }
    : s.env,
}));
const hosted = HOST_FILTER ? scenes.filter((s) => s.host === HOST_FILTER) : scenes;
if (HOST_FILTER) {
  const dropped = scenes.length - hosted.length;
  if (dropped) console.log(`  not on this host: ${dropped} scene(s)`);
  scenes.length = 0;
  scenes.push(...hosted);
}
const hostsUsed = new Set(scenes.map((s) => s.host));
console.log(`render ${only ? `${scenes.length} named scene(s)` : `${TIER}: ${scenes.length} scene(s)`} on ${BACKEND}`
  + ` (host: ${[...hostsUsed].sort().join(' + ')})`);

/**
 * A host with no page serves 404s, the driving surface never appears, and every
 * scene fails with "cannot read properties of undefined". Say which build is
 * missing instead. `git submodule deinit` empties the editor's dist among other
 * ignored output, so this is reachable without anyone deleting anything.
 */
function requirePage(page, host, build) {
  if (existsSync(page)) return;
  console.error(`verify-render: the ${host} host has no page at ${path.relative(ROOT, page)}.`);
  console.error(`  build it:  ${build}`);
  process.exit(2);
}

// Build each host this run needs, then check the wasm it will actually SERVE:
// --no-build leaves a host's output as it stands, and a freshly built public/
// copy it never staged is evidence about a binary no scene below is judging.
if (hostsUsed.has('engine')) {
  if (!NO_BUILD) {
    const built = runTool('node', ['tools/render-host/build.mjs'], { cwd: ROOT, encoding: 'utf8' });
    if (built.status !== 0) {
      console.error(`verify-render: render-host build failed\n${(built.stderr || built.stdout || '').slice(-800)}`);
      process.exit(1);
    }
  }
  requireCurrentEngine(ROOT, path.join(ROOT, 'build', 'render-host', 'wasm'));
  requirePage(path.join(ROOT, 'build', 'render-host', 'index.html'), 'engine',
    'node tools/render-host/build.mjs');
}
if (hostsUsed.has('editor')) {
  if (!existsSync(DESKTOP)) {
    console.error('verify-render: these scenes name the editor host, which is not checked out'
      + ' (desktop/ is an optional submodule).');
    process.exit(2);
  }
  if (!NO_BUILD) {
    const built = runTool('pnpm', ['exec', 'vite', 'build'], { cwd: DESKTOP, encoding: 'utf8' });
    if (built.status !== 0) {
      console.error(`verify-render: vite build failed\n${(built.stderr || built.stdout || '').slice(-800)}`);
      process.exit(1);
    }
  }
  requireCurrentEngine(ROOT, path.join(DESKTOP, NO_BUILD ? 'dist' : 'public', 'wasm'));
  requirePage(path.join(DESKTOP, 'dist', 'headless.html'), 'editor',
    'pnpm --filter @estella/editor exec vite build');
}

const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;

function runScene(scene) {
  const RUNNER = 'tools/render-host/run.mjs';
  const watchdog = watchdogFor(scene.host);
  const at = Date.now();
  const r = runElectron([RUNNER], {
    cwd: ROOT,
    encoding: 'utf8',
    timeout: watchdog + LAUNCH_MARGIN_MS,
    env: {
      ESTELLA_VERIFY_TIMEOUT_MS: String(watchdog),
      ESTELLA_VERIFY_HOST: scene.host,
      ...scene.env,
    },
  });
  const out = `${r.stdout || ''}${r.stderr || ''}`;
  // DRIVE_RESULT is the run's own verdict: printed once the page loaded, the
  // engine booted and the driver finished. Whether it is there answers "did a
  // measurement happen" far better than anything in the log text can.
  const drive = /^DRIVE_RESULT (\{.*\})$/m.exec(out);
  let result = null;
  try {
    result = drive ? JSON.parse(drive[1]) : null;
  } catch {
    result = null; // truncated line — treat as no verdict
  }
  return {
    ok: r.status === 0,
    out,
    ms: Date.now() - at,
    measured: resultMeasured(result),
    drew: result?.capture?.rendered,
    verdict: r.timedOut
      ? 'the run never reached its own watchdog and was killed'
      : ((r.stdout || '').match(/\[verify:render\][^\n]*/) ?? [''])[0].replace('[verify:render] ', ''),
  };
}

// Published assets a scene draws (tools/corpora.mjs), fetched and imported before
// any scene runs: a download inside a scene's watchdog would be timed as rendering.
const corporaNeeded = [...new Set(selected.flatMap((s) => s.corpora ?? []))];
if (corporaNeeded.length) {
  const { importedCorpus } = await import('./corpora.mjs');
  for (const id of corporaNeeded) await importedCorpus(id);
}

const failed = [];
const unmeasured = [];
const startedAt = Date.now();
let retried = 0;
// A runner with no GPU for this backend fails every scene the same way, six
// attempts apiece — an hour to say nothing, with the rest blamed on the budget.
// Once this many have died with it down and none has drawn, no scene can change it.
const DEAD_GPU_GIVE_UP = 3;
let deadInARow = 0;
let drewSomething = false;
let noGpu = false;
for (const [index, scene] of scenes.entries()) {
  const outOfBudget = BUDGET_MS && Date.now() - startedAt >= BUDGET_MS;
  noGpu = !drewSomething && deadInARow >= DEAD_GPU_GIVE_UP;
  if (outOfBudget || noGpu) {
    for (const rest of scenes.slice(index)) unmeasured.push(rest.id);
    break;
  }
  // A scene that TAKES the GPU away on purpose owns its own device-loss result;
  // everything else shares one retry policy with the golden runner, or the same
  // dead runner is a retry in one and a broken engine in the other.
  let run;
  if (scene.env.ESTELLA_VERIFY_DEVICE_LOSS) {
    run = runScene(scene);
  } else {
    // `gpuNeverCameUp` matches a message this runner prints on success too, so
    // every failure reads as "the GPU died" and the real reason never reaches the
    // log — a host failing half its launches looks exactly like a flaky runner.
    let firstFailure = null;
    const attempt = retryOnDeadGpu(
      () => { const r = runScene(scene); return { ...r, output: r.out }; },
      (died) => {
        retried++;
        console.log(`↻ ${scene.id.padEnd(28)} ${died
          ? 'the GPU process died before it drew'
          : 'a blank frame on the run after a GPU death'}; measuring again`);
      },
    );
    if (attempt.retried && !firstFailure) {
      firstFailure = (attempt.out || '').split('\n')
        .filter((l) => /error|fail|refus|throw|undefined|cannot|timeout|watchdog/i.test(l))
        .slice(-3);
      if (firstFailure.length) {
        console.log(`  └ what the retried run(s) said: ${firstFailure.join(' | ').slice(0, 400)}`);
      }
    }
    run = attempt;
  }
  if (run.ok) drewSomething = true; else failed.push(scene.id);
  deadInARow = run.gpuDied ? deadInARow + 1 : 0;
  const verdict = run.launchFailed ? launchNeverHappenedVerdict('the scene')
    : run.gpuDied ? deadGpuVerdict('the scene')
      : run.verdict;
  console.log(`${run.ok ? '✓' : '✗'} ${scene.id.padEnd(28)} ${seconds(run.ms).padStart(7)}  ${verdict}`);
  if (!run.ok) {
    for (const l of run.out.split('\n').slice(-8)) if (l.trim()) console.log(`    ${l}`);
  }
}

const measured = scenes.length - unmeasured.length;
console.log(`\nrender ${TIER}: ${measured - failed.length}/${measured} scene(s) drew what they claim`
  + ` in ${seconds(Date.now() - startedAt)}`
  + (retried ? `, ${retried} measured twice (the GPU had died)` : ''));
if (failed.length) for (const id of failed) console.log(`  ✗ ${id}`);
if (unmeasured.length) {
  console.log(noGpu
    ? `\nthe runner never gave ${BACKEND} a GPU: ${measured} scene(s) died with it down and `
      + `${unmeasured.length} were not run — this is a verdict about the RUNNER, not the engine`
    : `\nthe ${BUDGET} minute budget ran out: ${unmeasured.length} scene(s) never ran`);
  for (const id of unmeasured) console.log(`  · ${id}`);
}
if (failed.length || unmeasured.length) process.exit(1);