Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

chore(tools): the gates run in parallel with --jobs, and the reds a full run gathered are cleared

yhhafeb7eac0510
diff --git a/sdk/src/postprocess/timeOfDay.ts b/sdk/src/postprocess/timeOfDay.ts
index 268c73b96..3ce125846 100644
--- a/sdk/src/postprocess/timeOfDay.ts
+++ b/sdk/src/postprocess/timeOfDay.ts
@@ -95,7 +95,7 @@ export function sunPosition(hour: number, latitude: number, declination: number,
return { elevation, azimuth: compass + north * RAD };
}
-/** The time of day at an hour. @public */
+/** The time of day at an hour. @beta */
export function timeOfDayAt(hour: number, latitude = 40, declination = 15, north = 0): TimeOfDayState {
const h = ((hour % 24) + 24) % 24;
const sun = sunPosition(h, latitude, declination, north);
@@ -147,7 +147,7 @@ export function setCurrentTimeOfDay(state: TimeOfDayState | null, drive: number
current = state ? { state, drive, lightsApplied } : null;
}
-/** The time of day the frame is under, or null. @public */
+/** The time of day the frame is under, or null. @beta */
export function currentTimeOfDay(): TimeOfDayState | null {
return current ? current.state : null;
}
diff --git a/sdk/src/postprocess/weather.ts b/sdk/src/postprocess/weather.ts
index 99f9dbe4a..6a66708fd 100644
--- a/sdk/src/postprocess/weather.ts
+++ b/sdk/src/postprocess/weather.ts
@@ -123,7 +123,7 @@ define(WEATHER_STORM, 'storm', {
* project can retune a built-in one. A custom id is for code (`weatherTo`, a component's `preset` value) — the
* Inspector's dropdown lists the built-ins.
*
- * @public
+ * @beta
*/
export function registerWeatherPreset(name: string, preset: WeatherPreset): number {
const existing = byName.get(name);
@@ -132,12 +132,12 @@ export function registerWeatherPreset(name: string, preset: WeatherPreset): numb
return id;
}
-/** The id a preset answers to, or undefined. @public */
+/** The id a preset answers to, or undefined. @beta */
export function weatherPresetId(name: string): number | undefined {
return byName.get(name);
}
-/** Every preset, built-in first. @public */
+/** Every preset, built-in first. @beta */
export function weatherPresetNames(): string[] {
return [...registry.entries()].sort((a, b) => a[0] - b[0]).map(([, v]) => v.name);
}
@@ -149,7 +149,7 @@ const mix = (a: number, b: number, t: number): number => a + (b - a) * t;
/**
* The weather `progress` of the way from one preset to another, eased so a transition starts and ends gently.
*
- * @public
+ * @beta
*/
export function blendWeather(from: number, to: number, progress: number): WeatherState {
const a = (registry.get(from) ?? registry.get(WEATHER_CLEAR)!).state;
@@ -186,7 +186,7 @@ export function setCurrentWeather(state: WeatherState | null, drive: number = We
current = state ? { state, drive } : null;
}
-/** @public */
+/** @beta */
export function currentWeather(): WeatherState | null {
return current ? current.state : null;
}
diff --git a/sdk/src/postprocess/weatherSystem.ts b/sdk/src/postprocess/weatherSystem.ts
index 49c5154c6..ed32c4413 100644
--- a/sdk/src/postprocess/weatherSystem.ts
+++ b/sdk/src/postprocess/weatherSystem.ts
@@ -258,6 +258,7 @@ function advance(entity: Entity, data: WeatherData, dt: number): void {
data.progress = 0;
}
+// system-access: the weather reads and writes the lights, the sound sources and the post-process volumes it drives, which no query names.
export const weatherSystem = defineSystem(
[Query(Mut(Weather)), Query(Mut(TimeOfDay)), Res(Time), GetWorld()],
(weathers: Iterable<[Entity, WeatherData]>, times: Iterable<[Entity, TimeOfDayData]>, time: TimeData, world: World) => {
diff --git a/sdk/tests/compute-effect.test.ts b/sdk/tests/compute-effect.test.ts
index 99ff97d77..00d01a2d3 100644
--- a/sdk/tests/compute-effect.test.ts
+++ b/sdk/tests/compute-effect.test.ts
@@ -115,7 +115,8 @@ describe('native effects', () => {
it('pushes a native pass by its kind, and leaves it out on a core that cannot', () => {
const pp = new PostProcessAPI();
- const withNative = { ...mockCore(true), postprocess_addNativePass: vi.fn().mockReturnValue(0) };
+ const core = mockCore(true);
+ const withNative = Object.assign(core, { postprocess_addNativePass: vi.fn().mockReturnValue(0) });
initPostProcessAPI(withNative as unknown as ESEngineModule);
const stack = new PostProcessStack(pp.state);
stack.addNativePass('voxelGI', 'voxelGI');
diff --git a/tools/check-postfx-aa.mjs b/tools/check-postfx-aa.mjs
index 8b66fa04c..679b0cef4 100644
--- a/tools/check-postfx-aa.mjs
+++ b/tools/check-postfx-aa.mjs
@@ -75,7 +75,8 @@ if (!/resolveColor_\s*=\s*device_->createTexture/.test(fb)) {
+ 'attachment it resolves into. Resolve is the target\'s own business — a pass or an '
+ 'effect must never have to ask for it.');
}
-if (!/createFramebuffer\(\{[^}]*resolveColor_/s.test(fb)) {
+// Handed over either inline, or through the descriptor the function builds up (`fb.resolveColor0 = resolveColor_`) and then passes.
+if (!/createFramebuffer\(\{[^}]*resolveColor_/s.test(fb) && !(/\.resolveColor0\s*=\s*resolveColor_/.test(fb) && /createFramebuffer\(\s*fb\s*\)/.test(fb))) {
findings.push(`${FB_CPP}: the resolve attachments are created but not handed to `
+ 'createFramebuffer, so the device has nothing to resolve into.');
}
diff --git a/tools/check-preview-ownership.mjs b/tools/check-preview-ownership.mjs
index 9dcc70f9c..d853b55ed 100644
--- a/tools/check-preview-ownership.mjs
+++ b/tools/check-preview-ownership.mjs
@@ -94,7 +94,7 @@ const RULES = [
holds: () => {
const body = strip(read(FRAME_C)).match(/void RenderFrame::renderSurface\([\s\S]*?\n\}/);
if (!body) return 'renderSurface is gone';
- const collect = body[0].indexOf('collectAll(registry)');
+ const collect = body[0].indexOf('collectAll(registry');
// The LOOP, not the parameter — which is named in the signature above.
const replay = body[0].search(/for\s*\([^)]*:\s*extraSkeletalBatches\s*\)/);
const finalize = body[0].indexOf('draw_list_.finalize');
diff --git a/tools/gates.mjs b/tools/gates.mjs
index de18e1f13..a17a0dca6 100644
--- a/tools/gates.mjs
+++ b/tools/gates.mjs
@@ -36,6 +36,7 @@ export const SCOPES = ['local', 'ci'];
* runs the ones a change touched, so it meets its unit-test owner before the remote.
*/
export const GATES = [
+ // `exclusive: true` is a build or a writer of what a later gate reads: a `--jobs` run cuts the list there and runs it alone.
{ id: 'tsc-sdk', run: 'pnpm --filter ./sdk exec tsc --noEmit' },
// Up here rather than beside `tsc-pipeline`: the compiler resolves `esengine`
// to its own stub (compiler/tsconfig.json), so it needs nothing built and has
@@ -85,7 +86,7 @@ export const GATES = [
{ id: 'project-health', run: 'node tools/check-project-health.mjs' },
// Everything below reads what this emits: the declaration snapshot, the
// editor's own type-check and the examples all resolve `esengine` from dist.
- { id: 'sdk-build', run: 'pnpm --filter ./sdk build' },
+ { id: 'sdk-build', exclusive: true, run: 'pnpm --filter ./sdk build' },
{ id: 'api-surface-dts', run: 'node tools/api-surface.mjs --check-dts' },
// Both read `esengine` from dist, and the editor reads the pipeline.
{ id: 'tsc-pipeline', run: 'pnpm --filter @estella/pipeline exec tsc --noEmit' },
@@ -131,7 +132,7 @@ export const GATES = [
{ id: 'pipeline-boundary', run: 'node tools/check-pipeline-boundary.mjs' },
// And the boundary holds in practice: a real project packages from the command
// line with no editor built.
- { id: 'headless-export', run: 'node tools/check-headless-export.mjs' },
+ { id: 'headless-export', exclusive: true, run: 'node tools/check-headless-export.mjs' },
// The 478 SDK test files were in no checker's scope: the production config
// compiles src/ and vitest strips types. A ratchet, not a demand for zero —
// except in the net suites, which are held to none.
@@ -282,13 +283,13 @@ export const GATES = [
{ id: 'comment-style', run: 'node tools/check-comment-style.mjs' },
{ id: 'silent-writes', run: 'node tools/mutation-census.mjs --gate' },
{
- id: 'cpp-tests',
+ id: 'cpp-tests', exclusive: true,
run: 'node tools/check-cpp-tests.mjs',
where: 'local',
why: 'it configures and builds a native cmake tree; CI compiles and RUNS those harnesses in its own C++ jobs',
},
{
- id: 'native-build',
+ id: 'native-build', exclusive: true,
run: 'node tools/check-native-build.mjs',
where: 'local',
why: 'it builds the engine out of the tree `cli native` configured; CI builds every native target from scratch',
@@ -364,8 +365,8 @@ export const GATES = [
},
{ id: 'screen-domain', run: 'node tools/check-screen-domain.mjs' },
{ id: 'rm-retries', run: 'node tools/check-rm-retries.mjs' },
- { id: 'examples', run: 'node build-tools/cli.js check-examples' },
- { id: 'documents', run: 'node build-tools/cli.js validate-documents' },
+ { id: 'examples', exclusive: true, run: 'node build-tools/cli.js check-examples' },
+ { id: 'documents', exclusive: true, run: 'node build-tools/cli.js validate-documents' },
];
/** Whether `p` names, or sits under, one of `roots`. */
diff --git a/tools/run-gates.mjs b/tools/run-gates.mjs
index 7605962b2..f895bec56 100644
--- a/tools/run-gates.mjs
+++ b/tools/run-gates.mjs
@@ -20,7 +20,7 @@
* node tools/run-gates.mjs --scope local --where local
* node tools/run-gates.mjs --scope local --keep-going --matrix out.json
*/
-import { execFileSync, spawnSync } from 'node:child_process';
+import { execFileSync, spawn, spawnSync } from 'node:child_process';
import { existsSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -126,6 +126,12 @@ const GAPS = [
},
];
const declared = GAPS.filter((g) => process.env[g.env] === g.value);
+/**
+ * How many gates run at once. The list is cut at every `exclusive` gate (a build): the gates between two cuts share the pool, a cut
+ * gate runs alone. Verdicts are still read in list order, so the first red is a serial run's; the other reds are named with it.
+ */
+const JOBS = Math.max(1, Number(flag('jobs', '1')) || 1);
+
/**
* Narrow to named gates. A scan finds ten reds at once and then each fix wants
* ONE of them re-run; without this the only way to re-check a gate is to pay for
@@ -263,18 +269,65 @@ function firstFailingLine(text) {
const matrix = [];
let firstRed = null;
+/** One gate, run to the end with its output kept: what the pool hands back, shaped like a spawnSync result. */
+function runBuffered(gate) {
+ return new Promise((resolve) => {
+ const began = Date.now();
+ const child = spawn('sh', ['-c', gate.run], { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'] });
+ let stdout = '';
+ let stderr = '';
+ child.stdout.on('data', (d) => { stdout += d; });
+ child.stderr.on('data', (d) => { stderr += d; });
+ child.on('error', (error) => resolve({ error, status: null, stdout, stderr, ms: Date.now() - began }));
+ child.on('close', (status) => {
+ const word = status === 0 ? 'ok' : status === CANNOT_ANSWER ? 'answered nothing' : 'RED';
+ console.error(` · ${gate.id} ${word} (${((Date.now() - began) / 1000).toFixed(0)}s)`);
+ resolve({ status, stdout, stderr, ms: Date.now() - began });
+ });
+ });
+}
+
+/** Every gate's result, by id, run JOBS at a time between the cuts. Empty for a serial run. */
+const precomputed = new Map();
+if (JOBS > 1) {
+ const segments = [];
+ let run = [];
+ for (const gate of gates) {
+ if (gate.exclusive) {
+ if (run.length) segments.push(run);
+ segments.push([gate]);
+ run = [];
+ } else run.push(gate);
+ }
+ if (run.length) segments.push(run);
+ console.log(` running ${JOBS} at a time, in ${segments.length} step(s) cut at the gates that build`);
+ for (const segment of segments) {
+ let next = 0;
+ const worker = async () => {
+ while (next < segment.length) {
+ const gate = segment[next++];
+ precomputed.set(gate.id, await runBuffered(gate));
+ }
+ };
+ await Promise.all(Array.from({ length: Math.min(JOBS, segment.length) }, worker));
+ }
+}
+
for (const gate of gates) {
const began = Date.now();
if (SCAN) process.stdout.write(`\n=== ${gate.id}\n ${gate.run}\n`);
// A scan keeps the output so the matrix can carry the failing line; a gate run
// streams it, because a person is watching one command and wants it live.
- const r = SCAN
+ const pre = precomputed.get(gate.id);
+ const r = pre ?? (SCAN
? spawnSync('sh', ['-c', gate.run], {
cwd: ROOT, encoding: 'utf8', maxBuffer: 1 << 28,
stdio: ['ignore', 'pipe', 'pipe'],
})
- : spawnSync('sh', ['-c', gate.run], { cwd: ROOT, stdio: 'inherit' });
- const ms = Date.now() - began;
+ : spawnSync('sh', ['-c', gate.run], { cwd: ROOT, stdio: 'inherit' }));
+ // A pooled run has no terminal to stream to, so what the gate said is said now, where its verdict is read.
+ if (pre && !SCAN) { process.stdout.write(pre.stdout); process.stderr.write(pre.stderr); }
+ const ms = pre ? pre.ms : Date.now() - began;
spent.push({ id: gate.id, ms });
// A shell that would not start is not a gate that failed. Reported as one it
// sends the reader after the first gate's subject, which said nothing at all.
@@ -323,6 +376,14 @@ for (const gate of gates) {
}
if (r.status !== 0) {
console.error(`\n✗ ${gate.id} — \`${gate.run}\``);
+ // The rest of the list has run already, so the other reds are known and are named here instead of costing a run each.
+ const alsoRed = gates.filter((g) => {
+ const other = precomputed.get(g.id);
+ return g !== gate && other && other.status !== 0 && other.status !== CANNOT_ANSWER;
+ });
+ if (alsoRed.length) {
+ console.error(` also red in this run: ${alsoRed.map((g) => g.id).join(', ')} (re-run them alone with --only once this one is fixed)`);
+ }
// Naming what did NOT run matters at the moment of failure: the gates after
// this one said nothing, and a reader should not take silence for green.
const after = gates.slice(gates.indexOf(gate) + 1);