Skip to content

Write HTML. Render video. Built for agents.

fix(engine): auto worker count stays within the heap budget (#5149)

Miguel Ángel9e0d05d45064
diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts
index 685014a8bd..923631c959 100644
--- a/packages/cli/src/telemetry/events.ts
+++ b/packages/cli/src/telemetry/events.ts
@@ -342,8 +342,8 @@ export function trackRenderComplete(
catalogUsage?: CatalogUsage;
workers?: number;
// Worker auto-sizing provenance (RenderPerfSummary.workerSizing). Answers
- // "why N workers?" fleet-wide, and validates the advisory per-worker heap
- // budget before it's enforced (field OOM: 6 auto workers on a 24GB/4GB-heap
+ // "why N workers?" fleet-wide, and reports the per-worker heap budget
+ // that caps auto sizing (field OOM: 6 auto workers on a 24GB/4GB-heap
// machine — see computeWorkerSizing in @hyperframes/engine).
workersBoundBy?: string;
workersCpuBased?: number;
diff --git a/packages/engine/src/services/parallelCoordinator.test.ts b/packages/engine/src/services/parallelCoordinator.test.ts
index 414902db5a..365ce3ffbc 100644
--- a/packages/engine/src/services/parallelCoordinator.test.ts
+++ b/packages/engine/src/services/parallelCoordinator.test.ts
@@ -1,4 +1,6 @@
-import { describe, it, expect, vi } from "vitest";
+import { cpus, totalmem } from "os";
+import { getHeapStatistics } from "v8";
+import { afterEach, describe, it, expect, vi } from "vitest";
import {
calculateOptimalWorkers,
computeWorkerSizing,
@@ -128,7 +130,43 @@ describe("calculateOptimalWorkers", () => {
});
});
+vi.mock("os", async (importOriginal) => {
+ const os = await importOriginal<typeof import("os")>();
+ return { ...os, default: os, cpus: vi.fn(os.cpus), totalmem: vi.fn(os.totalmem) };
+});
+vi.mock("v8", async (importOriginal) => {
+ const v8 = await importOriginal<typeof import("v8")>();
+ return { ...v8, default: v8, getHeapStatistics: vi.fn(v8.getHeapStatistics) };
+});
+
describe("computeWorkerSizing", () => {
+ afterEach(() => {
+ vi.mocked(cpus).mockRestore();
+ vi.mocked(totalmem).mockRestore();
+ vi.mocked(getHeapStatistics).mockRestore();
+ });
+
+ // The field case: a 24GB 14-core Mac, Node's default ~4GB heap, a long 1080x1920 project.
+ it("never auto-picks more workers than the V8 heap can feed", () => {
+ vi.mocked(cpus).mockReturnValue(
+ Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),
+ );
+ vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
+ vi.mocked(getHeapStatistics).mockReturnValue({
+ heap_size_limit: 4192 * 1024 ** 2,
+ } as ReturnType<typeof getHeapStatistics>);
+ const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" });
+ expect(sizing.heapBasedWorkers).toBe(4);
+ expect(sizing.workers).toBe(4);
+ expect(sizing.boundBy).toBe("heap");
+ expect(sizing.exceedsHeapAdvisory).toBe(false);
+ expect(computeWorkerSizing(1300, 6, { concurrency: "auto" }).workers).toBe(6);
+ vi.mocked(getHeapStatistics).mockReturnValue({
+ heap_size_limit: 8192 * 1024 ** 2,
+ } as ReturnType<typeof getHeapStatistics>);
+ expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).workers).toBe(5);
+ });
+
it("matches calculateOptimalWorkers and reports every constraint", () => {
const config = { concurrency: "auto" as const };
const sizing = computeWorkerSizing(900, undefined, config);
diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts
index 457f22b631..dc8240af10 100644
--- a/packages/engine/src/services/parallelCoordinator.ts
+++ b/packages/engine/src/services/parallelCoordinator.ts
@@ -141,10 +141,8 @@ const MEMORY_PER_WORKER_MB = 1536;
const HEAP_RESERVED_MB = 1024;
// Parent-process V8 heap consumed per worker (protocol buffers + in-flight
// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB
-// default heap ⇒ >~500MB/worker + base. ponytail: advisory-only until the
-// workers_heap_* telemetry added alongside this constant validates the figure
-// — enforcing a guessed budget could silently cut worker counts fleet-wide.
-// TODO(PRINFRA-341): decide enforcement after ~2 weeks of fleet soak.
+// default heap ⇒ >~500MB/worker + base. Caps auto sizing; an explicit
+// `--workers N` is still the operator's call.
const HEAP_PER_WORKER_MB = 640;
const MIN_WORKERS = 1;
const MAX_WORKER_DIAGNOSTIC_LINES = 8;
@@ -275,7 +273,8 @@ export type WorkerSizingBound =
| "frames"
| "max_workers"
| "min_parallel_floor"
- | "contention";
+ | "contention"
+ | "heap";
/**
* Full provenance of a worker-sizing decision. Threaded into render
@@ -290,9 +289,8 @@ export interface WorkerSizing {
frameBasedWorkers: number;
effectiveMaxWorkers: number;
/**
- * ADVISORY, not enforced (see HEAP_PER_WORKER_MB): how many workers the
- * parent process's V8 heap could feed. Compare against `workers` in
- * telemetry to validate the budget before enforcement.
+ * How many workers the parent process's V8 heap can feed (see
+ * HEAP_PER_WORKER_MB); auto sizing never exceeds it.
*/
heapBasedWorkers: number;
/** V8 `heap_size_limit` for the parent process, MB. */
@@ -300,7 +298,7 @@ export interface WorkerSizing {
totalMemoryMb: number;
cpuCount: number;
captureCostMultiplier: number;
- /** true when the chosen count exceeds the advisory heap budget. */
+ /** true when the chosen count exceeds the heap budget (only an explicit request can). */
exceedsHeapAdvisory: boolean;
}
@@ -405,6 +403,11 @@ export function computeWorkerSizing(
}
}
+ if (finalWorkers > heapBasedWorkers) {
+ finalWorkers = heapBasedWorkers;
+ boundBy = "heap";
+ }
+
return finish(finalWorkers, boundBy, effectiveMaxWorkers);
}
diff --git a/packages/producer/src/services/render/captureCost.ts b/packages/producer/src/services/render/captureCost.ts
index db00538d92..c7ceb1c728 100644
--- a/packages/producer/src/services/render/captureCost.ts
+++ b/packages/producer/src/services/render/captureCost.ts
@@ -116,11 +116,8 @@ function combineCaptureCostEstimates(
* - Auto-sized renders only (`requestedWorkers === undefined`) — the field
* failure was auto sizing, and an explicit `--workers N` is the operator's
* own call.
- * - Not enforced as a cap yet — the per-worker budget constant is derived
- * from one field report; the `workers_heap_*` telemetry emitted with the
- * sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in
- * @hyperframes/engine's parallelCoordinator). The message gives the
- * operator the actionable knobs today.
+ * - Auto sizing is capped at the heap budget (computeWorkerSizing), so the
+ * warning cannot fire today; it stays for a budget that stops being a cap.
*
* Pure so the message shape + firing condition are unit-testable with a
* synthetic `WorkerSizing` (the real one depends on the host's heap).
diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts
index 467d9f8967..d7842e5176 100644
--- a/packages/producer/src/services/renderOrchestrator.ts
+++ b/packages/producer/src/services/renderOrchestrator.ts
@@ -455,9 +455,8 @@ export interface RenderPerfSummary {
/**
* Provenance of the auto worker-sizing decision (undefined when the
* htmlInCanvas / low-memory pins short-circuited sizing). `boundBy` names
- * the binding constraint; the heap fields are the advisory budget being
- * validated by fleet telemetry before enforcement — see
- * `computeWorkerSizing` in @hyperframes/engine.
+ * the binding constraint; the heap fields are the budget that caps auto
+ * sizing — see `computeWorkerSizing` in @hyperframes/engine.
*/
workerSizing?: WorkerSizing;
chunkedEncode: boolean;