Skip to content

Write HTML. Render video. Built for agents.

fix(producer): disk-space error suggests --workers 1 only when one worker would stream (#5934)

Miguel Ángelf0ef4cb9d479
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index 4969f96905..dcfce67382 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -1323,8 +1323,11 @@ path's.
Renders still land on the disk path when the output is `png-sequence` or `gif`,
when HDR or shader-transition layering is in play, and when streaming is
switched off. If a render fails preflight asking for tens of GB of temporary
-frame storage, the message names which of those applied; the usual fixes are
-`--workers 1`, `--low-memory-mode`, or freeing disk.
+frame storage, the message names the condition that applied and what takes that
+render off disk. `--workers 1` is offered only for a multi-worker render that one
+worker would stream; it is slower, so it is a fix for that render, not a default.
+For `png-sequence` or `gif` output, or with streaming switched off, one worker
+stores frames too: free disk, render to mp4, or undo the switch.
| Environment variable | Default | Effect |
| ------------------------------------------------ | ------- | ----------------------------------------------------------------------------------------- |
diff --git a/packages/producer/src/services/render/stages/captureStage.test.ts b/packages/producer/src/services/render/stages/captureStage.test.ts
index f8341a8cd3..169098437a 100644
--- a/packages/producer/src/services/render/stages/captureStage.test.ts
+++ b/packages/producer/src/services/render/stages/captureStage.test.ts
@@ -43,6 +43,7 @@ vi.mock("node:fs", async (importOriginal) => {
import {
assertDiskCaptureHeadroom,
createDiskCaptureProjection,
+ diskCaptureRemedy,
diskCaptureShortfallError,
measureCaptureFrameBytes,
estimateDiskCaptureBytes,
@@ -88,34 +89,14 @@ describe("disk capture capacity", () => {
it("fails before capture when estimated frames exceed available headroom", () => {
expect(() =>
- assertDiskCaptureHeadroom("/render/captured-frames", 10, pngOptions, () => 800_000),
- ).toThrow(/may need ~0\.8 MB.*0\.8 MB is free.*--low-memory-mode/s);
- });
-
- it("tells the user which routes still use disk capture, not a dead env var", () => {
- const landscape = {
- width: 1920,
- height: 1080,
- fps: { num: 30, den: 1 },
- format: "png" as const,
- };
- let message = "";
- try {
- assertDiskCaptureHeadroom("/tmp/frames", 9000, landscape, () => 10 * 1e9);
- } catch (error) {
- message = error instanceof Error ? error.message : String(error);
- }
- expect(message).toContain("Disk capture may need");
- expect(message).toContain("--workers 1");
- expect(message).toContain("--low-memory-mode");
- // Every override that can land an otherwise-streaming render here, so the
- // message explains the case the reader is actually in. The duration cap
- // matters because it sends even a --workers 1 render to disk, which the
- // pre-Phase-1 wording told them to "fix" with --workers 1.
- expect(message).toContain("HF_CAPTURE_PARALLEL_STREAM=true");
- expect(message).toContain("PRODUCER_STREAMING_ENCODE_DURATION_CAP_ENABLED=true");
- expect(message).toContain("png-sequence");
- expect(message).not.toContain("PRODUCER_STREAMING_ENCODE_MAX_DURATION_SECONDS if streaming");
+ assertDiskCaptureHeadroom(
+ "/render/captured-frames",
+ 10,
+ pngOptions,
+ "Free up disk space.",
+ () => 800_000,
+ ),
+ ).toThrow(/may need ~0\.8 MB.*0\.8 MB is free.*Free up disk space\./s);
});
it("exposes the same 90% headroom decision to fallback planning", () => {
@@ -161,6 +142,36 @@ describe("disk capture capacity", () => {
});
});
+describe("diskCaptureRemedy", () => {
+ it("offers --workers 1 when one worker would stream this render", () => {
+ const remedy = diskCaptureRemedy(DEFAULT_CONFIG, "mp4", 60, 4);
+ expect(remedy).toContain("captures with 4 workers");
+ expect(remedy).toContain("re-run this render with --workers 1");
+ });
+
+ it("never offers --workers 1 where one worker would still store every frame", () => {
+ const capped = { ...DEFAULT_CONFIG, streamingEncodeDurationCapEnabled: true };
+ const cases = [
+ [DEFAULT_CONFIG, "mp4", 60, 1, "Free up disk space."],
+ [DEFAULT_CONFIG, "png-sequence", 60, 4, "png-sequence output stores every frame"],
+ [DEFAULT_CONFIG, "gif", 60, 1, "gif output stores every frame"],
+ [
+ { ...DEFAULT_CONFIG, enableStreamingEncode: false },
+ "mp4",
+ 60,
+ 4,
+ "PRODUCER_ENABLE_STREAMING_ENCODE=false",
+ ],
+ [capped, "mp4", 600, 4, "--low-memory-mode, which streams past it"],
+ ] as const;
+ for (const [cfg, format, duration, workers, condition] of cases) {
+ const remedy = diskCaptureRemedy(cfg, format, duration, workers);
+ expect(remedy).toContain(condition);
+ expect(remedy).not.toContain("--workers 1");
+ }
+ });
+});
+
describe("createDiskCaptureProjection", () => {
it("throws a measured shortfall once ten frames show the remainder cannot fit", () => {
const measure = vi.fn(() => 10e6);
@@ -168,6 +179,7 @@ describe("createDiskCaptureProjection", () => {
framesDir: "/frames",
totalFrames: 1010,
measureBytes: measure,
+ remedy: "Free up disk space.",
freeDiskBytes: () => 500e6,
});
check(9);
@@ -184,6 +196,7 @@ describe("createDiskCaptureProjection", () => {
framesDir: "/frames",
totalFrames: 1010,
measureBytes: () => bytes,
+ remedy: "Free up disk space.",
freeDiskBytes: () => 500e6,
});
expect(() => check(10)).not.toThrow();
@@ -214,6 +227,7 @@ describe("createDiskCaptureProjection", () => {
framesDir: "/frames",
totalFrames: 1010,
measureBytes: () => 10e6,
+ remedy: "Free up disk space.",
freeDiskBytes: () => 5000e6,
});
expect(() => check(10)).not.toThrow();
@@ -227,7 +241,7 @@ describe("runCaptureStage measured disk projection (both branches)", () => {
fileServer: { url: "http://localhost:0" } as never,
workDir,
framesDir,
- job: createRenderJob({ fps: { num: 30, den: 1 }, quality: "draft" }),
+ job: { ...createRenderJob({ fps: { num: 30, den: 1 }, quality: "draft" }), duration: 20 / 30 },
totalFrames: 20,
cfg: DEFAULT_CONFIG,
plan: { workerCount, forceScreenshot: false },
@@ -277,15 +291,15 @@ describe("runCaptureStage measured disk projection (both branches)", () => {
writeFileSync(join(framesDir, formatCaptureFrameName(index, "jpg")), oneMb);
});
- await expect(runCaptureStage(makeInput(1, workDir, framesDir))).rejects.toThrow(
- /measured from the first 10 frames/,
- );
+ const error = await runCaptureStage(makeInput(1, workDir, framesDir)).catch((e: Error) => e);
+ expect(String(error)).toMatch(/measured from the first 10 frames.*Free up disk space\./s);
+ expect(String(error)).not.toContain("--workers 1");
expect(captureFrame).toHaveBeenCalledTimes(10);
expect(createCaptureSession).toHaveBeenCalledTimes(1);
});
it("a measured shortfall is never classified as a retryable browser death", () => {
- const shortfall = diskCaptureShortfallError(10e9, 1e9, "/frames", "measured");
+ const shortfall = diskCaptureShortfallError(10e9, 1e9, "/frames", "measured", "");
const kind = classifyCaptureFailure(shortfall);
expect(kind).not.toBe("transient_browser");
expect(isTransientCaptureRetryEligible(kind, [{ startFrame: 10, endFrame: 20 }], 0)).toBe(
@@ -321,7 +335,7 @@ describe("runCaptureStage measured disk projection (both branches)", () => {
);
await expect(runCaptureStage(makeInput(2, workDir, framesDir))).rejects.toThrow(
- /measured from the first 10 frames/,
+ /measured from the first 10 frames.*re-run this render with --workers 1/s,
);
});
});
diff --git a/packages/producer/src/services/render/stages/captureStage.ts b/packages/producer/src/services/render/stages/captureStage.ts
index 21f4a19238..a1963b9b26 100644
--- a/packages/producer/src/services/render/stages/captureStage.ts
+++ b/packages/producer/src/services/render/stages/captureStage.ts
@@ -60,11 +60,13 @@ import type { FileServerHandle } from "../../fileServer.js";
import type { ProducerLogger } from "../../../logger.js";
import {
executeDiskCaptureWithAdaptiveRetry,
+ explainStreamingEncodeGate,
findMissingFrameRanges,
isTransientCaptureRetryEligible,
sampleDirectoryBytes,
type CaptureAttemptSummary,
type ProgressCallback,
+ type RenderConfig,
type RenderJob,
} from "../../renderOrchestrator.js";
import { wrapCaptureStageError } from "../captureStageError.js";
@@ -199,11 +201,46 @@ export function inspectDiskCaptureHeadroom(
return { available: false, estimatedBytes, freeBytes };
}
+/**
+ * Why this render stores frames and what would take it off disk. `--workers 1`
+ * is offered only when a one-worker run of the same render would stream.
+ */
+export function diskCaptureRemedy(
+ cfg: EngineConfig,
+ outputFormat: NonNullable<RenderConfig["format"]>,
+ durationSeconds: number,
+ workerCount: number,
+): string {
+ const oneWorker = explainStreamingEncodeGate(cfg, outputFormat, 1, durationSeconds);
+ if (workerCount > 1 && oneWorker.enabled) {
+ return (
+ `This render stores frames because it captures with ${workerCount} workers. ` +
+ "Free up disk space, or re-run this render with --workers 1 to stream frames into the encoder instead. " +
+ "One worker is slower, so keep the default worker count for renders that fit on disk."
+ );
+ }
+ switch (oneWorker.reason) {
+ case "format_excluded":
+ return `${outputFormat} output stores every frame at any worker count. Free up disk space or render to mp4.`;
+ case "disabled_by_config":
+ return "PRODUCER_ENABLE_STREAMING_ENCODE=false sends every render to disk. Free up disk space or unset it.";
+ case "duration_cap":
+ return (
+ "The streaming duration cap (PRODUCER_STREAMING_ENCODE_DURATION_CAP_ENABLED=true) sends renders longer " +
+ "than PRODUCER_STREAMING_ENCODE_MAX_DURATION_SECONDS to disk at any worker count. Free up disk space, " +
+ "unset the cap, or use --low-memory-mode, which streams past it on one worker."
+ );
+ default:
+ return "Free up disk space.";
+ }
+}
+
export function diskCaptureShortfallError(
needBytes: number,
freeBytes: number,
framesDir: string,
basis: "estimate" | "measured",
+ remedy: string,
): Error {
const basisNote =
basis === "measured"
@@ -212,16 +249,7 @@ export function diskCaptureShortfallError(
return new Error(
`Disk capture may need ~${(needBytes / 1e6).toFixed(1)} MB of temporary frame storage${basisNote}, ` +
`but only ${(freeBytes / 1e6).toFixed(1)} MB is free at ${framesDir}. ` +
- "Disk capture stores every frame as an image file (JPEG, or PNG for alpha). mp4/mov renders stream instead: " +
- "always at --workers 1, and multi-worker on Linux BeginFrame capture. " +
- "This render landed on disk because the output is png-sequence or gif, because it " +
- "is multi-worker screenshot capture (opt in with HF_CAPTURE_PARALLEL_STREAM=true), " +
- "or because streaming was switched off: HF_CAPTURE_PARALLEL_STREAM=false, " +
- "PRODUCER_ENABLE_STREAMING_ENCODE=false, or the duration cap " +
- "(PRODUCER_STREAMING_ENCODE_DURATION_CAP_ENABLED=true) on a render longer than " +
- "PRODUCER_STREAMING_ENCODE_MAX_DURATION_SECONDS. HDR and shader-transition renders " +
- "never reach this check; they run their own compositor. " +
- "Re-run with --workers 1, use --low-memory-mode, or free up disk space.",
+ `Disk capture stores every frame as an image file (JPEG, or PNG for alpha). ${remedy}`,
);
}
@@ -229,6 +257,7 @@ export function assertDiskCaptureHeadroom(
framesDir: string,
totalFrames: number,
captureOptions: CaptureOptions,
+ remedy: string,
freeDiskBytes?: (path: string) => number | null,
): void {
const headroom = inspectDiskCaptureHeadroom(
@@ -243,6 +272,7 @@ export function assertDiskCaptureHeadroom(
headroom.freeBytes,
framesDir,
"estimate",
+ remedy,
);
}
@@ -269,9 +299,10 @@ export function createDiskCaptureProjection(input: {
framesDir: string;
totalFrames: number;
measureBytes: () => number;
+ remedy: string;
freeDiskBytes?: (path: string) => number | null;
}): (capturedFrames: number) => void {
- const { framesDir, totalFrames, measureBytes, freeDiskBytes = freeDiskBytesAt } = input;
+ const { framesDir, totalFrames, measureBytes, remedy, freeDiskBytes = freeDiskBytesAt } = input;
let checked = false;
return (capturedFrames) => {
if (checked || capturedFrames < DISK_PROJECTION_SAMPLE_FRAMES) return;
@@ -283,7 +314,7 @@ export function createDiskCaptureProjection(input: {
if (remaining <= 0 || freeBytes === null) return;
const needBytes = Math.ceil((measured / capturedFrames) * remaining);
if (needBytes > freeBytes * 0.9) {
- throw diskCaptureShortfallError(needBytes, freeBytes, framesDir, "measured");
+ throw diskCaptureShortfallError(needBytes, freeBytes, framesDir, "measured", remedy);
}
};
}
@@ -346,11 +377,13 @@ export async function runCaptureStage(input: CaptureStageInput): Promise<Capture
}
const captureOptions = buildCaptureOptions();
- assertDiskCaptureHeadroom(framesDir, totalFrames, captureOptions);
+ const remedy = diskCaptureRemedy(cfg, job.config.format ?? "mp4", job.duration ?? 0, workerCount);
+ assertDiskCaptureHeadroom(framesDir, totalFrames, captureOptions, remedy);
const checkDiskProjection = createDiskCaptureProjection({
framesDir,
totalFrames,
measureBytes: () => measureCaptureFrameBytes(workDir, framesDir),
+ remedy,
});
if (workerCount > 1) {
diff --git a/packages/producer/src/services/render/stages/captureStreamingStage.test.ts b/packages/producer/src/services/render/stages/captureStreamingStage.test.ts
index de9f578c49..3914fb151a 100644
--- a/packages/producer/src/services/render/stages/captureStreamingStage.test.ts
+++ b/packages/producer/src/services/render/stages/captureStreamingStage.test.ts
@@ -207,6 +207,7 @@ mock.module("../../renderOrchestrator.js", () => ({
closeHdrVideoFrameSource: () => {},
createHdrPerfCollector: () => ({}),
executeDiskCaptureWithAdaptiveRetry: async () => [],
+ explainStreamingEncodeGate: () => ({ enabled: true, reason: "single_worker" }),
findMissingFrameRanges: () => [],
isTransientCaptureRetryEligible: () => false,
sampleDirectoryBytes: () => 0,