Skip to content

Write HTML. Render video. Built for agents.

fix(producer): a render no longer publishes a short or wrong-size video silently (#5978)

Miguel Ángel8942aa23bdaa
diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts
index 24b19c919b..cbca3e3e2c 100644
--- a/packages/engine/src/services/chunkEncoder.ts
+++ b/packages/engine/src/services/chunkEncoder.ts
@@ -258,7 +258,8 @@ export function buildEncoderArgs(
options = { ...options, hdr: undefined };
}
- const args: string[] = [...inputArgs, "-r", fpsToFfmpegArg(fps)];
+ // Without -xerror, ffmpeg ends the video at the first frame file it cannot read and still exits 0.
+ const args: string[] = ["-xerror", ...inputArgs, "-r", fpsToFfmpegArg(fps)];
const shouldUseGpu = useGpu && gpuEncoder !== null;
if (codec === "h264" || codec === "h265") {
@@ -510,6 +511,25 @@ export function buildEncoderArgs(
const framesReader = (onFrames?: (frames: number) => void) =>
onFrames && ffmpegStatsReader(({ frames }) => frames !== undefined && onFrames(frames));
+
+/** ffmpeg's frame read error, which the encoder summary pushes out of the stderr tail, and where it stopped. */
+function frameReadFailure(
+ stderr: string,
+ framePattern: string,
+ startNumber: number,
+ framesRead: number,
+ frameCount: number,
+): string {
+ const lines = stderr.match(/^.*(?:Error during demuxing|Could not open file).*$/gm) ?? [];
+ const readError = [...new Set(lines.map((line) => line.trim()))].join("; ");
+ if (!readError) return "";
+ const name = framePattern.replace(/%0(\d+)d/, (_, width: string) =>
+ String(startNumber + framesRead).padStart(Number(width), "0"),
+ );
+ const at = framesRead < frameCount ? ` at ${name} (${framesRead} of ${frameCount} read)` : "";
+ return `[FFmpeg] could not read a frame${at}: ${readError}\n`;
+}
+
const secondsReader = (onSeconds?: (seconds: number) => void) =>
onSeconds && ffmpegStatsReader(({ seconds }) => seconds !== undefined && onSeconds(seconds));
@@ -550,10 +570,14 @@ export async function encodeFramesFromDir(
const inputArgs = ["-framerate", fpsToFfmpegArg(options.fps), "-i", inputPath];
const args = buildEncoderArgs(options, inputArgs, outputPath, gpuEncoder);
const encodeTimeout = config?.ffmpegEncodeTimeout ?? DEFAULT_CONFIG.ffmpegEncodeTimeout;
+ let framesRead = 0;
const result = await runFfmpeg(args, {
signal,
timeout: encodeTimeout,
- onStderr: framesReader(onFramesEncoded),
+ onStderr: framesReader((frames) => {
+ framesRead = frames;
+ onFramesEncoded?.(frames);
+ }),
});
if (result.terminationReason === "abort") {
return {
@@ -573,7 +597,8 @@ export async function encodeFramesFromDir(
framesEncoded: 0,
fileSize: 0,
error: appendEncodeTimeoutMessage(
- formatFfmpegError(result.exitCode, result.stderr),
+ frameReadFailure(result.stderr, framePattern, 0, framesRead, frameCount) +
+ formatFfmpegError(result.exitCode, result.stderr),
result.terminationReason === "deadline",
encodeTimeout,
),
@@ -721,19 +746,27 @@ export async function encodeFramesChunkedConcat(
if (options.useGpu) gpuEncoder = await getCachedGpuEncoder();
const args = buildEncoderArgs(options, inputArgs, chunkPath, gpuEncoder);
const encodeTimeout = config?.ffmpegEncodeTimeout ?? DEFAULT_CONFIG.ffmpegEncodeTimeout;
+ let framesRead = 0;
const processResult = await runFfmpeg(args, {
signal,
timeout: encodeTimeout,
- onStderr: framesReader(
- onFramesEncoded && ((frames) => onFramesEncoded(startNumber + frames)),
- ),
+ onStderr: framesReader((frames) => {
+ framesRead = frames;
+ onFramesEncoded?.(startNumber + frames);
+ }),
});
const chunkResult = {
success: processResult.success,
error: processResult.success
? undefined
: appendEncodeTimeoutMessage(
- `Chunk ${i} encode failed: ${processResult.stderr.slice(-400)}`,
+ frameReadFailure(
+ processResult.stderr,
+ framePattern,
+ startNumber,
+ framesRead,
+ framesInChunk,
+ ) + `Chunk ${i} encode failed: ${processResult.stderr.slice(-400)}`,
processResult.terminationReason === "deadline",
encodeTimeout,
),
diff --git a/packages/engine/src/services/chunkEncoder.unreadableFrame.test.ts b/packages/engine/src/services/chunkEncoder.unreadableFrame.test.ts
new file mode 100644
index 0000000000..2ac931c323
--- /dev/null
+++ b/packages/engine/src/services/chunkEncoder.unreadableFrame.test.ts
@@ -0,0 +1,83 @@
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+import { encodeFramesChunkedConcat, encodeFramesFromDir } from "./chunkEncoder.js";
+import type { EncoderOptions } from "./chunkEncoder.types.js";
+
+const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
+
+const options: EncoderOptions = {
+ fps: { num: 25, den: 1 },
+ width: 32,
+ height: 32,
+ codec: "h264",
+ preset: "ultrafast",
+ quality: 30,
+};
+
+describe.skipIf(!HAS_FFMPEG)("frame-sequence encode with an unreadable frame", () => {
+ let framesDir: string;
+
+ // A directory under a frame's name: ffmpeg sees it in the sequence but cannot read it, for any user.
+ function writeFrames(count: number, unreadable: number): void {
+ const result = spawnSync(getFfmpegBinary(), [
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=size=32x32:rate=25",
+ "-frames:v",
+ String(count),
+ "-start_number",
+ "0",
+ join(framesDir, "frame_%06d.jpg"),
+ ]);
+ expect(result.status).toBe(0);
+ const name = `frame_${String(unreadable).padStart(6, "0")}.jpg`;
+ rmSync(join(framesDir, name));
+ mkdirSync(join(framesDir, name));
+ }
+
+ beforeEach(() => {
+ framesDir = mkdtempSync(join(tmpdir(), "hf-unreadable-frame-"));
+ });
+
+ afterEach(() => {
+ rmSync(framesDir, { recursive: true, force: true });
+ });
+
+ it("fails at that frame instead of encoding a shorter video", async () => {
+ writeFrames(20, 10);
+
+ const result = await encodeFramesFromDir(
+ framesDir,
+ "frame_%06d.jpg",
+ join(framesDir, "out", "video.mp4"),
+ options,
+ );
+
+ expect(result.success).toBe(false);
+ expect(result.error).toContain("frame_000010.jpg");
+ expect(result.error).toMatch(/Error during demuxing: \S/);
+ });
+
+ it("fails a chunked encode at that frame too", async () => {
+ writeFrames(40, 15);
+
+ const result = await encodeFramesChunkedConcat(
+ framesDir,
+ "frame_%06d.jpg",
+ join(framesDir, "out", "video.mp4"),
+ options,
+ 30,
+ );
+
+ expect(result.success).toBe(false);
+ expect(result.error).toContain("frame_000015.jpg");
+ expect(result.error).toMatch(/Error during demuxing: \S/);
+ });
+});
diff --git a/packages/engine/src/services/streamStartOffset.test.ts b/packages/engine/src/services/streamStartOffset.test.ts
index 2d87f1f7c7..58d13c461d 100644
--- a/packages/engine/src/services/streamStartOffset.test.ts
+++ b/packages/engine/src/services/streamStartOffset.test.ts
@@ -93,7 +93,7 @@ describe.skipIf(!HAS_FFMPEG)(
ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]);
}, 60_000);
- afterAll(() => rmSync(dir, { recursive: true, force: true }));
+ afterAll(() => rmSync(dir, { recursive: true, force: true }), 30_000);
const flatLane = {
target: "rate",
diff --git a/packages/producer/src/services/render/artifactTransaction.test.ts b/packages/producer/src/services/render/artifactTransaction.test.ts
index e4b9b35902..950a579ef7 100644
--- a/packages/producer/src/services/render/artifactTransaction.test.ts
+++ b/packages/producer/src/services/render/artifactTransaction.test.ts
@@ -291,6 +291,49 @@ describe("ArtifactTransaction", () => {
transaction.rollback();
});
+ it("rejects a video encoded at a different size than the render asked for", async () => {
+ // A 4K export of a 1080x1920 HDR project once published a 1080x1920 file as done.
+ const dir = tempDir();
+ const probe: ArtifactDurationProbe = async () => ({
+ durationSeconds: 2,
+ width: 1080,
+ height: 1920,
+ });
+ const transaction = new ArtifactTransaction(join(dir, "render.mp4"), "file", undefined, probe);
+ writeFileSync(transaction.stagingPath, "fake-mp4-bytes-that-are-non-empty");
+
+ await expect(
+ transaction.validate({
+ expectedDurationSeconds: 2,
+ expectedWidth: 2160,
+ expectedHeight: 3840,
+ }),
+ ).rejects.toThrow("expected 2160x3840, probed 1080x1920");
+
+ transaction.rollback();
+ });
+
+ it("accepts the one-pixel pad a 4:2:0 encode gives an odd side", async () => {
+ const dir = tempDir();
+ const probe: ArtifactDurationProbe = async () => ({
+ durationSeconds: 2,
+ width: 1082,
+ height: 724,
+ });
+ const transaction = new ArtifactTransaction(join(dir, "render.mp4"), "file", undefined, probe);
+ writeFileSync(transaction.stagingPath, "fake-mp4-bytes-that-are-non-empty");
+
+ await expect(
+ transaction.validate({
+ expectedDurationSeconds: 2,
+ expectedWidth: 1081,
+ expectedHeight: 724,
+ }),
+ ).resolves.toBeUndefined();
+
+ transaction.rollback();
+ });
+
it("rejects a video whose probed frame count is far short of the captured frame count", async () => {
// #3395 field packet frame shortfalls (148 of 300, 1240 of 1566) are
// both well beyond the single-frame tolerance, so the frame-count check
@@ -502,6 +545,12 @@ describe("buildArtifactExpectation", () => {
}
});
+ it("carries the requested output size", () => {
+ expect(
+ buildArtifactExpectation({ ...base, outputFormat: "mp4", width: 2160, height: 3840 }),
+ ).toMatchObject({ expectedWidth: 2160, expectedHeight: 3840 });
+ });
+
it("names the video playlist and requires the master for hls", () => {
expect(buildArtifactExpectation({ ...base, outputFormat: "hls" })).toEqual({
expectedDurationSeconds: 4,
diff --git a/packages/producer/src/services/render/artifactTransaction.ts b/packages/producer/src/services/render/artifactTransaction.ts
index 296a63a817..0005579e56 100644
--- a/packages/producer/src/services/render/artifactTransaction.ts
+++ b/packages/producer/src/services/render/artifactTransaction.ts
@@ -42,6 +42,8 @@ const defaultFileSystem: ArtifactTransactionFileSystem = {
export interface ArtifactDurationProbeResult {
durationSeconds: number;
frames?: number;
+ width?: number;
+ height?: number;
}
/**
@@ -68,6 +70,8 @@ async function defaultArtifactDurationProbe(path: string): Promise<ArtifactDurat
return {
durationSeconds: meta.durationSeconds,
frames: meta.frames,
+ width: meta.width,
+ height: meta.height,
};
}
@@ -97,6 +101,9 @@ export interface ArtifactValidationExpectation {
expectedDurationSeconds: number;
fps?: number;
expectedFrames?: number;
+ /** The output size the render was asked for; a route that ignores it must not publish. */
+ expectedWidth?: number;
+ expectedHeight?: number;
toleranceSeconds?: number;
/** Path relative to the staging directory to run the duration probe against. */
probeTarget?: string;
@@ -122,6 +129,8 @@ export function buildArtifactExpectation(input: {
durationSeconds: number;
fps: number;
expectedFrames: number | undefined;
+ width?: number;
+ height?: number;
}): ArtifactValidationExpectation | undefined {
if (input.outputFormat === "png-sequence" || input.outputFormat === "gif") return undefined;
if (!Number.isFinite(input.durationSeconds) || input.durationSeconds <= 0) return undefined;
@@ -129,6 +138,8 @@ export function buildArtifactExpectation(input: {
expectedDurationSeconds: input.durationSeconds,
fps: input.fps,
expectedFrames: input.expectedFrames,
+ expectedWidth: input.width,
+ expectedHeight: input.height,
};
if (input.outputFormat !== "hls") return expectation;
return {
@@ -239,6 +250,23 @@ function assertFrameCountWithinTolerance(
);
}
+function assertSizeMatches(
+ expected: ArtifactValidationExpectation,
+ probed: ArtifactDurationProbeResult,
+ stagingPath: string,
+): void {
+ const { expectedWidth: width, expectedHeight: height } = expected;
+ if (width === undefined || height === undefined) return;
+ if (probed.width === undefined || probed.height === undefined) return;
+ // A 4:2:0 encode pads an odd side up by one pixel.
+ const fits = (want: number, got: number): boolean => got === want || got === want + (want % 2);
+ if (fits(width, probed.width) && fits(height, probed.height)) return;
+ throw new Error(
+ `Render artifact has the wrong size: expected ${width}x${height}, ` +
+ `probed ${probed.width}x${probed.height}. Artifact: ${stagingPath}`,
+ );
+}
+
/**
* Stages a render beside its final destination and promotes only a validated
* artifact. File promotion uses one atomic replacement rename, so an existing
@@ -342,6 +370,7 @@ export class ArtifactTransaction {
// above carries the gate on its own.
assertFrameCountWithinTolerance(expected.expectedFrames, probed.frames, probePath);
}
+ assertSizeMatches(expected, probed, probePath);
}
private async runDurationProbe(probePath: string): Promise<ArtifactDurationProbeResult> {
diff --git a/packages/producer/src/services/render/outputScaleRoute.test.ts b/packages/producer/src/services/render/outputScaleRoute.test.ts
index 306b953210..b84caf0474 100644
--- a/packages/producer/src/services/render/outputScaleRoute.test.ts
+++ b/packages/producer/src/services/render/outputScaleRoute.test.ts
@@ -11,6 +11,12 @@ describe("assertOutputScaleSupported", () => {
expect(() => assertOutputScaleSupported(scale, "hdr_layered")).toThrow(/hdr_layered/);
});
+ it("names the ways out that apply: --sdr for HDR footage, mp4 for shader transitions", () => {
+ expect(() => assertOutputScaleSupported(2, "hdr_layered")).toThrow(/add --sdr/);
+ expect(() => assertOutputScaleSupported(2, "hdr_layered")).toThrow(/render as mp4/);
+ expect(() => assertOutputScaleSupported(2, "hdr_layered")).not.toThrow(/page-side compositing/);
+ });
+
it("lets every route run at the composition's size, and the screenshot routes scale", () => {
for (const route of ["sdr_streaming", "sdr_disk", "sdr_segmented", "hdr_layered"] as const)
expect(() => assertOutputScaleSupported(1, route)).not.toThrow();
diff --git a/packages/producer/src/services/render/outputScaleRoute.ts b/packages/producer/src/services/render/outputScaleRoute.ts
index 6b3256ccf9..f601850dd6 100644
--- a/packages/producer/src/services/render/outputScaleRoute.ts
+++ b/packages/producer/src/services/render/outputScaleRoute.ts
@@ -26,6 +26,7 @@ export function assertOutputScaleSupported(
if (reason === null) return;
throw new Error(
`[Render] --resolution other than the composition's size is not supported on the "${route}" ` +
- `capture route: ${reason}. Render at the composition's size, or as mp4 with page-side compositing.`,
+ `capture route: ${reason}. Render at the composition's size; or, for HDR footage, add --sdr to ` +
+ "render this size in standard colour; or, for shader transitions, render as mp4.",
);
}
diff --git a/packages/producer/src/services/render/stages/encodeStage.test.ts b/packages/producer/src/services/render/stages/encodeStage.test.ts
index 816e55985d..74858dab71 100644
--- a/packages/producer/src/services/render/stages/encodeStage.test.ts
+++ b/packages/producer/src/services/render/stages/encodeStage.test.ts
@@ -129,6 +129,7 @@ describe("gif encode args", () => {
it("builds the palettegen pass with diff statistics", () => {
expect(buildGifPalettegenArgs(input)).toEqual([
"-y",
+ "-xerror",
"-framerate",
"15",
"-reinit_filter",
@@ -144,6 +145,7 @@ describe("gif encode args", () => {
it("builds the paletteuse pass with Sierra dithering and loop count", () => {
expect(buildGifPaletteuseArgs({ ...input, loop: 3 })).toEqual([
"-y",
+ "-xerror",
"-framerate",
"15",
"-reinit_filter",
diff --git a/packages/producer/src/services/render/stages/gifEncodeArgs.ts b/packages/producer/src/services/render/stages/gifEncodeArgs.ts
index b420cf2847..5ed5160872 100644
--- a/packages/producer/src/services/render/stages/gifEncodeArgs.ts
+++ b/packages/producer/src/services/render/stages/gifEncodeArgs.ts
@@ -18,9 +18,11 @@ function fpsToFfmpegArg(fps: Fps): string {
const KEEP_FILTER_GRAPH_WHEN_FRAMES_DROP_ALPHA = ["-reinit_filter", "0"];
const WRITE_EVERY_FRAME_WHOLE = ["-gifflags", "0"];
+const FAIL_ON_UNREADABLE_FRAME_INSTEAD_OF_ENDING_THERE = ["-xerror"];
function framesInput(input: GifEncodeArgsInput, fpsArg: string): string[] {
return [
+ ...FAIL_ON_UNREADABLE_FRAME_INSTEAD_OF_ENDING_THERE,
"-framerate",
fpsArg,
...KEEP_FILTER_GRAPH_WHEN_FRAMES_DROP_ALPHA,
diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts
index 57be38faef..6af2f91518 100644
--- a/packages/producer/src/services/renderOrchestrator.test.ts
+++ b/packages/producer/src/services/renderOrchestrator.test.ts
@@ -98,6 +98,20 @@ describe("resolveRenderWorkDirPrefix", () => {
join("C:/Temp", "hf-render-"),
);
});
+
+ it("keeps macOS frames out of the project folder, which a sync client may be watching", () => {
+ const outputPath = join("/Users/me/Nextcloud/project", "renders", "final.mp4");
+
+ expect(resolveRenderWorkDirPrefix(outputPath, "job-id", "darwin", "/var/folders/x/T")).toBe(
+ join("/var/folders/x/T", "hf-render-"),
+ );
+ });
+
+ it("keeps Linux frames beside the output, where a container's small /tmp cannot fill", () => {
+ expect(resolveRenderWorkDirPrefix("/work/renders/final.mp4", "job-id", "linux", "/tmp")).toBe(
+ join("/work/renders", "work-job-id-"),
+ );
+ });
});
describe("captureAttemptMadeProgress", () => {
diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts
index c86407d7c9..c67efe3c99 100644
--- a/packages/producer/src/services/renderOrchestrator.ts
+++ b/packages/producer/src/services/renderOrchestrator.ts
@@ -1071,7 +1071,7 @@ export function resolveRenderWorkDirPrefix(
platform: NodeJS.Platform = process.platform,
systemTempDir: string = tmpdir(),
): string {
- if (platform === "win32") return join(systemTempDir, "hf-render-");
+ if (platform === "win32" || platform === "darwin") return join(systemTempDir, "hf-render-");
return join(dirname(outputPath), `work-${jobId}-`);
}
@@ -3560,6 +3560,7 @@ async function executeRenderPipeline(input: {
// comps that route to the layered/HDR paths.)
const nativeHdrIds = new Set([...nativeHdrVideoIds, ...nativeHdrImageIds]);
const hasHdrContent = Boolean(effectiveHdr && nativeHdrIds.size > 0);
+ if (hasHdrContent) assertOutputScaleSupported(deviceScaleFactor, "hdr_layered");
// DE priority inversion eligibility — evaluated BEFORE capture calibration
// because when every multi-worker resolution would be inverted to 1 anyway,
// the calibration stage (a throwaway Chrome launch + timeline-spread sample
@@ -5084,6 +5085,8 @@ async function executeRenderPipeline(input: {
durationSeconds: job.duration,
fps: fpsToNumber(job.config.fps),
expectedFrames: captureTotalFrames,
+ width: outputWidth,
+ height: outputHeight,
}),
);