Skip to content

Write HTML. Render video. Built for agents.

fix(engine): exports carry full colour tags with every encoder, SDR and HDR (#5742)

Miguel Ángelea742c501880
diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.ts
index 74e5e278a2..4433c23bcd 100644
--- a/packages/engine/src/services/chunkEncoder.test.ts
+++ b/packages/engine/src/services/chunkEncoder.test.ts
@@ -1226,7 +1226,9 @@ describe("buildEncoderArgs color space", () => {
"out.mp4",
"nvenc",
);
- expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
+ expect(args[args.indexOf("-vf") + 1]).toBe(
+ "setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc:range=tv,pad=ceil(iw/2)*2:ceil(ih/2)*2",
+ );
});
it("converts alpha ProRes to BT.709 without an even-dim pad", () => {
@@ -1897,6 +1899,53 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
}
}, 30_000);
+ // VideoToolbox, like a libx264/libx265 given no colour options, writes only the colour its frames carry.
+ const streamTagsThrough = (args: string[], encoder: string[]): string => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-frame-tags-"));
+ try {
+ const out = join(dir, "out.mp4");
+ const vfAt = args.indexOf("-vf");
+ const vf = vfAt === -1 ? [] : ["-vf", args[vfAt + 1]!];
+ const source = ["-f", "lavfi", "-i", "color=c=0x808080:s=64x16:d=0.1"];
+ const encode = spawnSync(ffmpeg, ["-y", "-v", "error", ...source, ...vf, ...encoder, out]);
+ expect(encode.status).toBe(0);
+ const entries = "stream=color_space,color_transfer,color_primaries";
+ const probe = ["-v", "error", "-show_entries", entries, "-of", "csv=p=0", "--", out];
+ return spawnSync(getFfprobeBinary(), probe).stdout.toString().trim();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ };
+ const tinyFrame = { fps: { num: 30, den: 1 }, width: 64, height: 16, preset: "ultrafast" };
+
+ it("tags BT.709 on the frames, so an encoder that reads only them tags the stream", () => {
+ const args = buildEncoderArgs(
+ { ...tinyFrame, codec: "h264", quality: 23 },
+ ["-i", "in"],
+ "o.mp4",
+ );
+
+ expect(streamTagsThrough(args, ["-c:v", "libx264", "-preset", "ultrafast"])).toBe(
+ "bt709,bt709,bt709",
+ );
+ });
+
+ it.each([
+ ["pq", "smpte2084", null],
+ ["hlg", "arib-std-b67", null],
+ ["pq", "smpte2084", "videotoolbox"],
+ ] as const)("tags HDR %s (%s) on the frames for encoder %s", (transfer, trc, gpu) => {
+ const args = buildEncoderArgs(
+ { ...tinyFrame, codec: "h265", quality: 23, hdr: { transfer }, useGpu: gpu !== null },
+ ["-i", "in"],
+ "o.mp4",
+ gpu,
+ );
+ const x265 = ["-c:v", "libx265", "-pix_fmt", "yuv420p10le", "-x265-params", "log-level=error"];
+
+ expect(streamTagsThrough(args, x265)).toBe(`bt2020nc,${trc},bt2020`);
+ });
+
it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) mp4", () => {
const dir = mkdtempSync(join(tmpdir(), "hf-sdr-right-edge-"));
try {
diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts
index 98cd10aa08..24b19c919b 100644
--- a/packages/engine/src/services/chunkEncoder.ts
+++ b/packages/engine/src/services/chunkEncoder.ts
@@ -29,7 +29,7 @@ import {
buildVideoToolboxRateControlArgs,
mapPresetForGpuEncoder,
} from "../utils/gpuEncoder.js";
-import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
+import { type HdrTransfer, getHdrEncoderColorParams, hdrFrameTagsFilter } from "../utils/hdr.js";
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
import {
SDR_CAPTURE_TO_BT709_FILTER,
@@ -460,8 +460,9 @@ export function buildEncoderArgs(
// Range conversion: Chrome's full-range capture → limited/TV range; SDR also
// converts to the BT.709 matrix it is tagged with.
+ const hdrTags = options.hdr ? hdrFrameTagsFilter(options.hdr.transfer) : undefined;
const sdrFilter = options.hdr ? undefined : SDR_CAPTURE_TO_BT709_FILTER;
- const captureFilter = sdrFilter ?? "scale=in_range=pc:out_range=tv";
+ const captureFilter = sdrFilter ?? `scale=in_range=pc:out_range=tv,${hdrTags}`;
if (gpuEncoder === "vaapi") {
// vaapi already runs `format=nv12,hwupload`; the nv12 conversion aligns
// odd dimensions before upload, so only prepend the colour conversion.
@@ -474,7 +475,12 @@ export function buildEncoderArgs(
// encoder. They hit the same "height not divisible by 2" abort as
// libx264 on an odd-sized 4:2:0 canvas, so pad odd dimensions up to even
// on the software side before the encode.
- const vf = withEvenDimensionPad(sdrFilter ?? "", pixelFormat, options.width, options.height);
+ const vf = withEvenDimensionPad(
+ sdrFilter ?? hdrTags ?? "",
+ pixelFormat,
+ options.width,
+ options.height,
+ );
if (vf) args.push("-vf", vf);
} else {
// The scale filter handles both 8-bit and 10-bit correctly. Pad odd
diff --git a/packages/engine/src/services/streamingEncoder.test.ts b/packages/engine/src/services/streamingEncoder.test.ts
index 74b9111873..0616ca47f8 100644
--- a/packages/engine/src/services/streamingEncoder.test.ts
+++ b/packages/engine/src/services/streamingEncoder.test.ts
@@ -1228,6 +1228,70 @@ describe("createFrameReorderBuffer abort (interleaved parallel drain)", () => {
});
});
+describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)(
+ "buildStreamingArgs frame tags",
+ () => {
+ // VideoToolbox, like a libx264/libx265 given no colour options, writes only the colour its frames carry.
+ const streamTagsThrough = (args: string[], encoder: string[]): string => {
+ const dir = mkdtempSync(join(tmpdir(), "se-frame-tags-"));
+ try {
+ const out = join(dir, "out.mp4");
+ const vfAt = args.indexOf("-vf");
+ const vf = vfAt === -1 ? [] : ["-vf", args[vfAt + 1]!];
+ const source = ["-f", "lavfi", "-i", "color=c=0x808080:s=64x16:d=0.1"];
+ const encode = spawnSync("ffmpeg", [
+ "-y",
+ "-v",
+ "error",
+ ...source,
+ ...vf,
+ ...encoder,
+ out,
+ ]);
+ expect(encode.status).toBe(0);
+ const entries = "stream=color_space,color_transfer,color_primaries";
+ const probe = ["-v", "error", "-show_entries", entries, "-of", "csv=p=0", "--", out];
+ return spawnSync("ffprobe", probe).stdout.toString().trim();
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ };
+ const tiny = { width: 64, height: 16 };
+
+ it.each([
+ ["captured frames", {}],
+ ["raw RGB frames", { rawInputFormat: "rgb48le" as const }],
+ ])("tags BT.709 on %s, so an encoder that reads only them tags the stream", (_, extra) => {
+ const args = buildStreamingArgs({ ...baseSdr, ...tiny, ...extra }, "o.mp4");
+
+ expect(streamTagsThrough(args, ["-c:v", "libx264", "-preset", "ultrafast"])).toBe(
+ "bt709,bt709,bt709",
+ );
+ });
+
+ it.each([
+ [baseHdrPq, "smpte2084", null],
+ [baseHdrHlg, "arib-std-b67", null],
+ [baseHdrPq, "smpte2084", "videotoolbox"],
+ ] as const)(
+ "tags raw HDR frames %#, so an encoder that reads only them tags the stream",
+ (base, trc, gpu) => {
+ const args = buildStreamingArgs({ ...base, ...tiny, useGpu: gpu !== null }, "o.mp4", gpu);
+ const x265 = [
+ "-c:v",
+ "libx265",
+ "-pix_fmt",
+ "yuv420p10le",
+ "-x265-params",
+ "log-level=error",
+ ];
+
+ expect(streamTagsThrough(args, x265)).toBe(`bt2020nc,${trc},bt2020`);
+ },
+ );
+ },
+);
+
describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)(
"buildStreamingArgs raw SDR colour",
() => {
diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.ts
index 605f450380..0e00b4d215 100644
--- a/packages/engine/src/services/streamingEncoder.ts
+++ b/packages/engine/src/services/streamingEncoder.ts
@@ -37,11 +37,10 @@ import {
isExternalFfmpegInterruption,
} from "../utils/runFfmpeg.js";
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
-import { getHdrEncoderColorParams } from "../utils/hdr.js";
+import { getHdrEncoderColorParams, hdrFrameTagsFilter } from "../utils/hdr.js";
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
import {
SDR_CAPTURE_TO_BT709_FILTER,
- SDR_RGB_TO_BT709_FILTER,
SDR_RGB_TO_TAGGED_BT709_FILTER,
} from "../utils/sdrCaptureColor.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
@@ -424,24 +423,28 @@ export function buildStreamingArgs(
);
}
- // Raw HDR input (from WebGPU pipeline) is already PQ-encoded, so only SDR gets a
- // conversion: raw sRGB or Chrome's BT.601 JPEG screenshots to BT.709 limited range.
- const sdrFilter = options.rawInputFormat
- ? SDR_RGB_TO_BT709_FILTER
- : SDR_CAPTURE_TO_BT709_FILTER;
- if (options.rawInputFormat && options.hdr) {
- // No filter needed — PQ data goes straight to encoder
- } else if (gpuEncoder === "vaapi") {
+ // Raw HDR input (from WebGPU pipeline) is already PQ-encoded and is only tagged; SDR is
+ // converted: raw sRGB or Chrome's BT.601 JPEG screenshots to tagged BT.709 limited range.
+ const frameFilter =
+ options.rawInputFormat && options.hdr
+ ? hdrFrameTagsFilter(options.hdr.transfer)
+ : options.rawInputFormat
+ ? SDR_RGB_TO_TAGGED_BT709_FILTER
+ : SDR_CAPTURE_TO_BT709_FILTER;
+ if (gpuEncoder === "vaapi") {
// vaapi already runs `format=nv12,hwupload`; the nv12 conversion aligns
// odd dimensions before upload, so only prepend the colour conversion.
const vfIdx = args.indexOf("-vf");
if (vfIdx !== -1) {
- args[vfIdx + 1] = `${sdrFilter},${args[vfIdx + 1]}`;
+ args[vfIdx + 1] = `${frameFilter},${args[vfIdx + 1]}`;
}
} else {
// Pad odd dimensions up to even so 4:2:0 encoders (software and
// nvenc/videotoolbox/qsv/amf) don't abort with "height not divisible by 2".
- args.push("-vf", withEvenDimensionPad(sdrFilter, pixelFormat, options.width, options.height));
+ args.push(
+ "-vf",
+ withEvenDimensionPad(frameFilter, pixelFormat, options.width, options.height),
+ );
}
// Fixed timescale for consistent A/V timing across platforms.
diff --git a/packages/engine/src/utils/hdr.ts b/packages/engine/src/utils/hdr.ts
index 5a58e302f1..df0b75744e 100644
--- a/packages/engine/src/utils/hdr.ts
+++ b/packages/engine/src/utils/hdr.ts
@@ -106,6 +106,11 @@ export function getHdrEncoderColorParams(
};
}
+export function hdrFrameTagsFilter(transfer: HdrTransfer): string {
+ const { colorPrimaries, colorTrc, colorspace } = getHdrEncoderColorParams(transfer);
+ return `setparams=color_primaries=${colorPrimaries}:color_trc=${colorTrc}:colorspace=${colorspace}:range=tv`;
+}
+
export interface CompositionHdrInfo {
hasHdr: boolean;
dominantTransfer: HdrTransfer | null;
diff --git a/packages/engine/src/utils/sdrCaptureColor.ts b/packages/engine/src/utils/sdrCaptureColor.ts
index fcb9977362..425a59a9a6 100644
--- a/packages/engine/src/utils/sdrCaptureColor.ts
+++ b/packages/engine/src/utils/sdrCaptureColor.ts
@@ -1,5 +1,5 @@
export const SDR_RGB_TO_BT709_FILTER = "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor";
-export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor+accurate_rnd,format=gbrp,${SDR_RGB_TO_BT709_FILTER}`;
-
export const SDR_RGB_TO_TAGGED_BT709_FILTER = `${SDR_RGB_TO_BT709_FILTER},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv`;
+
+export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor+accurate_rnd,format=gbrp,${SDR_RGB_TO_TAGGED_BT709_FILTER}`;