Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): exports carry full colour tags with every encoder, SDR and HDR (#5742)
Miguel Ángelea742c501880
diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.tsindex 74e5e278a2..4433c23bcd 100644--- a/packages/engine/src/services/chunkEncoder.test.ts+++ b/packages/engine/src/services/chunkEncoder.test.ts@@ -1226,7 +1226,9 @@ describe("buildEncoderArgs color space", () => {"out.mp4","nvenc",);- expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");+ expect(args[args.indexOf("-vf") + 1]).toBe(+ "setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc:range=tv,pad=ceil(iw/2)*2:ceil(ih/2)*2",+ );});it("converts alpha ProRes to BT.709 without an even-dim pad", () => {@@ -1897,6 +1899,53 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {}}, 30_000);+ // VideoToolbox, like a libx264/libx265 given no colour options, writes only the colour its frames carry.+ const streamTagsThrough = (args: string[], encoder: string[]): string => {+ const dir = mkdtempSync(join(tmpdir(), "hf-frame-tags-"));+ try {+ const out = join(dir, "out.mp4");+ const vfAt = args.indexOf("-vf");+ const vf = vfAt === -1 ? [] : ["-vf", args[vfAt + 1]!];+ const source = ["-f", "lavfi", "-i", "color=c=0x808080:s=64x16:d=0.1"];+ const encode = spawnSync(ffmpeg, ["-y", "-v", "error", ...source, ...vf, ...encoder, out]);+ expect(encode.status).toBe(0);+ const entries = "stream=color_space,color_transfer,color_primaries";+ const probe = ["-v", "error", "-show_entries", entries, "-of", "csv=p=0", "--", out];+ return spawnSync(getFfprobeBinary(), probe).stdout.toString().trim();+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ };+ const tinyFrame = { fps: { num: 30, den: 1 }, width: 64, height: 16, preset: "ultrafast" };++ it("tags BT.709 on the frames, so an encoder that reads only them tags the stream", () => {+ const args = buildEncoderArgs(+ { ...tinyFrame, codec: "h264", quality: 23 },+ ["-i", "in"],+ "o.mp4",+ );++ expect(streamTagsThrough(args, ["-c:v", "libx264", "-preset", "ultrafast"])).toBe(+ "bt709,bt709,bt709",+ );+ });++ it.each([+ ["pq", "smpte2084", null],+ ["hlg", "arib-std-b67", null],+ ["pq", "smpte2084", "videotoolbox"],+ ] as const)("tags HDR %s (%s) on the frames for encoder %s", (transfer, trc, gpu) => {+ const args = buildEncoderArgs(+ { ...tinyFrame, codec: "h265", quality: 23, hdr: { transfer }, useGpu: gpu !== null },+ ["-i", "in"],+ "o.mp4",+ gpu,+ );+ const x265 = ["-c:v", "libx265", "-pix_fmt", "yuv420p10le", "-x265-params", "log-level=error"];++ expect(streamTagsThrough(args, x265)).toBe(`bt2020nc,${trc},bt2020`);+ });+it("keeps the rightmost 8 columns of a 1080-wide (16n + 8) mp4", () => {const dir = mkdtempSync(join(tmpdir(), "hf-sdr-right-edge-"));try {diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.tsindex 98cd10aa08..24b19c919b 100644--- a/packages/engine/src/services/chunkEncoder.ts+++ b/packages/engine/src/services/chunkEncoder.ts@@ -29,7 +29,7 @@ import {buildVideoToolboxRateControlArgs,mapPresetForGpuEncoder,} from "../utils/gpuEncoder.js";-import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";+import { type HdrTransfer, getHdrEncoderColorParams, hdrFrameTagsFilter } from "../utils/hdr.js";import { withEvenDimensionPad } from "../utils/evenDimensions.js";import {SDR_CAPTURE_TO_BT709_FILTER,@@ -460,8 +460,9 @@ export function buildEncoderArgs(// Range conversion: Chrome's full-range capture → limited/TV range; SDR also// converts to the BT.709 matrix it is tagged with.+ const hdrTags = options.hdr ? hdrFrameTagsFilter(options.hdr.transfer) : undefined;const sdrFilter = options.hdr ? undefined : SDR_CAPTURE_TO_BT709_FILTER;- const captureFilter = sdrFilter ?? "scale=in_range=pc:out_range=tv";+ const captureFilter = sdrFilter ?? `scale=in_range=pc:out_range=tv,${hdrTags}`;if (gpuEncoder === "vaapi") {// vaapi already runs `format=nv12,hwupload`; the nv12 conversion aligns// odd dimensions before upload, so only prepend the colour conversion.@@ -474,7 +475,12 @@ export function buildEncoderArgs(// encoder. They hit the same "height not divisible by 2" abort as// libx264 on an odd-sized 4:2:0 canvas, so pad odd dimensions up to even// on the software side before the encode.- const vf = withEvenDimensionPad(sdrFilter ?? "", pixelFormat, options.width, options.height);+ const vf = withEvenDimensionPad(+ sdrFilter ?? hdrTags ?? "",+ pixelFormat,+ options.width,+ options.height,+ );if (vf) args.push("-vf", vf);} else {// The scale filter handles both 8-bit and 10-bit correctly. Pad odddiff --git a/packages/engine/src/services/streamingEncoder.test.ts b/packages/engine/src/services/streamingEncoder.test.tsindex 74b9111873..0616ca47f8 100644--- a/packages/engine/src/services/streamingEncoder.test.ts+++ b/packages/engine/src/services/streamingEncoder.test.ts@@ -1228,6 +1228,70 @@ describe("createFrameReorderBuffer abort (interleaved parallel drain)", () => {});});+describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)(+ "buildStreamingArgs frame tags",+ () => {+ // VideoToolbox, like a libx264/libx265 given no colour options, writes only the colour its frames carry.+ const streamTagsThrough = (args: string[], encoder: string[]): string => {+ const dir = mkdtempSync(join(tmpdir(), "se-frame-tags-"));+ try {+ const out = join(dir, "out.mp4");+ const vfAt = args.indexOf("-vf");+ const vf = vfAt === -1 ? [] : ["-vf", args[vfAt + 1]!];+ const source = ["-f", "lavfi", "-i", "color=c=0x808080:s=64x16:d=0.1"];+ const encode = spawnSync("ffmpeg", [+ "-y",+ "-v",+ "error",+ ...source,+ ...vf,+ ...encoder,+ out,+ ]);+ expect(encode.status).toBe(0);+ const entries = "stream=color_space,color_transfer,color_primaries";+ const probe = ["-v", "error", "-show_entries", entries, "-of", "csv=p=0", "--", out];+ return spawnSync("ffprobe", probe).stdout.toString().trim();+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ };+ const tiny = { width: 64, height: 16 };++ it.each([+ ["captured frames", {}],+ ["raw RGB frames", { rawInputFormat: "rgb48le" as const }],+ ])("tags BT.709 on %s, so an encoder that reads only them tags the stream", (_, extra) => {+ const args = buildStreamingArgs({ ...baseSdr, ...tiny, ...extra }, "o.mp4");++ expect(streamTagsThrough(args, ["-c:v", "libx264", "-preset", "ultrafast"])).toBe(+ "bt709,bt709,bt709",+ );+ });++ it.each([+ [baseHdrPq, "smpte2084", null],+ [baseHdrHlg, "arib-std-b67", null],+ [baseHdrPq, "smpte2084", "videotoolbox"],+ ] as const)(+ "tags raw HDR frames %#, so an encoder that reads only them tags the stream",+ (base, trc, gpu) => {+ const args = buildStreamingArgs({ ...base, ...tiny, useGpu: gpu !== null }, "o.mp4", gpu);+ const x265 = [+ "-c:v",+ "libx265",+ "-pix_fmt",+ "yuv420p10le",+ "-x265-params",+ "log-level=error",+ ];++ expect(streamTagsThrough(args, x265)).toBe(`bt2020nc,${trc},bt2020`);+ },+ );+ },+);+describe.skipIf(spawnSync("ffmpeg", ["-version"]).status !== 0)("buildStreamingArgs raw SDR colour",() => {diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.tsindex 605f450380..0e00b4d215 100644--- a/packages/engine/src/services/streamingEncoder.ts+++ b/packages/engine/src/services/streamingEncoder.ts@@ -37,11 +37,10 @@ import {isExternalFfmpegInterruption,} from "../utils/runFfmpeg.js";import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";-import { getHdrEncoderColorParams } from "../utils/hdr.js";+import { getHdrEncoderColorParams, hdrFrameTagsFilter } from "../utils/hdr.js";import { withEvenDimensionPad } from "../utils/evenDimensions.js";import {SDR_CAPTURE_TO_BT709_FILTER,- SDR_RGB_TO_BT709_FILTER,SDR_RGB_TO_TAGGED_BT709_FILTER,} from "../utils/sdrCaptureColor.js";import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";@@ -424,24 +423,28 @@ export function buildStreamingArgs();}- // Raw HDR input (from WebGPU pipeline) is already PQ-encoded, so only SDR gets a- // conversion: raw sRGB or Chrome's BT.601 JPEG screenshots to BT.709 limited range.- const sdrFilter = options.rawInputFormat- ? SDR_RGB_TO_BT709_FILTER- : SDR_CAPTURE_TO_BT709_FILTER;- if (options.rawInputFormat && options.hdr) {- // No filter needed — PQ data goes straight to encoder- } else if (gpuEncoder === "vaapi") {+ // Raw HDR input (from WebGPU pipeline) is already PQ-encoded and is only tagged; SDR is+ // converted: raw sRGB or Chrome's BT.601 JPEG screenshots to tagged BT.709 limited range.+ const frameFilter =+ options.rawInputFormat && options.hdr+ ? hdrFrameTagsFilter(options.hdr.transfer)+ : options.rawInputFormat+ ? SDR_RGB_TO_TAGGED_BT709_FILTER+ : SDR_CAPTURE_TO_BT709_FILTER;+ if (gpuEncoder === "vaapi") {// vaapi already runs `format=nv12,hwupload`; the nv12 conversion aligns// odd dimensions before upload, so only prepend the colour conversion.const vfIdx = args.indexOf("-vf");if (vfIdx !== -1) {- args[vfIdx + 1] = `${sdrFilter},${args[vfIdx + 1]}`;+ args[vfIdx + 1] = `${frameFilter},${args[vfIdx + 1]}`;}} else {// Pad odd dimensions up to even so 4:2:0 encoders (software and// nvenc/videotoolbox/qsv/amf) don't abort with "height not divisible by 2".- args.push("-vf", withEvenDimensionPad(sdrFilter, pixelFormat, options.width, options.height));+ args.push(+ "-vf",+ withEvenDimensionPad(frameFilter, pixelFormat, options.width, options.height),+ );}// Fixed timescale for consistent A/V timing across platforms.diff --git a/packages/engine/src/utils/hdr.ts b/packages/engine/src/utils/hdr.tsindex 5a58e302f1..df0b75744e 100644--- a/packages/engine/src/utils/hdr.ts+++ b/packages/engine/src/utils/hdr.ts@@ -106,6 +106,11 @@ export function getHdrEncoderColorParams(};}+export function hdrFrameTagsFilter(transfer: HdrTransfer): string {+ const { colorPrimaries, colorTrc, colorspace } = getHdrEncoderColorParams(transfer);+ return `setparams=color_primaries=${colorPrimaries}:color_trc=${colorTrc}:colorspace=${colorspace}:range=tv`;+}+export interface CompositionHdrInfo {hasHdr: boolean;dominantTransfer: HdrTransfer | null;diff --git a/packages/engine/src/utils/sdrCaptureColor.ts b/packages/engine/src/utils/sdrCaptureColor.tsindex fcb9977362..425a59a9a6 100644--- a/packages/engine/src/utils/sdrCaptureColor.ts+++ b/packages/engine/src/utils/sdrCaptureColor.ts@@ -1,5 +1,5 @@export const SDR_RGB_TO_BT709_FILTER = "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor";-export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor+accurate_rnd,format=gbrp,${SDR_RGB_TO_BT709_FILTER}`;-export const SDR_RGB_TO_TAGGED_BT709_FILTER = `${SDR_RGB_TO_BT709_FILTER},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv`;++export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor+accurate_rnd,format=gbrp,${SDR_RGB_TO_TAGGED_BT709_FILTER}`;