Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): encode and tag ProRes mov output as BT.709 (#5382)
Diego Lópezc8e105dc9281
diff --git a/packages/cli/src/background-removal/pipeline.test.ts b/packages/cli/src/background-removal/pipeline.test.tsindex afb8ad7fc5..4504e396bb 100644--- a/packages/cli/src/background-removal/pipeline.test.ts+++ b/packages/cli/src/background-removal/pipeline.test.ts@@ -79,8 +79,15 @@ describe("background-removal/pipeline — buildEncoderArgs", () => {},);- it.each(["mov", "png"] as const)("keeps the %s output color path", (format) => {- const args = buildEncoderArgs(format, 64, 64, 30, `/tmp/cutout.${format}`);+ it("converts raw RGBA to the tagged BT.709 matrix for the ProRes mov", () => {+ const args = buildEncoderArgs("mov", 64, 64, 30, "/tmp/cutout.mov");+ const filter = args[args.indexOf("-vf") + 1];+ expect(filter).toContain("out_color_matrix=bt709");+ expect(filter).toContain("setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709");+ });++ it("keeps the png output color path", () => {+ const args = buildEncoderArgs("png", 64, 64, 30, "/tmp/cutout.png");expect(args).not.toContain("-vf");});diff --git a/packages/cli/src/background-removal/pipeline.ts b/packages/cli/src/background-removal/pipeline.tsindex 9a49c19c0e..9ead622129 100644--- a/packages/cli/src/background-removal/pipeline.ts+++ b/packages/cli/src/background-removal/pipeline.ts@@ -23,6 +23,7 @@ import { type Device, type ModelId } from "./manager.js";import {DEFAULT_VP9_CPU_USED,SDR_RGB_TO_BT709_FILTER,+ SDR_RGB_TO_TAGGED_BT709_FILTER,renderProvenanceArgs,} from "@hyperframes/engine";@@ -192,6 +193,8 @@ export function buildEncoderArgs(if (format === "mov") {return [...base,+ "-vf",+ SDR_RGB_TO_TAGGED_BT709_FILTER,"-c:v","prores_ks","-profile:v",diff --git a/packages/cli/src/background-removal/pipeline.windowsHide.test.ts b/packages/cli/src/background-removal/pipeline.windowsHide.test.tsindex 69eb558c47..05a3a5e22b 100644--- a/packages/cli/src/background-removal/pipeline.windowsHide.test.ts+++ b/packages/cli/src/background-removal/pipeline.windowsHide.test.ts@@ -36,6 +36,8 @@ vi.mock("./inference.js", () => ({vi.mock("@hyperframes/engine", () => ({DEFAULT_VP9_CPU_USED: 4,SDR_RGB_TO_BT709_FILTER: "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor",+ SDR_RGB_TO_TAGGED_BT709_FILTER:+ "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",renderProvenanceArgs: () => [],extractMediaMetadata: async () => ({ width: 1, height: 1, fps: 1, durationSeconds: 1 }),}));diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.tsindex 9a0c01280d..49ff970f48 100644--- a/packages/engine/src/index.ts+++ b/packages/engine/src/index.ts@@ -61,7 +61,10 @@ export {type EngineConfig,type ExtractCacheDirResolution,} from "./config.js";-export { SDR_RGB_TO_BT709_FILTER } from "./utils/sdrCaptureColor.js";+export {+ SDR_RGB_TO_BT709_FILTER,+ SDR_RGB_TO_TAGGED_BT709_FILTER,+} from "./utils/sdrCaptureColor.js";export {DEFAULT_VP9_CPU_USED,MAX_VP9_CPU_USED,diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.tsindex 3ad3dbb566..7e263cfb7b 100644--- a/packages/engine/src/services/chunkEncoder.test.ts+++ b/packages/engine/src/services/chunkEncoder.test.ts@@ -14,8 +14,11 @@ import {resolveLockedGopSize,} from "./chunkEncoder.js";import { renderProvenanceArgs } from "../utils/renderProvenance.js";-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";-import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";+import {+ SDR_CAPTURE_TO_BT709_FILTER,+ SDR_RGB_TO_TAGGED_BT709_FILTER,+} from "../utils/sdrCaptureColor.js";+import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js";const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"]).status === 0;@@ -1226,7 +1229,7 @@ describe("buildEncoderArgs color space", () => {expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");});- it("leaves alpha ProRes untouched (no even-dim pad)", () => {+ it("converts alpha ProRes to BT.709 without an even-dim pad", () => {const args = buildEncoderArgs({...baseOptions,@@ -1238,7 +1241,7 @@ describe("buildEncoderArgs color space", () => {inputArgs,"out.mov",);- expect(args.indexOf("-vf")).toBe(-1);+ expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER);expect(args.join(" ")).not.toContain("pad=");});@@ -1820,26 +1823,27 @@ describe("buildConcatArgs", () => {});describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {+ const ffmpeg = getFfmpegBinary();+ const pixelAt = (file: string, decode: string, x: number, format = "rgb24"): number[] => [+ ...spawnSync(ffmpeg, [+ "-v",+ "error",+ "-i",+ file,+ "-vf",+ `${decode}format=${format},crop=1:1:${x}:8`,+ "-frames:v",+ "1",+ "-f",+ "rawvideo",+ "-",+ ]).stdout,+ ];+// Chrome captures are BT.601 JPEGs. A direct YUV-to-YUV scale keeps that matrix under the BT.709// tag (ffmpeg 7 and older) or tints greys (8 and newer), so the encode goes through RGB.it("delivers Chrome's JPEG colours in the BT.709 the mp4 is tagged with", () => {- const ffmpeg = getFfmpegBinary();const dir = mkdtempSync(join(tmpdir(), "hf-sdr-colour-"));- const rgbAt = (file: string, decode: string, x: number): number[] => [- ...spawnSync(ffmpeg, [- "-v",- "error",- "-i",- file,- "-vf",- `${decode}format=rgb24,crop=1:1:${x}:8`,- "-frames:v",- "1",- "-f",- "rawvideo",- "-",- ]).stdout,- ];try {for (const color of ["0xC83C28", "0xFE0000", "0x101010", "0x2050E0"]) {const jpg = join(dir, "frame.jpg");@@ -1879,8 +1883,8 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {[8, 2],[32, 2],] as const) {- const captured = rgbAt(jpg, "", x);- const delivered = rgbAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x);+ const captured = pixelAt(jpg, "", x);+ const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x);const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!)));expect(worst,@@ -1892,4 +1896,70 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {rmSync(dir, { recursive: true, force: true });}}, 30_000);++ // Untagged HD reads as BT.709 in QuickTime, so ProRes has to be BT.709 too. MOV captures are+ // RGBA PNGs, and the conversion must leave their alpha plane alone. The tags come from+ // setparams because prores_ks ignores -color_* (ffmpeg 5.1 writes none, 8.1 only the matrix).+ it("delivers PNG colours and alpha in the BT.709 the ProRes mov is tagged with", () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-prores-colour-"));+ const png = join(dir, "frame.png");+ const out = join(dir, "out.mov");+ try {+ for (const color of ["0xC83C28", "0x00FF00", "0x101010"]) {+ const synth = spawnSync(ffmpeg, [+ "-v",+ "error",+ "-y",+ "-f",+ "lavfi",+ "-i",+ `color=c=${color}@0.5:s=64x16,format=rgba`,+ "-frames:v",+ "1",+ png,+ ]);+ expect(synth.status).toBe(0);+ const args = buildEncoderArgs(+ {+ fps: { num: 30, den: 1 },+ width: 64,+ height: 16,+ codec: "prores",+ preset: "4444",+ quality: 23,+ pixelFormat: "yuva444p10le",+ },+ ["-i", png],+ out,+ );+ expect(spawnSync(ffmpeg, args).status).toBe(0);++ const captured = pixelAt(png, "", 8, "rgba");+ const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", 8, "rgba");+ const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!)));+ expect(worst, `${color}: capture ${captured} delivered ${delivered}`).toBeLessThanOrEqual(+ 2,+ );+ }+ // Read the first frame, not the stream: a prores_ks mov's stream-level range depends on+ // the ffprobe version (7.1 and 8.0 report unknown), while every version tags the frames.+ const tags = spawnSync(getFfprobeBinary(), [+ "-v",+ "error",+ "-select_streams",+ "v:0",+ "-read_intervals",+ "%+#1",+ "-show_entries",+ "frame=color_range,color_space,color_primaries,color_transfer",+ "-of",+ "csv=p=0",+ "--",+ out,+ ]);+ expect(tags.stdout.toString().trim()).toBe("tv,bt709,bt709,bt709");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+ }, 30_000);});diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.tsindex 777f3b2dbc..e733eb7d26 100644--- a/packages/engine/src/services/chunkEncoder.ts+++ b/packages/engine/src/services/chunkEncoder.ts@@ -31,7 +31,10 @@ import {} from "../utils/gpuEncoder.js";import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";import { withEvenDimensionPad } from "../utils/evenDimensions.js";-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";+import {+ SDR_CAPTURE_TO_BT709_FILTER,+ SDR_RGB_TO_TAGGED_BT709_FILTER,+} from "../utils/sdrCaptureColor.js";import {describeFfmpegFailure,ffmpegStatsReader,@@ -408,6 +411,7 @@ export function buildEncoderArgs(}} else if (codec === "prores") {args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");+ args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER);args.push("-pix_fmt", pixelFormat);appendRenderProvenanceArgs(args, outputPath);return [...args, "-y", outputPath];diff --git a/packages/engine/src/services/streamingEncoder.test.ts b/packages/engine/src/services/streamingEncoder.test.tsindex fbe6267252..74b9111873 100644--- a/packages/engine/src/services/streamingEncoder.test.ts+++ b/packages/engine/src/services/streamingEncoder.test.ts@@ -26,7 +26,10 @@ import {} from "./streamingEncoder.js";import { DEFAULT_HDR10_MASTERING } from "../utils/hdr.js";import { type GpuEncoder } from "../utils/gpuEncoder.js";-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";+import {+ SDR_CAPTURE_TO_BT709_FILTER,+ SDR_RGB_TO_TAGGED_BT709_FILTER,+} from "../utils/sdrCaptureColor.js";const baseHdrPq: StreamingEncoderOptions = {fps: { num: 30, den: 1 },@@ -241,6 +244,15 @@ describe("buildStreamingArgs", () => {`${SDR_CAPTURE_TO_BT709_FILTER},pad=ceil(iw/2)*2:ceil(ih/2)*2`,);});++ it("converts ProRes to the BT.709 it is tagged with and keeps the alpha format", () => {+ const args = buildStreamingArgs(+ { ...baseVp9, codec: "prores", preset: "4444", pixelFormat: "yuva444p10le" },+ "/tmp/out.mov",+ );+ expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER);+ expect(args[args.indexOf("-pix_fmt") + 1]).toBe("yuva444p10le");+ });});describe("output path", () => {diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.tsindex 6c379c1351..605f450380 100644--- a/packages/engine/src/services/streamingEncoder.ts+++ b/packages/engine/src/services/streamingEncoder.ts@@ -39,7 +39,11 @@ import {import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";import { getHdrEncoderColorParams } from "../utils/hdr.js";import { withEvenDimensionPad } from "../utils/evenDimensions.js";-import { SDR_CAPTURE_TO_BT709_FILTER, SDR_RGB_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";+import {+ SDR_CAPTURE_TO_BT709_FILTER,+ SDR_RGB_TO_BT709_FILTER,+ SDR_RGB_TO_TAGGED_BT709_FILTER,+} from "../utils/sdrCaptureColor.js";import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";import { fpsToFfmpegArg, fpsToNumber, type Fps } from "@hyperframes/core";import { appendVp9CpuUsedArg } from "./vp9Options.js";@@ -385,6 +389,7 @@ export function buildStreamingArgs(}} else if (codec === "prores") {args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");+ args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER);args.push("-pix_fmt", pixelFormat);appendRenderProvenanceArgs(args, outputPath);return [...args, "-y", outputPath];diff --git a/packages/engine/src/utils/sdrCaptureColor.ts b/packages/engine/src/utils/sdrCaptureColor.tsindex 3a8e240144..7cc94e0fb6 100644--- a/packages/engine/src/utils/sdrCaptureColor.ts+++ b/packages/engine/src/utils/sdrCaptureColor.ts@@ -1,3 +1,5 @@export const SDR_RGB_TO_BT709_FILTER = "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor";export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor,format=gbrp,${SDR_RGB_TO_BT709_FILTER}`;++export const SDR_RGB_TO_TAGGED_BT709_FILTER = `${SDR_RGB_TO_BT709_FILTER},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv`;