Skip to content

Write HTML. Render video. Built for agents.

fix(engine): encode and tag ProRes mov output as BT.709 (#5382)

Diego Lópezc8e105dc9281
diff --git a/packages/cli/src/background-removal/pipeline.test.ts b/packages/cli/src/background-removal/pipeline.test.ts
index afb8ad7fc5..4504e396bb 100644
--- a/packages/cli/src/background-removal/pipeline.test.ts
+++ b/packages/cli/src/background-removal/pipeline.test.ts
@@ -79,8 +79,15 @@ describe("background-removal/pipeline — buildEncoderArgs", () => {
},
);
- it.each(["mov", "png"] as const)("keeps the %s output color path", (format) => {
- const args = buildEncoderArgs(format, 64, 64, 30, `/tmp/cutout.${format}`);
+ it("converts raw RGBA to the tagged BT.709 matrix for the ProRes mov", () => {
+ const args = buildEncoderArgs("mov", 64, 64, 30, "/tmp/cutout.mov");
+ const filter = args[args.indexOf("-vf") + 1];
+ expect(filter).toContain("out_color_matrix=bt709");
+ expect(filter).toContain("setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709");
+ });
+
+ it("keeps the png output color path", () => {
+ const args = buildEncoderArgs("png", 64, 64, 30, "/tmp/cutout.png");
expect(args).not.toContain("-vf");
});
diff --git a/packages/cli/src/background-removal/pipeline.ts b/packages/cli/src/background-removal/pipeline.ts
index 9a49c19c0e..9ead622129 100644
--- a/packages/cli/src/background-removal/pipeline.ts
+++ b/packages/cli/src/background-removal/pipeline.ts
@@ -23,6 +23,7 @@ import { type Device, type ModelId } from "./manager.js";
import {
DEFAULT_VP9_CPU_USED,
SDR_RGB_TO_BT709_FILTER,
+ SDR_RGB_TO_TAGGED_BT709_FILTER,
renderProvenanceArgs,
} from "@hyperframes/engine";
@@ -192,6 +193,8 @@ export function buildEncoderArgs(
if (format === "mov") {
return [
...base,
+ "-vf",
+ SDR_RGB_TO_TAGGED_BT709_FILTER,
"-c:v",
"prores_ks",
"-profile:v",
diff --git a/packages/cli/src/background-removal/pipeline.windowsHide.test.ts b/packages/cli/src/background-removal/pipeline.windowsHide.test.ts
index 69eb558c47..05a3a5e22b 100644
--- a/packages/cli/src/background-removal/pipeline.windowsHide.test.ts
+++ b/packages/cli/src/background-removal/pipeline.windowsHide.test.ts
@@ -36,6 +36,8 @@ vi.mock("./inference.js", () => ({
vi.mock("@hyperframes/engine", () => ({
DEFAULT_VP9_CPU_USED: 4,
SDR_RGB_TO_BT709_FILTER: "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor",
+ SDR_RGB_TO_TAGGED_BT709_FILTER:
+ "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
renderProvenanceArgs: () => [],
extractMediaMetadata: async () => ({ width: 1, height: 1, fps: 1, durationSeconds: 1 }),
}));
diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts
index 9a0c01280d..49ff970f48 100644
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -61,7 +61,10 @@ export {
type EngineConfig,
type ExtractCacheDirResolution,
} from "./config.js";
-export { SDR_RGB_TO_BT709_FILTER } from "./utils/sdrCaptureColor.js";
+export {
+ SDR_RGB_TO_BT709_FILTER,
+ SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "./utils/sdrCaptureColor.js";
export {
DEFAULT_VP9_CPU_USED,
MAX_VP9_CPU_USED,
diff --git a/packages/engine/src/services/chunkEncoder.test.ts b/packages/engine/src/services/chunkEncoder.test.ts
index 3ad3dbb566..7e263cfb7b 100644
--- a/packages/engine/src/services/chunkEncoder.test.ts
+++ b/packages/engine/src/services/chunkEncoder.test.ts
@@ -14,8 +14,11 @@ import {
resolveLockedGopSize,
} from "./chunkEncoder.js";
import { renderProvenanceArgs } from "../utils/renderProvenance.js";
-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
-import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+import {
+ SDR_CAPTURE_TO_BT709_FILTER,
+ SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "../utils/sdrCaptureColor.js";
+import { getFfmpegBinary, getFfprobeBinary } from "../utils/ffmpegBinaries.js";
const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"]).status === 0;
@@ -1226,7 +1229,7 @@ describe("buildEncoderArgs color space", () => {
expect(args[args.indexOf("-vf") + 1]).toBe("pad=ceil(iw/2)*2:ceil(ih/2)*2");
});
- it("leaves alpha ProRes untouched (no even-dim pad)", () => {
+ it("converts alpha ProRes to BT.709 without an even-dim pad", () => {
const args = buildEncoderArgs(
{
...baseOptions,
@@ -1238,7 +1241,7 @@ describe("buildEncoderArgs color space", () => {
inputArgs,
"out.mov",
);
- expect(args.indexOf("-vf")).toBe(-1);
+ expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER);
expect(args.join(" ")).not.toContain("pad=");
});
@@ -1820,26 +1823,27 @@ describe("buildConcatArgs", () => {
});
describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
+ const ffmpeg = getFfmpegBinary();
+ const pixelAt = (file: string, decode: string, x: number, format = "rgb24"): number[] => [
+ ...spawnSync(ffmpeg, [
+ "-v",
+ "error",
+ "-i",
+ file,
+ "-vf",
+ `${decode}format=${format},crop=1:1:${x}:8`,
+ "-frames:v",
+ "1",
+ "-f",
+ "rawvideo",
+ "-",
+ ]).stdout,
+ ];
+
// Chrome captures are BT.601 JPEGs. A direct YUV-to-YUV scale keeps that matrix under the BT.709
// tag (ffmpeg 7 and older) or tints greys (8 and newer), so the encode goes through RGB.
it("delivers Chrome's JPEG colours in the BT.709 the mp4 is tagged with", () => {
- const ffmpeg = getFfmpegBinary();
const dir = mkdtempSync(join(tmpdir(), "hf-sdr-colour-"));
- const rgbAt = (file: string, decode: string, x: number): number[] => [
- ...spawnSync(ffmpeg, [
- "-v",
- "error",
- "-i",
- file,
- "-vf",
- `${decode}format=rgb24,crop=1:1:${x}:8`,
- "-frames:v",
- "1",
- "-f",
- "rawvideo",
- "-",
- ]).stdout,
- ];
try {
for (const color of ["0xC83C28", "0xFE0000", "0x101010", "0x2050E0"]) {
const jpg = join(dir, "frame.jpg");
@@ -1879,8 +1883,8 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
[8, 2],
[32, 2],
] as const) {
- const captured = rgbAt(jpg, "", x);
- const delivered = rgbAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x);
+ const captured = pixelAt(jpg, "", x);
+ const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", x);
const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!)));
expect(
worst,
@@ -1892,4 +1896,70 @@ describe.skipIf(!HAS_FFMPEG)("buildEncoderArgs SDR colour", () => {
rmSync(dir, { recursive: true, force: true });
}
}, 30_000);
+
+ // Untagged HD reads as BT.709 in QuickTime, so ProRes has to be BT.709 too. MOV captures are
+ // RGBA PNGs, and the conversion must leave their alpha plane alone. The tags come from
+ // setparams because prores_ks ignores -color_* (ffmpeg 5.1 writes none, 8.1 only the matrix).
+ it("delivers PNG colours and alpha in the BT.709 the ProRes mov is tagged with", () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-prores-colour-"));
+ const png = join(dir, "frame.png");
+ const out = join(dir, "out.mov");
+ try {
+ for (const color of ["0xC83C28", "0x00FF00", "0x101010"]) {
+ const synth = spawnSync(ffmpeg, [
+ "-v",
+ "error",
+ "-y",
+ "-f",
+ "lavfi",
+ "-i",
+ `color=c=${color}@0.5:s=64x16,format=rgba`,
+ "-frames:v",
+ "1",
+ png,
+ ]);
+ expect(synth.status).toBe(0);
+ const args = buildEncoderArgs(
+ {
+ fps: { num: 30, den: 1 },
+ width: 64,
+ height: 16,
+ codec: "prores",
+ preset: "4444",
+ quality: 23,
+ pixelFormat: "yuva444p10le",
+ },
+ ["-i", png],
+ out,
+ );
+ expect(spawnSync(ffmpeg, args).status).toBe(0);
+
+ const captured = pixelAt(png, "", 8, "rgba");
+ const delivered = pixelAt(out, "scale=in_color_matrix=bt709:in_range=tv,", 8, "rgba");
+ const worst = Math.max(...delivered.map((v, i) => Math.abs(v - captured[i]!)));
+ expect(worst, `${color}: capture ${captured} delivered ${delivered}`).toBeLessThanOrEqual(
+ 2,
+ );
+ }
+ // Read the first frame, not the stream: a prores_ks mov's stream-level range depends on
+ // the ffprobe version (7.1 and 8.0 report unknown), while every version tags the frames.
+ const tags = spawnSync(getFfprobeBinary(), [
+ "-v",
+ "error",
+ "-select_streams",
+ "v:0",
+ "-read_intervals",
+ "%+#1",
+ "-show_entries",
+ "frame=color_range,color_space,color_primaries,color_transfer",
+ "-of",
+ "csv=p=0",
+ "--",
+ out,
+ ]);
+ expect(tags.stdout.toString().trim()).toBe("tv,bt709,bt709,bt709");
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }, 30_000);
});
diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts
index 777f3b2dbc..e733eb7d26 100644
--- a/packages/engine/src/services/chunkEncoder.ts
+++ b/packages/engine/src/services/chunkEncoder.ts
@@ -31,7 +31,10 @@ import {
} from "../utils/gpuEncoder.js";
import { type HdrTransfer, getHdrEncoderColorParams } from "../utils/hdr.js";
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
+import {
+ SDR_CAPTURE_TO_BT709_FILTER,
+ SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "../utils/sdrCaptureColor.js";
import {
describeFfmpegFailure,
ffmpegStatsReader,
@@ -408,6 +411,7 @@ export function buildEncoderArgs(
}
} else if (codec === "prores") {
args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");
+ args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER);
args.push("-pix_fmt", pixelFormat);
appendRenderProvenanceArgs(args, outputPath);
return [...args, "-y", outputPath];
diff --git a/packages/engine/src/services/streamingEncoder.test.ts b/packages/engine/src/services/streamingEncoder.test.ts
index fbe6267252..74b9111873 100644
--- a/packages/engine/src/services/streamingEncoder.test.ts
+++ b/packages/engine/src/services/streamingEncoder.test.ts
@@ -26,7 +26,10 @@ import {
} from "./streamingEncoder.js";
import { DEFAULT_HDR10_MASTERING } from "../utils/hdr.js";
import { type GpuEncoder } from "../utils/gpuEncoder.js";
-import { SDR_CAPTURE_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
+import {
+ SDR_CAPTURE_TO_BT709_FILTER,
+ SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "../utils/sdrCaptureColor.js";
const baseHdrPq: StreamingEncoderOptions = {
fps: { num: 30, den: 1 },
@@ -241,6 +244,15 @@ describe("buildStreamingArgs", () => {
`${SDR_CAPTURE_TO_BT709_FILTER},pad=ceil(iw/2)*2:ceil(ih/2)*2`,
);
});
+
+ it("converts ProRes to the BT.709 it is tagged with and keeps the alpha format", () => {
+ const args = buildStreamingArgs(
+ { ...baseVp9, codec: "prores", preset: "4444", pixelFormat: "yuva444p10le" },
+ "/tmp/out.mov",
+ );
+ expect(args[args.indexOf("-vf") + 1]).toBe(SDR_RGB_TO_TAGGED_BT709_FILTER);
+ expect(args[args.indexOf("-pix_fmt") + 1]).toBe("yuva444p10le");
+ });
});
describe("output path", () => {
diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.ts
index 6c379c1351..605f450380 100644
--- a/packages/engine/src/services/streamingEncoder.ts
+++ b/packages/engine/src/services/streamingEncoder.ts
@@ -39,7 +39,11 @@ import {
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
import { getHdrEncoderColorParams } from "../utils/hdr.js";
import { withEvenDimensionPad } from "../utils/evenDimensions.js";
-import { SDR_CAPTURE_TO_BT709_FILTER, SDR_RGB_TO_BT709_FILTER } from "../utils/sdrCaptureColor.js";
+import {
+ SDR_CAPTURE_TO_BT709_FILTER,
+ SDR_RGB_TO_BT709_FILTER,
+ SDR_RGB_TO_TAGGED_BT709_FILTER,
+} from "../utils/sdrCaptureColor.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { fpsToFfmpegArg, fpsToNumber, type Fps } from "@hyperframes/core";
import { appendVp9CpuUsedArg } from "./vp9Options.js";
@@ -385,6 +389,7 @@ export function buildStreamingArgs(
}
} else if (codec === "prores") {
args.push("-c:v", "prores_ks", "-profile:v", preset, "-vendor", "apl0");
+ args.push("-vf", SDR_RGB_TO_TAGGED_BT709_FILTER);
args.push("-pix_fmt", pixelFormat);
appendRenderProvenanceArgs(args, outputPath);
return [...args, "-y", outputPath];
diff --git a/packages/engine/src/utils/sdrCaptureColor.ts b/packages/engine/src/utils/sdrCaptureColor.ts
index 3a8e240144..7cc94e0fb6 100644
--- a/packages/engine/src/utils/sdrCaptureColor.ts
+++ b/packages/engine/src/utils/sdrCaptureColor.ts
@@ -1,3 +1,5 @@
export const SDR_RGB_TO_BT709_FILTER = "scale=out_color_matrix=bt709:out_range=tv:flags=neighbor";
export const SDR_CAPTURE_TO_BT709_FILTER = `scale=in_color_matrix=bt601:in_range=pc:flags=neighbor,format=gbrp,${SDR_RGB_TO_BT709_FILTER}`;
+
+export const SDR_RGB_TO_TAGGED_BT709_FILTER = `${SDR_RGB_TO_BT709_FILTER},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv`;