Skip to content

Write HTML. Render video. Built for agents.

fix(studio): tone-map HDR preview proxies with the render tone map (#4606)

Miguel Ángel5257dd581f7c
diff --git a/packages/core/src/hdrToneMap.test.ts b/packages/core/src/hdrToneMap.test.ts
new file mode 100644
index 0000000000..3e084b66ab
--- /dev/null
+++ b/packages/core/src/hdrToneMap.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+import { hdrToSdrToneMapFilter } from "./hdrToneMap";
+
+const PQ = { colorSpace: "bt2020nc", colorPrimaries: "bt2020", colorTransfer: "smpte2084" };
+const TONE_MAP =
+ "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv";
+
+describe("hdrToSdrToneMapFilter", () => {
+ it("leaves every frame its own tags when the first frame carries them all", () => {
+ expect(hdrToSdrToneMapFilter(PQ, PQ)).toBe(TONE_MAP);
+ });
+
+ it("sets the probed tags the first frame lacks", () => {
+ expect(hdrToSdrToneMapFilter(PQ, {})).toBe(
+ `setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084,${TONE_MAP}`,
+ );
+ expect(hdrToSdrToneMapFilter(PQ, { colorTransfer: "arib-std-b67" })).toBe(
+ `setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`,
+ );
+ });
+
+ it("treats a missing matrix or primaries as BT.2020 and leaves an unknown transfer to the frames", () => {
+ expect(
+ hdrToSdrToneMapFilter(
+ { colorSpace: "unknown", colorTransfer: "arib-std-b67" },
+ { colorSpace: "unknown", colorPrimaries: "unknown", colorTransfer: "arib-std-b67" },
+ ),
+ ).toBe(`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`);
+ expect(hdrToSdrToneMapFilter({ colorPrimaries: "reserved" }, {})).toBe(
+ `setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`,
+ );
+ });
+});
diff --git a/packages/core/src/hdrToneMap.ts b/packages/core/src/hdrToneMap.ts
new file mode 100644
index 0000000000..5cee5ccdd8
--- /dev/null
+++ b/packages/core/src/hdrToneMap.ts
@@ -0,0 +1,41 @@
+export interface ToneMapSourceColour {
+ colorSpace?: string;
+ colorPrimaries?: string;
+ colorTransfer?: string;
+}
+
+function known(value: string | undefined): string | undefined {
+ return value && value !== "unknown" && value !== "reserved" ? value : undefined;
+}
+
+/**
+ * HDR to SDR BT.709 with zscale and hable. zscale reads each frame's own tags, so only those the
+ * first frame lacks are set: from ffprobe, which may see the container's, else BT.2020.
+ */
+export function hdrToSdrToneMapFilter(
+ probed: ToneMapSourceColour,
+ firstFrame: ToneMapSourceColour,
+): string {
+ const fill = (key: keyof ToneMapSourceColour, fallback?: string) =>
+ known(firstFrame[key]) ? undefined : (known(probed[key]) ?? fallback);
+ const tags = [
+ ["colorspace", fill("colorSpace", "bt2020nc")],
+ ["color_primaries", fill("colorPrimaries", "bt2020")],
+ ["color_trc", fill("colorTransfer")],
+ ].filter(([, value]) => value);
+ const setTags = tags.length ? `setparams=${tags.map((tag) => tag.join("=")).join(":")},` : "";
+ return `${setTags}zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`;
+}
+
+/** ffmpeg args printing the first shown frame's tags; ffprobe's packet-bounded reads miss edit-list pre-roll. */
+export function firstFrameColourArgs(videoPath: string): string[] {
+ return [
+ ...["-hide_banner", "-nostats", "-i", videoPath, "-map", "0:v:0", "-frames:v", "1"],
+ ...["-vf", "showinfo", "-f", "null", "-"],
+ ];
+}
+
+export function parseFirstFrameColour(stderr: string): ToneMapSourceColour {
+ const tags = / color_space:(\S+) color_primaries:(\S+) color_trc:(\S+)/.exec(stderr);
+ return tags ? { colorSpace: tags[1], colorPrimaries: tags[2], colorTransfer: tags[3] } : {};
+}
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 268999cf8a..31a4898f3e 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -252,6 +252,12 @@ export {
type ResolvedHfColorGrading,
} from "./colorGrading";
export { parseCubeLut, CubeLutParseError, type ParseCubeLutOptions } from "./colorLuts";
+export {
+ firstFrameColourArgs,
+ hdrToSdrToneMapFilter,
+ parseFirstFrameColour,
+ type ToneMapSourceColour,
+} from "./hdrToneMap";
// Inline scripts
export {
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index be8fe4973c..dbaa4948a3 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -28,13 +28,15 @@ import {
readMediaStart,
toFps,
type FpsInput,
+ firstFrameColourArgs,
+ hdrToSdrToneMapFilter,
+ parseFirstFrameColour,
} from "@hyperframes/core";
import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
import {
extractFinalVideoFrameTimestamp,
extractMediaMetadata,
- type VideoColorSpace,
type VideoMetadata,
} from "../utils/ffprobe.js";
import {
@@ -220,8 +222,6 @@ export interface ExtractionOptions {
const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
-const HDR_TO_SDR_TONEMAP_FILTER =
- "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv";
const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";
const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";
@@ -776,37 +776,10 @@ export function parseImageElements(html: string): ImageElement[] {
return images;
}
-/** zscale reads each frame's own tags; set only those the first frame lacks, from ffprobe or BT.2020. */
-function hdrFrameColourTags(metadata: VideoMetadata, frame: Partial<VideoColorSpace>): string[] {
- const known = (value: string | undefined) =>
- value && value !== "unknown" && value !== "reserved" ? value : undefined;
- const probed = metadata.colorSpace;
- const fill = (key: keyof VideoColorSpace, fallback?: string) =>
- known(frame[key]) ? undefined : (known(probed?.[key]) ?? fallback);
- const tags = [
- ["colorspace", fill("colorSpace", "bt2020nc")],
- ["color_primaries", fill("colorPrimaries", "bt2020")],
- ["color_trc", fill("colorTransfer")],
- ].filter(([, value]) => value);
- return tags.length
- ? [`setparams=${tags.map(([key, value]) => `${key}=${value}`).join(":")}`]
- : [];
-}
-
-/** The first shown frame's tags, read by ffmpeg, which decodes past the edit-list pre-roll ffprobe counts. */
-async function readFirstFrameColour(
- videoPath: string,
- signal?: AbortSignal,
-): Promise<Partial<VideoColorSpace>> {
- const result = await runFfmpeg(
- [
- ...["-hide_banner", "-nostats", "-i", videoPath, "-map", "0:v:0", "-frames:v", "1"],
- ...["-vf", "showinfo", "-f", "null", "-"],
- ],
- { signal },
- );
- const tags = / color_space:(\S+) color_primaries:(\S+) color_trc:(\S+)/.exec(result.stderr);
- return tags ? { colorSpace: tags[1], colorPrimaries: tags[2], colorTransfer: tags[3] } : {};
+/** The first shown frame's colour tags; see {@link firstFrameColourArgs}. */
+async function readFirstFrameColour(videoPath: string, signal?: AbortSignal) {
+ const result = await runFfmpeg(firstFrameColourArgs(videoPath), { signal });
+ return parseFirstFrameColour(result.stderr);
}
/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */
@@ -984,7 +957,7 @@ export async function extractVideoFramesRange(
}
if (toneMappedToSdr) {
const frameColour = await readFirstFrameColour(videoPath, signal);
- vfFilters.push(...hdrFrameColourTags(metadata, frameColour), HDR_TO_SDR_TONEMAP_FILTER);
+ vfFilters.push(hdrToSdrToneMapFilter(metadata.colorSpace ?? {}, frameColour));
}
if (toneMappedToSdr || (!isHdr && !options.sdrToHdrTransfer)) {
vfFilters.push(
diff --git a/packages/studio-server/src/helpers/mediaMetadata.ts b/packages/studio-server/src/helpers/mediaMetadata.ts
index 28edfe7f02..d99b0f53aa 100644
--- a/packages/studio-server/src/helpers/mediaMetadata.ts
+++ b/packages/studio-server/src/helpers/mediaMetadata.ts
@@ -1,5 +1,10 @@
import { execFile } from "node:child_process";
import { extname } from "node:path";
+import {
+ firstFrameColourArgs,
+ parseFirstFrameColour,
+ type ToneMapSourceColour,
+} from "@hyperframes/core";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
export interface FfprobeRunResult {
@@ -177,6 +182,22 @@ export function classifyMediaColor(stream: FfprobeStream | null | undefined): Me
};
}
+/** The first shown frame's colour tags, which zscale reads per frame. Empty when unread. */
+export async function probeFirstFrameColour(
+ filePath: string,
+ runner: FfprobeRunner = execFileRunner,
+): Promise<ToneMapSourceColour> {
+ const ffmpegPath =
+ findFfBinary("ffmpeg", { configuredMustExist: true }) ??
+ (runner === execFileRunner ? undefined : "ffmpeg");
+ if (!ffmpegPath) return {};
+ const result = await runner(ffmpegPath, firstFrameColourArgs(filePath), {
+ timeout: 15_000,
+ maxBuffer: 1024 * 1024,
+ });
+ return result.status === 0 ? parseFirstFrameColour(String(result.stderr)) : {};
+}
+
export async function probeMediaMetadata(
filePath: string,
runner: FfprobeRunner = execFileRunner,
diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.ts
index 963148658e..3d8078f103 100644
--- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts
+++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts
@@ -1,3 +1,4 @@
+import { execFileSync, spawnSync } from "node:child_process";
import { EventEmitter } from "node:events";
import {
existsSync,
@@ -11,9 +12,39 @@ import {
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
+import { hdrToSdrToneMapFilter } from "@hyperframes/core";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
import { afterEach, describe, expect, it, vi } from "vitest";
const FFMPEG_PATH = "/usr/bin/ffmpeg";
+const realFfmpeg = findFfBinary("ffmpeg");
+const realFfmpegFilters = realFfmpeg
+ ? spawnSync(realFfmpeg, ["-hide_banner", "-filters"], { encoding: "utf8" }).stdout
+ : "";
+const canToneMapForReal =
+ Boolean(findFfBinary("ffprobe")) && /\szscale\s/.test(realFfmpegFilters ?? "");
+
+function meanLuma(path: string, filters = ""): number {
+ const stats = execFileSync(
+ realFfmpeg!,
+ [
+ "-v",
+ "error",
+ "-i",
+ path,
+ "-frames:v",
+ "1",
+ "-vf",
+ `${filters}signalstats,metadata=print:file=-`,
+ "-f",
+ "null",
+ "-",
+ ],
+ { encoding: "utf8" },
+ );
+ return Number(/YAVG=([\d.]+)/.exec(stats)?.[1]);
+}
+
// Mirrors MAX_CONCURRENT_TRANSCODES in proxyTranscoder.ts (not exported —
// this test file and the module are authored together).
const MAX_CONCURRENT = 2;
@@ -80,6 +111,7 @@ async function loadModule(
spawn: SpawnImpl,
ffmpegPath: string | undefined,
isHdr = false,
+ hdrTransfer: string | null = isHdr ? "pq" : null,
): Promise<typeof import("./proxyTranscoder.js")> {
vi.resetModules();
vi.doMock("node:child_process", () => {
@@ -92,8 +124,9 @@ async function loadModule(
vi.doMock("./mediaMetadata.js", () => ({
probeMediaMetadata: async () => ({
kind: "video",
- color: { isHdr },
+ color: { isHdr, hdrTransfer },
}),
+ probeFirstFrameColour: async () => ({}),
}));
return import("./proxyTranscoder.js");
}
@@ -235,6 +268,7 @@ describe("resolveProxy", () => {
const filter = calls[1]!.args[filterIndex + 1];
expect(filter).toContain("tonemap=");
expect(filter).toContain("bt709");
+ expect(filter).toContain("out_color_matrix=bt709");
succeed(calls[1]!);
await result;
@@ -277,6 +311,112 @@ describe("resolveProxy", () => {
expect(calls).toHaveLength(1);
});
+ it("does not tone-map BT.2020 footage without a PQ or HLG transfer, as renders do", async () => {
+ const { spawn, calls } = createSpawnSpy();
+ const { resolveProxy } = await loadModule(spawn, FFMPEG_PATH, true, "unknown");
+ const projectDir = tmpProject();
+ const sourcePath = join(projectDir, "bt2020-sdr.mov");
+ writeFileSync(sourcePath, "source-bytes");
+
+ const result = resolveProxy(projectDir, sourcePath);
+ await flush();
+
+ expect(calls).toHaveLength(1);
+ expect(calls[0]!.args).toContain("-vf");
+ expect(calls[0]!.args[calls[0]!.args.indexOf("-vf") + 1]).not.toContain("tonemap=");
+ succeed(calls[0]!);
+ await result;
+ });
+
+ it.skipIf(!canToneMapForReal)(
+ "tone-maps real HLG footage that tags only its transfer, as renders do",
+ async () => {
+ vi.resetModules();
+ vi.doUnmock("./mediaMetadata.js");
+ const { resolveProxy } = await import("./proxyTranscoder.js");
+ const projectDir = tmpProject();
+ const sourcePath = join(projectDir, "hlg-transfer-only.mp4");
+ // Primaries and matrix "unspecified" (2), transfer HLG (18).
+ const hlgTransferOnly =
+ "h264_metadata=colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2";
+ execFileSync(realFfmpeg!, [
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=s=64x36:d=0.2",
+ "-c:v",
+ "libx264",
+ "-pix_fmt",
+ "yuv420p",
+ "-bsf:v",
+ hlgTransferOnly,
+ sourcePath,
+ ]);
+
+ const proxyPath = await resolveProxy(projectDir, sourcePath);
+ // Reference: the render tone map's RGB, converted with BT.709. Mean luma is ~53 there;
+ // skipping the tone map gives ~60, a BT.601 conversion ~56.
+ const reference = meanLuma(
+ sourcePath,
+ `${hdrToSdrToneMapFilter({ colorTransfer: "arib-std-b67" }, { colorTransfer: "arib-std-b67" })},format=gbrp,scale=in_range=pc:out_color_matrix=bt709:out_range=tv,format=yuv420p,`,
+ );
+ expect(Math.abs(meanLuma(proxyPath) - reference)).toBeLessThan(1);
+ },
+ 60_000,
+ );
+
+ it.skipIf(!canToneMapForReal)(
+ "tone-maps each frame of a real mixed HLG and PQ stream with its own transfer",
+ async () => {
+ vi.resetModules();
+ vi.doUnmock("./mediaMetadata.js");
+ const { resolveProxy } = await import("./proxyTranscoder.js");
+ const projectDir = tmpProject();
+ const segment = (transfer: number) =>
+ execFileSync(realFfmpeg!, [
+ ...["-v", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:r=25:d=0.2"],
+ // No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times.
+ ...["-c:v", "libx264", "-bf", "0", "-pix_fmt", "yuv420p", "-bsf:v"],
+ `h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,
+ ...["-f", "h264", "-"],
+ ]);
+ const mux = (name: string, stream: Buffer) => {
+ const raw = join(projectDir, `${name}.h264`);
+ const path = join(projectDir, `${name}.mp4`);
+ writeFileSync(raw, stream);
+ execFileSync(realFfmpeg!, [
+ ...["-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw],
+ ...["-c", "copy", path],
+ ]);
+ return path;
+ };
+ const pq = segment(16);
+ const mixedProxy = await resolveProxy(
+ projectDir,
+ mux("hlg-then-pq", Buffer.concat([segment(18), pq])),
+ );
+ const pqProxy = await resolveProxy(projectDir, mux("pq-only", pq));
+
+ // Frame 5 is the first PQ frame: tone-mapped as PQ it lands near 42 dB, read as HLG near 30.
+ const psnrAt = (n: number) => {
+ const stats = spawnSync(
+ realFfmpeg!,
+ [
+ ...["-i", mixedProxy, "-i", pqProxy, "-lavfi"],
+ `[0:v]select=eq(n\\,${n}),setpts=PTS-STARTPTS[a];[1:v]select=eq(n\\,0),setpts=PTS-STARTPTS[b];[a][b]psnr`,
+ ...["-frames:v", "1", "-f", "null", "-"],
+ ],
+ { encoding: "utf8" },
+ ).stderr;
+ return Number(/average:([\d.]+)/.exec(stats)?.[1] ?? 0);
+ };
+ expect(psnrAt(5)).toBeGreaterThan(36);
+ },
+ 60_000,
+ );
+
it("dedupes two concurrent same-key calls to one spawn", async () => {
const { spawn, calls } = createSpawnSpy();
const { resolveProxy } = await loadModule(spawn, FFMPEG_PATH);
diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.ts
index 358c8b64ed..845a44ba6c 100644
--- a/packages/studio-server/src/helpers/proxyTranscoder.ts
+++ b/packages/studio-server/src/helpers/proxyTranscoder.ts
@@ -2,8 +2,9 @@ import { spawn } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import { existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs";
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { hdrToSdrToneMapFilter } from "@hyperframes/core";
import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
-import { probeMediaMetadata } from "./mediaMetadata.js";
+import { probeFirstFrameColour, probeMediaMetadata } from "./mediaMetadata.js";
import { cleanupProxyCache } from "./proxyCache.js";
import { PROXY_VARIANT_CONFIG, type ProxyVariant } from "./mediaCodecMap.js";
import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js";
@@ -25,7 +26,7 @@ import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js";
* entry still lands for the next request.
*/
-export const PROXY_PARAMS_VERSION = "v4";
+export const PROXY_PARAMS_VERSION = "v5";
const CACHE_DIR_NAME = ".transcode-cache";
@@ -327,21 +328,21 @@ async function runFfmpeg(
if (!ffmpegPath) {
throw new FfmpegUnavailableError();
}
- // The HDR tonemap filters discard alpha. VP8 is the alpha-preserving proxy
- // variant, so retain its source color values instead of making it opaque.
- if (metadata.color.isHdr && variant !== "vp8") await ensureHdrFilters(ffmpegPath);
+ const keepsAlpha = variant === "vp8";
+ const { hdrTransfer } = metadata.color;
+ const toneMap = (hdrTransfer === "pq" || hdrTransfer === "hlg") && !keepsAlpha;
+ if (toneMap) await ensureHdrFilters(ffmpegPath);
+ const firstFrame = toneMap ? await probeFirstFrameColour(sourcePath) : {};
const evenScale = "scale=trunc(iw/2)*2:trunc(ih/2)*2";
- const pixelFormat = variant === "vp8" ? "yuva420p" : "yuv420p";
- const videoFilter =
- metadata.color.isHdr && variant !== "vp8"
- ? [
- "zscale=t=linear:npl=100",
- "tonemap=hable:desat=0",
- "zscale=p=bt709:t=bt709:m=bt709:r=tv",
- evenScale,
- `format=${pixelFormat}`,
- ].join(",")
- : [evenScale, `format=${pixelFormat}`].join(",");
+ const pixelFormat = keepsAlpha ? "yuva420p" : "yuv420p";
+ // The tone map ends in RGB; older ffmpeg (seen on 5.1) converts it with BT.601 unless the matrix is named.
+ const videoFilter = toneMap
+ ? [
+ hdrToSdrToneMapFilter(metadata.color, firstFrame),
+ `${evenScale}:out_color_matrix=bt709:out_range=tv`,
+ `format=${pixelFormat}`,
+ ].join(",")
+ : [evenScale, `format=${pixelFormat}`].join(",");
return new Promise((resolvePromise, reject) => {
const commonArgs = ["-y", "-i", sourcePath, "-vf", videoFilter];