Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(studio): tone-map HDR preview proxies with the render tone map (#4606)
Miguel Ángel5257dd581f7c
diff --git a/packages/core/src/hdrToneMap.test.ts b/packages/core/src/hdrToneMap.test.tsnew file mode 100644index 0000000000..3e084b66ab--- /dev/null+++ b/packages/core/src/hdrToneMap.test.ts@@ -0,0 +1,33 @@+import { describe, expect, it } from "vitest";+import { hdrToSdrToneMapFilter } from "./hdrToneMap";++const PQ = { colorSpace: "bt2020nc", colorPrimaries: "bt2020", colorTransfer: "smpte2084" };+const TONE_MAP =+ "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv";++describe("hdrToSdrToneMapFilter", () => {+ it("leaves every frame its own tags when the first frame carries them all", () => {+ expect(hdrToSdrToneMapFilter(PQ, PQ)).toBe(TONE_MAP);+ });++ it("sets the probed tags the first frame lacks", () => {+ expect(hdrToSdrToneMapFilter(PQ, {})).toBe(+ `setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084,${TONE_MAP}`,+ );+ expect(hdrToSdrToneMapFilter(PQ, { colorTransfer: "arib-std-b67" })).toBe(+ `setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`,+ );+ });++ it("treats a missing matrix or primaries as BT.2020 and leaves an unknown transfer to the frames", () => {+ expect(+ hdrToSdrToneMapFilter(+ { colorSpace: "unknown", colorTransfer: "arib-std-b67" },+ { colorSpace: "unknown", colorPrimaries: "unknown", colorTransfer: "arib-std-b67" },+ ),+ ).toBe(`setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`);+ expect(hdrToSdrToneMapFilter({ colorPrimaries: "reserved" }, {})).toBe(+ `setparams=colorspace=bt2020nc:color_primaries=bt2020,${TONE_MAP}`,+ );+ });+});diff --git a/packages/core/src/hdrToneMap.ts b/packages/core/src/hdrToneMap.tsnew file mode 100644index 0000000000..5cee5ccdd8--- /dev/null+++ b/packages/core/src/hdrToneMap.ts@@ -0,0 +1,41 @@+export interface ToneMapSourceColour {+ colorSpace?: string;+ colorPrimaries?: string;+ colorTransfer?: string;+}++function known(value: string | undefined): string | undefined {+ return value && value !== "unknown" && value !== "reserved" ? value : undefined;+}++/**+ * HDR to SDR BT.709 with zscale and hable. zscale reads each frame's own tags, so only those the+ * first frame lacks are set: from ffprobe, which may see the container's, else BT.2020.+ */+export function hdrToSdrToneMapFilter(+ probed: ToneMapSourceColour,+ firstFrame: ToneMapSourceColour,+): string {+ const fill = (key: keyof ToneMapSourceColour, fallback?: string) =>+ known(firstFrame[key]) ? undefined : (known(probed[key]) ?? fallback);+ const tags = [+ ["colorspace", fill("colorSpace", "bt2020nc")],+ ["color_primaries", fill("colorPrimaries", "bt2020")],+ ["color_trc", fill("colorTransfer")],+ ].filter(([, value]) => value);+ const setTags = tags.length ? `setparams=${tags.map((tag) => tag.join("=")).join(":")},` : "";+ return `${setTags}zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv`;+}++/** ffmpeg args printing the first shown frame's tags; ffprobe's packet-bounded reads miss edit-list pre-roll. */+export function firstFrameColourArgs(videoPath: string): string[] {+ return [+ ...["-hide_banner", "-nostats", "-i", videoPath, "-map", "0:v:0", "-frames:v", "1"],+ ...["-vf", "showinfo", "-f", "null", "-"],+ ];+}++export function parseFirstFrameColour(stderr: string): ToneMapSourceColour {+ const tags = / color_space:(\S+) color_primaries:(\S+) color_trc:(\S+)/.exec(stderr);+ return tags ? { colorSpace: tags[1], colorPrimaries: tags[2], colorTransfer: tags[3] } : {};+}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 268999cf8a..31a4898f3e 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -252,6 +252,12 @@ export {type ResolvedHfColorGrading,} from "./colorGrading";export { parseCubeLut, CubeLutParseError, type ParseCubeLutOptions } from "./colorLuts";+export {+ firstFrameColourArgs,+ hdrToSdrToneMapFilter,+ parseFirstFrameColour,+ type ToneMapSourceColour,+} from "./hdrToneMap";// Inline scriptsexport {diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex be8fe4973c..dbaa4948a3 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -28,13 +28,15 @@ import {readMediaStart,toFps,type FpsInput,+ firstFrameColourArgs,+ hdrToSdrToneMapFilter,+ parseFirstFrameColour,} from "@hyperframes/core";import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";import {extractFinalVideoFrameTimestamp,extractMediaMetadata,- type VideoColorSpace,type VideoMetadata,} from "../utils/ffprobe.js";import {@@ -220,8 +222,6 @@ export interface ExtractionOptions {const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;const GC_STALENESS_MS = 24 * 60 * 60 * 1000;const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";-const HDR_TO_SDR_TONEMAP_FILTER =- "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv";const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";@@ -776,37 +776,10 @@ export function parseImageElements(html: string): ImageElement[] {return images;}-/** zscale reads each frame's own tags; set only those the first frame lacks, from ffprobe or BT.2020. */-function hdrFrameColourTags(metadata: VideoMetadata, frame: Partial<VideoColorSpace>): string[] {- const known = (value: string | undefined) =>- value && value !== "unknown" && value !== "reserved" ? value : undefined;- const probed = metadata.colorSpace;- const fill = (key: keyof VideoColorSpace, fallback?: string) =>- known(frame[key]) ? undefined : (known(probed?.[key]) ?? fallback);- const tags = [- ["colorspace", fill("colorSpace", "bt2020nc")],- ["color_primaries", fill("colorPrimaries", "bt2020")],- ["color_trc", fill("colorTransfer")],- ].filter(([, value]) => value);- return tags.length- ? [`setparams=${tags.map(([key, value]) => `${key}=${value}`).join(":")}`]- : [];-}--/** The first shown frame's tags, read by ffmpeg, which decodes past the edit-list pre-roll ffprobe counts. */-async function readFirstFrameColour(- videoPath: string,- signal?: AbortSignal,-): Promise<Partial<VideoColorSpace>> {- const result = await runFfmpeg(- [- ...["-hide_banner", "-nostats", "-i", videoPath, "-map", "0:v:0", "-frames:v", "1"],- ...["-vf", "showinfo", "-f", "null", "-"],- ],- { signal },- );- const tags = / color_space:(\S+) color_primaries:(\S+) color_trc:(\S+)/.exec(result.stderr);- return tags ? { colorSpace: tags[1], colorPrimaries: tags[2], colorTransfer: tags[3] } : {};+/** The first shown frame's colour tags; see {@link firstFrameColourArgs}. */+async function readFirstFrameColour(videoPath: string, signal?: AbortSignal) {+ const result = await runFfmpeg(firstFrameColourArgs(videoPath), { signal });+ return parseFirstFrameColour(result.stderr);}/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */@@ -984,7 +957,7 @@ export async function extractVideoFramesRange(}if (toneMappedToSdr) {const frameColour = await readFirstFrameColour(videoPath, signal);- vfFilters.push(...hdrFrameColourTags(metadata, frameColour), HDR_TO_SDR_TONEMAP_FILTER);+ vfFilters.push(hdrToSdrToneMapFilter(metadata.colorSpace ?? {}, frameColour));}if (toneMappedToSdr || (!isHdr && !options.sdrToHdrTransfer)) {vfFilters.push(diff --git a/packages/studio-server/src/helpers/mediaMetadata.ts b/packages/studio-server/src/helpers/mediaMetadata.tsindex 28edfe7f02..d99b0f53aa 100644--- a/packages/studio-server/src/helpers/mediaMetadata.ts+++ b/packages/studio-server/src/helpers/mediaMetadata.ts@@ -1,5 +1,10 @@import { execFile } from "node:child_process";import { extname } from "node:path";+import {+ firstFrameColourArgs,+ parseFirstFrameColour,+ type ToneMapSourceColour,+} from "@hyperframes/core";import { findFfBinary } from "@hyperframes/parsers/ff-binaries";export interface FfprobeRunResult {@@ -177,6 +182,22 @@ export function classifyMediaColor(stream: FfprobeStream | null | undefined): Me};}+/** The first shown frame's colour tags, which zscale reads per frame. Empty when unread. */+export async function probeFirstFrameColour(+ filePath: string,+ runner: FfprobeRunner = execFileRunner,+): Promise<ToneMapSourceColour> {+ const ffmpegPath =+ findFfBinary("ffmpeg", { configuredMustExist: true }) ??+ (runner === execFileRunner ? undefined : "ffmpeg");+ if (!ffmpegPath) return {};+ const result = await runner(ffmpegPath, firstFrameColourArgs(filePath), {+ timeout: 15_000,+ maxBuffer: 1024 * 1024,+ });+ return result.status === 0 ? parseFirstFrameColour(String(result.stderr)) : {};+}+export async function probeMediaMetadata(filePath: string,runner: FfprobeRunner = execFileRunner,diff --git a/packages/studio-server/src/helpers/proxyTranscoder.test.ts b/packages/studio-server/src/helpers/proxyTranscoder.test.tsindex 963148658e..3d8078f103 100644--- a/packages/studio-server/src/helpers/proxyTranscoder.test.ts+++ b/packages/studio-server/src/helpers/proxyTranscoder.test.ts@@ -1,3 +1,4 @@+import { execFileSync, spawnSync } from "node:child_process";import { EventEmitter } from "node:events";import {existsSync,@@ -11,9 +12,39 @@ import {} from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";+import { hdrToSdrToneMapFilter } from "@hyperframes/core";+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";import { afterEach, describe, expect, it, vi } from "vitest";const FFMPEG_PATH = "/usr/bin/ffmpeg";+const realFfmpeg = findFfBinary("ffmpeg");+const realFfmpegFilters = realFfmpeg+ ? spawnSync(realFfmpeg, ["-hide_banner", "-filters"], { encoding: "utf8" }).stdout+ : "";+const canToneMapForReal =+ Boolean(findFfBinary("ffprobe")) && /\szscale\s/.test(realFfmpegFilters ?? "");++function meanLuma(path: string, filters = ""): number {+ const stats = execFileSync(+ realFfmpeg!,+ [+ "-v",+ "error",+ "-i",+ path,+ "-frames:v",+ "1",+ "-vf",+ `${filters}signalstats,metadata=print:file=-`,+ "-f",+ "null",+ "-",+ ],+ { encoding: "utf8" },+ );+ return Number(/YAVG=([\d.]+)/.exec(stats)?.[1]);+}+// Mirrors MAX_CONCURRENT_TRANSCODES in proxyTranscoder.ts (not exported —// this test file and the module are authored together).const MAX_CONCURRENT = 2;@@ -80,6 +111,7 @@ async function loadModule(spawn: SpawnImpl,ffmpegPath: string | undefined,isHdr = false,+ hdrTransfer: string | null = isHdr ? "pq" : null,): Promise<typeof import("./proxyTranscoder.js")> {vi.resetModules();vi.doMock("node:child_process", () => {@@ -92,8 +124,9 @@ async function loadModule(vi.doMock("./mediaMetadata.js", () => ({probeMediaMetadata: async () => ({kind: "video",- color: { isHdr },+ color: { isHdr, hdrTransfer },}),+ probeFirstFrameColour: async () => ({}),}));return import("./proxyTranscoder.js");}@@ -235,6 +268,7 @@ describe("resolveProxy", () => {const filter = calls[1]!.args[filterIndex + 1];expect(filter).toContain("tonemap=");expect(filter).toContain("bt709");+ expect(filter).toContain("out_color_matrix=bt709");succeed(calls[1]!);await result;@@ -277,6 +311,112 @@ describe("resolveProxy", () => {expect(calls).toHaveLength(1);});+ it("does not tone-map BT.2020 footage without a PQ or HLG transfer, as renders do", async () => {+ const { spawn, calls } = createSpawnSpy();+ const { resolveProxy } = await loadModule(spawn, FFMPEG_PATH, true, "unknown");+ const projectDir = tmpProject();+ const sourcePath = join(projectDir, "bt2020-sdr.mov");+ writeFileSync(sourcePath, "source-bytes");++ const result = resolveProxy(projectDir, sourcePath);+ await flush();++ expect(calls).toHaveLength(1);+ expect(calls[0]!.args).toContain("-vf");+ expect(calls[0]!.args[calls[0]!.args.indexOf("-vf") + 1]).not.toContain("tonemap=");+ succeed(calls[0]!);+ await result;+ });++ it.skipIf(!canToneMapForReal)(+ "tone-maps real HLG footage that tags only its transfer, as renders do",+ async () => {+ vi.resetModules();+ vi.doUnmock("./mediaMetadata.js");+ const { resolveProxy } = await import("./proxyTranscoder.js");+ const projectDir = tmpProject();+ const sourcePath = join(projectDir, "hlg-transfer-only.mp4");+ // Primaries and matrix "unspecified" (2), transfer HLG (18).+ const hlgTransferOnly =+ "h264_metadata=colour_primaries=2:transfer_characteristics=18:matrix_coefficients=2";+ execFileSync(realFfmpeg!, [+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ "testsrc2=s=64x36:d=0.2",+ "-c:v",+ "libx264",+ "-pix_fmt",+ "yuv420p",+ "-bsf:v",+ hlgTransferOnly,+ sourcePath,+ ]);++ const proxyPath = await resolveProxy(projectDir, sourcePath);+ // Reference: the render tone map's RGB, converted with BT.709. Mean luma is ~53 there;+ // skipping the tone map gives ~60, a BT.601 conversion ~56.+ const reference = meanLuma(+ sourcePath,+ `${hdrToSdrToneMapFilter({ colorTransfer: "arib-std-b67" }, { colorTransfer: "arib-std-b67" })},format=gbrp,scale=in_range=pc:out_color_matrix=bt709:out_range=tv,format=yuv420p,`,+ );+ expect(Math.abs(meanLuma(proxyPath) - reference)).toBeLessThan(1);+ },+ 60_000,+ );++ it.skipIf(!canToneMapForReal)(+ "tone-maps each frame of a real mixed HLG and PQ stream with its own transfer",+ async () => {+ vi.resetModules();+ vi.doUnmock("./mediaMetadata.js");+ const { resolveProxy } = await import("./proxyTranscoder.js");+ const projectDir = tmpProject();+ const segment = (transfer: number) =>+ execFileSync(realFfmpeg!, [+ ...["-v", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:r=25:d=0.2"],+ // No B-frames: genpts stamps a raw stream in decode order, so reordered frames get wrong times.+ ...["-c:v", "libx264", "-bf", "0", "-pix_fmt", "yuv420p", "-bsf:v"],+ `h264_metadata=colour_primaries=9:transfer_characteristics=${transfer}:matrix_coefficients=9`,+ ...["-f", "h264", "-"],+ ]);+ const mux = (name: string, stream: Buffer) => {+ const raw = join(projectDir, `${name}.h264`);+ const path = join(projectDir, `${name}.mp4`);+ writeFileSync(raw, stream);+ execFileSync(realFfmpeg!, [+ ...["-v", "error", "-fflags", "+genpts", "-r", "25", "-f", "h264", "-i", raw],+ ...["-c", "copy", path],+ ]);+ return path;+ };+ const pq = segment(16);+ const mixedProxy = await resolveProxy(+ projectDir,+ mux("hlg-then-pq", Buffer.concat([segment(18), pq])),+ );+ const pqProxy = await resolveProxy(projectDir, mux("pq-only", pq));++ // Frame 5 is the first PQ frame: tone-mapped as PQ it lands near 42 dB, read as HLG near 30.+ const psnrAt = (n: number) => {+ const stats = spawnSync(+ realFfmpeg!,+ [+ ...["-i", mixedProxy, "-i", pqProxy, "-lavfi"],+ `[0:v]select=eq(n\\,${n}),setpts=PTS-STARTPTS[a];[1:v]select=eq(n\\,0),setpts=PTS-STARTPTS[b];[a][b]psnr`,+ ...["-frames:v", "1", "-f", "null", "-"],+ ],+ { encoding: "utf8" },+ ).stderr;+ return Number(/average:([\d.]+)/.exec(stats)?.[1] ?? 0);+ };+ expect(psnrAt(5)).toBeGreaterThan(36);+ },+ 60_000,+ );+it("dedupes two concurrent same-key calls to one spawn", async () => {const { spawn, calls } = createSpawnSpy();const { resolveProxy } = await loadModule(spawn, FFMPEG_PATH);diff --git a/packages/studio-server/src/helpers/proxyTranscoder.ts b/packages/studio-server/src/helpers/proxyTranscoder.tsindex 358c8b64ed..845a44ba6c 100644--- a/packages/studio-server/src/helpers/proxyTranscoder.ts+++ b/packages/studio-server/src/helpers/proxyTranscoder.ts@@ -2,8 +2,9 @@ import { spawn } from "node:child_process";import { createHash, randomUUID } from "node:crypto";import { existsSync, renameSync, statSync, unlinkSync, utimesSync } from "node:fs";import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";+import { hdrToSdrToneMapFilter } from "@hyperframes/core";import { findFfBinary } from "@hyperframes/parsers/ff-binaries";-import { probeMediaMetadata } from "./mediaMetadata.js";+import { probeFirstFrameColour, probeMediaMetadata } from "./mediaMetadata.js";import { cleanupProxyCache } from "./proxyCache.js";import { PROXY_VARIANT_CONFIG, type ProxyVariant } from "./mediaCodecMap.js";import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js";@@ -25,7 +26,7 @@ import { mkdirWithinProject, realpath, realProjectRoot } from "./safePath.js";* entry still lands for the next request.*/-export const PROXY_PARAMS_VERSION = "v4";+export const PROXY_PARAMS_VERSION = "v5";const CACHE_DIR_NAME = ".transcode-cache";@@ -327,21 +328,21 @@ async function runFfmpeg(if (!ffmpegPath) {throw new FfmpegUnavailableError();}- // The HDR tonemap filters discard alpha. VP8 is the alpha-preserving proxy- // variant, so retain its source color values instead of making it opaque.- if (metadata.color.isHdr && variant !== "vp8") await ensureHdrFilters(ffmpegPath);+ const keepsAlpha = variant === "vp8";+ const { hdrTransfer } = metadata.color;+ const toneMap = (hdrTransfer === "pq" || hdrTransfer === "hlg") && !keepsAlpha;+ if (toneMap) await ensureHdrFilters(ffmpegPath);+ const firstFrame = toneMap ? await probeFirstFrameColour(sourcePath) : {};const evenScale = "scale=trunc(iw/2)*2:trunc(ih/2)*2";- const pixelFormat = variant === "vp8" ? "yuva420p" : "yuv420p";- const videoFilter =- metadata.color.isHdr && variant !== "vp8"- ? [- "zscale=t=linear:npl=100",- "tonemap=hable:desat=0",- "zscale=p=bt709:t=bt709:m=bt709:r=tv",- evenScale,- `format=${pixelFormat}`,- ].join(",")- : [evenScale, `format=${pixelFormat}`].join(",");+ const pixelFormat = keepsAlpha ? "yuva420p" : "yuv420p";+ // The tone map ends in RGB; older ffmpeg (seen on 5.1) converts it with BT.601 unless the matrix is named.+ const videoFilter = toneMap+ ? [+ hdrToSdrToneMapFilter(metadata.color, firstFrame),+ `${evenScale}:out_color_matrix=bt709:out_range=tv`,+ `format=${pixelFormat}`,+ ].join(",")+ : [evenScale, `format=${pixelFormat}`].join(",");return new Promise((resolvePromise, reject) => {const commonArgs = ["-y", "-i", sourcePath, "-vf", videoFilter];