Skip to content

Write HTML. Render video. Built for agents.

fix(core): keep the colour grade on a graded video's first rendered frame (#5575)

Miguel Ángel95837a3df7c1
diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.ts
index 014eb8826d..dd27d64a16 100644
--- a/packages/core/src/runtime/colorGrading.test.ts
+++ b/packages/core/src/runtime/colorGrading.test.ts
@@ -1041,6 +1041,29 @@ describe("createColorGradingRuntime", () => {
expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();
});
+ it("grades a clip's first producer frame when the DOM observer runs before it decodes", () => {
+ const video = makeDrawableVideo();
+ video.style.visibility = "hidden";
+ document.body.appendChild(video);
+ runtime = createColorGradingRuntime();
+
+ let decoded = false;
+ const frame = document.createElement("img");
+ frame.id = "__render_frame_hero-video__";
+ frame.className = "__render_frame__";
+ frame.style.visibility = "visible";
+ Object.defineProperty(frame, "complete", { get: () => decoded, configurable: true });
+ Object.defineProperty(frame, "naturalWidth", { get: () => (decoded ? 640 : 0) });
+ Object.defineProperty(frame, "naturalHeight", { get: () => (decoded ? 360 : 0) });
+ video.after(frame);
+ runtime.refresh();
+ decoded = true;
+
+ expect(runtime.redraw()).toBe(1);
+ expect(texImage2DCalls.at(-1)).toContain(frame);
+ expect(frame.nextSibling).toBe(document.querySelector("[data-hf-color-grading-canvas]"));
+ });
+
it("does not recreate an inactive clip from its hidden producer frame", () => {
const { video, canvas } = startRuntimeWithVideo();
const frame = document.createElement("img");
diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.ts
index f0c5ae4a32..f72079ba96 100644
--- a/packages/core/src/runtime/colorGrading.ts
+++ b/packages/core/src/runtime/colorGrading.ts
@@ -2494,7 +2494,7 @@ function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null
function hasInjectedRenderFrame(element: ColorGradingMediaElement): boolean {
if (!isVideoElement(element)) return false;
- const frame = findRenderFrameImage(element);
+ const frame = findInjectedRenderFrame(element);
if (!frame) return false;
const style = window.getComputedStyle(frame);
return style.display !== "none" && style.visibility !== "hidden";
diff --git a/packages/producer/scripts/test-classification.mjs b/packages/producer/scripts/test-classification.mjs
index 2aa4f1673b..0bf0e4f5d2 100644
--- a/packages/producer/scripts/test-classification.mjs
+++ b/packages/producer/scripts/test-classification.mjs
@@ -33,6 +33,7 @@ const INTEGRATION_TEST_FILES = new Set([
"src/services/render/hlsRender.integration.test.ts",
"src/services/render/motionBlurVideo.integration.test.ts",
"src/services/render/renderCancel.integration.test.ts",
+ "src/services/render/segmentedColorGrading.integration.test.ts",
"src/services/render/stages/compileStage.mediaType.test.ts",
"src/services/render/stages/gifEncodeArgs.test.ts",
"src/services/videoFrameBorderClip.test.ts",
diff --git a/packages/producer/src/services/render/segmentedColorGrading.integration.test.ts b/packages/producer/src/services/render/segmentedColorGrading.integration.test.ts
new file mode 100644
index 0000000000..57356f9698
--- /dev/null
+++ b/packages/producer/src/services/render/segmentedColorGrading.integration.test.ts
@@ -0,0 +1,195 @@
+// A graded <video>, rendered for real (#5548): the first frame a capture session draws inside the
+// clip (a segment's or a parallel worker's), and the clip's own first frame, keep the grade.
+// Needs Chrome + ffmpeg: integration lane.
+
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { getFfmpegBinary, resolveConfig } from "@hyperframes/engine";
+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";
+
+const FFMPEG = getFfmpegBinary();
+const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"], { encoding: "utf-8" }).status === 0;
+
+const WIDTH = 160;
+const HEIGHT = 90;
+const FPS = 30;
+const FRAME_COUNT = 60;
+const CLIP_FIRST = 15;
+const CLIP_LAST = 44;
+// The smallest segment size. With two workers, both the second segment and the second worker's
+// frame range start here, mid-clip.
+const SECOND_SESSION_FIRST = 30;
+const CLIP_GREY = 128;
+
+const COMPOSITION = `<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <style>
+ * { margin: 0; padding: 0; box-sizing: border-box; }
+ body { overflow: hidden; background: #000; }
+ #clip { position: absolute; left: 0; top: 0; width: ${WIDTH}px; height: ${HEIGHT}px; }
+ </style>
+</head>
+<body>
+ <div
+ id="root"
+ data-composition-id="segmented-color-grading"
+ data-start="0"
+ data-duration="${FRAME_COUNT / FPS}"
+ data-width="${WIDTH}"
+ data-height="${HEIGHT}"
+ data-fps="${FPS}"
+ >
+ <video id="clip" class="clip" src="assets/clip.mp4" data-start="${CLIP_FIRST / FPS}"
+ data-duration="${(CLIP_LAST + 1 - CLIP_FIRST) / FPS}" data-track-index="0" muted playsinline
+ data-color-grading='{"adjust":{"exposure":-1}}'></video>
+ </div>
+ <script>window.__timelines = window.__timelines || {};</script>
+</body>
+</html>
+`;
+
+let root: string;
+
+/** Mean RGB level of every frame of an encoded video, 0-255. */
+function frameLevels(videoPath: string): number[] {
+ const raw = spawnSync(
+ FFMPEG,
+ ["-v", "error", "-i", videoPath, "-f", "rawvideo", "-pix_fmt", "gray", "-"],
+ {
+ maxBuffer: 64 * 1024 * 1024,
+ },
+ );
+ if (raw.status !== 0) throw new Error(`decode failed: ${raw.stderr}`);
+ const frameBytes = WIDTH * HEIGHT;
+ const levels: number[] = [];
+ for (let offset = 0; offset + frameBytes <= raw.stdout.length; offset += frameBytes) {
+ let sum = 0;
+ for (let i = offset; i < offset + frameBytes; i++) sum += raw.stdout[i]!;
+ levels.push(sum / frameBytes);
+ }
+ return levels;
+}
+
+interface Render {
+ levels: number[];
+ phases: string[];
+ workerCount: number;
+}
+
+async function render(name: string, segmented: boolean, workers: number): Promise<Render> {
+ const phases: string[] = [];
+ let workerCount = 0;
+ const job = createRenderJob({
+ fps: FPS,
+ quality: "draft",
+ format: "mp4",
+ hdrMode: "force-sdr",
+ workers,
+ logger: {
+ info(message: string, meta?: Record<string, unknown>) {
+ if (message !== "[Render:trace]") return;
+ if (typeof meta?.phase === "string") phases.push(meta.phase);
+ if (typeof meta?.workerCount === "number")
+ workerCount = Math.max(workerCount, meta.workerCount);
+ },
+ warn() {},
+ error() {},
+ debug() {},
+ },
+ producerConfig: resolveConfig({ browserGpuMode: "software" }),
+ });
+ const saved = {
+ capture: process.env.HF_SEGMENTED_CAPTURE,
+ frames: process.env.HF_SEGMENT_FRAMES,
+ };
+ process.env.HF_SEGMENTED_CAPTURE = String(segmented);
+ process.env.HF_SEGMENT_FRAMES = String(SECOND_SESSION_FIRST);
+ try {
+ const output = join(root, `${name}.mp4`);
+ await executeRenderJob(job, root, output);
+ return { levels: frameLevels(output), phases, workerCount };
+ } finally {
+ if (saved.capture === undefined) delete process.env.HF_SEGMENTED_CAPTURE;
+ else process.env.HF_SEGMENTED_CAPTURE = saved.capture;
+ if (saved.frames === undefined) delete process.env.HF_SEGMENT_FRAMES;
+ else process.env.HF_SEGMENT_FRAMES = saved.frames;
+ }
+}
+
+describe.skipIf(!HAS_FFMPEG)(
+ "color-graded video at a capture session's first frame — real render (#5548)",
+ () => {
+ let reference: Render;
+ let segmented: Render;
+ let parallel: Render;
+
+ beforeAll(async () => {
+ root = mkdtempSync(join(tmpdir(), "hf-segmented-grading-"));
+ mkdirSync(join(root, "assets"), { recursive: true });
+ writeFileSync(join(root, "index.html"), COMPOSITION);
+ const clip = spawnSync(
+ FFMPEG,
+ [
+ "-v",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ `color=c=0x808080:size=${WIDTH}x${HEIGHT}:rate=${FPS}:duration=2`,
+ "-pix_fmt",
+ "yuv420p",
+ "-c:v",
+ "libx264",
+ "-y",
+ join(root, "assets", "clip.mp4"),
+ ],
+ { encoding: "utf-8" },
+ );
+ if (clip.status !== 0) throw new Error(`clip fixture failed: ${clip.stderr}`);
+
+ reference = await render("reference", false, 1);
+ segmented = await render("segmented", true, 2);
+ parallel = await render("parallel", false, 2);
+ }, 600_000);
+
+ afterAll(() => {
+ if (root) rmSync(root, { recursive: true, force: true });
+ });
+
+ it("renders each shape on the route it asks for", () => {
+ expect(segmented.phases).toContain("capture_segmented");
+ expect(reference.phases).not.toContain("capture_segmented");
+ expect(parallel.phases).not.toContain("capture_segmented");
+ expect(reference.workerCount).toBe(1);
+ expect(segmented.workerCount).toBe(2);
+ expect(parallel.workerCount).toBe(2);
+ for (const { levels } of [reference, segmented, parallel]) {
+ expect(levels).toHaveLength(FRAME_COUNT);
+ }
+ });
+
+ it.each([
+ ["a segment", () => segmented],
+ ["a parallel worker", () => parallel],
+ ])("renders the first frame of %s inside the clip like one session does", (_, shape) => {
+ const level = shape().levels[SECOND_SESSION_FIRST]!;
+ expect(Math.abs(level - reference.levels[SECOND_SESSION_FIRST]!)).toBeLessThan(3);
+ });
+
+ it("grades every frame of the clip, including its first", () => {
+ const graded = reference.levels[CLIP_FIRST + 5]!;
+ // Non-vacuous only if the grade visibly moves the clip off its source grey.
+ expect(CLIP_GREY - graded).toBeGreaterThan(30);
+ for (const { levels } of [reference, segmented, parallel]) {
+ for (let i = CLIP_FIRST; i <= CLIP_LAST; i++) {
+ expect(Math.abs(levels[i]! - graded), `frame ${i}`).toBeLessThan(3);
+ }
+ }
+ });
+ },
+);
diff --git a/scripts/test-reachability.json b/scripts/test-reachability.json
index 86c7aae277..bbf5feddd9 100644
--- a/scripts/test-reachability.json
+++ b/scripts/test-reachability.json
@@ -61,7 +61,7 @@
],
"sources": {
"packages/producer/scripts/run-test-lane.mjs": "638885e14480d03290803389325a655f3749442167d784e07bb3af3404598af7",
- "packages/producer/scripts/test-classification.mjs": "cdfa4035d0e2c0b1fea4d4c5b4697e0287d48c4474ab105f5e3d1ce839cca500",
+ "packages/producer/scripts/test-classification.mjs": "11102d43fdae5ab611c2f840e3d7fb4f367683fbddf9dfc770e0a623deb6aef2",
".github/workflows/ci.yml#job#producer-source-tests": "45da72aa6cf08cdfd5bcf5311e89f12af4842f9eebc6735df19da8c02f4d4864",
"packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1",
"packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",