Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): keep the colour grade on a graded video's first rendered frame (#5575)
Miguel Ángel95837a3df7c1
diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.tsindex 014eb8826d..dd27d64a16 100644--- a/packages/core/src/runtime/colorGrading.test.ts+++ b/packages/core/src/runtime/colorGrading.test.ts@@ -1041,6 +1041,29 @@ describe("createColorGradingRuntime", () => {expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();});+ it("grades a clip's first producer frame when the DOM observer runs before it decodes", () => {+ const video = makeDrawableVideo();+ video.style.visibility = "hidden";+ document.body.appendChild(video);+ runtime = createColorGradingRuntime();++ let decoded = false;+ const frame = document.createElement("img");+ frame.id = "__render_frame_hero-video__";+ frame.className = "__render_frame__";+ frame.style.visibility = "visible";+ Object.defineProperty(frame, "complete", { get: () => decoded, configurable: true });+ Object.defineProperty(frame, "naturalWidth", { get: () => (decoded ? 640 : 0) });+ Object.defineProperty(frame, "naturalHeight", { get: () => (decoded ? 360 : 0) });+ video.after(frame);+ runtime.refresh();+ decoded = true;++ expect(runtime.redraw()).toBe(1);+ expect(texImage2DCalls.at(-1)).toContain(frame);+ expect(frame.nextSibling).toBe(document.querySelector("[data-hf-color-grading-canvas]"));+ });+it("does not recreate an inactive clip from its hidden producer frame", () => {const { video, canvas } = startRuntimeWithVideo();const frame = document.createElement("img");diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.tsindex f0c5ae4a32..f72079ba96 100644--- a/packages/core/src/runtime/colorGrading.ts+++ b/packages/core/src/runtime/colorGrading.ts@@ -2494,7 +2494,7 @@ function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | nullfunction hasInjectedRenderFrame(element: ColorGradingMediaElement): boolean {if (!isVideoElement(element)) return false;- const frame = findRenderFrameImage(element);+ const frame = findInjectedRenderFrame(element);if (!frame) return false;const style = window.getComputedStyle(frame);return style.display !== "none" && style.visibility !== "hidden";diff --git a/packages/producer/scripts/test-classification.mjs b/packages/producer/scripts/test-classification.mjsindex 2aa4f1673b..0bf0e4f5d2 100644--- a/packages/producer/scripts/test-classification.mjs+++ b/packages/producer/scripts/test-classification.mjs@@ -33,6 +33,7 @@ const INTEGRATION_TEST_FILES = new Set(["src/services/render/hlsRender.integration.test.ts","src/services/render/motionBlurVideo.integration.test.ts","src/services/render/renderCancel.integration.test.ts",+ "src/services/render/segmentedColorGrading.integration.test.ts","src/services/render/stages/compileStage.mediaType.test.ts","src/services/render/stages/gifEncodeArgs.test.ts","src/services/videoFrameBorderClip.test.ts",diff --git a/packages/producer/src/services/render/segmentedColorGrading.integration.test.ts b/packages/producer/src/services/render/segmentedColorGrading.integration.test.tsnew file mode 100644index 0000000000..57356f9698--- /dev/null+++ b/packages/producer/src/services/render/segmentedColorGrading.integration.test.ts@@ -0,0 +1,195 @@+// A graded <video>, rendered for real (#5548): the first frame a capture session draws inside the+// clip (a segment's or a parallel worker's), and the clip's own first frame, keep the grade.+// Needs Chrome + ffmpeg: integration lane.++import { spawnSync } from "node:child_process";+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { getFfmpegBinary, resolveConfig } from "@hyperframes/engine";+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";++const FFMPEG = getFfmpegBinary();+const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"], { encoding: "utf-8" }).status === 0;++const WIDTH = 160;+const HEIGHT = 90;+const FPS = 30;+const FRAME_COUNT = 60;+const CLIP_FIRST = 15;+const CLIP_LAST = 44;+// The smallest segment size. With two workers, both the second segment and the second worker's+// frame range start here, mid-clip.+const SECOND_SESSION_FIRST = 30;+const CLIP_GREY = 128;++const COMPOSITION = `<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8" />+ <style>+ * { margin: 0; padding: 0; box-sizing: border-box; }+ body { overflow: hidden; background: #000; }+ #clip { position: absolute; left: 0; top: 0; width: ${WIDTH}px; height: ${HEIGHT}px; }+ </style>+</head>+<body>+ <div+ id="root"+ data-composition-id="segmented-color-grading"+ data-start="0"+ data-duration="${FRAME_COUNT / FPS}"+ data-width="${WIDTH}"+ data-height="${HEIGHT}"+ data-fps="${FPS}"+ >+ <video id="clip" class="clip" src="assets/clip.mp4" data-start="${CLIP_FIRST / FPS}"+ data-duration="${(CLIP_LAST + 1 - CLIP_FIRST) / FPS}" data-track-index="0" muted playsinline+ data-color-grading='{"adjust":{"exposure":-1}}'></video>+ </div>+ <script>window.__timelines = window.__timelines || {};</script>+</body>+</html>+`;++let root: string;++/** Mean RGB level of every frame of an encoded video, 0-255. */+function frameLevels(videoPath: string): number[] {+ const raw = spawnSync(+ FFMPEG,+ ["-v", "error", "-i", videoPath, "-f", "rawvideo", "-pix_fmt", "gray", "-"],+ {+ maxBuffer: 64 * 1024 * 1024,+ },+ );+ if (raw.status !== 0) throw new Error(`decode failed: ${raw.stderr}`);+ const frameBytes = WIDTH * HEIGHT;+ const levels: number[] = [];+ for (let offset = 0; offset + frameBytes <= raw.stdout.length; offset += frameBytes) {+ let sum = 0;+ for (let i = offset; i < offset + frameBytes; i++) sum += raw.stdout[i]!;+ levels.push(sum / frameBytes);+ }+ return levels;+}++interface Render {+ levels: number[];+ phases: string[];+ workerCount: number;+}++async function render(name: string, segmented: boolean, workers: number): Promise<Render> {+ const phases: string[] = [];+ let workerCount = 0;+ const job = createRenderJob({+ fps: FPS,+ quality: "draft",+ format: "mp4",+ hdrMode: "force-sdr",+ workers,+ logger: {+ info(message: string, meta?: Record<string, unknown>) {+ if (message !== "[Render:trace]") return;+ if (typeof meta?.phase === "string") phases.push(meta.phase);+ if (typeof meta?.workerCount === "number")+ workerCount = Math.max(workerCount, meta.workerCount);+ },+ warn() {},+ error() {},+ debug() {},+ },+ producerConfig: resolveConfig({ browserGpuMode: "software" }),+ });+ const saved = {+ capture: process.env.HF_SEGMENTED_CAPTURE,+ frames: process.env.HF_SEGMENT_FRAMES,+ };+ process.env.HF_SEGMENTED_CAPTURE = String(segmented);+ process.env.HF_SEGMENT_FRAMES = String(SECOND_SESSION_FIRST);+ try {+ const output = join(root, `${name}.mp4`);+ await executeRenderJob(job, root, output);+ return { levels: frameLevels(output), phases, workerCount };+ } finally {+ if (saved.capture === undefined) delete process.env.HF_SEGMENTED_CAPTURE;+ else process.env.HF_SEGMENTED_CAPTURE = saved.capture;+ if (saved.frames === undefined) delete process.env.HF_SEGMENT_FRAMES;+ else process.env.HF_SEGMENT_FRAMES = saved.frames;+ }+}++describe.skipIf(!HAS_FFMPEG)(+ "color-graded video at a capture session's first frame — real render (#5548)",+ () => {+ let reference: Render;+ let segmented: Render;+ let parallel: Render;++ beforeAll(async () => {+ root = mkdtempSync(join(tmpdir(), "hf-segmented-grading-"));+ mkdirSync(join(root, "assets"), { recursive: true });+ writeFileSync(join(root, "index.html"), COMPOSITION);+ const clip = spawnSync(+ FFMPEG,+ [+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ `color=c=0x808080:size=${WIDTH}x${HEIGHT}:rate=${FPS}:duration=2`,+ "-pix_fmt",+ "yuv420p",+ "-c:v",+ "libx264",+ "-y",+ join(root, "assets", "clip.mp4"),+ ],+ { encoding: "utf-8" },+ );+ if (clip.status !== 0) throw new Error(`clip fixture failed: ${clip.stderr}`);++ reference = await render("reference", false, 1);+ segmented = await render("segmented", true, 2);+ parallel = await render("parallel", false, 2);+ }, 600_000);++ afterAll(() => {+ if (root) rmSync(root, { recursive: true, force: true });+ });++ it("renders each shape on the route it asks for", () => {+ expect(segmented.phases).toContain("capture_segmented");+ expect(reference.phases).not.toContain("capture_segmented");+ expect(parallel.phases).not.toContain("capture_segmented");+ expect(reference.workerCount).toBe(1);+ expect(segmented.workerCount).toBe(2);+ expect(parallel.workerCount).toBe(2);+ for (const { levels } of [reference, segmented, parallel]) {+ expect(levels).toHaveLength(FRAME_COUNT);+ }+ });++ it.each([+ ["a segment", () => segmented],+ ["a parallel worker", () => parallel],+ ])("renders the first frame of %s inside the clip like one session does", (_, shape) => {+ const level = shape().levels[SECOND_SESSION_FIRST]!;+ expect(Math.abs(level - reference.levels[SECOND_SESSION_FIRST]!)).toBeLessThan(3);+ });++ it("grades every frame of the clip, including its first", () => {+ const graded = reference.levels[CLIP_FIRST + 5]!;+ // Non-vacuous only if the grade visibly moves the clip off its source grey.+ expect(CLIP_GREY - graded).toBeGreaterThan(30);+ for (const { levels } of [reference, segmented, parallel]) {+ for (let i = CLIP_FIRST; i <= CLIP_LAST; i++) {+ expect(Math.abs(levels[i]! - graded), `frame ${i}`).toBeLessThan(3);+ }+ }+ });+ },+);diff --git a/scripts/test-reachability.json b/scripts/test-reachability.jsonindex 86c7aae277..bbf5feddd9 100644--- a/scripts/test-reachability.json+++ b/scripts/test-reachability.json@@ -61,7 +61,7 @@],"sources": {"packages/producer/scripts/run-test-lane.mjs": "638885e14480d03290803389325a655f3749442167d784e07bb3af3404598af7",- "packages/producer/scripts/test-classification.mjs": "cdfa4035d0e2c0b1fea4d4c5b4697e0287d48c4474ab105f5e3d1ce839cca500",+ "packages/producer/scripts/test-classification.mjs": "11102d43fdae5ab611c2f840e3d7fb4f367683fbddf9dfc770e0a623deb6aef2",".github/workflows/ci.yml#job#producer-source-tests": "45da72aa6cf08cdfd5bcf5311e89f12af4842f9eebc6735df19da8c02f4d4864","packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1","packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",