Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): grade an image that appears after its scene starts hidden (#6012)
Miguel Ángel087451837b16
diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.tsindex fb15a82b64..66b241fe9e 100644--- a/packages/core/src/runtime/colorGrading.test.ts+++ b/packages/core/src/runtime/colorGrading.test.ts@@ -1080,6 +1080,54 @@ describe("createColorGradingRuntime", () => {expect(canvas.style.opacity).toBe("1");});+ it.each([+ ["a seek", "redraw"],+ ["a playback tick", "redrawAnimated"],+ ] as const)("grades an image whose ancestor starts hidden once %s shows it", (_, pass) => {+ // A layer a timeline reveals, or a sub-composition host that starts later.+ const scene = document.createElement("div");+ scene.style.visibility = "hidden";+ const image = makeDrawableImage();+ scene.appendChild(image);+ document.body.appendChild(scene);+ runtime = createColorGradingRuntime();++ runtime[pass]();+ expect(runtime.isGraded(image)).toBe(false);+ expect(webglContexts()).toBe(0);++ scene.style.visibility = "visible";+ runtime[pass]();++ expect(runtime.isGraded(image)).toBe(true);+ expect(image.nextElementSibling?.hasAttribute("data-hf-color-grading-canvas")).toBe(true);+ expect(image.style.getPropertyValue("opacity")).toBe("0");+ });++ it.each([+ ["shown", "visible"],+ ["still hidden", "hidden"],+ ])("keeps a live grade switched off while %s, though its saved attribute grades", (_, start) => {+ const scene = document.createElement("div");+ scene.style.visibility = start;+ const image = makeDrawableImage();+ image.style.removeProperty("--hf-color-grading-blur");+ image.setAttribute(+ HF_COLOR_GRADING_ATTR,+ serializeHfColorGrading({ adjust: { exposure: 0.5 } }),+ );+ scene.appendChild(image);+ document.body.appendChild(scene);+ runtime = createColorGradingRuntime();++ runtime.setGrading(image, {});+ scene.style.visibility = "visible";+ runtime.redraw();+ runtime.redrawAnimated();++ expect(runtime.isGraded(image)).toBe(false);+ });+it("updates canvas opacity through grading's own hide when CSS animation progresses (#3329)", () => {const video = makeDrawableVideo();// Simulate a CSS entrance animation: the authored opacity is empty (nodiff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.tsindex 02740790b0..110774fc1c 100644--- a/packages/core/src/runtime/colorGrading.ts+++ b/packages/core/src/runtime/colorGrading.ts@@ -2697,6 +2697,10 @@ function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | nullreturn frame && isDrawableSource(frame) ? frame : null;}+function isShownForColorGrading(element: ColorGradingMediaElement): boolean {+ return isVisibleForColorGrading(element) || hasInjectedRenderFrame(element);+}+function hasInjectedRenderFrame(element: ColorGradingMediaElement): boolean {if (!isVideoElement(element)) return false;const frame = findInjectedRenderFrame(element);@@ -3831,6 +3835,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {}): RuntimeColorGradingApi {const entries = new WeakMap<ColorGradingMediaElement, ColorGradingEntry>();const trackedElements = new Set<ColorGradingMediaElement>();+ const hiddenAtRefresh = new Set<ColorGradingMediaElement>();const idleRenderers: ColorGradingRenderer[] = [];let observer: MutationObserver | null = null;let previewRenderer: ColorGradingPreviewRenderer | null = null;@@ -3887,6 +3892,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {grading: ResolvedHfColorGrading,source: EntrySource,): boolean => {+ hiddenAtRefresh.delete(element);const existing = entries.get(element);if (existing) {existing.grading = grading;@@ -3966,6 +3972,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {};const removeElement = (element: ColorGradingMediaElement): void => {+ hiddenAtRefresh.delete(element);const entry = entries.get(element);if (!entry) return;const renderer = detachEntry(entry);@@ -3983,16 +3990,18 @@ export function createColorGradingRuntime(pausedMediaLease?: {const refresh = (): number => {if (destroyed) return 0;const attributeElements = new Set<ColorGradingMediaElement>();+ hiddenAtRefresh.clear();const nodes = document.querySelectorAll(COLOR_GRADING_SOURCE_SELECTOR);nodes.forEach((node) => {if (!isColorGradingMediaElement(node)) return;attributeElements.add(node);const grading = readColorGradingAttribute(node);if (isRuntimeColorGradingActive(node, grading)) {- if (isVisibleForColorGrading(node) || hasInjectedRenderFrame(node)) {+ if (isShownForColorGrading(node)) {upsert(node, grading, "attribute");} else {removeElement(node);+ hiddenAtRefresh.add(node);}} else {removeElement(node);@@ -4011,8 +4020,20 @@ export function createColorGradingRuntime(pausedMediaLease?: {return trackedElements.size;};+ const adoptShownMedia = (): void => {+ for (const element of hiddenAtRefresh) {+ if (element.isConnected && !isShownForColorGrading(element)) continue;+ hiddenAtRefresh.delete(element);+ const grading = readColorGradingAttribute(element);+ if (element.isConnected && isRuntimeColorGradingActive(element, grading)) {+ upsert(element, grading, "attribute");+ }+ }+ };+const redraw = (): number => {if (destroyed) return 0;+ adoptShownMedia();let drawn = 0;for (const element of trackedElements) {const entry = entries.get(element);@@ -4039,6 +4060,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {const redrawAnimated = (): number => {if (destroyed) return 0;+ adoptShownMedia();let drawn = 0;for (const element of trackedElements) {const entry = entries.get(element);@@ -4111,7 +4133,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {}const grading = readColorGradingAttribute(element);if (isRuntimeColorGradingActive(element, grading)) {- if (!isVisibleForColorGrading(element) && !hasInjectedRenderFrame(element)) {+ if (!isShownForColorGrading(element)) {return { state: "pending", message: "Waiting for visible media" };}return { state: "unavailable", message: "WebGL unavailable" };@@ -4179,6 +4201,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {observer?.disconnect();observer = null;for (const element of trackedElements) removeElement(element);+ hiddenAtRefresh.clear();for (const renderer of idleRenderers) destroyProgramResources(renderer);idleRenderers.length = 0;shared?.gl.getExtension("WEBGL_lose_context")?.loseContext();diff --git a/packages/producer/src/services/render/lateVisibleImageGrading.integration.test.ts b/packages/producer/src/services/render/lateVisibleImageGrading.integration.test.tsnew file mode 100644index 0000000000..a775751fe8--- /dev/null+++ b/packages/producer/src/services/render/lateVisibleImageGrading.integration.test.ts@@ -0,0 +1,146 @@+// A graded <img> that is hidden when grading first looks, rendered for real: it is graded from the+// frame it appears on. One project per shape, since grading another element re-scans the page and+// would hide the bug. Needs Chrome + ffmpeg: integration lane.++import { spawnSync } from "node:child_process";+import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join, resolve } from "node:path";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { resolveConfig } from "@hyperframes/engine";+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";+import { FFMPEG, HAS_FFMPEG, frameLevels } from "./frameLevelsTestFixtures.js";++const WIDTH = 64;+const HEIGHT = 36;+const FPS = 30;+const FRAME_COUNT = 60;+const REVEAL_FRAME = 30;+const REVEAL = REVEAL_FRAME / FPS;+const DURATION = FRAME_COUNT / FPS;+const SOURCE_GREY = 128;+const IMAGE = `<img src="assets/grey.png" data-color-grading='{"adjust":{"exposure":-1}}'+ style="position:absolute;inset:0;width:100%;height:100%" />`;+const SIZE = `data-width="${WIDTH}" data-height="${HEIGHT}"`;++const index = (scene: string) => `<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8" />+ <script src="assets/gsap.min.js"></script>+ <style>* { margin: 0; padding: 0; } body { overflow: hidden; background: #000; }</style>+</head>+<body>+ <div id="root" data-composition-id="main" data-start="0" data-duration="${DURATION}" ${SIZE}+ data-fps="${FPS}">+ ${scene}+ </div>+ <script>window.__timelines["main"] = gsap.timeline({ paused: true });</script>+</body>+</html>+`;++const subComposition = (id: string, duration: number, body: string, script = "") => `<!DOCTYPE html>+<html>+<body>+<template id="${id}-template">+ <div data-composition-id="${id}" ${SIZE} data-duration="${duration}">${body}</div>+ ${script}+</template>+</body>+</html>+`;++const subCompositionHost = (id: string, start: number, duration: number) =>+ `<div id="s-${id}" data-composition-id="${id}" data-composition-src="compositions/${id}.html"+ data-start="${start}" data-duration="${duration}" data-track-index="0" ${SIZE}></div>`;++const SHAPES = {+ "a still image in a timed insert mid-timeline": {+ index: index(`<div class="clip" data-start="${REVEAL}" data-duration="${DURATION - REVEAL}"+ data-track-index="0" style="position:absolute;inset:0">${IMAGE}</div>`),+ },+ "an image in a layer its timeline reveals from visibility:hidden": {+ index: index(subCompositionHost("layer", 0, DURATION)),+ layer: subComposition(+ "layer",+ DURATION,+ `<div id="hidden-layer" style="position:absolute;inset:0;visibility:hidden">${IMAGE}</div>`,+ `<script>+ (() => {+ const tl = gsap.timeline({ paused: true });+ tl.set("#hidden-layer", { visibility: "visible" }, ${REVEAL});+ tl.set({}, {}, ${DURATION});+ window.__timelines["layer"] = tl;+ })();+ </script>`,+ ),+ },+ "an image in a sub-composition whose host starts later": {+ index: index(subCompositionHost("late", REVEAL, DURATION - REVEAL)),+ late: subComposition("late", DURATION - REVEAL, IMAGE),+ },+} satisfies Record<string, Record<string, string>>;++let root: string;++describe.skipIf(!HAS_FFMPEG)("color-graded <img> hidden at first, then shown — real render", () => {+ const levels = new Map<string, number[]>();++ beforeAll(async () => {+ root = mkdtempSync(join(tmpdir(), "hf-late-image-grading-"));+ const grey = join(root, "grey.png");+ const fixture = spawnSync(+ FFMPEG,+ ["-v", "error", "-f", "lavfi", "-i", `color=c=0x808080:size=${WIDTH}x${HEIGHT}`].concat([+ "-frames:v",+ "1",+ "-y",+ grey,+ ]),+ { encoding: "utf-8" },+ );+ if (fixture.status !== 0) throw new Error(`image fixture failed: ${fixture.stderr}`);+ for (const [shape, files] of Object.entries(SHAPES)) {+ const project = mkdtempSync(join(root, "shape-"));+ mkdirSync(join(project, "assets"));+ mkdirSync(join(project, "compositions"));+ copyFileSync(grey, join(project, "assets/grey.png"));+ copyFileSync(+ resolve(import.meta.dirname, "../../../../core/node_modules/gsap/dist/gsap.min.js"),+ join(project, "assets/gsap.min.js"),+ );+ for (const [name, html] of Object.entries(files)) {+ writeFileSync(+ join(project, name === "index" ? "index.html" : `compositions/${name}.html`),+ html,+ );+ }+ const job = createRenderJob({+ fps: FPS,+ quality: "draft",+ format: "mp4",+ hdrMode: "force-sdr",+ workers: 1,+ producerConfig: resolveConfig({ browserGpuMode: "software" }),+ });+ const output = join(project, "out.mp4");+ await executeRenderJob(job, project, output);+ levels.set(shape, frameLevels(output, WIDTH, HEIGHT));+ }+ }, 600_000);++ afterAll(() => {+ if (root) rmSync(root, { recursive: true, force: true });+ });++ it.each(Object.keys(SHAPES))("grades %s from the frame it appears on", (shape) => {+ const shown = levels.get(shape)!;+ expect(shown).toHaveLength(FRAME_COUNT);+ expect(shown[REVEAL_FRAME - 5], "hidden before the reveal").toBeLessThan(3);+ for (let i = REVEAL_FRAME; i < FRAME_COUNT; i++) {+ // Ungraded, the frame shows the source grey; the grade pulls it far below.+ expect(SOURCE_GREY - shown[i]!, `frame ${i}`).toBeGreaterThan(30);+ }+ });+});