Skip to content

Write HTML. Render video. Built for agents.

fix(core): grade an image that appears after its scene starts hidden (#6012)

Miguel Ángel087451837b16
diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.ts
index fb15a82b64..66b241fe9e 100644
--- a/packages/core/src/runtime/colorGrading.test.ts
+++ b/packages/core/src/runtime/colorGrading.test.ts
@@ -1080,6 +1080,54 @@ describe("createColorGradingRuntime", () => {
expect(canvas.style.opacity).toBe("1");
});
+ it.each([
+ ["a seek", "redraw"],
+ ["a playback tick", "redrawAnimated"],
+ ] as const)("grades an image whose ancestor starts hidden once %s shows it", (_, pass) => {
+ // A layer a timeline reveals, or a sub-composition host that starts later.
+ const scene = document.createElement("div");
+ scene.style.visibility = "hidden";
+ const image = makeDrawableImage();
+ scene.appendChild(image);
+ document.body.appendChild(scene);
+ runtime = createColorGradingRuntime();
+
+ runtime[pass]();
+ expect(runtime.isGraded(image)).toBe(false);
+ expect(webglContexts()).toBe(0);
+
+ scene.style.visibility = "visible";
+ runtime[pass]();
+
+ expect(runtime.isGraded(image)).toBe(true);
+ expect(image.nextElementSibling?.hasAttribute("data-hf-color-grading-canvas")).toBe(true);
+ expect(image.style.getPropertyValue("opacity")).toBe("0");
+ });
+
+ it.each([
+ ["shown", "visible"],
+ ["still hidden", "hidden"],
+ ])("keeps a live grade switched off while %s, though its saved attribute grades", (_, start) => {
+ const scene = document.createElement("div");
+ scene.style.visibility = start;
+ const image = makeDrawableImage();
+ image.style.removeProperty("--hf-color-grading-blur");
+ image.setAttribute(
+ HF_COLOR_GRADING_ATTR,
+ serializeHfColorGrading({ adjust: { exposure: 0.5 } }),
+ );
+ scene.appendChild(image);
+ document.body.appendChild(scene);
+ runtime = createColorGradingRuntime();
+
+ runtime.setGrading(image, {});
+ scene.style.visibility = "visible";
+ runtime.redraw();
+ runtime.redrawAnimated();
+
+ expect(runtime.isGraded(image)).toBe(false);
+ });
+
it("updates canvas opacity through grading's own hide when CSS animation progresses (#3329)", () => {
const video = makeDrawableVideo();
// Simulate a CSS entrance animation: the authored opacity is empty (no
diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.ts
index 02740790b0..110774fc1c 100644
--- a/packages/core/src/runtime/colorGrading.ts
+++ b/packages/core/src/runtime/colorGrading.ts
@@ -2697,6 +2697,10 @@ function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null
return frame && isDrawableSource(frame) ? frame : null;
}
+function isShownForColorGrading(element: ColorGradingMediaElement): boolean {
+ return isVisibleForColorGrading(element) || hasInjectedRenderFrame(element);
+}
+
function hasInjectedRenderFrame(element: ColorGradingMediaElement): boolean {
if (!isVideoElement(element)) return false;
const frame = findInjectedRenderFrame(element);
@@ -3831,6 +3835,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
}): RuntimeColorGradingApi {
const entries = new WeakMap<ColorGradingMediaElement, ColorGradingEntry>();
const trackedElements = new Set<ColorGradingMediaElement>();
+ const hiddenAtRefresh = new Set<ColorGradingMediaElement>();
const idleRenderers: ColorGradingRenderer[] = [];
let observer: MutationObserver | null = null;
let previewRenderer: ColorGradingPreviewRenderer | null = null;
@@ -3887,6 +3892,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
grading: ResolvedHfColorGrading,
source: EntrySource,
): boolean => {
+ hiddenAtRefresh.delete(element);
const existing = entries.get(element);
if (existing) {
existing.grading = grading;
@@ -3966,6 +3972,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
};
const removeElement = (element: ColorGradingMediaElement): void => {
+ hiddenAtRefresh.delete(element);
const entry = entries.get(element);
if (!entry) return;
const renderer = detachEntry(entry);
@@ -3983,16 +3990,18 @@ export function createColorGradingRuntime(pausedMediaLease?: {
const refresh = (): number => {
if (destroyed) return 0;
const attributeElements = new Set<ColorGradingMediaElement>();
+ hiddenAtRefresh.clear();
const nodes = document.querySelectorAll(COLOR_GRADING_SOURCE_SELECTOR);
nodes.forEach((node) => {
if (!isColorGradingMediaElement(node)) return;
attributeElements.add(node);
const grading = readColorGradingAttribute(node);
if (isRuntimeColorGradingActive(node, grading)) {
- if (isVisibleForColorGrading(node) || hasInjectedRenderFrame(node)) {
+ if (isShownForColorGrading(node)) {
upsert(node, grading, "attribute");
} else {
removeElement(node);
+ hiddenAtRefresh.add(node);
}
} else {
removeElement(node);
@@ -4011,8 +4020,20 @@ export function createColorGradingRuntime(pausedMediaLease?: {
return trackedElements.size;
};
+ const adoptShownMedia = (): void => {
+ for (const element of hiddenAtRefresh) {
+ if (element.isConnected && !isShownForColorGrading(element)) continue;
+ hiddenAtRefresh.delete(element);
+ const grading = readColorGradingAttribute(element);
+ if (element.isConnected && isRuntimeColorGradingActive(element, grading)) {
+ upsert(element, grading, "attribute");
+ }
+ }
+ };
+
const redraw = (): number => {
if (destroyed) return 0;
+ adoptShownMedia();
let drawn = 0;
for (const element of trackedElements) {
const entry = entries.get(element);
@@ -4039,6 +4060,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
const redrawAnimated = (): number => {
if (destroyed) return 0;
+ adoptShownMedia();
let drawn = 0;
for (const element of trackedElements) {
const entry = entries.get(element);
@@ -4111,7 +4133,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
}
const grading = readColorGradingAttribute(element);
if (isRuntimeColorGradingActive(element, grading)) {
- if (!isVisibleForColorGrading(element) && !hasInjectedRenderFrame(element)) {
+ if (!isShownForColorGrading(element)) {
return { state: "pending", message: "Waiting for visible media" };
}
return { state: "unavailable", message: "WebGL unavailable" };
@@ -4179,6 +4201,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
observer?.disconnect();
observer = null;
for (const element of trackedElements) removeElement(element);
+ hiddenAtRefresh.clear();
for (const renderer of idleRenderers) destroyProgramResources(renderer);
idleRenderers.length = 0;
shared?.gl.getExtension("WEBGL_lose_context")?.loseContext();
diff --git a/packages/producer/src/services/render/lateVisibleImageGrading.integration.test.ts b/packages/producer/src/services/render/lateVisibleImageGrading.integration.test.ts
new file mode 100644
index 0000000000..a775751fe8
--- /dev/null
+++ b/packages/producer/src/services/render/lateVisibleImageGrading.integration.test.ts
@@ -0,0 +1,146 @@
+// A graded <img> that is hidden when grading first looks, rendered for real: it is graded from the
+// frame it appears on. One project per shape, since grading another element re-scans the page and
+// would hide the bug. Needs Chrome + ffmpeg: integration lane.
+
+import { spawnSync } from "node:child_process";
+import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { resolveConfig } from "@hyperframes/engine";
+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";
+import { FFMPEG, HAS_FFMPEG, frameLevels } from "./frameLevelsTestFixtures.js";
+
+const WIDTH = 64;
+const HEIGHT = 36;
+const FPS = 30;
+const FRAME_COUNT = 60;
+const REVEAL_FRAME = 30;
+const REVEAL = REVEAL_FRAME / FPS;
+const DURATION = FRAME_COUNT / FPS;
+const SOURCE_GREY = 128;
+const IMAGE = `<img src="assets/grey.png" data-color-grading='{"adjust":{"exposure":-1}}'
+ style="position:absolute;inset:0;width:100%;height:100%" />`;
+const SIZE = `data-width="${WIDTH}" data-height="${HEIGHT}"`;
+
+const index = (scene: string) => `<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8" />
+ <script src="assets/gsap.min.js"></script>
+ <style>* { margin: 0; padding: 0; } body { overflow: hidden; background: #000; }</style>
+</head>
+<body>
+ <div id="root" data-composition-id="main" data-start="0" data-duration="${DURATION}" ${SIZE}
+ data-fps="${FPS}">
+ ${scene}
+ </div>
+ <script>window.__timelines["main"] = gsap.timeline({ paused: true });</script>
+</body>
+</html>
+`;
+
+const subComposition = (id: string, duration: number, body: string, script = "") => `<!DOCTYPE html>
+<html>
+<body>
+<template id="${id}-template">
+ <div data-composition-id="${id}" ${SIZE} data-duration="${duration}">${body}</div>
+ ${script}
+</template>
+</body>
+</html>
+`;
+
+const subCompositionHost = (id: string, start: number, duration: number) =>
+ `<div id="s-${id}" data-composition-id="${id}" data-composition-src="compositions/${id}.html"
+ data-start="${start}" data-duration="${duration}" data-track-index="0" ${SIZE}></div>`;
+
+const SHAPES = {
+ "a still image in a timed insert mid-timeline": {
+ index: index(`<div class="clip" data-start="${REVEAL}" data-duration="${DURATION - REVEAL}"
+ data-track-index="0" style="position:absolute;inset:0">${IMAGE}</div>`),
+ },
+ "an image in a layer its timeline reveals from visibility:hidden": {
+ index: index(subCompositionHost("layer", 0, DURATION)),
+ layer: subComposition(
+ "layer",
+ DURATION,
+ `<div id="hidden-layer" style="position:absolute;inset:0;visibility:hidden">${IMAGE}</div>`,
+ `<script>
+ (() => {
+ const tl = gsap.timeline({ paused: true });
+ tl.set("#hidden-layer", { visibility: "visible" }, ${REVEAL});
+ tl.set({}, {}, ${DURATION});
+ window.__timelines["layer"] = tl;
+ })();
+ </script>`,
+ ),
+ },
+ "an image in a sub-composition whose host starts later": {
+ index: index(subCompositionHost("late", REVEAL, DURATION - REVEAL)),
+ late: subComposition("late", DURATION - REVEAL, IMAGE),
+ },
+} satisfies Record<string, Record<string, string>>;
+
+let root: string;
+
+describe.skipIf(!HAS_FFMPEG)("color-graded <img> hidden at first, then shown — real render", () => {
+ const levels = new Map<string, number[]>();
+
+ beforeAll(async () => {
+ root = mkdtempSync(join(tmpdir(), "hf-late-image-grading-"));
+ const grey = join(root, "grey.png");
+ const fixture = spawnSync(
+ FFMPEG,
+ ["-v", "error", "-f", "lavfi", "-i", `color=c=0x808080:size=${WIDTH}x${HEIGHT}`].concat([
+ "-frames:v",
+ "1",
+ "-y",
+ grey,
+ ]),
+ { encoding: "utf-8" },
+ );
+ if (fixture.status !== 0) throw new Error(`image fixture failed: ${fixture.stderr}`);
+ for (const [shape, files] of Object.entries(SHAPES)) {
+ const project = mkdtempSync(join(root, "shape-"));
+ mkdirSync(join(project, "assets"));
+ mkdirSync(join(project, "compositions"));
+ copyFileSync(grey, join(project, "assets/grey.png"));
+ copyFileSync(
+ resolve(import.meta.dirname, "../../../../core/node_modules/gsap/dist/gsap.min.js"),
+ join(project, "assets/gsap.min.js"),
+ );
+ for (const [name, html] of Object.entries(files)) {
+ writeFileSync(
+ join(project, name === "index" ? "index.html" : `compositions/${name}.html`),
+ html,
+ );
+ }
+ const job = createRenderJob({
+ fps: FPS,
+ quality: "draft",
+ format: "mp4",
+ hdrMode: "force-sdr",
+ workers: 1,
+ producerConfig: resolveConfig({ browserGpuMode: "software" }),
+ });
+ const output = join(project, "out.mp4");
+ await executeRenderJob(job, project, output);
+ levels.set(shape, frameLevels(output, WIDTH, HEIGHT));
+ }
+ }, 600_000);
+
+ afterAll(() => {
+ if (root) rmSync(root, { recursive: true, force: true });
+ });
+
+ it.each(Object.keys(SHAPES))("grades %s from the frame it appears on", (shape) => {
+ const shown = levels.get(shape)!;
+ expect(shown).toHaveLength(FRAME_COUNT);
+ expect(shown[REVEAL_FRAME - 5], "hidden before the reveal").toBeLessThan(3);
+ for (let i = REVEAL_FRAME; i < FRAME_COUNT; i++) {
+ // Ungraded, the frame shows the source grey; the grade pulls it far below.
+ expect(SOURCE_GREY - shown[i]!, `frame ${i}`).toBeGreaterThan(30);
+ }
+ });
+});