Skip to content

Write HTML. Render video. Built for agents.

fix(core): color grading applies to canvas compositions (#5860)

Miguel Ángel5658f700650b
diff --git a/docs/guides/color-grading.mdx b/docs/guides/color-grading.mdx
index a0a8a49d75..7abca7425a 100644
--- a/docs/guides/color-grading.mdx
+++ b/docs/guides/color-grading.mdx
@@ -166,8 +166,9 @@ Grading picks pixels by value, never by position: no face tracking, region
tracking, rotoscoping or masks — an HSL selection is a color qualifier, not a
shape. To treat one part of a frame, split it into its own media layer and grade
that layer, as in the [implementation reference](/reference/color-grading). It
-targets media elements only, so a whole scene including HTML text cannot be
-graded.
+targets `<video>`, `<img>` and `<canvas>` elements only, so a whole scene
+including HTML text cannot be graded. A `<canvas>` is graded at its drawing-buffer
+size (`width`/`height`) and re-read on every seek and playback frame.
SDR images and video are the supported case; 4K works at a higher preview and
render cost. An HDR source — iPhone HDR, HLG, Dolby Vision-style — gets an SDR
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index 826681b293..d8d1dcc1b5 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -481,7 +481,7 @@ npx hyperframes media-treatment --selector '#hero' --clear
| `--all` | Print the exhaustive capability catalog |
| `--project <dir>` | Project directory (default: current directory) |
| `--file <path>` | Composition file (default `index.html`) |
-| `--selector <css>` | CSS selector; every matched `<img>` or `<video>` gets the patch |
+| `--selector <css>` | CSS selector; every matched `<img>`, `<video>` or `<canvas>` gets the patch |
| `--selector-index <n>` | Zero-based match index to patch only one of several matches |
| `--grading <json>` | Validated colour-grading patch |
| `--apply` | Apply the patch |
diff --git a/docs/reference/lint-rules.mdx b/docs/reference/lint-rules.mdx
index 6ccda0a2c8..d198134be1 100644
--- a/docs/reference/lint-rules.mdx
+++ b/docs/reference/lint-rules.mdx
@@ -554,9 +554,11 @@ Source: `packages/lint/src/rules/media.ts`.
### `color_grading_non_media`
-**Error.** `data-color-grading` is set on an element other than `<video>` or `<img>`, such as a wrapper or an element with a CSS background. The shader runtime only grades real `<video>` and `<img>` elements, so the attribute has no effect.
+**Error.** `data-color-grading` is set on an element other than `<video>`, `<img>` or `<canvas>`, such as a
+wrapper or an element with a CSS background. The shader runtime only grades real `<video>`, `<img>` and
+`<canvas>` elements, so the attribute has no effect.
-**Fix:** Move the attribute to the `<video>` or `<img>` element itself.
+**Fix:** Move the attribute to the `<video>`, `<img>` or `<canvas>` element itself.
### `duplicate_media_discovery_risk`
diff --git a/packages/cli/src/commands/media-treatment.test.ts b/packages/cli/src/commands/media-treatment.test.ts
index c7dd916f1d..ec3aef7ccb 100644
--- a/packages/cli/src/commands/media-treatment.test.ts
+++ b/packages/cli/src/commands/media-treatment.test.ts
@@ -598,7 +598,20 @@ describe("applyMediaTreatmentToHtml", () => {
selector: "#hero",
grading: { preset: "warm-daylight" },
}),
- ).toThrow(/requires an <img> or <video>/);
+ ).toThrow(/requires a <video>, <img> or <canvas>/);
+ });
+
+ it("grades a canvas", () => {
+ const result = applyMediaTreatmentToHtml(
+ `<canvas id="drawn" width="640" height="360"></canvas>`,
+ {
+ selector: "#drawn",
+ grading: { preset: "warm-daylight" },
+ },
+ );
+
+ expect(result.tag).toBe("canvas");
+ expect(result.html).toContain("data-color-grading=");
});
it("rejects unknown keys instead of silently dropping agent mistakes", () => {
diff --git a/packages/cli/src/commands/media-treatment.ts b/packages/cli/src/commands/media-treatment.ts
index 34c94e59a2..2af21733e2 100644
--- a/packages/cli/src/commands/media-treatment.ts
+++ b/packages/cli/src/commands/media-treatment.ts
@@ -10,6 +10,7 @@ import {
serializeHfColorGrading,
} from "@hyperframes/core";
import {
+ COLOR_GRADING_SOURCE_TAGS,
isColorGradingVariableRef,
validateColorGradingContract,
} from "@hyperframes/parsers/color-grading-contract";
@@ -344,9 +345,11 @@ interface ApplyMediaTreatmentOptions {
clear?: boolean;
}
+type GradingSourceTag = (typeof COLOR_GRADING_SOURCE_TAGS)[number];
+
interface MediaTreatmentTargetResult {
selectorIndex: number;
- tag: "img" | "video";
+ tag: GradingSourceTag;
changed: boolean;
value: string | null;
before: unknown;
@@ -443,7 +446,7 @@ function queryIncludingTemplates(root: Document | Element, selector: string): El
return [];
}
-type MediaTarget = { element: Element; selectorIndex: number; tag: "img" | "video" };
+type MediaTarget = { element: Element; selectorIndex: number; tag: GradingSourceTag };
function selectMediaElements(
document: Document,
@@ -463,9 +466,11 @@ function selectMediaElements(
const indexes = selectorIndex === undefined ? matches.keys() : [selectorIndex];
return Array.from(indexes, (index) => {
const element = matches[index]!;
- const tag = element.tagName.toLowerCase();
- if (tag !== "img" && tag !== "video") {
- throw new Error(`Color grading requires an <img> or <video>; selector matched <${tag}>`);
+ const tag = element.tagName.toLowerCase() as GradingSourceTag;
+ if (!COLOR_GRADING_SOURCE_TAGS.includes(tag)) {
+ throw new Error(
+ `Color grading requires a <video>, <img> or <canvas>; selector matched <${tag}>`,
+ );
}
if (!isHTMLElement(element)) {
throw new Error(`Color grading requires an HTML <${tag}>; selector matched one inside <svg>`);
diff --git a/packages/core/src/colorGrading.ts b/packages/core/src/colorGrading.ts
index 77eea925dc..cd53746841 100644
--- a/packages/core/src/colorGrading.ts
+++ b/packages/core/src/colorGrading.ts
@@ -13,6 +13,7 @@ import {
COLOR_GRADING_LUT_KEYS,
COLOR_GRADING_MAX_CURVE_POINTS,
COLOR_GRADING_MAX_SECONDARIES,
+ COLOR_GRADING_SOURCE_TAGS,
COLOR_GRADING_TOP_LEVEL_KEYS,
COLOR_GRADING_WHEEL_KEYS,
} from "@hyperframes/parsers/color-grading-contract";
@@ -41,6 +42,7 @@ export const COLOR_GRADING_AUTHORED_OPACITY_ATTR = "data-hf-authored-opacity";
export const HF_COLOR_GRADING_CANVAS_ID_PREFIX = "__hf_color_grading_";
export const HF_COLOR_GRADING_COLOR_SPACE = COLOR_GRADING_COLOR_SPACE;
+export const HF_COLOR_GRADING_SOURCE_TAGS = COLOR_GRADING_SOURCE_TAGS;
export type HfColorGradingPresetId =
| "neutral"
diff --git a/packages/core/src/editing/affordances.test.ts b/packages/core/src/editing/affordances.test.ts
index 86db7d912c..7ded2cb870 100644
--- a/packages/core/src/editing/affordances.test.ts
+++ b/packages/core/src/editing/affordances.test.ts
@@ -172,6 +172,11 @@ describe("resolveEditingAffordances — sections", () => {
expect(s).toMatchObject({ media: true, colorGrading: true });
});
+ it("canvas: colorGrading", () => {
+ const s = resolveEditingAffordances(baseFacts({ tag: "canvas" })).sections;
+ expect(s).toMatchObject({ media: false, colorGrading: true });
+ });
+
it("editable text on a plain element: text section", () => {
const s = resolveEditingAffordances(baseFacts({ hasEditableText: true })).sections;
expect(s.text).toBe(true);
diff --git a/packages/core/src/editing/affordances.ts b/packages/core/src/editing/affordances.ts
index 238a17a6b2..85bf006629 100644
--- a/packages/core/src/editing/affordances.ts
+++ b/packages/core/src/editing/affordances.ts
@@ -1,4 +1,5 @@
import { HF_AUDIO_GROUP_TAG } from "../audioGroups.js";
+import { HF_COLOR_GRADING_SOURCE_TAGS } from "../colorGrading.js";
/**
* Pure, DOM-free editing-affordance resolution. Single source of truth for what
* the studio's edit panel (and any SDK consumer) surfaces per selected element:
@@ -221,7 +222,7 @@ export function resolveEditingSections(facts: EditableElementFacts): EditingSect
media: facts.tag === "video" || facts.tag === "audio" || facts.tag === "img",
audioFx:
facts.tag === "audio" || isAudioBus || (facts.tag === "video" && facts.hasAudio === true),
- colorGrading: facts.tag === "video" || facts.tag === "img",
+ colorGrading: (HF_COLOR_GRADING_SOURCE_TAGS as readonly string[]).includes(facts.tag),
// A bus has no clip range at all — no `data-start`, no duration, and its
// automation clock is composition time — so Start/Duration/End would be
// editing nothing. Audio keeps its timing: an `<audio>` clip is placed on
diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.ts
index 15ae9871e9..2af15f05c8 100644
--- a/packages/core/src/runtime/colorGrading.test.ts
+++ b/packages/core/src/runtime/colorGrading.test.ts
@@ -856,6 +856,54 @@ describe("createColorGradingRuntime", () => {
expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.7);
});
+ describe("a graded canvas", () => {
+ function makeGradedCanvas(width: number, height: number): HTMLCanvasElement {
+ const canvas = document.createElement("canvas");
+ canvas.id = "drawn";
+ canvas.width = width;
+ canvas.height = height;
+ canvas.setAttribute(
+ HF_COLOR_GRADING_ATTR,
+ serializeHfColorGrading({ adjust: { exposure: 0.5 } }),
+ );
+ Object.defineProperty(canvas, "offsetWidth", { value: 640, configurable: true });
+ Object.defineProperty(canvas, "offsetHeight", { value: 360, configurable: true });
+ canvas.getBoundingClientRect = () =>
+ ({ width: 640, height: 360, left: 0, top: 0, right: 640, bottom: 360 }) as DOMRect;
+ document.body.appendChild(canvas);
+ return canvas;
+ }
+ const canvasUploads = (canvas: HTMLCanvasElement) =>
+ texImage2DCalls.filter((args) => args.length === 6 && args[5] === canvas).length;
+
+ it("grades its pixels and re-uploads them on every transport tick and seek", () => {
+ const canvas = makeGradedCanvas(320, 180);
+ runtime = createColorGradingRuntime();
+
+ expect(runtime.isGraded(canvas)).toBe(true);
+ expect(runtime.getStatus("drawn")).toMatchObject({ state: "active" });
+ expect(canvas.style.getPropertyValue("opacity")).toBe("0");
+ expect(canvasUploads(canvas)).toBe(1);
+
+ expect(runtime.redrawAnimated()).toBe(1);
+ expect(runtime.redraw()).toBe(1);
+ expect(canvasUploads(canvas)).toBe(3);
+ });
+
+ it("waits for a drawing buffer, whatever its CSS box", () => {
+ const canvas = makeGradedCanvas(0, 0);
+ runtime = createColorGradingRuntime();
+
+ expect(runtime.getStatus("drawn")).toMatchObject({ state: "pending" });
+ expect(canvasUploads(canvas)).toBe(0);
+
+ canvas.width = 320;
+ canvas.height = 180;
+ expect(runtime.redraw()).toBe(1);
+ expect(canvasUploads(canvas)).toBe(1);
+ });
+ });
+
it("redraws animated held video frames without duplicating active video draws", () => {
const video = makeDrawableVideo();
video.style.setProperty("--hf-color-grading-blur", "0.1");
diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.ts
index de2c963574..2a11118ebe 100644
--- a/packages/core/src/runtime/colorGrading.ts
+++ b/packages/core/src/runtime/colorGrading.ts
@@ -5,6 +5,7 @@ import {
HF_COLOR_GRADING_CANVAS_ID_PREFIX,
HF_COLOR_GRADING_DETAIL_KEYS,
HF_COLOR_GRADING_EFFECT_KEYS,
+ HF_COLOR_GRADING_SOURCE_TAGS,
hasHfColorGradingHueCurveValues,
hasHfColorGradingRgbCurveValues,
hasHfColorGradingSecondaryValues,
@@ -42,9 +43,15 @@ import { copyMediaVisualStyles } from "../inline-scripts/parityContract";
import { readVariablesForElement } from "./variableScope";
import { swallow } from "./diagnostics";
import { findInjectedRenderFrame } from "./renderFrameSibling";
-import { isElementNode, isHtmlElement, isImageElement, isVideoElement } from "./domRealm";
+import {
+ isCanvasElement,
+ isElementNode,
+ isHtmlElement,
+ isImageElement,
+ isVideoElement,
+} from "./domRealm";
-type ColorGradingMediaElement = HTMLVideoElement | HTMLImageElement;
+type ColorGradingMediaElement = HTMLVideoElement | HTMLImageElement | HTMLCanvasElement;
type EntrySource = "attribute" | "live";
@@ -1462,8 +1469,15 @@ const KUWAHARA_RESOLVE_FRAGMENT_SHADER = [
"}",
].join("\n");
+const COLOR_GRADING_SOURCE_SELECTOR = HF_COLOR_GRADING_SOURCE_TAGS.map(
+ (tag) => `${tag}[${HF_COLOR_GRADING_ATTR}]`,
+).join(", ");
+
function isColorGradingMediaElement(value: Element): value is ColorGradingMediaElement {
- return isVideoElement(value) || isImageElement(value);
+ return (
+ isHtmlElement(value) &&
+ (HF_COLOR_GRADING_SOURCE_TAGS as readonly string[]).includes(value.localName)
+ );
}
function isVisibleForColorGrading(element: ColorGradingMediaElement): boolean {
@@ -1781,7 +1795,7 @@ function hashStringSeed(value: string): number {
function seedForElement(element: ColorGradingMediaElement): number {
const key =
element.id ||
- element.currentSrc ||
+ (!isCanvasElement(element) && element.currentSrc) ||
element.getAttribute("src") ||
`${element.tagName}:${Array.prototype.indexOf.call(element.parentNode?.children ?? [], element)}`;
return hashStringSeed(key);
@@ -2462,31 +2476,22 @@ function resolveTarget(
}
function readSourceSize(source: TexImageSource): { width: number; height: number } | null {
- if (isVideoElement(source)) {
- return source.videoWidth > 0 && source.videoHeight > 0
- ? { width: source.videoWidth, height: source.videoHeight }
- : null;
- }
- if (isImageElement(source)) {
- return source.naturalWidth > 0 && source.naturalHeight > 0
- ? { width: source.naturalWidth, height: source.naturalHeight }
- : null;
- }
- return null;
+ const [width, height] = isVideoElement(source)
+ ? [source.videoWidth, source.videoHeight]
+ : isImageElement(source)
+ ? [source.naturalWidth, source.naturalHeight]
+ : isCanvasElement(source)
+ ? [source.width, source.height]
+ : [0, 0];
+ return width > 0 && height > 0 ? { width, height } : null;
}
function isDrawableSource(source: TexImageSource): boolean {
- if (isVideoElement(source)) {
- return (
- source.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA &&
- source.videoWidth > 0 &&
- source.videoHeight > 0
- );
- }
- if (isImageElement(source)) {
- return source.complete && source.naturalWidth > 0 && source.naturalHeight > 0;
+ if (isVideoElement(source) && source.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
+ return false;
}
- return false;
+ if (isImageElement(source) && !source.complete) return false;
+ return readSourceSize(source) !== null;
}
function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null {
@@ -3468,6 +3473,14 @@ function addListener(
entry.cleanup.push(() => target.removeEventListener(type, listener));
}
+function redrawsOnTransportTick(element: ColorGradingMediaElement): boolean {
+ if (isCanvasElement(element)) return true;
+ if (isVideoElement(element) && !element.paused && !element.ended) return false;
+ return ANIMATED_GRADING_PROPERTIES.some(
+ (property) => readAnimatedValue(element, property) !== null,
+ );
+}
+
function cancelScheduledFrame(entry: ColorGradingEntry): void {
if (entry.animationFrame !== null) {
window.cancelAnimationFrame(entry.animationFrame);
@@ -3759,9 +3772,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
const refresh = (): number => {
if (destroyed) return 0;
const attributeElements = new Set<ColorGradingMediaElement>();
- const nodes = document.querySelectorAll(
- `video[${HF_COLOR_GRADING_ATTR}], img[${HF_COLOR_GRADING_ATTR}]`,
- );
+ const nodes = document.querySelectorAll(COLOR_GRADING_SOURCE_SELECTOR);
nodes.forEach((node) => {
if (!isColorGradingMediaElement(node)) return;
attributeElements.add(node);
@@ -3821,12 +3832,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
for (const element of trackedElements) {
const entry = entries.get(element);
if (!entry) continue;
- const hasAnimatedProperty = ANIMATED_GRADING_PROPERTIES.some(
- (property) => readAnimatedValue(element, property) !== null,
- );
- if (!hasAnimatedProperty) continue;
- if (isVideoElement(element) && !element.paused && !element.ended) continue;
- if (drawEntry(entry)) drawn += 1;
+ if (redrawsOnTransportTick(element) && drawEntry(entry)) drawn += 1;
}
return drawn;
};
diff --git a/packages/lint/src/rules/media.test.ts b/packages/lint/src/rules/media.test.ts
index bc9bf8b07a..97ad6fd1aa 100644
--- a/packages/lint/src/rules/media.test.ts
+++ b/packages/lint/src/rules/media.test.ts
@@ -233,6 +233,18 @@ describe("media rules", () => {
);
});
+ it("accepts color grading on a canvas", async () => {
+ const html = `
+<html><body>
+ <div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
+ <canvas id="drawn" width="1920" height="1080" data-color-grading='{"preset":"skin-soft"}'></canvas>
+ </div>
+ <script>window.__timelines = {};</script>
+</body></html>`;
+ const result = await lintHyperframeHtml(html);
+ expect(result.findings.filter((f) => f.code.startsWith("color_grading_"))).toEqual([]);
+ });
+
it("reports warning for media with preload=none", async () => {
const html = `
<html><body>
diff --git a/packages/lint/src/rules/media.ts b/packages/lint/src/rules/media.ts
index 6de99ba668..b2d5b9fb66 100644
--- a/packages/lint/src/rules/media.ts
+++ b/packages/lint/src/rules/media.ts
@@ -11,7 +11,10 @@ import {
mediaTimeWindow,
mediaWindowsOverlap,
} from "../utils";
-import { validateColorGradingContract } from "@hyperframes/parsers/color-grading-contract";
+import {
+ COLOR_GRADING_SOURCE_TAGS,
+ validateColorGradingContract,
+} from "@hyperframes/parsers/color-grading-contract";
import { extractMediaSrcMutations } from "@hyperframes/parsers/composition";
import { parseHTML } from "linkedom";
import { findLinkedClipFindings } from "./linkedClips";
@@ -494,11 +497,11 @@ export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> =
snippet: truncateSnippet(tag.raw),
});
};
- if (tag.name !== "video" && tag.name !== "img") {
+ if (!(COLOR_GRADING_SOURCE_TAGS as readonly string[]).includes(tag.name)) {
report(
"color_grading_non_media",
- `data-color-grading on <${tag.name}> has no effect. The shader runtime only grades real <video> and <img> elements.`,
- "Move the grading attribute to the real <video> or <img> media element. Do not attach it to a wrapper or CSS background.",
+ `data-color-grading on <${tag.name}> has no effect. The shader runtime only grades real <video>, <img> and <canvas> elements.`,
+ "Move the grading attribute to the real <video>, <img> or <canvas> element. Do not attach it to a wrapper or CSS background.",
);
continue;
}
diff --git a/packages/parsers/src/colorGradingContract.ts b/packages/parsers/src/colorGradingContract.ts
index b7ef09cca9..14dfe4e668 100644
--- a/packages/parsers/src/colorGradingContract.ts
+++ b/packages/parsers/src/colorGradingContract.ts
@@ -2,6 +2,7 @@ export const COLOR_GRADING_CONTRACT_VERSION = 2;
export const COLOR_GRADING_COLOR_SPACE = "rec709";
export const COLOR_GRADING_MAX_CURVE_POINTS = 16;
export const COLOR_GRADING_MAX_SECONDARIES = 4;
+export const COLOR_GRADING_SOURCE_TAGS = ["video", "img", "canvas"] as const;
export const COLOR_GRADING_ADVANCED_LIMITS = {
hueDegrees: { min: 0, max: 360, inclusiveMax: false },
unit: { min: 0, max: 1 },