Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): color grading applies to canvas compositions (#5860)
Miguel Ángel5658f700650b
diff --git a/docs/guides/color-grading.mdx b/docs/guides/color-grading.mdxindex a0a8a49d75..7abca7425a 100644--- a/docs/guides/color-grading.mdx+++ b/docs/guides/color-grading.mdx@@ -166,8 +166,9 @@ Grading picks pixels by value, never by position: no face tracking, regiontracking, rotoscoping or masks — an HSL selection is a color qualifier, not ashape. To treat one part of a frame, split it into its own media layer and gradethat layer, as in the [implementation reference](/reference/color-grading). It-targets media elements only, so a whole scene including HTML text cannot be-graded.+targets `<video>`, `<img>` and `<canvas>` elements only, so a whole scene+including HTML text cannot be graded. A `<canvas>` is graded at its drawing-buffer+size (`width`/`height`) and re-read on every seek and playback frame.SDR images and video are the supported case; 4K works at a higher preview andrender cost. An HDR source — iPhone HDR, HLG, Dolby Vision-style — gets an SDRdiff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdxindex 826681b293..d8d1dcc1b5 100644--- a/docs/packages/cli.mdx+++ b/docs/packages/cli.mdx@@ -481,7 +481,7 @@ npx hyperframes media-treatment --selector '#hero' --clear| `--all` | Print the exhaustive capability catalog || `--project <dir>` | Project directory (default: current directory) || `--file <path>` | Composition file (default `index.html`) |-| `--selector <css>` | CSS selector; every matched `<img>` or `<video>` gets the patch |+| `--selector <css>` | CSS selector; every matched `<img>`, `<video>` or `<canvas>` gets the patch || `--selector-index <n>` | Zero-based match index to patch only one of several matches || `--grading <json>` | Validated colour-grading patch || `--apply` | Apply the patch |diff --git a/docs/reference/lint-rules.mdx b/docs/reference/lint-rules.mdxindex 6ccda0a2c8..d198134be1 100644--- a/docs/reference/lint-rules.mdx+++ b/docs/reference/lint-rules.mdx@@ -554,9 +554,11 @@ Source: `packages/lint/src/rules/media.ts`.### `color_grading_non_media`-**Error.** `data-color-grading` is set on an element other than `<video>` or `<img>`, such as a wrapper or an element with a CSS background. The shader runtime only grades real `<video>` and `<img>` elements, so the attribute has no effect.+**Error.** `data-color-grading` is set on an element other than `<video>`, `<img>` or `<canvas>`, such as a+wrapper or an element with a CSS background. The shader runtime only grades real `<video>`, `<img>` and+`<canvas>` elements, so the attribute has no effect.-**Fix:** Move the attribute to the `<video>` or `<img>` element itself.+**Fix:** Move the attribute to the `<video>`, `<img>` or `<canvas>` element itself.### `duplicate_media_discovery_risk`diff --git a/packages/cli/src/commands/media-treatment.test.ts b/packages/cli/src/commands/media-treatment.test.tsindex c7dd916f1d..ec3aef7ccb 100644--- a/packages/cli/src/commands/media-treatment.test.ts+++ b/packages/cli/src/commands/media-treatment.test.ts@@ -598,7 +598,20 @@ describe("applyMediaTreatmentToHtml", () => {selector: "#hero",grading: { preset: "warm-daylight" },}),- ).toThrow(/requires an <img> or <video>/);+ ).toThrow(/requires a <video>, <img> or <canvas>/);+ });++ it("grades a canvas", () => {+ const result = applyMediaTreatmentToHtml(+ `<canvas id="drawn" width="640" height="360"></canvas>`,+ {+ selector: "#drawn",+ grading: { preset: "warm-daylight" },+ },+ );++ expect(result.tag).toBe("canvas");+ expect(result.html).toContain("data-color-grading=");});it("rejects unknown keys instead of silently dropping agent mistakes", () => {diff --git a/packages/cli/src/commands/media-treatment.ts b/packages/cli/src/commands/media-treatment.tsindex 34c94e59a2..2af21733e2 100644--- a/packages/cli/src/commands/media-treatment.ts+++ b/packages/cli/src/commands/media-treatment.ts@@ -10,6 +10,7 @@ import {serializeHfColorGrading,} from "@hyperframes/core";import {+ COLOR_GRADING_SOURCE_TAGS,isColorGradingVariableRef,validateColorGradingContract,} from "@hyperframes/parsers/color-grading-contract";@@ -344,9 +345,11 @@ interface ApplyMediaTreatmentOptions {clear?: boolean;}+type GradingSourceTag = (typeof COLOR_GRADING_SOURCE_TAGS)[number];+interface MediaTreatmentTargetResult {selectorIndex: number;- tag: "img" | "video";+ tag: GradingSourceTag;changed: boolean;value: string | null;before: unknown;@@ -443,7 +446,7 @@ function queryIncludingTemplates(root: Document | Element, selector: string): Elreturn [];}-type MediaTarget = { element: Element; selectorIndex: number; tag: "img" | "video" };+type MediaTarget = { element: Element; selectorIndex: number; tag: GradingSourceTag };function selectMediaElements(document: Document,@@ -463,9 +466,11 @@ function selectMediaElements(const indexes = selectorIndex === undefined ? matches.keys() : [selectorIndex];return Array.from(indexes, (index) => {const element = matches[index]!;- const tag = element.tagName.toLowerCase();- if (tag !== "img" && tag !== "video") {- throw new Error(`Color grading requires an <img> or <video>; selector matched <${tag}>`);+ const tag = element.tagName.toLowerCase() as GradingSourceTag;+ if (!COLOR_GRADING_SOURCE_TAGS.includes(tag)) {+ throw new Error(+ `Color grading requires a <video>, <img> or <canvas>; selector matched <${tag}>`,+ );}if (!isHTMLElement(element)) {throw new Error(`Color grading requires an HTML <${tag}>; selector matched one inside <svg>`);diff --git a/packages/core/src/colorGrading.ts b/packages/core/src/colorGrading.tsindex 77eea925dc..cd53746841 100644--- a/packages/core/src/colorGrading.ts+++ b/packages/core/src/colorGrading.ts@@ -13,6 +13,7 @@ import {COLOR_GRADING_LUT_KEYS,COLOR_GRADING_MAX_CURVE_POINTS,COLOR_GRADING_MAX_SECONDARIES,+ COLOR_GRADING_SOURCE_TAGS,COLOR_GRADING_TOP_LEVEL_KEYS,COLOR_GRADING_WHEEL_KEYS,} from "@hyperframes/parsers/color-grading-contract";@@ -41,6 +42,7 @@ export const COLOR_GRADING_AUTHORED_OPACITY_ATTR = "data-hf-authored-opacity";export const HF_COLOR_GRADING_CANVAS_ID_PREFIX = "__hf_color_grading_";export const HF_COLOR_GRADING_COLOR_SPACE = COLOR_GRADING_COLOR_SPACE;+export const HF_COLOR_GRADING_SOURCE_TAGS = COLOR_GRADING_SOURCE_TAGS;export type HfColorGradingPresetId =| "neutral"diff --git a/packages/core/src/editing/affordances.test.ts b/packages/core/src/editing/affordances.test.tsindex 86db7d912c..7ded2cb870 100644--- a/packages/core/src/editing/affordances.test.ts+++ b/packages/core/src/editing/affordances.test.ts@@ -172,6 +172,11 @@ describe("resolveEditingAffordances — sections", () => {expect(s).toMatchObject({ media: true, colorGrading: true });});+ it("canvas: colorGrading", () => {+ const s = resolveEditingAffordances(baseFacts({ tag: "canvas" })).sections;+ expect(s).toMatchObject({ media: false, colorGrading: true });+ });+it("editable text on a plain element: text section", () => {const s = resolveEditingAffordances(baseFacts({ hasEditableText: true })).sections;expect(s.text).toBe(true);diff --git a/packages/core/src/editing/affordances.ts b/packages/core/src/editing/affordances.tsindex 238a17a6b2..85bf006629 100644--- a/packages/core/src/editing/affordances.ts+++ b/packages/core/src/editing/affordances.ts@@ -1,4 +1,5 @@import { HF_AUDIO_GROUP_TAG } from "../audioGroups.js";+import { HF_COLOR_GRADING_SOURCE_TAGS } from "../colorGrading.js";/*** Pure, DOM-free editing-affordance resolution. Single source of truth for what* the studio's edit panel (and any SDK consumer) surfaces per selected element:@@ -221,7 +222,7 @@ export function resolveEditingSections(facts: EditableElementFacts): EditingSectmedia: facts.tag === "video" || facts.tag === "audio" || facts.tag === "img",audioFx:facts.tag === "audio" || isAudioBus || (facts.tag === "video" && facts.hasAudio === true),- colorGrading: facts.tag === "video" || facts.tag === "img",+ colorGrading: (HF_COLOR_GRADING_SOURCE_TAGS as readonly string[]).includes(facts.tag),// A bus has no clip range at all — no `data-start`, no duration, and its// automation clock is composition time — so Start/Duration/End would be// editing nothing. Audio keeps its timing: an `<audio>` clip is placed ondiff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.tsindex 15ae9871e9..2af15f05c8 100644--- a/packages/core/src/runtime/colorGrading.test.ts+++ b/packages/core/src/runtime/colorGrading.test.ts@@ -856,6 +856,54 @@ describe("createColorGradingRuntime", () => {expect(lastUniform1f).toHaveBeenCalledWith("u_blur", 0.7);});+ describe("a graded canvas", () => {+ function makeGradedCanvas(width: number, height: number): HTMLCanvasElement {+ const canvas = document.createElement("canvas");+ canvas.id = "drawn";+ canvas.width = width;+ canvas.height = height;+ canvas.setAttribute(+ HF_COLOR_GRADING_ATTR,+ serializeHfColorGrading({ adjust: { exposure: 0.5 } }),+ );+ Object.defineProperty(canvas, "offsetWidth", { value: 640, configurable: true });+ Object.defineProperty(canvas, "offsetHeight", { value: 360, configurable: true });+ canvas.getBoundingClientRect = () =>+ ({ width: 640, height: 360, left: 0, top: 0, right: 640, bottom: 360 }) as DOMRect;+ document.body.appendChild(canvas);+ return canvas;+ }+ const canvasUploads = (canvas: HTMLCanvasElement) =>+ texImage2DCalls.filter((args) => args.length === 6 && args[5] === canvas).length;++ it("grades its pixels and re-uploads them on every transport tick and seek", () => {+ const canvas = makeGradedCanvas(320, 180);+ runtime = createColorGradingRuntime();++ expect(runtime.isGraded(canvas)).toBe(true);+ expect(runtime.getStatus("drawn")).toMatchObject({ state: "active" });+ expect(canvas.style.getPropertyValue("opacity")).toBe("0");+ expect(canvasUploads(canvas)).toBe(1);++ expect(runtime.redrawAnimated()).toBe(1);+ expect(runtime.redraw()).toBe(1);+ expect(canvasUploads(canvas)).toBe(3);+ });++ it("waits for a drawing buffer, whatever its CSS box", () => {+ const canvas = makeGradedCanvas(0, 0);+ runtime = createColorGradingRuntime();++ expect(runtime.getStatus("drawn")).toMatchObject({ state: "pending" });+ expect(canvasUploads(canvas)).toBe(0);++ canvas.width = 320;+ canvas.height = 180;+ expect(runtime.redraw()).toBe(1);+ expect(canvasUploads(canvas)).toBe(1);+ });+ });+it("redraws animated held video frames without duplicating active video draws", () => {const video = makeDrawableVideo();video.style.setProperty("--hf-color-grading-blur", "0.1");diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.tsindex de2c963574..2a11118ebe 100644--- a/packages/core/src/runtime/colorGrading.ts+++ b/packages/core/src/runtime/colorGrading.ts@@ -5,6 +5,7 @@ import {HF_COLOR_GRADING_CANVAS_ID_PREFIX,HF_COLOR_GRADING_DETAIL_KEYS,HF_COLOR_GRADING_EFFECT_KEYS,+ HF_COLOR_GRADING_SOURCE_TAGS,hasHfColorGradingHueCurveValues,hasHfColorGradingRgbCurveValues,hasHfColorGradingSecondaryValues,@@ -42,9 +43,15 @@ import { copyMediaVisualStyles } from "../inline-scripts/parityContract";import { readVariablesForElement } from "./variableScope";import { swallow } from "./diagnostics";import { findInjectedRenderFrame } from "./renderFrameSibling";-import { isElementNode, isHtmlElement, isImageElement, isVideoElement } from "./domRealm";+import {+ isCanvasElement,+ isElementNode,+ isHtmlElement,+ isImageElement,+ isVideoElement,+} from "./domRealm";-type ColorGradingMediaElement = HTMLVideoElement | HTMLImageElement;+type ColorGradingMediaElement = HTMLVideoElement | HTMLImageElement | HTMLCanvasElement;type EntrySource = "attribute" | "live";@@ -1462,8 +1469,15 @@ const KUWAHARA_RESOLVE_FRAGMENT_SHADER = ["}",].join("\n");+const COLOR_GRADING_SOURCE_SELECTOR = HF_COLOR_GRADING_SOURCE_TAGS.map(+ (tag) => `${tag}[${HF_COLOR_GRADING_ATTR}]`,+).join(", ");+function isColorGradingMediaElement(value: Element): value is ColorGradingMediaElement {- return isVideoElement(value) || isImageElement(value);+ return (+ isHtmlElement(value) &&+ (HF_COLOR_GRADING_SOURCE_TAGS as readonly string[]).includes(value.localName)+ );}function isVisibleForColorGrading(element: ColorGradingMediaElement): boolean {@@ -1781,7 +1795,7 @@ function hashStringSeed(value: string): number {function seedForElement(element: ColorGradingMediaElement): number {const key =element.id ||- element.currentSrc ||+ (!isCanvasElement(element) && element.currentSrc) ||element.getAttribute("src") ||`${element.tagName}:${Array.prototype.indexOf.call(element.parentNode?.children ?? [], element)}`;return hashStringSeed(key);@@ -2462,31 +2476,22 @@ function resolveTarget(}function readSourceSize(source: TexImageSource): { width: number; height: number } | null {- if (isVideoElement(source)) {- return source.videoWidth > 0 && source.videoHeight > 0- ? { width: source.videoWidth, height: source.videoHeight }- : null;- }- if (isImageElement(source)) {- return source.naturalWidth > 0 && source.naturalHeight > 0- ? { width: source.naturalWidth, height: source.naturalHeight }- : null;- }- return null;+ const [width, height] = isVideoElement(source)+ ? [source.videoWidth, source.videoHeight]+ : isImageElement(source)+ ? [source.naturalWidth, source.naturalHeight]+ : isCanvasElement(source)+ ? [source.width, source.height]+ : [0, 0];+ return width > 0 && height > 0 ? { width, height } : null;}function isDrawableSource(source: TexImageSource): boolean {- if (isVideoElement(source)) {- return (- source.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA &&- source.videoWidth > 0 &&- source.videoHeight > 0- );- }- if (isImageElement(source)) {- return source.complete && source.naturalWidth > 0 && source.naturalHeight > 0;+ if (isVideoElement(source) && source.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {+ return false;}- return false;+ if (isImageElement(source) && !source.complete) return false;+ return readSourceSize(source) !== null;}function findRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null {@@ -3468,6 +3473,14 @@ function addListener(entry.cleanup.push(() => target.removeEventListener(type, listener));}+function redrawsOnTransportTick(element: ColorGradingMediaElement): boolean {+ if (isCanvasElement(element)) return true;+ if (isVideoElement(element) && !element.paused && !element.ended) return false;+ return ANIMATED_GRADING_PROPERTIES.some(+ (property) => readAnimatedValue(element, property) !== null,+ );+}+function cancelScheduledFrame(entry: ColorGradingEntry): void {if (entry.animationFrame !== null) {window.cancelAnimationFrame(entry.animationFrame);@@ -3759,9 +3772,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {const refresh = (): number => {if (destroyed) return 0;const attributeElements = new Set<ColorGradingMediaElement>();- const nodes = document.querySelectorAll(- `video[${HF_COLOR_GRADING_ATTR}], img[${HF_COLOR_GRADING_ATTR}]`,- );+ const nodes = document.querySelectorAll(COLOR_GRADING_SOURCE_SELECTOR);nodes.forEach((node) => {if (!isColorGradingMediaElement(node)) return;attributeElements.add(node);@@ -3821,12 +3832,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {for (const element of trackedElements) {const entry = entries.get(element);if (!entry) continue;- const hasAnimatedProperty = ANIMATED_GRADING_PROPERTIES.some(- (property) => readAnimatedValue(element, property) !== null,- );- if (!hasAnimatedProperty) continue;- if (isVideoElement(element) && !element.paused && !element.ended) continue;- if (drawEntry(entry)) drawn += 1;+ if (redrawsOnTransportTick(element) && drawEntry(entry)) drawn += 1;}return drawn;};diff --git a/packages/lint/src/rules/media.test.ts b/packages/lint/src/rules/media.test.tsindex bc9bf8b07a..97ad6fd1aa 100644--- a/packages/lint/src/rules/media.test.ts+++ b/packages/lint/src/rules/media.test.ts@@ -233,6 +233,18 @@ describe("media rules", () => {);});+ it("accepts color grading on a canvas", async () => {+ const html = `+<html><body>+ <div id="root" data-composition-id="c1" data-width="1920" data-height="1080">+ <canvas id="drawn" width="1920" height="1080" data-color-grading='{"preset":"skin-soft"}'></canvas>+ </div>+ <script>window.__timelines = {};</script>+</body></html>`;+ const result = await lintHyperframeHtml(html);+ expect(result.findings.filter((f) => f.code.startsWith("color_grading_"))).toEqual([]);+ });+it("reports warning for media with preload=none", async () => {const html = `<html><body>diff --git a/packages/lint/src/rules/media.ts b/packages/lint/src/rules/media.tsindex 6de99ba668..b2d5b9fb66 100644--- a/packages/lint/src/rules/media.ts+++ b/packages/lint/src/rules/media.ts@@ -11,7 +11,10 @@ import {mediaTimeWindow,mediaWindowsOverlap,} from "../utils";-import { validateColorGradingContract } from "@hyperframes/parsers/color-grading-contract";+import {+ COLOR_GRADING_SOURCE_TAGS,+ validateColorGradingContract,+} from "@hyperframes/parsers/color-grading-contract";import { extractMediaSrcMutations } from "@hyperframes/parsers/composition";import { parseHTML } from "linkedom";import { findLinkedClipFindings } from "./linkedClips";@@ -494,11 +497,11 @@ export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> =snippet: truncateSnippet(tag.raw),});};- if (tag.name !== "video" && tag.name !== "img") {+ if (!(COLOR_GRADING_SOURCE_TAGS as readonly string[]).includes(tag.name)) {report("color_grading_non_media",- `data-color-grading on <${tag.name}> has no effect. The shader runtime only grades real <video> and <img> elements.`,- "Move the grading attribute to the real <video> or <img> media element. Do not attach it to a wrapper or CSS background.",+ `data-color-grading on <${tag.name}> has no effect. The shader runtime only grades real <video>, <img> and <canvas> elements.`,+ "Move the grading attribute to the real <video>, <img> or <canvas> element. Do not attach it to a wrapper or CSS background.",);continue;}diff --git a/packages/parsers/src/colorGradingContract.ts b/packages/parsers/src/colorGradingContract.tsindex b7ef09cca9..14dfe4e668 100644--- a/packages/parsers/src/colorGradingContract.ts+++ b/packages/parsers/src/colorGradingContract.ts@@ -2,6 +2,7 @@ export const COLOR_GRADING_CONTRACT_VERSION = 2;export const COLOR_GRADING_COLOR_SPACE = "rec709";export const COLOR_GRADING_MAX_CURVE_POINTS = 16;export const COLOR_GRADING_MAX_SECONDARIES = 4;+export const COLOR_GRADING_SOURCE_TAGS = ["video", "img", "canvas"] as const;export const COLOR_GRADING_ADVANCED_LIMITS = {hueDegrees: { min: 0, max: 360, inclusiveMax: false },unit: { min: 0, max: 1 },