Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(runtime): graded pictures stay graded in sandboxed players (#5383)
Miguel Ángel684c038483cd
diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.tsindex afa313f1a5..f7916e93a1 100644--- a/packages/core/src/runtime/colorGrading.test.ts+++ b/packages/core/src/runtime/colorGrading.test.ts@@ -200,6 +200,7 @@ describe("createColorGradingRuntime", () => {runtime?.destroy();runtime = null;vi.unstubAllGlobals();+ vi.restoreAllMocks();getContextSpy.mockRestore();delete window.__hfVariables;delete window.__hfVariablesByComp;@@ -719,6 +720,82 @@ describe("createColorGradingRuntime", () => {expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 1);});+ describe("a picture served by URL", () => {+ const uploads = () => texImage2DCalls.map((call) => call[5]);++ /** The graded picture, and the pictures the runtime creates afterwards, still loading. */+ function servedPicture(): { image: HTMLImageElement; created: HTMLImageElement[] } {+ const image = makeDrawableImage();+ Object.defineProperty(image, "currentSrc", { value: "http://127.0.0.1/p/cutout.png" });+ document.body.appendChild(image);+ const created: HTMLImageElement[] = [];+ const createElement = document.createElement.bind(document);+ vi.spyOn(document, "createElement").mockImplementation(+ (tag: string, options?: ElementCreationOptions) => {+ const element = createElement(tag, options);+ if (element instanceof HTMLImageElement) {+ Object.defineProperty(element, "complete", { value: false, configurable: true });+ created.push(element);+ }+ return element;+ },+ );+ return { image, created };+ }++ function settle(copy: HTMLImageElement | undefined, naturalWidth: number): void {+ if (!copy) throw new Error("Expected a CORS copy of the picture");+ Object.defineProperty(copy, "complete", { value: true });+ Object.defineProperty(copy, "naturalWidth", { value: naturalWidth });+ copy.dispatchEvent(new Event(naturalWidth > 0 ? "load" : "error"));+ }++ it("grades the picture itself in a document with an origin", () => {+ const { image, created } = servedPicture();+ runtime = createColorGradingRuntime();++ expect(created).toEqual([]);+ expect(uploads().at(-1)).toBe(image);+ });++ it("grades a CORS copy in an opaque document, whose own no-cors load WebGL can't read", () => {+ vi.stubGlobal("origin", "null");+ const { image, created } = servedPicture();+ runtime = createColorGradingRuntime();++ const [copy] = created;+ expect([copy?.src, copy?.crossOrigin]).toEqual([+ "http://127.0.0.1/p/cutout.png",+ "anonymous",+ ]);+ expect(uploads()).not.toContain(image);+ for (let tick = 0; tick < 5; tick += 1) runtime.redrawAnimated();+ settle(copy, 640);+ expect(uploads().filter((upload) => upload === copy)).toHaveLength(1);+ expect(uploads()).not.toContain(image);+ });++ it("redraws a grading started while another's copy was loading", () => {+ vi.stubGlobal("origin", "null");+ const { created } = servedPicture();+ createColorGradingRuntime().destroy();+ runtime = createColorGradingRuntime();++ expect(created).toHaveLength(1);+ settle(created[0], 640);+ expect(uploads().at(-1)).toBe(created[0]);+ });++ it("grades the picture itself when an opaque document's CORS copy is refused, as before", () => {+ vi.stubGlobal("origin", "null");+ const { image, created } = servedPicture();+ runtime = createColorGradingRuntime();++ settle(created[0], 0);+ expect(uploads().at(-1)).toBe(image);+ });+ });+it("redraws animated still images from the transport tick", () => {const image = makeDrawableImage();document.body.appendChild(image);diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.tsindex 1ced515bb6..f2b932afb7 100644--- a/packages/core/src/runtime/colorGrading.ts+++ b/packages/core/src/runtime/colorGrading.ts@@ -2495,6 +2495,40 @@ function keepCanvasAboveSource(entry: ColorGradingEntry, source: HTMLImageElemen}}+const corsCopiesForOpaqueDocuments = new WeakMap<+ HTMLImageElement,+ { image: HTMLImageElement; waiting: Set<ColorGradingEntry> }+>();++function readablePixels(+ source: TexImageSource,+ waiter: ColorGradingEntry | null,+): TexImageSource | null {+ if (!isImageElement(source) || window.origin !== "null" || source.crossOrigin !== null)+ return source;+ if (!/^https?:/i.test(source.currentSrc)) return source;+ let copy = corsCopiesForOpaqueDocuments.get(source);+ if (copy?.image.src !== source.currentSrc) {+ const next = {+ image: source.ownerDocument.createElement("img"),+ waiting: new Set<ColorGradingEntry>(),+ };+ const settle = () => {+ const waiting = [...next.waiting];+ next.waiting.clear();+ waiting.forEach(drawEntry);+ };+ next.image.crossOrigin = "anonymous";+ next.image.addEventListener("load", settle, { once: true });+ next.image.addEventListener("error", settle, { once: true });+ next.image.src = source.currentSrc;+ corsCopiesForOpaqueDocuments.set(source, (copy = next));+ }+ if (copy.image.complete) return copy.image.naturalWidth > 0 ? copy.image : source;+ if (waiter) copy.waiting.add(waiter);+ return null;+}+function getDrawableSource(element: ColorGradingMediaElement): TexImageSource | null {if (isVideoElement(element)) {const renderFrame = findRenderFrameImage(element);@@ -3053,7 +3087,8 @@ function bindProgramTextures(function drawEntry(entry: ColorGradingEntry): boolean {if (entry.destroyed || entry.contextLost) return false;const source = getDrawableSource(entry.element);- if (!source) {+ const pixels = source && readablePixels(source, entry);+ if (!source || !pixels) {if (!entry.hasDrawn) entry.canvas.style.display = "none";return false;}@@ -3112,7 +3147,7 @@ function drawEntry(entry: ColorGradingEntry): boolean {const lut = ensureEntryLut(entry);// Browser media elements are top-left oriented; WebGL texture coordinates// are bottom-left oriented unless the upload is flipped.- uploadSourceTexture(gl, program.texture, source);+ uploadSourceTexture(gl, program.texture, pixels);const hasAnimatedKuwahara =readAnimatedValue(entry.element, ANIMATED_KUWAHARA_PROPERTY) !== null;const prepared = prepareEffectTextures(entry, grading, layout, uv, {@@ -3212,7 +3247,8 @@ function preparePreviewFrame(useMediaTime: boolean,): PreviewFrame | null {const source = getDrawableSource(element);- if (!source) return null;+ const pixels = source && readablePixels(source, null);+ if (!source || !pixels) return null;const sourceSize = readSourceSize(source);if (!sourceSize) return null;const dimensions = previewDimensions(element, sourceSize, maxDimension);@@ -3228,7 +3264,7 @@ function preparePreviewFrame(style.objectFit,style.objectPosition,);- uploadSourceTexture(renderer.gl, renderer.program.texture, source);+ uploadSourceTexture(renderer.gl, renderer.program.texture, pixels);return {dimensions,uv,