Skip to content

Write HTML. Render video. Built for agents.

fix(runtime): graded pictures stay graded in sandboxed players (#5383)

Miguel Ángel684c038483cd
diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.ts
index afa313f1a5..f7916e93a1 100644
--- a/packages/core/src/runtime/colorGrading.test.ts
+++ b/packages/core/src/runtime/colorGrading.test.ts
@@ -200,6 +200,7 @@ describe("createColorGradingRuntime", () => {
runtime?.destroy();
runtime = null;
vi.unstubAllGlobals();
+ vi.restoreAllMocks();
getContextSpy.mockRestore();
delete window.__hfVariables;
delete window.__hfVariablesByComp;
@@ -719,6 +720,82 @@ describe("createColorGradingRuntime", () => {
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 1);
});
+ describe("a picture served by URL", () => {
+ const uploads = () => texImage2DCalls.map((call) => call[5]);
+
+ /** The graded picture, and the pictures the runtime creates afterwards, still loading. */
+ function servedPicture(): { image: HTMLImageElement; created: HTMLImageElement[] } {
+ const image = makeDrawableImage();
+ Object.defineProperty(image, "currentSrc", { value: "http://127.0.0.1/p/cutout.png" });
+ document.body.appendChild(image);
+ const created: HTMLImageElement[] = [];
+ const createElement = document.createElement.bind(document);
+ vi.spyOn(document, "createElement").mockImplementation(
+ (tag: string, options?: ElementCreationOptions) => {
+ const element = createElement(tag, options);
+ if (element instanceof HTMLImageElement) {
+ Object.defineProperty(element, "complete", { value: false, configurable: true });
+ created.push(element);
+ }
+ return element;
+ },
+ );
+ return { image, created };
+ }
+
+ function settle(copy: HTMLImageElement | undefined, naturalWidth: number): void {
+ if (!copy) throw new Error("Expected a CORS copy of the picture");
+ Object.defineProperty(copy, "complete", { value: true });
+ Object.defineProperty(copy, "naturalWidth", { value: naturalWidth });
+ copy.dispatchEvent(new Event(naturalWidth > 0 ? "load" : "error"));
+ }
+
+ it("grades the picture itself in a document with an origin", () => {
+ const { image, created } = servedPicture();
+ runtime = createColorGradingRuntime();
+
+ expect(created).toEqual([]);
+ expect(uploads().at(-1)).toBe(image);
+ });
+
+ it("grades a CORS copy in an opaque document, whose own no-cors load WebGL can't read", () => {
+ vi.stubGlobal("origin", "null");
+ const { image, created } = servedPicture();
+ runtime = createColorGradingRuntime();
+
+ const [copy] = created;
+ expect([copy?.src, copy?.crossOrigin]).toEqual([
+ "http://127.0.0.1/p/cutout.png",
+ "anonymous",
+ ]);
+ expect(uploads()).not.toContain(image);
+ for (let tick = 0; tick < 5; tick += 1) runtime.redrawAnimated();
+ settle(copy, 640);
+ expect(uploads().filter((upload) => upload === copy)).toHaveLength(1);
+ expect(uploads()).not.toContain(image);
+ });
+
+ it("redraws a grading started while another's copy was loading", () => {
+ vi.stubGlobal("origin", "null");
+ const { created } = servedPicture();
+ createColorGradingRuntime().destroy();
+ runtime = createColorGradingRuntime();
+
+ expect(created).toHaveLength(1);
+ settle(created[0], 640);
+ expect(uploads().at(-1)).toBe(created[0]);
+ });
+
+ it("grades the picture itself when an opaque document's CORS copy is refused, as before", () => {
+ vi.stubGlobal("origin", "null");
+ const { image, created } = servedPicture();
+ runtime = createColorGradingRuntime();
+
+ settle(created[0], 0);
+ expect(uploads().at(-1)).toBe(image);
+ });
+ });
+
it("redraws animated still images from the transport tick", () => {
const image = makeDrawableImage();
document.body.appendChild(image);
diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.ts
index 1ced515bb6..f2b932afb7 100644
--- a/packages/core/src/runtime/colorGrading.ts
+++ b/packages/core/src/runtime/colorGrading.ts
@@ -2495,6 +2495,40 @@ function keepCanvasAboveSource(entry: ColorGradingEntry, source: HTMLImageElemen
}
}
+const corsCopiesForOpaqueDocuments = new WeakMap<
+ HTMLImageElement,
+ { image: HTMLImageElement; waiting: Set<ColorGradingEntry> }
+>();
+
+function readablePixels(
+ source: TexImageSource,
+ waiter: ColorGradingEntry | null,
+): TexImageSource | null {
+ if (!isImageElement(source) || window.origin !== "null" || source.crossOrigin !== null)
+ return source;
+ if (!/^https?:/i.test(source.currentSrc)) return source;
+ let copy = corsCopiesForOpaqueDocuments.get(source);
+ if (copy?.image.src !== source.currentSrc) {
+ const next = {
+ image: source.ownerDocument.createElement("img"),
+ waiting: new Set<ColorGradingEntry>(),
+ };
+ const settle = () => {
+ const waiting = [...next.waiting];
+ next.waiting.clear();
+ waiting.forEach(drawEntry);
+ };
+ next.image.crossOrigin = "anonymous";
+ next.image.addEventListener("load", settle, { once: true });
+ next.image.addEventListener("error", settle, { once: true });
+ next.image.src = source.currentSrc;
+ corsCopiesForOpaqueDocuments.set(source, (copy = next));
+ }
+ if (copy.image.complete) return copy.image.naturalWidth > 0 ? copy.image : source;
+ if (waiter) copy.waiting.add(waiter);
+ return null;
+}
+
function getDrawableSource(element: ColorGradingMediaElement): TexImageSource | null {
if (isVideoElement(element)) {
const renderFrame = findRenderFrameImage(element);
@@ -3053,7 +3087,8 @@ function bindProgramTextures(
function drawEntry(entry: ColorGradingEntry): boolean {
if (entry.destroyed || entry.contextLost) return false;
const source = getDrawableSource(entry.element);
- if (!source) {
+ const pixels = source && readablePixels(source, entry);
+ if (!source || !pixels) {
if (!entry.hasDrawn) entry.canvas.style.display = "none";
return false;
}
@@ -3112,7 +3147,7 @@ function drawEntry(entry: ColorGradingEntry): boolean {
const lut = ensureEntryLut(entry);
// Browser media elements are top-left oriented; WebGL texture coordinates
// are bottom-left oriented unless the upload is flipped.
- uploadSourceTexture(gl, program.texture, source);
+ uploadSourceTexture(gl, program.texture, pixels);
const hasAnimatedKuwahara =
readAnimatedValue(entry.element, ANIMATED_KUWAHARA_PROPERTY) !== null;
const prepared = prepareEffectTextures(entry, grading, layout, uv, {
@@ -3212,7 +3247,8 @@ function preparePreviewFrame(
useMediaTime: boolean,
): PreviewFrame | null {
const source = getDrawableSource(element);
- if (!source) return null;
+ const pixels = source && readablePixels(source, null);
+ if (!source || !pixels) return null;
const sourceSize = readSourceSize(source);
if (!sourceSize) return null;
const dimensions = previewDimensions(element, sourceSize, maxDimension);
@@ -3228,7 +3264,7 @@ function preparePreviewFrame(
style.objectFit,
style.objectPosition,
);
- uploadSourceTexture(renderer.gl, renderer.program.texture, source);
+ uploadSourceTexture(renderer.gl, renderer.program.texture, pixels);
return {
dimensions,
uv,