Skip to content

Write HTML. Render video. Built for agents.

perf(core): every graded clip draws through one WebGL context, not one per clip (#5809)

Miguel Ángelaba5eae23001
diff --git a/packages/core/src/runtime/colorGrading.test.ts b/packages/core/src/runtime/colorGrading.test.ts
index dd27d64a16..15ae9871e9 100644
--- a/packages/core/src/runtime/colorGrading.test.ts
+++ b/packages/core/src/runtime/colorGrading.test.ts
@@ -122,6 +122,11 @@ function createMockWebGl(
} as unknown as WebGLRenderingContext;
}
+// Each clip shows its pixels on a 2D canvas, copied from the one shared WebGL canvas.
+function mock2d(): CanvasRenderingContext2D {
+ return { clearRect: vi.fn(), drawImage: vi.fn() } as unknown as CanvasRenderingContext2D;
+}
+
function makeDrawableVideo(): HTMLVideoElement {
const video = document.createElement("video");
video.id = "hero-video";
@@ -182,6 +187,8 @@ describe("createColorGradingRuntime", () => {
let getContextSpy: ReturnType<typeof vi.spyOn>;
let runtime: RuntimeColorGradingApi | null = null;
+ const webglContexts = () => getContextSpy.mock.calls.filter(([type]) => type === "webgl").length;
+
beforeEach(() => {
document.body.innerHTML = "";
lastUniform1f = null;
@@ -192,7 +199,7 @@ describe("createColorGradingRuntime", () => {
getContextSpy = vi
.spyOn(HTMLCanvasElement.prototype, "getContext")
.mockImplementation((type: string) =>
- type === "webgl" ? createMockWebGl() : null,
+ type === "webgl" ? createMockWebGl() : type === "2d" ? mock2d() : null,
) as ReturnType<typeof vi.spyOn>;
});
@@ -355,11 +362,41 @@ describe("createColorGradingRuntime", () => {
const recreated = document.querySelector<HTMLCanvasElement>("[data-hf-color-grading-canvas]");
expect(recreated).not.toBeNull();
expect(recreated).toBe(canvas);
- expect(getContextSpy).toHaveBeenCalledTimes(1);
+ expect(webglContexts()).toBe(1);
expect(video.style.getPropertyValue("opacity")).toBe("0");
expect(video.style.getPropertyPriority("opacity")).toBe("important");
});
+ it("draws every graded clip through one WebGL context, however often a seek run shows and hides them", () => {
+ const clips = Array.from({ length: 6 }, (_, i) => {
+ const video = makeDrawableVideo();
+ video.id = `clip-${i}`;
+ video.style.display = "none";
+ document.body.appendChild(video);
+ return video;
+ });
+ runtime = createColorGradingRuntime();
+
+ // Each seek shows two neighbouring clips and hides the rest, as scenes come and go.
+ for (const at of [0, 2, 4, 1, 3, 5, 0, 4]) {
+ clips.forEach((video, i) => {
+ const shown = i === at || i === (at + 1) % clips.length;
+ video.style.display = shown ? "block" : "none";
+ runtime?.setSourceVisibility(video, shown);
+ });
+ }
+
+ expect(webglContexts()).toBe(1);
+ const shared = getContextSpy.mock.instances[0];
+ const copies = getContextSpy.mock.results
+ .filter((_, i) => getContextSpy.mock.calls[i]?.[0] === "2d")
+ .flatMap(
+ ({ value }) => (value as { drawImage: ReturnType<typeof vi.fn> }).drawImage.mock.calls,
+ );
+ expect(copies.length).toBeGreaterThan(0);
+ expect(copies.every(([source]) => source === shared)).toBe(true);
+ });
+
it("defers WebGL setup for hidden attributed media until it becomes visible", () => {
const video = makeDrawableVideo();
video.style.display = "none";
@@ -367,7 +404,7 @@ describe("createColorGradingRuntime", () => {
runtime = createColorGradingRuntime();
- expect(getContextSpy).not.toHaveBeenCalled();
+ expect(webglContexts()).toBe(0);
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBeNull();
expect(runtime.getStatus(video)).toEqual({
state: "pending",
@@ -376,7 +413,7 @@ describe("createColorGradingRuntime", () => {
video.style.display = "block";
expect(runtime.setSourceVisibility(video, true)).toBe(true);
- expect(getContextSpy).toHaveBeenCalledTimes(1);
+ expect(webglContexts()).toBe(1);
expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();
});
@@ -406,12 +443,12 @@ describe("createColorGradingRuntime", () => {
{ id: "warm", dataUrl: "data:image/png;base64,preview" },
],
});
- expect(getContextSpy).toHaveBeenCalledTimes(1);
+ expect(webglContexts()).toBe(1);
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBeNull();
expect(video.style.opacity).toBe("");
await runtime.renderPreviews("#hero-video", [{ id: "mono", grading: "mono-clean" }]);
- expect(getContextSpy).toHaveBeenCalledTimes(1);
+ expect(webglContexts()).toBe(1);
const retinaBatch = await runtime.renderPreviews(
"#hero-video",
@@ -717,7 +754,7 @@ describe("createColorGradingRuntime", () => {
runtime = createColorGradingRuntime();
- expect(getContextSpy).toHaveBeenCalledTimes(1);
+ expect(webglContexts()).toBe(1);
if (!lastUniform1f) throw new Error("Expected WebGL uniform calls");
expect(lastUniform1f).toHaveBeenCalledWith("u_intensity", 0);
expect(lastUniform1f).toHaveBeenCalledWith("u_kuwahara", 0);
@@ -1037,7 +1074,7 @@ describe("createColorGradingRuntime", () => {
runtime = createColorGradingRuntime();
- expect(getContextSpy).toHaveBeenCalledTimes(1);
+ expect(webglContexts()).toBe(1);
expect(document.querySelector("[data-hf-color-grading-canvas]")).not.toBeNull();
});
@@ -1079,7 +1116,7 @@ describe("createColorGradingRuntime", () => {
expect(runtime.setSourceVisibility(video, false)).toBe(true);
runtime.refresh();
expect(canvas.isConnected).toBe(false);
- expect(getContextSpy).toHaveBeenCalledTimes(1);
+ expect(webglContexts()).toBe(1);
expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(false);
});
@@ -1329,7 +1366,11 @@ describe("createColorGradingRuntime", () => {
it("falls back to the untreated shader output when half-float targets are unavailable", () => {
getContextSpy.mockImplementation((type: string) =>
- type === "webgl" ? createMockWebGl({ halfFloatSupported: false }) : null,
+ type === "webgl"
+ ? createMockWebGl({ halfFloatSupported: false })
+ : type === "2d"
+ ? mock2d()
+ : null,
);
const video = makeDrawableVideo();
video.setAttribute(
@@ -1785,7 +1826,11 @@ describe("createColorGradingRuntime", () => {
it("reports media texture upload failures in runtime status", () => {
getContextSpy.mockImplementation((type: string) =>
- type === "webgl" ? createMockWebGl({ failMediaUpload: true }) : null,
+ type === "webgl"
+ ? createMockWebGl({ failMediaUpload: true })
+ : type === "2d"
+ ? mock2d()
+ : null,
);
const video = makeDrawableVideo();
document.body.appendChild(video);
@@ -1800,9 +1845,10 @@ describe("createColorGradingRuntime", () => {
it("falls back to the source media on WebGL context loss and redraws after restore", () => {
const { video, canvas } = startRuntimeWithVideo();
+ const shared = getContextSpy.mock.instances[0] as HTMLCanvasElement;
const lost = new Event("webglcontextlost", { cancelable: true });
- canvas.dispatchEvent(lost);
+ shared.dispatchEvent(lost);
expect(lost.defaultPrevented).toBe(true);
expect(canvas.style.display).toBe("none");
@@ -1813,7 +1859,7 @@ describe("createColorGradingRuntime", () => {
message: "WebGL context lost",
});
- canvas.dispatchEvent(new Event("webglcontextrestored"));
+ shared.dispatchEvent(new Event("webglcontextrestored"));
expect(runtime?.getStatus("#hero-video").state).toBe("active");
expect(video.hasAttribute("data-hf-color-grading-source-hidden")).toBe(true);
diff --git a/packages/core/src/runtime/colorGrading.ts b/packages/core/src/runtime/colorGrading.ts
index f72079ba96..de2c963574 100644
--- a/packages/core/src/runtime/colorGrading.ts
+++ b/packages/core/src/runtime/colorGrading.ts
@@ -192,6 +192,7 @@ interface EffectRenderState {
interface ColorGradingRenderer extends EffectRenderState {
canvas: HTMLCanvasElement;
+ glCanvas: HTMLCanvasElement;
}
interface ColorGradingEntry extends ColorGradingRenderer {
@@ -1657,6 +1658,11 @@ function createProgramInfo(canvas: HTMLCanvasElement): {
premultipliedAlpha: false,
});
if (!gl) return null;
+ const program = createProgramResources(gl);
+ return program ? { gl, program } : null;
+}
+
+function createProgramResources(gl: WebGLRenderingContext): ProgramInfo | null {
const program = createProgram(gl);
const texture = createTexture(gl);
const lutTexture = createTexture(gl, gl.NEAREST);
@@ -1674,57 +1680,54 @@ function createProgramInfo(canvas: HTMLCanvasElement): {
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]), gl.STATIC_DRAW);
return {
- gl,
- program: {
- program,
- texture,
- lutTexture,
- advancedTexture,
- advancedSignature: null,
- quad,
- position: gl.getAttribLocation(program, "a_pos"),
- source: gl.getUniformLocation(program, "u_source"),
- blurSource: gl.getUniformLocation(program, "u_blurSource"),
- bloomSource: gl.getUniformLocation(program, "u_bloomSource"),
- kuwaharaSource: gl.getUniformLocation(program, "u_kuwaharaSource"),
- lut: gl.getUniformLocation(program, "u_lut"),
- advanced: gl.getUniformLocation(program, "u_advanced"),
- resolution: gl.getUniformLocation(program, "u_resolution"),
- uvScale: gl.getUniformLocation(program, "u_uvScale"),
- uvOffset: gl.getUniformLocation(program, "u_uvOffset"),
- blurReady: gl.getUniformLocation(program, "u_blurReady"),
- bloomReady: gl.getUniformLocation(program, "u_bloomReady"),
- kuwaharaReady: gl.getUniformLocation(program, "u_kuwaharaReady"),
- lutEnabled: gl.getUniformLocation(program, "u_lutEnabled"),
- lutSize: gl.getUniformLocation(program, "u_lutSize"),
- lutTextureSize: gl.getUniformLocation(program, "u_lutTextureSize"),
- lutDomainMin: gl.getUniformLocation(program, "u_lutDomainMin"),
- lutDomainMax: gl.getUniformLocation(program, "u_lutDomainMax"),
- lutIntensity: gl.getUniformLocation(program, "u_lutIntensity"),
- shadowWheel: gl.getUniformLocation(program, "u_shadowWheel"),
- midtoneWheel: gl.getUniformLocation(program, "u_midtoneWheel"),
- highlightWheel: gl.getUniformLocation(program, "u_highlightWheel"),
- rgbCurvesEnabled: gl.getUniformLocation(program, "u_rgbCurvesEnabled"),
- hueCurvesEnabled: gl.getUniformLocation(program, "u_hueCurvesEnabled"),
- secondaryCount: gl.getUniformLocation(program, "u_secondaryCount"),
- adjustUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_ADJUST_KEYS),
- detailUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_DETAIL_KEYS),
- effectUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_EFFECT_KEYS),
- grainSeed: gl.getUniformLocation(program, "u_grainSeed"),
- effectTime: gl.getUniformLocation(program, "u_effectTime"),
- paletteSize: gl.getUniformLocation(program, "u_paletteSize"),
- palette0: gl.getUniformLocation(program, "u_palette0"),
- palette1: gl.getUniformLocation(program, "u_palette1"),
- palette2: gl.getUniformLocation(program, "u_palette2"),
- palette3: gl.getUniformLocation(program, "u_palette3"),
- palette4: gl.getUniformLocation(program, "u_palette4"),
- palette5: gl.getUniformLocation(program, "u_palette5"),
- intensity: gl.getUniformLocation(program, "u_intensity"),
- compareEnabled: gl.getUniformLocation(program, "u_compareEnabled"),
- comparePosition: gl.getUniformLocation(program, "u_comparePosition"),
- compareSoftness: gl.getUniformLocation(program, "u_compareSoftness"),
- compareLineWidth: gl.getUniformLocation(program, "u_compareLineWidth"),
- },
+ program,
+ texture,
+ lutTexture,
+ advancedTexture,
+ advancedSignature: null,
+ quad,
+ position: gl.getAttribLocation(program, "a_pos"),
+ source: gl.getUniformLocation(program, "u_source"),
+ blurSource: gl.getUniformLocation(program, "u_blurSource"),
+ bloomSource: gl.getUniformLocation(program, "u_bloomSource"),
+ kuwaharaSource: gl.getUniformLocation(program, "u_kuwaharaSource"),
+ lut: gl.getUniformLocation(program, "u_lut"),
+ advanced: gl.getUniformLocation(program, "u_advanced"),
+ resolution: gl.getUniformLocation(program, "u_resolution"),
+ uvScale: gl.getUniformLocation(program, "u_uvScale"),
+ uvOffset: gl.getUniformLocation(program, "u_uvOffset"),
+ blurReady: gl.getUniformLocation(program, "u_blurReady"),
+ bloomReady: gl.getUniformLocation(program, "u_bloomReady"),
+ kuwaharaReady: gl.getUniformLocation(program, "u_kuwaharaReady"),
+ lutEnabled: gl.getUniformLocation(program, "u_lutEnabled"),
+ lutSize: gl.getUniformLocation(program, "u_lutSize"),
+ lutTextureSize: gl.getUniformLocation(program, "u_lutTextureSize"),
+ lutDomainMin: gl.getUniformLocation(program, "u_lutDomainMin"),
+ lutDomainMax: gl.getUniformLocation(program, "u_lutDomainMax"),
+ lutIntensity: gl.getUniformLocation(program, "u_lutIntensity"),
+ shadowWheel: gl.getUniformLocation(program, "u_shadowWheel"),
+ midtoneWheel: gl.getUniformLocation(program, "u_midtoneWheel"),
+ highlightWheel: gl.getUniformLocation(program, "u_highlightWheel"),
+ rgbCurvesEnabled: gl.getUniformLocation(program, "u_rgbCurvesEnabled"),
+ hueCurvesEnabled: gl.getUniformLocation(program, "u_hueCurvesEnabled"),
+ secondaryCount: gl.getUniformLocation(program, "u_secondaryCount"),
+ adjustUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_ADJUST_KEYS),
+ detailUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_DETAIL_KEYS),
+ effectUniforms: createFloatUniformBindings(gl, program, HF_COLOR_GRADING_EFFECT_KEYS),
+ grainSeed: gl.getUniformLocation(program, "u_grainSeed"),
+ effectTime: gl.getUniformLocation(program, "u_effectTime"),
+ paletteSize: gl.getUniformLocation(program, "u_paletteSize"),
+ palette0: gl.getUniformLocation(program, "u_palette0"),
+ palette1: gl.getUniformLocation(program, "u_palette1"),
+ palette2: gl.getUniformLocation(program, "u_palette2"),
+ palette3: gl.getUniformLocation(program, "u_palette3"),
+ palette4: gl.getUniformLocation(program, "u_palette4"),
+ palette5: gl.getUniformLocation(program, "u_palette5"),
+ intensity: gl.getUniformLocation(program, "u_intensity"),
+ compareEnabled: gl.getUniformLocation(program, "u_compareEnabled"),
+ comparePosition: gl.getUniformLocation(program, "u_comparePosition"),
+ compareSoftness: gl.getUniformLocation(program, "u_compareSoftness"),
+ compareLineWidth: gl.getUniformLocation(program, "u_compareLineWidth"),
};
}
@@ -1936,11 +1939,10 @@ function destroyMainProgramResources(gl: WebGLRenderingContext, program: Program
}
function replaceProgramResources(entry: ColorGradingEntry): boolean {
- const created = createProgramInfo(entry.canvas);
- if (!created) return false;
+ const program = createProgramResources(entry.gl);
+ if (!program) return false;
destroyProgramResources(entry);
- entry.gl = created.gl;
- entry.program = created.program;
+ entry.program = program;
entry.lut = null;
entry.lutLoadingSrc = null;
entry.lutError = null;
@@ -3100,24 +3102,76 @@ function bindProgramTextures(
}
// fallow-ignore-next-line complexity
-function drawEntry(entry: ColorGradingEntry): boolean {
- if (entry.destroyed || entry.contextLost) return false;
+function fitSharedCanvas(entry: ColorGradingEntry, size: { width: number; height: number }): void {
+ if (entry.glCanvas === entry.canvas) return;
+ if (entry.glCanvas.width < size.width) entry.glCanvas.width = size.width;
+ if (entry.glCanvas.height < size.height) entry.glCanvas.height = size.height;
+}
+
+function presentEntry(entry: ColorGradingEntry, size: { width: number; height: number }): void {
+ if (entry.glCanvas === entry.canvas) return;
+ const context = entry.canvas.getContext("2d");
+ if (!context) throw new Error("Canvas 2D is unavailable");
+ const { width, height } = size;
+ context.clearRect(0, 0, width, height);
+ context.drawImage(
+ entry.glCanvas,
+ 0,
+ entry.glCanvas.height - height,
+ width,
+ height,
+ 0,
+ 0,
+ width,
+ height,
+ );
+}
+
+type DrawableFrame = {
+ source: TexImageSource;
+ pixels: NonNullable<ReturnType<typeof readablePixels>>;
+ size: { width: number; height: number };
+};
+
+function drawableFrame(entry: ColorGradingEntry): DrawableFrame | null {
const source = getDrawableSource(entry.element);
const pixels = source && readablePixels(source, entry);
if (!source || !pixels) {
if (!entry.hasDrawn) entry.canvas.style.display = "none";
- return false;
+ return null;
}
- const sourceSize = readSourceSize(source);
- if (!sourceSize) return false;
- const styleSource = isHtmlElement(source) ? source : entry.element;
- const sourceOpacity = entry.element.style.getPropertyValue("opacity");
- const sourceOpacityPriority = entry.element.style.getPropertyPriority("opacity");
- const hiddenByColorGrading =
- entry.sourceHidden && sourceOpacity === "0" && sourceOpacityPriority === "important";
+ const size = readSourceSize(source);
+ return size ? { source, pixels, size } : null;
+}
+
+function restoreAuthoredOpacity(entry: ColorGradingEntry): void {
+ if (entry.sourceInlineOpacity === null) {
+ entry.element.style.removeProperty("opacity");
+ return;
+ }
+ entry.element.style.setProperty(
+ "opacity",
+ entry.sourceInlineOpacity,
+ entry.sourceInlineOpacityPriority || "",
+ );
+}
+
+function isHiddenByColorGrading(entry: ColorGradingEntry): boolean {
+ const { style } = entry.element;
+ return (
+ entry.sourceHidden &&
+ style.getPropertyValue("opacity") === "0" &&
+ style.getPropertyPriority("opacity") === "important"
+ );
+}
+
+function syncSourceAppearance(
+ entry: ColorGradingEntry,
+ source: TexImageSource,
+ injectedFrameSource: boolean,
+): void {
+ const hiddenByColorGrading = isHiddenByColorGrading(entry);
const sourceVisibility = entry.element.style.getPropertyValue("visibility");
- const injectedFrameSource = isRenderFrameImage(source);
- if (injectedFrameSource) keepCanvasAboveSource(entry, source);
// When color grading hid the source (opacity:0 !important), temporarily
// restore the authored inline opacity so getComputedStyle reflects the CSS
// animation's current value instead of our own hide. Without this, a CSS
@@ -3125,29 +3179,79 @@ function drawEntry(entry: ColorGradingEntry): boolean {
// the entire render because the !important blocks all animation effects on
// the property (#3329). The restore–read–rehide is synchronous, so no
// repaint occurs between the style writes.
- if (hiddenByColorGrading && !injectedFrameSource) {
- if (entry.sourceInlineOpacity !== null) {
- entry.element.style.setProperty(
- "opacity",
- entry.sourceInlineOpacity,
- entry.sourceInlineOpacityPriority || "",
- );
- } else {
- entry.element.style.removeProperty("opacity");
- }
- }
- const computed = window.getComputedStyle(injectedFrameSource ? source : entry.element);
+ const restoresOpacity = hiddenByColorGrading && !injectedFrameSource;
+ if (restoresOpacity) restoreAuthoredOpacity(entry);
+ const computed = window.getComputedStyle(
+ injectedFrameSource ? (source as HTMLImageElement) : entry.element,
+ );
// Grading never writes `visibility`, so the source's computed visibility
// always tracks the clip window and has to be re-read every frame.
entry.sourceOpacityForCanvas = computed.opacity || "1";
- if (hiddenByColorGrading && !injectedFrameSource) {
- entry.element.style.setProperty("opacity", "0", "important");
- }
+ if (restoresOpacity) entry.element.style.setProperty("opacity", "0", "important");
entry.sourceVisibleForCanvas =
(injectedFrameSource || sourceVisibility !== "hidden") && computed.visibility !== "hidden";
- const layout = updateCanvasLayout(entry, styleSource);
- if (!layout) return false;
+}
+
+function canonicalFrameTime(element: ColorGradingMediaElement): number {
+ const runtimeTime = (window as WindowWithColorGrading).__player?.getTime?.();
+ if (typeof runtimeTime === "number" && Number.isFinite(runtimeTime))
+ return Math.max(0, runtimeTime);
+ return isVideoElement(element) ? Math.max(0, element.currentTime) : 0;
+}
+
+function renderGradedFrame(
+ entry: ColorGradingEntry,
+ pixels: DrawableFrame["pixels"],
+ layout: { width: number; height: number },
+ uv: ReturnType<typeof calculateObjectFitUv>,
+): void {
+ const { gl, program } = entry;
+ fitSharedCanvas(entry, layout);
+ const grading = readAnimatedGrading(entry);
+ const lut = ensureEntryLut(entry);
+ // Browser media elements are top-left oriented; WebGL texture coordinates
+ // are bottom-left oriented unless the upload is flipped.
+ uploadSourceTexture(gl, program.texture, pixels);
+ const hasAnimatedKuwahara = readAnimatedValue(entry.element, ANIMATED_KUWAHARA_PROPERTY) !== null;
+ const prepared = prepareEffectTextures(entry, grading, layout, uv, {
+ preserveKuwahara: hasAnimatedKuwahara,
+ });
+ gl.bindFramebuffer(gl.FRAMEBUFFER, null);
+ gl.viewport(0, 0, layout.width, layout.height);
+ gl.useProgram(program.program);
+ bindProgramTextures(gl, program, prepared);
+ const frameTime = canonicalFrameTime(entry.element);
+ const grainSeed = entry.grainSeed + Math.floor(frameTime * 60);
+ applyUniforms(
+ gl,
+ program,
+ grading,
+ lut,
+ prepared.blurReady,
+ prepared.bloomReady,
+ prepared.kuwaharaReady,
+ entry.compare,
+ layout,
+ uv,
+ grainSeed,
+ frameTime,
+ );
+ drawFullscreenQuad(gl, program);
+ presentEntry(entry, layout);
+}
+
+function placeCanvas(
+ entry: ColorGradingEntry,
+ source: TexImageSource,
+ sourceSize: { width: number; height: number },
+): {
+ layout: { width: number; height: number };
+ uv: ReturnType<typeof calculateObjectFitUv>;
+} | null {
+ const styleSource = isHtmlElement(source) ? source : entry.element;
+ const layout = updateCanvasLayout(entry, styleSource);
+ if (!layout) return null;
const style = window.getComputedStyle(styleSource);
const uv = calculateObjectFitUv(
layout.width,
@@ -3157,46 +3261,22 @@ function drawEntry(entry: ColorGradingEntry): boolean {
style.objectFit,
style.objectPosition,
);
- const { gl, program } = entry;
- try {
- const grading = readAnimatedGrading(entry);
- const lut = ensureEntryLut(entry);
- // Browser media elements are top-left oriented; WebGL texture coordinates
- // are bottom-left oriented unless the upload is flipped.
- uploadSourceTexture(gl, program.texture, pixels);
- const hasAnimatedKuwahara =
- readAnimatedValue(entry.element, ANIMATED_KUWAHARA_PROPERTY) !== null;
- const prepared = prepareEffectTextures(entry, grading, layout, uv, {
- preserveKuwahara: hasAnimatedKuwahara,
- });
+ return { layout, uv };
+}
- gl.bindFramebuffer(gl.FRAMEBUFFER, null);
- gl.viewport(0, 0, layout.width, layout.height);
- gl.useProgram(program.program);
- bindProgramTextures(gl, program, prepared);
- const runtimeTime = (window as WindowWithColorGrading).__player?.getTime?.();
- const frameTime =
- typeof runtimeTime === "number" && Number.isFinite(runtimeTime)
- ? Math.max(0, runtimeTime)
- : isVideoElement(entry.element)
- ? Math.max(0, entry.element.currentTime)
- : 0;
- const grainSeed = entry.grainSeed + Math.floor(frameTime * 60);
- applyUniforms(
- gl,
- program,
- grading,
- lut,
- prepared.blurReady,
- prepared.bloomReady,
- prepared.kuwaharaReady,
- entry.compare,
- layout,
- uv,
- grainSeed,
- frameTime,
- );
- drawFullscreenQuad(gl, program);
+function drawEntry(entry: ColorGradingEntry): boolean {
+ if (entry.destroyed || entry.contextLost) return false;
+ const frame = drawableFrame(entry);
+ if (!frame) return false;
+ const { source, pixels, size: sourceSize } = frame;
+ const injectedFrameSource = isRenderFrameImage(source);
+ if (injectedFrameSource) keepCanvasAboveSource(entry, source);
+ syncSourceAppearance(entry, source, injectedFrameSource);
+ const placed = placeCanvas(entry, source, sourceSize);
+ if (!placed) return false;
+ const { layout, uv } = placed;
+ try {
+ renderGradedFrame(entry, pixels, layout, uv);
hideSourceElement(entry);
if (injectedFrameSource) hideFrameBorder(entry, source);
entry.hasDrawn = true;
@@ -3231,6 +3311,7 @@ function createPreviewRenderer(): ColorGradingPreviewRenderer | null {
return created
? {
canvas,
+ glCanvas: canvas,
...created,
lut: null,
effectTargets: null,
@@ -3433,24 +3514,6 @@ function installEntryListeners(entry: ColorGradingEntry): void {
addListener(entry, entry.element, "play", () => scheduleVideoDraw(entry));
addListener(entry, entry.element, "pause", redraw);
}
- addListener(entry, entry.canvas, "webglcontextlost", (event) => {
- event.preventDefault();
- entry.contextLost = true;
- entry.drawError = "WebGL context lost";
- entry.canvas.style.display = "none";
- restoreSourceElement(entry);
- });
- addListener(entry, entry.canvas, "webglcontextrestored", () => {
- entry.contextLost = false;
- if (!replaceProgramResources(entry)) {
- entry.contextLost = true;
- entry.drawError = "WebGL context restore failed";
- restoreSourceElement(entry);
- return;
- }
- entry.drawError = null;
- drawEntry(entry);
- });
if (typeof ResizeObserver !== "undefined") {
entry.resizeObserver = new ResizeObserver(redraw);
entry.resizeObserver.observe(entry.element);
@@ -3503,6 +3566,7 @@ function detachEntry(entry: ColorGradingEntry): ColorGradingRenderer | null {
}
return {
canvas: entry.canvas,
+ glCanvas: entry.glCanvas,
gl: entry.gl,
program: entry.program,
effectTargets: null,
@@ -3548,6 +3612,51 @@ export function createColorGradingRuntime(pausedMediaLease?: {
let previewRenderer: ColorGradingPreviewRenderer | null = null;
let previewQueue = Promise.resolve();
let destroyed = false;
+ let shared: { canvas: HTMLCanvasElement; gl: WebGLRenderingContext } | null = null;
+ const trackedEntries = () =>
+ [...trackedElements].flatMap((element) => entries.get(element) ?? []);
+
+ const sharedContext = () => {
+ if (shared) return shared;
+ const canvas = document.createElement("canvas");
+ const gl = canvas.getContext("webgl", { alpha: true, premultipliedAlpha: false });
+ if (!gl) return null;
+ canvas.addEventListener("webglcontextlost", (event) => {
+ event.preventDefault();
+ idleRenderers.length = 0;
+ for (const entry of trackedEntries()) {
+ entry.contextLost = true;
+ entry.drawError = "WebGL context lost";
+ entry.canvas.style.display = "none";
+ restoreSourceElement(entry);
+ }
+ });
+ canvas.addEventListener("webglcontextrestored", () => {
+ for (const entry of trackedEntries()) {
+ entry.contextLost = !replaceProgramResources(entry);
+ entry.drawError = entry.contextLost ? "WebGL context restore failed" : null;
+ if (entry.contextLost) restoreSourceElement(entry);
+ else drawEntry(entry);
+ }
+ });
+ shared = { canvas, gl };
+ return shared;
+ };
+
+ const newRenderer = (element: ColorGradingMediaElement): ColorGradingRenderer | null => {
+ const context = sharedContext();
+ const program = context && createProgramResources(context.gl);
+ if (!context || !program) return null;
+ return {
+ canvas: makeCanvas(element),
+ glCanvas: context.canvas,
+ gl: context.gl,
+ program,
+ effectTargets: null,
+ kuwaharaTargets: null,
+ effectError: null,
+ };
+ };
const upsert = (
element: ColorGradingMediaElement,
@@ -3566,20 +3675,9 @@ export function createColorGradingRuntime(pausedMediaLease?: {
if (renderer) {
attachCanvas(renderer.canvas, element);
} else {
- const canvas = makeCanvas(element);
- const created = createProgramInfo(canvas);
- if (!created) {
- canvas.remove();
- return false;
- }
- renderer = {
- canvas,
- gl: created.gl,
- program: created.program,
- effectTargets: null,
- kuwaharaTargets: null,
- effectError: null,
- };
+ const created = newRenderer(element);
+ if (!created) return false;
+ renderer = created;
}
const entry: ColorGradingEntry = {
element,
@@ -3649,7 +3747,7 @@ export function createColorGradingRuntime(pausedMediaLease?: {
const renderer = detachEntry(entry);
if (renderer) {
if (entry.contextLost || idleRenderers.length >= MAX_IDLE_COLOR_GRADING_RENDERERS) {
- destroyProgramResources(renderer, true);
+ destroyProgramResources(renderer);
} else {
idleRenderers.push(renderer);
}
@@ -3864,8 +3962,10 @@ export function createColorGradingRuntime(pausedMediaLease?: {
observer?.disconnect();
observer = null;
for (const element of trackedElements) removeElement(element);
- for (const renderer of idleRenderers) destroyProgramResources(renderer, true);
+ for (const renderer of idleRenderers) destroyProgramResources(renderer);
idleRenderers.length = 0;
+ shared?.gl.getExtension("WEBGL_lose_context")?.loseContext();
+ shared = null;
if (previewRenderer) {
destroyProgramResources(previewRenderer, true);
previewRenderer = null;