π₯ Make videos programmatically with React
import {CanvasPool} from './canvas-pool.js';
import {groupByBackend} from './effect-internals.js';
import type {
EffectDefinition,
EffectDefinitionAndStack,
} from './effect-types.js';
import {getGpuDevice} from './gpu-device.js';
export type EffectChainState = {
readonly width: number;
readonly height: number;
pool: CanvasPool;
setupCache: WeakMap<
EffectDefinition<unknown, unknown>,
WeakMap<HTMLCanvasElement, unknown>
>;
cleanupRegistry: Array<{
definition: EffectDefinition<unknown, unknown>;
state: unknown;
target: HTMLCanvasElement;
}>;
currentRunId: number;
};
export const createEffectChainState = (
width: number,
height: number,
): EffectChainState => {
const state: EffectChainState = {
width,
height,
pool: new CanvasPool(width, height, (canvas) => {
// The GL objects these setups hold died with the context and stay
// invalid on a restored one, so the next run has to recreate them.
// While the context is lost, GL calls are no-ops, which makes this
// the moment to run the cleanups without side effects on the
// context's error state.
const dead = state.cleanupRegistry.filter(
(entry) => entry.target === canvas,
);
state.cleanupRegistry = state.cleanupRegistry.filter(
(entry) => entry.target !== canvas,
);
for (const entry of dead) {
state.setupCache.get(entry.definition)?.delete(canvas);
}
for (const entry of dead) {
entry.definition.cleanup(entry.state);
}
}),
setupCache: new WeakMap(),
cleanupRegistry: [],
currentRunId: 0,
};
return state;
};
// Cancels in-flight runs, releases per-effect resources and then the pool's
// canvases and contexts. The state must not be used afterwards.
export const cleanupEffectChainState = (state: EffectChainState): void => {
state.currentRunId++;
try {
for (const entry of state.cleanupRegistry) {
entry.definition.cleanup(entry.state);
}
} finally {
state.cleanupRegistry.length = 0;
state.pool.dispose();
}
};
const ensureSetup = <S>(
state: EffectChainState,
def: EffectDefinition<unknown, S>,
target: HTMLCanvasElement,
): S => {
const widened = def as EffectDefinition<unknown, unknown>;
let cacheForDefinition = state.setupCache.get(widened);
if (!cacheForDefinition) {
cacheForDefinition = new WeakMap<HTMLCanvasElement, unknown>();
state.setupCache.set(widened, cacheForDefinition);
}
if (cacheForDefinition.has(target)) {
return cacheForDefinition.get(target) as S;
}
const setupState = def.setup(target);
cacheForDefinition.set(target, setupState);
state.cleanupRegistry.push({definition: widened, state: setupState, target});
return setupState;
};
/** Final compositing target for an effect chain (layout canvas or transferred offscreen). */
export type EffectChainOutput = HTMLCanvasElement | OffscreenCanvas;
export type RunEffectChainOptions = {
readonly state: EffectChainState;
readonly source: CanvasImageSource;
readonly effects: EffectDefinitionAndStack<unknown>[];
readonly output: EffectChainOutput;
readonly width: number;
readonly height: number;
};
// Runs the effect pipeline imperatively. Returns `true` if the pipeline
// completed and wrote to `output`, `false` if it was superseded by a newer
// run (caller should not act on a stale result).
const runEffectChainWithoutResize = async (
{state, source, effects, output, width, height}: RunEffectChainOptions,
initialFlipSourceY: boolean,
runId: number,
): Promise<boolean> => {
const isCancelled = () => state.currentRunId !== runId;
// Bypass any effect with `disabled: true` before grouping by backend, so
// disabled effects don't create empty runs or force unnecessary backend
// transitions. The `disabled` flag is injected by `createEffect` and lives
// on `params` so it flows through code/drag override merging.
const enabledEffects = effects.filter(
(e) => !(e.params as {disabled?: boolean}).disabled,
);
const runs = groupByBackend(enabledEffects);
let currentImage: CanvasImageSource = source;
let lastTarget: HTMLCanvasElement | null = null;
if (runs.length === 0) {
// In-place pipeline (e.g. <HtmlInCanvas>: drawElementImage into the same
// surface, no further effects) β the bitmap is already on `output`.
if (source === output) {
return true;
}
const ctx = output.getContext('2d');
if (!ctx) {
throw new Error('Failed to acquire 2D context for output canvas');
}
ctx.clearRect(0, 0, width, height);
ctx.drawImage(currentImage, 0, 0, width, height);
return true;
}
let needsGpuDevice = false;
for (const run of runs) {
if (run.backend === 'webgpu') {
needsGpuDevice = true;
break;
}
}
const gpuDevice = needsGpuDevice ? await getGpuDevice() : null;
if (isCancelled()) {
return false;
}
// Canvas sources are DOM-oriented. Flip them when uploading into WebGL so
// texture coordinates match clip-space output. `ImageBitmap` bridges below
// opt out because they are already oriented for upload.
let flipWebGLSourceY = initialFlipSourceY;
// Only bridge bitmaps are owned here; the original source belongs to the caller.
let bridgeBitmap: ImageBitmap | null = null;
try {
for (let runIndex = 0; runIndex < runs.length; runIndex++) {
const run = runs[runIndex];
let canvasIndex: 0 | 1 = 0;
for (const eff of run.effects) {
const dst = state.pool.getCanvas(run.backend, canvasIndex);
const def = eff.definition as EffectDefinition<unknown, unknown>;
if (run.backend === 'webgl2') {
// Checked before `setup()`: a lost context cannot compile shaders,
// and this surfaces the dedicated error instead of an effect's own.
state.pool.assertContextNotLost(dst);
}
const setupState = ensureSetup(state, def, dst);
def.apply({
source: currentImage,
target: dst,
state: setupState,
params: eff.params,
width: state.width,
height: state.height,
gpuDevice,
flipSourceY: run.backend === 'webgl2' ? flipWebGLSourceY : false,
});
// Effects consume their source synchronously. Release it before the
// next bridge so a run never retains multiple temporary bitmaps.
bridgeBitmap?.close();
bridgeBitmap = null;
if (run.backend === 'webgl2') {
// Same-backend ping-pong passes feed the previous WebGL canvas back
// through `texImage2D()`. That source is still a DOM canvas, so the
// next upload also needs to be flipped.
flipWebGLSourceY = true;
}
currentImage = dst;
canvasIndex = canvasIndex === 0 ? 1 : 0;
}
lastTarget = (currentImage as HTMLCanvasElement | null) ?? lastTarget;
const nextRun = runs[runIndex + 1];
if (nextRun && nextRun.backend !== run.backend && lastTarget) {
// 2D β WebGL: pass the 2D canvas directly so `texImage2D` + `UNPACK_FLIP_Y`
// matches blur-only on a raw frame canvas. `createImageBitmap` here changes
// upload orientation and produced upside-down stacks (wave + blur).
if (run.backend === '2d' && nextRun.backend === 'webgl2') {
currentImage = lastTarget;
flipWebGLSourceY = true;
} else {
// Other bridges use `createImageBitmap` rather than passing the canvas
// straight through. A direct `drawImage(webglCanvas)` in the next
// backend's first effect forces an implicit GPU readback / finish on the
// consuming context, which empirically blows the per-frame vsync budget and
// halves the paint rate. `createImageBitmap` pipelines the GPU work.
bridgeBitmap = await createImageBitmap(lastTarget);
if (isCancelled()) {
return false;
}
currentImage = bridgeBitmap;
if (nextRun.backend === 'webgl2') {
flipWebGLSourceY = false;
}
}
}
}
} finally {
// Also release a bridge if setup/apply throws or the run is cancelled.
bridgeBitmap?.close();
}
if (!lastTarget) {
return true;
}
const outCtx = output.getContext('2d');
if (!outCtx) {
throw new Error('Failed to acquire 2D context for output canvas');
}
outCtx.clearRect(0, 0, width, height);
outCtx.drawImage(lastTarget, 0, 0, width, height);
return true;
};
// Keep ownership of the optional resized source until the existing chain settles.
export const runEffectChain = async (
options: RunEffectChainOptions,
): Promise<boolean> => {
const {state, source, width, height, effects} = options;
// Keep one ID across resizing and effect passes so callers can cancel the run.
const runId = ++state.currentRunId;
if (
(state.width === width && state.height === height) ||
!effects.some((effect) => !(effect.params as {disabled?: boolean}).disabled)
) {
return runEffectChainWithoutResize(options, true, runId);
}
const firstEnabledEffect = effects.find(
(effect) => !(effect.params as {disabled?: boolean}).disabled,
)!;
const resized = await createImageBitmap(source as ImageBitmapSource, {
resizeWidth: state.width,
resizeHeight: state.height,
// WebGL ignores UNPACK_FLIP_Y_WEBGL for ImageBitmap uploads.
imageOrientation:
firstEnabledEffect.definition.backend === 'webgl2' ? 'flipY' : 'none',
});
try {
if (state.currentRunId !== runId) {
return false;
}
return await runEffectChainWithoutResize(
{...options, source: resized},
false,
runId,
);
} finally {
resized.close();
}
};