Open source CapCut alternative.
TypeScript
"use client";
/**
* Painting one frame of the cut.
*
* This is the whole picture side of Cut in one place: how a clip's grade and
* look become pixels, how a transition's geometry joins two clips, how a
* region, a pan, a zoom, and a fade land on the frame. It knows nothing about
* where the pixels came from — a caller hands it a `Frame` and it draws.
*
* That indifference is the point. The live preview feeds it video elements it
* is steering in real time; an export feeds it decoded frames it pulled at an
* exact timestamp. Both draw through this same code, so what the editor shows
* and what the file contains cannot drift apart by one being taught something
* the other wasn't.
*
* Time is a parameter rather than a reading of the clock, which is what makes
* an export deterministic: film grain at 4.5 seconds is the same grain whether
* it was reached by playing there or by rendering the 135th frame.
*/
import { applyClipEffects, applyDetail, applyLutToImageData, applyMaskToCanvas, detailActive, drawStreak, elementLook, grainTile, hexToHlgHex, lookCssFilter, lookPost, maskComposite, paintStrokeInk, removalActive, type ClipColorRecipe, type OutputSpace } from "@donkeycut/effects-kit";
import { cameraCardLayout, traceCardPath, type CardLayout } from "./cameraCard";
import { applyDetailGpu } from "./detailGpu";
import { applyLutGpu } from "./gradeGpu";
import { clipRecipe, peekClipLut, requestClipLut, type ClipLut, type RecipeSource } from "./lutBuild";
import { createRasterCanvas } from "./raster";
import { clipLen } from "./timelineEnd";
import {
bandEdge,
CLOCK_SWEEP,
clampBand,
clampFeather,
featherAlpha,
shapeEase,
SLICE_TAPS,
sliceStreak,
takesBand,
takesFeather,
} from "./transitionShape";
import { clipCovers, clipFrameAt, clipPosed, clipZoom, contentRect, DEFAULT_BACKGROUND, isFullRect, rectOf, shadowInk, TRANSITION_BLUR } from "./types";
import type { ClipShadow, FrameRect, TransitionStyle, VideoClip } from "./types";
/** A clip's picture at some instant, or the reasons there isn't one.
*
* `missing` and `pending` differ in what they do to the frame: a clip with
* nothing to draw yields to whatever should be behind it, while a clip whose
* decoder just hasn't caught up keeps the last frame on screen. Painting black
* for the second one is what strobing looks like while skimming.
*/
export type Frame =
| { kind: "missing" }
| { kind: "pending" }
| {
kind: "ready";
image: CanvasImageSource;
width: number;
height: number;
/** What these pixels' code values mean, from the reader that decoded
* them. A frame held across a move to another file keeps its own;
* absent, the clip's `sourceProvider` answers. */
source?: RecipeSource;
};
export const MISSING_FRAME: Frame = { kind: "missing" };
export const PENDING_FRAME: Frame = { kind: "pending" };
/** Frame motion a transition or animation puts on a layer, in canvas pixels. */
export interface LayerFx {
dx?: number;
dy?: number;
/** Defocus as the layer lands, output px (an edge blur). */
blur?: number;
}
/** The two clips a transition joins, each with its ramps already resolved. */
export interface CrossJoin {
masterFrame: Frame;
masterClip: VideoClip;
masterAlpha: number;
masterZoom: number;
masterFx: { dx: number; dy: number; blur?: number };
incFrame: Frame;
incClip: VideoClip | undefined;
incAlpha: number;
incZoom: number;
}
/** Canvas kinds the compositor can draw onto and allocate beside. */
type Surface = HTMLCanvasElement | OffscreenCanvas;
type Ctx = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
/** The grain tile advances on this cadence, in seconds per step. */
const GRAIN_STEP = 0.08;
/** The source every clip renders as when nothing says otherwise: Rec.709
* display values, which need no conversion. */
const REC709_SOURCE: RecipeSource = { profile: "rec709" };
/** How many clips a compositor remembers the last drawn LUT for. */
const LAST_LUT_MAX = 64;
/** The posed picture's overhang grows in steps of this many px, so a streak
* that lengthens frame by frame keeps its padded scratches at one size. */
const PAD_STEP = 32;
const padStep = (px: number) => (px > 0 ? Math.ceil(px / PAD_STEP) * PAD_STEP : 0);
/** Whether the clip wears its effects on its framed picture. A keyed clip
* wears them on its source picture instead (see `gradedSource`). */
const framedEffects = (clip: VideoClip) => !!clip.effects?.length && !removalActive(clip.removal);
export class FrameCompositor {
/** Scratch buffers, kept for the compositor's life: the grade pass, the
* look's self-copy passes, and the vignette gradient. Allocating these per
* frame is the difference between a preview that holds 60fps and one that
* fights the garbage collector. */
private gradeCanvas: Surface | null = null;
private gradeLutScratch: Surface | null = null;
private lookScratch: Surface | null = null;
private vignetteCanvas: Surface | null = null;
/** The clip's own effects: the picture copies into `clipFxCanvas` and
* each effect runs over it, `clipFxScratch` holding its self-copies. */
private clipFxCanvas: Surface | null = null;
private clipFxScratch: Surface | null = null;
/** The masked/keyframed-layer pass: the layer draws into `layerScratch`,
* its mask's coverage paints into `maskScratch`, a keyframed pose blits
* through `poseScratch`, and the result composites back onto the frame.
* A streak smears into `smearScratch` and a subject matte lands in
* `matteScratch`, both at the posed picture's padded size. */
private layerScratch: Surface | null = null;
private maskScratch: Surface | null = null;
private poseScratch: Surface | null = null;
private smearScratch: Surface | null = null;
private matteScratch: Surface | null = null;
/** Where a shadow is cast before the picture goes down over it. */
private shadowScratch: Surface | null = null;
/** The removal pass: the keyed picture builds in `removalScratch`, its
* silhouette in `removalSil`, the stroke's ink in `removalInk`, and an AI
* matte lands in `removalMatte` on its way to the alpha multiply. */
private removalScratch: Surface | null = null;
private removalMatte: Surface | null = null;
private removalSil: Surface | null = null;
private removalInk: Surface | null = null;
/** The piece renderer's look bake: the post passes run over a copy in
* `removalLookA`, then land back inside the layer's own coverage through
* `removalLookB`. */
private removalLookA: Surface | null = null;
private removalLookB: Surface | null = null;
/** The pixel pass of the grade's spatial controls, when the GPU cannot
* run it: a scratch the CPU reads and writes. */
private detailScratch: Surface | null = null;
/** The camera card: the pop-out builds in `cardPop` (rows down to the
* band's fade), and a card drawn at partial alpha or under a look's post
* passes flattens in `cardLayer` first, with the post copy in `cardPost`. */
private cardPop: Surface | null = null;
private cardLayer: Surface | null = null;
private cardPost: Surface | null = null;
/** A soft-edged transition draws its incoming shot here, then fades it
* out along the edge before laying it over the outgoing one. */
private featherScratch: Surface | null = null;
/** The fade under the card edge, rebuilt only when the band moves. */
private cardFade: { ctx: Ctx; edge: number; bottom: number; fill: CanvasGradient } | null = null;
/** The card shadow's ink, rebuilt only when its opacity changes. */
private cardInk: { opacity: number; ink: string } | null = null;
/** Canvas px per design px for the card's lengths. Absent, the canvas's
* own short side sets it; the export's piece renderer draws one clip's box
* on a canvas of that box's size and names the frame's scale here. */
designScale: number | null = null;
/** Where a camera card's person matte comes from: the clip's baked matte
* at `at` timeline seconds as alpha coverage over its source frame, like
* `removalMatteProvider`. Null until the bake lands; the card draws
* without its pop-out until then. */
cardMatteProvider: ((clip: VideoClip, at: number) => CanvasImageSource | null) | null = null;
/** What a clip's code values mean — its source profile, and whatever the
* decode route adds — so the compositor can build the clip's color recipe.
* Absent, every clip is Rec.709. */
sourceProvider: ((clip: VideoClip) => RecipeSource) | null = null;
/** The project's delivery space, which the recipe bakes toward. */
output: OutputSpace = "sdr";
/**
* How a LUT that is not in hand is handled. `live` (the preview) never
* waits: the build goes off the thread and the clip draws through the last
* LUT it had, or plain, until it lands and `wake` repaints. `exact` (an
* export, a baked layer) builds on this thread, so every frame is drawn
* through its own LUT.
*/
colorMode: "live" | "exact" = "live";
/** Repaint request, fired when a LUT built off the thread lands. */
wake: (() => void) | null = null;
/** Per clip, the key of the LUT it last drew through: what stands in while
* a newer one builds. Bounded; the LUTs themselves live in the cache. */
private lastLut = new Map<string, string>();
/** Where a subject-masked clip's person matte comes from: the host hands a
* reader over the canvas as it stands (the layers beneath the clip), so
* the matte never includes the masked clip's own pixels. A computed frame
* with `alpha: null` means no person registered — empty coverage. Provider
* absent, or a null frame = subject masks draw the plain picture (no
* segmenter, or the matte pass that must not recurse). */
subjectMatteProvider:
| ((at: number) => { alpha: CanvasImageSource | null } | null)
| null = null;
/** Where a removal clip's baked AI matte comes from: an alpha image
* (subject opaque, backdrop transparent — the host converts a decoded
* luma frame) covering the clip's source frame at `at` timeline seconds.
* Null = no matte yet; the picture draws plain until the bake lands. */
removalMatteProvider: ((clip: VideoClip, at: number) => CanvasImageSource | null) | null = null;
/** Resolves a backdrop fill's image asset to a drawable picture. */
backdropImageProvider: ((assetId: string) => CanvasImageSource | null) | null = null;
/** A clip id whose removal is bypassed this frame — the panel's eye toggle
* showing the original picture. */
removalBypass: string | null = null;
/**
* The frame's own color, behind everything drawn into it. The project owns
* it, so the caller writes it before each frame; black keeps a compositor
* nobody told rendering the way it always did.
*/
background = DEFAULT_BACKGROUND;
constructor(private canvas: Surface) {}
/** Point the compositor at a different surface, keeping the scratch buffers. */
setCanvas(canvas: Surface) {
this.canvas = canvas;
}
get width() {
return this.canvas.width;
}
get height() {
return this.canvas.height;
}
private ctx(): Ctx | null {
return this.canvas.getContext("2d") as Ctx | null;
}
/** A scratch surface of exactly `w`×`h`, reused across frames. Returns
* whether it had to be resized — which also cleared it, since setting a
* canvas dimension wipes its contents. */
private scratch(
field:
| "gradeCanvas"
| "gradeLutScratch"
| "clipFxCanvas"
| "clipFxScratch"
| "lookScratch"
| "vignetteCanvas"
| "layerScratch"
| "maskScratch"
| "poseScratch"
| "smearScratch"
| "matteScratch"
| "shadowScratch"
| "removalScratch"
| "removalMatte"
| "removalSil"
| "removalInk"
| "removalLookA"
| "removalLookB"
| "detailScratch"
| "cardPop"
| "cardLayer"
| "cardPost"
| "featherScratch",
w: number,
h: number
): { surface: Surface; resized: boolean } {
let surface = this[field];
if (!surface) {
surface = createRasterCanvas(w, h);
this[field] = surface;
}
// Match both dimensions — a transition alternates two differently-sized
// sources through one scratch every frame, and a stale dimension leaves the
// previous clip's pixels in the draw.
const resized = surface.width !== w || surface.height !== h;
if (surface.width !== w) surface.width = w;
if (surface.height !== h) surface.height = h;
return { surface, resized };
}
/** The frame color as the composite holds it: as is in SDR, and mapped
* like every other sRGB graphic when the composite is HLG. One conversion
* per color, held for the frames that follow. */
private backgroundFill: { hex: string; output: OutputSpace; fill: string } | null = null;
private backgroundFor(): string {
if (this.output === "sdr") return this.background;
const held = this.backgroundFill;
if (held && held.hex === this.background && held.output === this.output) return held.fill;
const fill = hexToHlgHex(this.background);
this.backgroundFill = { hex: this.background, output: this.output, fill };
return fill;
}
clear() {
const ctx = this.ctx();
if (!ctx) return;
ctx.fillStyle = this.backgroundFor();
ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
}
/**
* Paint an `amount` (0..1) color veil (`rgb` = "r,g,b") — over the whole
* frame, or clipped to `rect` for a per-clip fade. Shared by the fade
* animations, the dip transitions, and the whole-video project fade so they
* can never drift apart.
*/
fillVeil(rgb: string, amount: number, rect?: FrameRect) {
if (amount <= 0) return;
const ctx = this.ctx();
if (!ctx) return;
const W = this.canvas.width;
const H = this.canvas.height;
ctx.fillStyle = `rgba(${rgb},${Math.min(1, amount).toFixed(3)})`;
if (rect) ctx.fillRect(rect.x * W, rect.y * H, rect.w * W, rect.h * H);
else ctx.fillRect(0, 0, W, H);
}
fillBlackVeil(amount: number, rect?: FrameRect) {
this.fillVeil("0,0,0", amount, rect);
}
/** The recipe a clip renders through: its source, its grade, the output.
* `frameSource` is the drawn frame's own, when its reader named one. */
recipeFor(clip: VideoClip | undefined, frameSource?: RecipeSource): ClipColorRecipe {
const source = frameSource ?? (clip && this.sourceProvider?.(clip)) ?? REC709_SOURCE;
return clipRecipe(source, clip?.grade, this.output, this.colorMode === "live");
}
/**
* The LUT a clip draws through this frame: its own when it is in hand,
* the last one it drew through while a newer build is out, and none for
* an identity recipe or a clip whose first LUT has not landed yet.
*/
private lutFor(clip: VideoClip | undefined, recipe: ClipColorRecipe): ClipLut | null {
const got = requestClipLut(recipe, {
exact: this.colorMode === "exact",
wake: this.wake ?? undefined,
});
if (got === null) {
if (clip) this.lastLut.delete(clip.id);
return null;
}
if (got) {
if (clip && this.lastLut.get(clip.id) !== got.key) {
this.lastLut.delete(clip.id);
this.lastLut.set(clip.id, got.key);
if (this.lastLut.size > LAST_LUT_MAX) {
const oldest = this.lastLut.keys().next().value;
if (oldest !== undefined) this.lastLut.delete(oldest);
}
}
return got;
}
const last = clip && this.lastLut.get(clip.id);
return last ? (peekClipLut(last) ?? null) : null;
}
/**
* The clip's picture with its color and look applied, or the raw image
* when there is nothing to apply. The color is one LUT per clip — source
* conversion, library LUT and grade baked together — applied on the GPU
* (the CPU where there is none), then the grade's sharpen and clarity as a
* spatial pass, then the look's color pass as a canvas filter, in the order
* the export's chain runs. Source alpha rides through every pass, so
* transparent stills keep their transparency. The look's post passes
* (vignette, grain, glow…) draw over the composited layer instead. A
* keyed clip wears its own effects here, in the picture's pixels, the way
* the export's keyed pieces bake them; every other clip wears them framed,
* in the mask and pose pass.
*/
private gradedSource(
frame: Extract<Frame, { kind: "ready" }>,
clip: VideoClip | undefined,
at: number
): CanvasImageSource {
const recipe = this.recipeFor(clip, frame.source);
const compiled = this.lutFor(clip, recipe);
const detail = detailActive(clip?.grade) ? clip!.grade! : null;
const lookCss = lookCssFilter(clip?.look, clip?.lookAmount);
const fx = clip?.effects?.length && removalActive(clip.removal) ? clip : null;
if (!compiled && !detail && !lookCss && !fx) return frame.image;
const graded = this.colorPasses(frame, compiled, detail, lookCss);
return fx ? (this.clipEffects(graded, frame.width, frame.height, fx, at) as CanvasImageSource) : graded;
}
/** The LUT, the spatial pass and the look's color pass, in that order. */
private colorPasses(
frame: Extract<Frame, { kind: "ready" }>,
compiled: ClipLut | null,
detail: VideoClip["grade"] | null,
lookCss: string
): CanvasImageSource {
const w = frame.width;
const h = frame.height;
let picture: CanvasImageSource = frame.image;
if (compiled) {
const gpu = applyLutGpu(picture, w, h, compiled.lut, compiled.key);
if (gpu) {
picture = gpu as CanvasImageSource;
} else {
const { surface: pixels } = this.scratch("gradeLutScratch", w, h);
const pctx = pixels.getContext("2d") as Ctx | null;
if (!pctx) return frame.image;
pctx.clearRect(0, 0, w, h);
pctx.drawImage(picture, 0, 0, w, h);
const img = pctx.getImageData(0, 0, w, h);
applyLutToImageData(img.data, compiled.lut);
pctx.putImageData(img, 0, 0);
picture = pixels as CanvasImageSource;
}
}
if (detail) {
const gpu = applyDetailGpu(picture, w, h, detail);
if (gpu) {
picture = gpu as CanvasImageSource;
} else {
const { surface: pixels } = this.scratch("detailScratch", w, h);
const pctx = pixels.getContext("2d") as Ctx | null;
if (!pctx) return picture;
pctx.clearRect(0, 0, w, h);
pctx.drawImage(picture, 0, 0, w, h);
const img = pctx.getImageData(0, 0, w, h);
applyDetail(img.data, w, h, detail);
pctx.putImageData(img, 0, 0);
picture = pixels as CanvasImageSource;
}
}
if (!lookCss) return picture;
const { surface: scratch } = this.scratch("gradeCanvas", w, h);
const ctx = scratch.getContext("2d") as Ctx | null;
// A context with no filter support draws the color without the look.
if (!ctx || !("filter" in ctx)) return picture;
ctx.clearRect(0, 0, w, h);
ctx.filter = lookCss;
ctx.drawImage(picture, 0, 0, w, h);
ctx.filter = "none";
return scratch;
}
/** The clip's effects over `picture`'s `src` rect, in order, on the
* effects' own clock: a color cycle turns from the clip's first frame (or
* from where a split left it) and a flash pops there. */
private clipEffects(
picture: CanvasImageSource,
w: number,
h: number,
clip: VideoClip,
at: number,
src = { x: 0, y: 0, w, h }
): Surface {
const { surface: canvas } = this.scratch("clipFxCanvas", w, h);
const { surface: scratch } = this.scratch("clipFxScratch", w, h);
const ctx = canvas.getContext("2d") as Ctx | null;
if (!ctx) return canvas;
ctx.setTransform(1, 0, 0, 1, 0, 0);
ctx.globalAlpha = 1;
ctx.globalCompositeOperation = "source-over";
ctx.clearRect(0, 0, w, h);
ctx.drawImage(picture, src.x, src.y, src.w, src.h, 0, 0, w, h);
// The effects' clock starts where the unsplit clip did, so a fade runs
// across all of it, as the export's does.
const from = clip.effectsFrom ?? 0;
applyClipEffects(canvas, scratch, clip.effects ?? [], at - clip.start + from, grainTile, from + clipLen(clip));
return canvas;
}
/**
* The clip's graded picture with its background removal applied: the baked
* AI matte the host provides multiplies an alpha into the frame, the
* stroke's ink goes down behind the
* silhouette, and the backdrop fills in behind everything. Runs in the
* clip's source pixel space, before the fit/fill draw, so the keyed layer
* rides zoom, pan, masks, poses and transitions like any other picture.
*/
/** The keyed layer alone, in source pixel space — what the ffmpeg export's
* piece renderer encodes so the engine composites the very pixels the
* preview draws. `bakeLookPost` folds the look's post passes (grain,
* vignette, glow, washes) into the layer for the overlay pieces, where the
* engine applies no look to an alpha layer: the passes run over a copy and
* land back inside the layer's own coverage, so the keyed-out hole stays
* transparent. */
removedLayer(
frame: Extract<Frame, { kind: "ready" }>,
clip: VideoClip | undefined,
at: number,
opts: { bakeLookPost?: boolean } = {}
): CanvasImageSource {
const out = this.removedSource(frame, clip, at);
if (!opts.bakeLookPost || !lookPost(clip?.look, clip?.lookAmount)) return out;
const w = frame.width;
const h = frame.height;
const { surface: processed } = this.scratch("removalLookA", w, h);
const pctx = processed.getContext("2d") as Ctx | null;
const { surface: baked } = this.scratch("removalLookB", w, h);
const bctx = baked.getContext("2d") as Ctx | null;
if (!pctx || !bctx) return out;
for (const c of [pctx, bctx]) {
c.setTransform(1, 0, 0, 1, 0, 0);
c.globalAlpha = 1;
c.globalCompositeOperation = "source-over";
c.clearRect(0, 0, w, h);
c.drawImage(out, 0, 0, w, h);
}
this.applyLookPost(clip, 0, 0, w, h, 1, at, { canvas: processed, ctx: pctx });
bctx.globalCompositeOperation = "source-atop";
bctx.drawImage(processed, 0, 0, w, h);
bctx.globalCompositeOperation = "source-over";
return baked as CanvasImageSource;
}
private removedSource(
frame: Extract<Frame, { kind: "ready" }>,
clip: VideoClip | undefined,
at: number
): CanvasImageSource {
const base = this.gradedSource(frame, clip, at);
const r = clip?.removal;
if (!r || !removalActive(r) || (clip && this.removalBypass === clip.id)) return base;
const w = frame.width;
const h = frame.height;
// The matte answers before any scratch is staged: while the bake is still
// running the picture stays plain — the stroke and backdrop wait with it —
// and a full-frame clear and composite per tick would be pure waste.
const matte = clip ? (this.removalMatteProvider?.(clip, at) ?? null) : null;
if (!matte) return base;
const { surface } = this.scratch("removalScratch", w, h);
const ctx = surface.getContext("2d") as Ctx | null;
if (!ctx) return base;
ctx.setTransform(1, 0, 0, 1, 0, 0);
ctx.globalAlpha = 1;
ctx.globalCompositeOperation = "source-over";
ctx.clearRect(0, 0, w, h);
// 1. The key: alpha into the picture. `invert` flips the direction — the
// selection is removed and the rest of the picture stays.
const inv = !!r.invert;
ctx.drawImage(base, 0, 0, w, h);
const { surface: cover } = this.scratch("removalMatte", w, h);
const cctx = cover.getContext("2d") as Ctx | null;
if (cctx) {
cctx.setTransform(1, 0, 0, 1, 0, 0);
cctx.clearRect(0, 0, w, h);
cctx.imageSmoothingEnabled = true;
cctx.drawImage(matte, 0, 0, w, h);
maskComposite(ctx, cover as CanvasImageSource, inv);
}
const tLocal = Math.max(0, at - (clip?.start ?? 0));
// Design px at the source frame's own short side: ink painted here is
// scaled with the picture into its box, so the stroke holds its design
// size wherever the clip lands.
const ds = Math.min(w, h) / 1080;
// 2. Stroke ink around the selection's silhouette, behind the picture.
// The ink always traces the selection — inverted, that is the hole the
// removal cut, whose boundary is the same curve, and the ink paints
// inset so it rims the hole where it shows through.
if (r.stroke) {
const { surface: sil } = this.scratch("removalSil", w, h);
const sctx = sil.getContext("2d") as Ctx | null;
const { surface: ink } = this.scratch("removalInk", w, h);
const ictx = ink.getContext("2d") as Ctx | null;
if (sctx && ictx) {
sctx.setTransform(1, 0, 0, 1, 0, 0);
sctx.globalCompositeOperation = "source-over";
sctx.clearRect(0, 0, w, h);
if (!inv) {
sctx.drawImage(surface, 0, 0);
} else {
sctx.drawImage(cover as CanvasImageSource, 0, 0);
}
sctx.globalCompositeOperation = "source-in";
sctx.fillStyle = "#ffffff";
sctx.fillRect(0, 0, w, h);
sctx.globalCompositeOperation = "source-over";
ictx.setTransform(1, 0, 0, 1, 0, 0);
ictx.globalCompositeOperation = "source-over";
ictx.clearRect(0, 0, w, h);
paintStrokeInk(ictx, sil as CanvasImageSource, w, h, r.stroke, tLocal, ds, inv);
// Color the white ink, then lay it under the picture.
ictx.globalCompositeOperation = "source-in";
ictx.fillStyle = r.stroke.color;
ictx.fillRect(0, 0, w, h);
ictx.globalCompositeOperation = "source-over";
ctx.globalCompositeOperation = "destination-over";
ctx.drawImage(ink, 0, 0);
ctx.globalCompositeOperation = "source-over";
}
}
// 3. Backdrop behind it all, covering the source frame.
const bd = r.backdrop;
if (bd && bd.kind !== "none") {
ctx.globalCompositeOperation = "destination-over";
if (bd.kind === "color" && bd.color) {
ctx.fillStyle = bd.color;
ctx.fillRect(0, 0, w, h);
} else if (bd.kind === "image" && bd.assetId) {
const img = this.backdropImageProvider?.(bd.assetId) ?? null;
const iw = img ? Number((img as { width?: number }).width ?? 0) : 0;
const ih = img ? Number((img as { height?: number }).height ?? 0) : 0;
if (img && iw > 0 && ih > 0) {
const s = Math.max(w / iw, h / ih);
ctx.drawImage(img, (w - iw * s) / 2, (h - ih * s) / 2, iw * s, ih * s);
}
}
ctx.globalCompositeOperation = "source-over";
}
return surface;
}
/**
* Draw a look's post passes over one composited layer's footprint (canvas
* pixels): vignette, animated grain, self-copy glow, color washes, chroma
* ghosts. `alpha` follows the layer so a dissolving clip's grain dissolves
* with it, and `at` drives the grain so the same second of the cut always
* grains the same way. Every buffer here is cached — no per-frame allocation
* or pixel reads.
*/
private applyLookPost(
clip: VideoClip | undefined,
rx: number,
ry: number,
rw: number,
rh: number,
alpha: number,
at: number,
// A different surface to paint over — the piece renderer bakes the post
// passes into the keyed layer itself. Default: the frame canvas.
target?: { canvas: Surface; ctx: Ctx }
) {
const post = lookPost(clip?.look, clip?.lookAmount);
if (!post || alpha <= 0 || rw <= 0 || rh <= 0) return;
const canvas = target?.canvas ?? this.canvas;
const ctx = target?.ctx ?? this.ctx();
if (!ctx || !("filter" in ctx)) return;
const W = canvas.width;
const H = canvas.height;
ctx.save();
ctx.beginPath();
ctx.rect(rx, ry, rw, rh);
ctx.clip();
if (post.glow) {
// Copy the just-drawn region through a blur (and a highlight isolation
// for halation) and lay it back over itself.
const { surface: scratch } = this.scratch("lookScratch", W, H);
const sctx = scratch.getContext("2d") as Ctx | null;
if (sctx && "filter" in sctx) {
const blurPx = Math.max(1, post.glow.blurFrac * H);
sctx.clearRect(rx, ry, rw, rh);
sctx.filter = post.glow.bright
? `contrast(2.5) brightness(0.55) saturate(1.4) sepia(0.35) blur(${blurPx}px)`
: `blur(${blurPx}px)`;
sctx.drawImage(canvas, rx, ry, rw, rh, rx, ry, rw, rh);
sctx.filter = "none";
ctx.globalAlpha = post.glow.alpha * alpha;
ctx.globalCompositeOperation = post.glow.mode;
ctx.drawImage(scratch, rx, ry, rw, rh, rx, ry, rw, rh);
}
}
if (post.ghost) {
// VHS chroma fringing: warm and cool copies of the region nudged apart.
const { surface: scratch } = this.scratch("lookScratch", W, H);
const sctx = scratch.getContext("2d") as Ctx | null;
if (sctx && "filter" in sctx) {
const shift = Math.max(1, post.ghost.shiftFrac * W);
sctx.clearRect(rx, ry, rw, rh);
sctx.filter = "sepia(1) saturate(4) hue-rotate(-40deg) brightness(0.55)";
sctx.drawImage(canvas, rx, ry, rw, rh, rx, ry, rw, rh);
sctx.filter = "none";
ctx.globalAlpha = post.ghost.alpha * alpha;
ctx.globalCompositeOperation = "lighter";
ctx.drawImage(scratch, rx, ry, rw, rh, rx + shift, ry, rw, rh);
sctx.clearRect(rx, ry, rw, rh);
sctx.filter = "sepia(1) saturate(4) hue-rotate(140deg) brightness(0.55)";
sctx.drawImage(canvas, rx, ry, rw, rh, rx, ry, rw, rh);
sctx.filter = "none";
ctx.drawImage(scratch, rx, ry, rw, rh, rx - shift, ry, rw, rh);
}
}
ctx.globalCompositeOperation = "source-over";
for (const wash of post.washes ?? []) {
ctx.globalAlpha = wash.alpha * alpha;
ctx.globalCompositeOperation = wash.mode;
ctx.fillStyle = wash.color;
ctx.fillRect(rx, ry, rw, rh);
}
if (post.grain) {
const tile = grainTile(Math.floor(at / GRAIN_STEP));
if (tile) {
ctx.globalAlpha = post.grain * alpha;
ctx.globalCompositeOperation = "overlay";
for (let y = ry; y < ry + rh; y += tile.height) {
for (let x = rx; x < rx + rw; x += tile.width) {
ctx.drawImage(tile, x, y);
}
}
}
}
if (post.vignette) {
// Cached radial gradient, rebuilt only when the footprint size changes.
const vw = Math.max(1, Math.round(rw));
const vh = Math.max(1, Math.round(rh));
const { surface: vg, resized } = this.scratch("vignetteCanvas", vw, vh);
if (resized) {
const vctx = vg.getContext("2d") as Ctx | null;
if (vctx) {
const g = vctx.createRadialGradient(
vw / 2,
vh / 2,
Math.min(vw, vh) * 0.35,
vw / 2,
vh / 2,
Math.hypot(vw, vh) / 2
);
g.addColorStop(0, "rgba(0,0,0,0)");
g.addColorStop(1, "rgba(0,0,0,1)");
vctx.clearRect(0, 0, vw, vh);
vctx.fillStyle = g;
vctx.fillRect(0, 0, vw, vh);
}
}
ctx.globalAlpha = post.vignette * alpha;
ctx.globalCompositeOperation = "source-over";
ctx.drawImage(vg, rx, ry, rw, rh);
}
ctx.restore();
ctx.globalAlpha = 1;
ctx.globalCompositeOperation = "source-over";
}
/** Whether the layer routes through the mask/pose pass at all. */
private needsFx(clip?: VideoClip): boolean {
return (
!!clip &&
(clipPosed(clip) ||
// A camera card casts its own shadow; the box style stands aside.
(!!clip.boxStyle?.shadow && !clip.card) ||
framedEffects(clip) ||
!!(clip.mask && (clip.mask.kind !== "subject" || this.subjectMatteProvider)))
);
}
/**
* Draw one masked or keyframed layer by re-entering `draw` against a
* transparent scratch (with the clip's mask and keys stripped so the inner
* call draws plainly), then compose in order: the geometry mask trims the
* layer in its own space (it rides the clip), the pose blits the result
* where the keys put it, and the person matte — anchored to the frame, so
* the person never travels with the clip — trims last. The mask and pose
* evaluate on the clip's own clock.
*/
private drawFx(
clip: VideoClip,
at: number,
fx: LayerFx | undefined,
draw: (plain: VideoClip) => void
) {
const ctx = this.ctx();
if (!ctx) return;
const W = this.canvas.width;
const H = this.canvas.height;
const tLocal = Math.max(0, at - clip.start);
const pose = clipPosed(clip) ? clipFrameAt(clip, tLocal, W / H) : null;
if (pose && pose.opacity <= 0.001) return;
// The keyed blur and the motion streak, in output px, as an element's.
const look = pose ? elementLook(pose, Math.min(W, H) / 1080) : null;
const mask = clip.mask;
const { surface: layer } = this.scratch("layerScratch", W, H);
const lctx = layer.getContext("2d") as Ctx | null;
if (!lctx) return;
lctx.clearRect(0, 0, W, H);
const prev = this.canvas;
this.canvas = layer;
try {
// The pose pass owns the turn and the fade; the inner draw takes the
// picture plain, or it would apply them a second time (and re-enter
// here forever).
draw({
...clip,
mask: undefined,
kf: undefined,
rotation: undefined,
opacity: undefined,
...(framedEffects(clip) ? { effects: undefined } : {}),
...(clip.boxStyle ? { boxStyle: { ...clip.boxStyle, shadow: undefined } } : {}),
});
} finally {
this.canvas = prev;
}
const rect = rectOf(clip);
// The clip's effects treat its framed picture, look and all, ahead of
// the mask and pose — the export's chain runs them on the framed segment
// the same way. Example: a vignette darkens the corners of the clip's box,
// wherever the source's own corners were cropped to.
if (framedEffects(clip)) {
const bx = Math.max(0, Math.floor(rect.x * W));
const by = Math.max(0, Math.floor(rect.y * H));
const bw = Math.min(W, Math.ceil((rect.x + rect.w) * W)) - bx;
const bh = Math.min(H, Math.ceil((rect.y + rect.h) * H)) - by;
if (bw > 0 && bh > 0) {
const worn = this.clipEffects(layer as CanvasImageSource, bw, bh, clip, at, { x: bx, y: by, w: bw, h: bh });
lctx.clearRect(bx, by, bw, bh);
lctx.drawImage(worn as CanvasImageSource, bx, by);
}
}
const { surface: cover } = this.scratch("maskScratch", W, H);
if (mask && mask.kind !== "subject") {
applyMaskToCanvas(
lctx,
cover,
mask,
tLocal,
{ width: W, height: H, scale: Math.min(W, H) / 1080 },
{ x: rect.x + rect.w / 2, y: rect.y + rect.h / 2 }
);
}
let out: Surface = layer;
let octx = lctx;
// The soft pass reads past the frame edge — half the streak and three
// blur radii — so the posed picture keeps that much overhang around the
// frame. Example: a clip zoomed 1.5x and streaking keeps its own picture
// at the border, where a frame-sized scratch would smear in a black rim.
let padX = 0;
let padY = 0;
if (pose && look) {
padX = padStep(Math.abs(look.streakX) / 2 + 3 * look.blur);
padY = padStep(Math.abs(look.streakY) / 2 + 3 * look.blur);
}
const PW = W + 2 * padX;
const PH = H + 2 * padY;
if (pose) {
const { surface: posed } = this.scratch("poseScratch", PW, PH);
const pctx = posed.getContext("2d") as Ctx | null;
if (!pctx) return;
pctx.setTransform(1, 0, 0, 1, 0, 0);
pctx.clearRect(0, 0, PW, PH);
pctx.save();
pctx.globalAlpha = Math.max(0, Math.min(1, pose.opacity));
pctx.translate(padX + pose.x * W, padY + pose.y * H);
pctx.rotate((pose.rotation * Math.PI) / 180);
pctx.scale(pose.scale, pose.scale);
pctx.translate(-(rect.x + rect.w / 2) * W, -(rect.y + rect.h / 2) * H);
pctx.drawImage(layer, 0, 0);
pctx.restore();
out = posed;
octx = pctx;
// Motion blur smears the posed picture along its streak; the layer
// scratch is free again by now, so it takes the smear.
if (look && look.taps >= 2) {
const { surface: smear } = this.scratch("smearScratch", PW, PH);
const sctx = smear.getContext("2d") as Ctx | null;
if (!sctx) return;
sctx.setTransform(1, 0, 0, 1, 0, 0);
sctx.globalAlpha = 1;
sctx.globalCompositeOperation = "source-over";
sctx.clearRect(0, 0, PW, PH);
drawStreak(sctx, posed as CanvasImageSource, 0, 0, look);
out = smear;
octx = sctx;
}
}
if (mask?.kind === "subject") {
// The person matte is the coverage: blur its edge by the feather and
// multiply it in (or out, inverted), after the pose so the person
// stays anchored to the frame. A computed frame with no person means
// empty coverage — a front-masked layer shows nothing (the export's
// black matte frames land the same way) and a behind-masked one shows
// whole.
const res = this.subjectMatteProvider?.(at) ?? null;
if (res && !res.alpha) {
if (!mask.invert) return;
} else if (res?.alpha) {
// The matte covers the frame; the overhang around it has no person.
const { surface: padded } = this.scratch("matteScratch", PW, PH);
const cctx = padded.getContext("2d") as Ctx;
cctx.clearRect(0, 0, PW, PH);
const feather = (mask.feather ?? 0) * (Math.min(W, H) / 1080);
if (feather > 0 && "filter" in cctx) cctx.filter = `blur(${feather / 2}px)`;
cctx.imageSmoothingEnabled = true;
cctx.drawImage(res.alpha, padX, padY, W, H);
cctx.filter = "none";
maskComposite(octx, padded as CanvasImageSource, mask.invert);
}
}
// Transition motion lands on the finished result, so a push carries the
// mask and pose along — the export translates the segment the same way.
const hasFx = !!fx && (!!fx.dx || !!fx.dy);
if (hasFx) {
ctx.save();
ctx.translate(Math.round(fx.dx ?? 0), Math.round(fx.dy ?? 0));
}
const shade = clip.card ? null : this.shadowOf(out, clip.boxStyle?.shadow, padX, padY);
// The keyed blur softens the picture and its shadow as they land, on
// top of any filter the caller set (a blur transition's own).
const soft = !!look && look.blur > 0 && "filter" in ctx;
const before = soft ? ctx.filter : "none";
if (soft) {
const own = `blur(${look.blur.toFixed(2)}px)`;
ctx.filter = before && before !== "none" ? `${before} ${own}` : own;
}
if (shade) ctx.drawImage(shade, 0, 0);
ctx.drawImage(out, -padX, -padY);
if (soft) {
ctx.filter = before;
}
if (hasFx) ctx.restore();
}
/**
* The shadow a finished layer casts: the layer blurred and thrown behind
* itself, with the layer's own shape punched back out. What is left is the
* shadow alone, so it can be laid down before the picture (here) or over the
* frame beside it (the export's painted PNG) and look the same either way.
* Null when the clip casts none.
*/
private shadowOf(layer: Surface, sh: ClipShadow | undefined, padX = 0, padY = 0): Surface | null {
const ctx = this.ctx();
if (!ctx || !sh) return null;
const W = this.canvas.width;
const H = this.canvas.height;
const { surface } = this.scratch("shadowScratch", W, H);
const sctx = surface.getContext("2d") as Ctx | null;
if (!sctx) return null;
sctx.setTransform(1, 0, 0, 1, 0, 0);
sctx.globalCompositeOperation = "source-over";
sctx.clearRect(0, 0, W, H);
// Shadow lengths are design px at the 1080 short side, like masks.
const ds = Math.min(W, H) / 1080;
sctx.save();
sctx.shadowColor = shadowInk(sh);
sctx.shadowBlur = Math.max(0, sh.blur) * ds;
sctx.shadowOffsetX = (sh.x ?? 0) * ds;
sctx.shadowOffsetY = (sh.y ?? 0) * ds;
sctx.drawImage(layer as CanvasImageSource, -padX, -padY);
sctx.restore();
sctx.globalCompositeOperation = "destination-out";
sctx.drawImage(layer as CanvasImageSource, -padX, -padY);
sctx.globalCompositeOperation = "source-over";
return surface;
}
/** Draw a frame into a sub-rectangle of the canvas (a split-screen half, an
* overlay's region), fitted or filled. */
drawIntoRect(
frame: Frame,
rect: FrameRect,
fill: boolean,
alpha: number,
at: number,
zoom = 1,
clip?: VideoClip
) {
if (frame.kind !== "ready") return;
if (this.needsFx(clip)) {
this.drawFx(clip!, at, undefined, (plain) =>
this.drawIntoRect(frame, rect, fill, alpha, at, zoom, plain)
);
return;
}
if (clip?.card) {
this.drawCard(frame, rect, alpha, at, zoom, clip);
return;
}
const ctx = this.ctx();
if (!ctx) return;
const W = this.canvas.width;
const H = this.canvas.height;
const rx = rect.x * W;
const ry = rect.y * H;
const rw = rect.w * W;
const rh = rect.h * H;
// The clip's own zoom rides on the transition's, so a cross zoom over a
// zoomed clip pushes in from where the clip already sits.
const {
x: dx,
y: dy,
w: dw,
h: dh,
} = contentRect(
{ x: rx, y: ry, w: rw, h: rh },
frame.width,
frame.height,
fill,
clipZoom(clip ?? {}) * zoom,
clip?.panX ?? 0,
clip?.panY ?? 0
);
const src = this.removedSource(frame, clip, at);
const bs = clip?.boxStyle;
// Box style lengths are design px at the 1080 short side, like masks.
const ds = Math.min(W, H) / 1080;
const rad = Math.max(0, (bs?.radius ?? 0) * ds);
const prevAlpha = ctx.globalAlpha;
ctx.globalAlpha = Math.max(0, Math.min(1, alpha));
const clipped = fill || dw > rw + 0.5 || dh > rh + 0.5 || rad > 0;
const flipped = !!(clip?.flipH || clip?.flipV);
if (clipped || flipped) {
ctx.save();
if (clipped) {
ctx.beginPath();
ctx.roundRect(rx, ry, rw, rh, rad);
ctx.clip();
}
if (flipped) {
// The mirror turns about the box center, so the framed picture stays
// in its box and the mask and pose passes see it where it was.
ctx.translate(rx + rw / 2, ry + rh / 2);
ctx.scale(clip?.flipH ? -1 : 1, clip?.flipV ? -1 : 1);
ctx.translate(-(rx + rw / 2), -(ry + rh / 2));
}
ctx.drawImage(src, dx, dy, dw, dh);
ctx.restore();
} else {
ctx.drawImage(src, dx, dy, dw, dh);
}
if (bs?.borderWidth) {
// Stroke inside the box edge, so the border never widens the region.
const bw = bs.borderWidth * ds;
ctx.strokeStyle = bs.borderColor ?? "#ffffff";
ctx.lineWidth = bw;
ctx.beginPath();
ctx.roundRect(rx + bw / 2, ry + bw / 2, rw - bw, rh - bw, Math.max(0, rad - bw / 2));
ctx.stroke();
}
ctx.globalAlpha = prevAlpha;
this.applyLookPost(clip, rx, ry, rw, rh, alpha, at);
}
/**
* Draw a camera-card clip into its box (see cameraCard.ts): the card's
* shadow, the picture through the card, and the matted head above the
* card's top edge, all from the one frame. A transition's zoom scales the
* whole layout about the box center inside the box, the way the export's
* engine zooms the card's rendered layer.
*/
private drawCard(
frame: Extract<Frame, { kind: "ready" }>,
rect: FrameRect,
alpha: number,
at: number,
zoom: number,
clip: VideoClip
) {
const ctx = this.ctx();
const card = clip.card;
if (!ctx || !card || alpha <= 0) return;
const W = this.canvas.width;
const H = this.canvas.height;
const ds = this.designScale ?? Math.min(W, H) / 1080;
const box = { x: rect.x * W, y: rect.y * H, w: rect.w * W, h: rect.h * H };
const L = cameraCardLayout(card, box, frame.width, frame.height, ds);
const src = this.removedSource(frame, clip, at);
// The matte is asked for only when the layout shows the head.
const matte = card.popOut ? (this.cardMatteProvider?.(clip, at) ?? null) : null;
const post = !!lookPost(clip.look, clip.lookAmount);
const a = Math.min(1, alpha);
if (a >= 0.999 && !post) {
this.paintCardShadow(ctx, L, box, zoom, card.shadow);
this.paintCardBody(ctx, L, box, zoom, src, matte, clip);
return;
}
// Flatten first: a fading card fades as one layer, the head over the
// card included, and a look's post passes stay inside what it shows.
const { surface: layer } = this.scratch("cardLayer", W, H);
const lctx = layer.getContext("2d") as Ctx | null;
if (!lctx) return;
lctx.setTransform(1, 0, 0, 1, 0, 0);
lctx.globalAlpha = 1;
lctx.globalCompositeOperation = "source-over";
lctx.clearRect(0, 0, W, H);
this.paintCardBody(lctx, L, box, zoom, src, matte, clip);
if (post) {
const { surface: copy } = this.scratch("cardPost", W, H);
const pctx = copy.getContext("2d") as Ctx | null;
if (pctx) {
pctx.setTransform(1, 0, 0, 1, 0, 0);
pctx.globalAlpha = 1;
pctx.globalCompositeOperation = "source-over";
pctx.clearRect(0, 0, W, H);
pctx.drawImage(layer, 0, 0);
this.applyLookPost(clip, box.x, box.y, box.w, box.h, 1, at, { canvas: copy, ctx: pctx });
lctx.globalCompositeOperation = "source-atop";
lctx.drawImage(copy, 0, 0);
}
}
// The shadow goes in under the finished picture.
lctx.globalCompositeOperation = "destination-over";
this.paintCardShadow(lctx, L, box, zoom, card.shadow);
lctx.globalCompositeOperation = "source-over";
const prev = ctx.globalAlpha;
ctx.globalAlpha = a;
ctx.drawImage(layer, 0, 0);
ctx.globalAlpha = prev;
}
/** Clip `c` to the box and scale the layout by `zoom` about its center. */
private enterCardBox(c: Ctx, box: { x: number; y: number; w: number; h: number }, zoom: number) {
c.beginPath();
c.rect(box.x, box.y, box.w, box.h);
c.clip();
if (zoom > 0 && Math.abs(zoom - 1) > 1e-4) {
const cx = box.x + box.w / 2;
const cy = box.y + box.h / 2;
c.translate(cx, cy);
c.scale(zoom, zoom);
c.translate(-cx, -cy);
}
}
/** The card's soft shadow, outside the card only: the card's own fill is
* clipped away and its shadow stays. */
private paintCardShadow(
c: Ctx,
L: CardLayout,
box: { x: number; y: number; w: number; h: number },
zoom: number,
opacity: number
) {
if (!(opacity > 0) || !(L.shadowBlur > 0)) return;
let ink = this.cardInk;
if (!ink || ink.opacity !== opacity) {
ink = { opacity, ink: `rgba(0,0,0,${Math.min(1, opacity).toFixed(3)})` };
this.cardInk = ink;
}
const reach = 2 * L.shadowBlur;
c.save();
this.enterCardBox(c, box, zoom);
c.beginPath();
c.rect(L.card.x - reach, L.card.y - reach, L.card.w + 2 * reach, L.card.h + 2 * reach);
traceCardPath(c, L.card, L.radius);
c.clip("evenodd");
// A canvas shadow ignores the transform, so the blur follows the zoom
// by hand.
c.shadowColor = ink.ink;
c.shadowBlur = L.shadowBlur * (zoom > 0 ? zoom : 1);
c.shadowOffsetX = 0;
c.shadowOffsetY = 0;
c.fillStyle = "#000000";
c.beginPath();
traceCardPath(c, L.card, L.radius);
c.fill();
c.restore();
}
/** The picture through the card, then the head above it. */
private paintCardBody(
c: Ctx,
L: CardLayout,
box: { x: number; y: number; w: number; h: number },
zoom: number,
src: CanvasImageSource,
matte: CanvasImageSource | null,
clip: VideoClip
) {
c.save();
this.enterCardBox(c, box, zoom);
c.save();
c.beginPath();
traceCardPath(c, L.card, L.radius);
c.clip();
this.drawCardPicture(c, src, L.picture, clip);
c.restore();
const pop = matte ? this.cardPopOut(L, src, matte, clip) : null;
if (pop) c.drawImage(pop, 0, 0);
c.restore();
}
/** The footage at its card placement, mirrored about its own center. */
private drawCardPicture(
c: Ctx,
img: CanvasImageSource,
P: { x: number; y: number; w: number; h: number },
clip: VideoClip
) {
if (clip.flipH || clip.flipV) {
c.save();
c.translate(P.x + P.w / 2, P.y + P.h / 2);
c.scale(clip.flipH ? -1 : 1, clip.flipV ? -1 : 1);
c.drawImage(img, -P.w / 2, -P.h / 2, P.w, P.h);
c.restore();
} else {
c.drawImage(img, P.x, P.y, P.w, P.h);
}
}
/**
* The head above the card: the picture again, multiplied by the person
* matte drawn through the same placement, kept over the card's columns
* from the box top down to the card's top edge and faded out over the
* feather below it. Built on
* the rows the band reaches; null when the band has no rows on the canvas.
*/
private cardPopOut(
L: CardLayout,
src: CanvasImageSource,
matte: CanvasImageSource,
clip: VideoClip
): Surface | null {
const W = this.canvas.width;
const rows = Math.min(this.canvas.height, Math.ceil(L.band.bottom));
const top = Math.max(0, Math.floor(L.band.top));
if (rows <= top || L.band.edge <= L.band.top) return null;
const { surface: pop } = this.scratch("cardPop", W, rows);
const p = pop.getContext("2d") as Ctx | null;
if (!p) return null;
p.setTransform(1, 0, 0, 1, 0, 0);
p.globalAlpha = 1;
p.globalCompositeOperation = "source-over";
p.clearRect(0, 0, W, rows);
p.save();
p.beginPath();
p.rect(L.band.left, top, L.band.right - L.band.left, rows - top);
p.clip();
this.drawCardPicture(p, src, L.picture, clip);
p.globalCompositeOperation = "destination-in";
p.imageSmoothingEnabled = true;
this.drawCardPicture(p, matte, L.picture, clip);
p.restore();
if (L.band.bottom > L.band.edge) {
let fade = this.cardFade;
if (!fade || fade.ctx !== p || fade.edge !== L.band.edge || fade.bottom !== L.band.bottom) {
const fill = p.createLinearGradient(0, L.band.edge, 0, L.band.bottom);
fill.addColorStop(0, "rgba(0,0,0,0)");
fill.addColorStop(1, "rgba(0,0,0,1)");
fade = { ctx: p, edge: L.band.edge, bottom: L.band.bottom, fill };
this.cardFade = fade;
}
p.globalCompositeOperation = "destination-out";
p.fillStyle = fade.fill;
p.fillRect(L.band.left, L.band.edge, L.band.right - L.band.left, L.band.bottom - L.band.edge);
p.globalCompositeOperation = "source-over";
}
return pop;
}
/**
* Draw one clip's layer over the frame, honouring its region, fit, pan, zoom,
* alpha and any frame motion a transition puts on it.
*
* `clear` says this layer owns the black behind it. A layer with nothing to
* draw takes the black with it; a layer whose picture merely hasn't arrived
* leaves the frame as it stands.
*/
drawLayer(
frame: Frame,
clip: VideoClip | undefined,
clear: boolean,
alpha: number,
at: number,
zoom = 1,
fx?: LayerFx
) {
const ctx = this.ctx();
if (!ctx) return;
const W = this.canvas.width;
const H = this.canvas.height;
const blank = () => {
if (clear) {
ctx.fillStyle = this.background;
ctx.fillRect(0, 0, W, H);
}
};
// A hidden clip plays nothing, and a clip with no picture at all yields the
// frame; both fill black only when they own the clear.
if (frame.kind === "missing" || clip?.hidden) return blank();
// Mid-seek there is no decodable frame; keep the previous one on the canvas
// instead of strobing black (matters while skimming).
if (frame.kind === "pending") return;
blank();
// An edge blur softens the whole layer as it lands, over any filter the
// caller set; the layer draws with the filter in place.
if (fx?.blur && fx.blur > 0.1 && "filter" in ctx) {
const before = ctx.filter;
const own = `blur(${fx.blur.toFixed(2)}px)`;
ctx.filter = before && before !== "none" ? `${before} ${own}` : own;
try {
this.drawLayer(frame, clip, false, alpha, at, zoom, { dx: fx.dx, dy: fx.dy });
} finally {
ctx.filter = before;
}
return;
}
if (this.needsFx(clip)) {
// The blank above already settled who owns the black; the layer itself
// draws through the mask/pose pass, and any transition motion lands on
// the finished result.
this.drawFx(clip!, at, fx, (plain) =>
this.drawLayer(frame, plain, false, alpha, at, zoom, undefined)
);
return;
}
const hasFx = !!fx && (!!fx.dx || !!fx.dy);
if (hasFx) {
// Motion runs in full-canvas space — the export translates the whole
// padded segment frame, so a regioned clip rides along with it.
// Whole-pixel translation: a fractional offset antialiases both frame
// edges and draws a hairline seam where a push's frames meet.
ctx.save();
ctx.translate(Math.round(fx.dx ?? 0), Math.round(fx.dy ?? 0));
}
// A regioned track-0 clip (split-screen half) draws into its rect over the
// black frame; the full-frame path below keeps the pan-crop behavior. A
// styled box (rounded corners, border) or a mirrored picture also routes
// through the rect path, which knows how to draw those at full frame too.
const rect = rectOf(clip ?? {});
if (!isFullRect(rect) || clip?.boxStyle || clip?.flipH || clip?.flipV || !!clip?.card) {
this.drawIntoRect(frame, rect, !!clip && clipCovers(clip), alpha, at, zoom, clip);
if (hasFx) ctx.restore();
return;
}
const fill = !!clip && clipCovers(clip);
const {
x: dx,
y: dy,
w: dw,
h: dh,
} = contentRect(
{ x: 0, y: 0, w: W, h: H },
frame.width,
frame.height,
fill,
clipZoom(clip ?? {}) * zoom,
clip?.panX ?? 0,
clip?.panY ?? 0
);
const prevAlpha = ctx.globalAlpha;
ctx.globalAlpha = Math.max(0, Math.min(1, alpha));
ctx.drawImage(this.removedSource(frame, clip, at), dx, dy, dw, dh);
ctx.globalAlpha = prevAlpha;
this.applyLookPost(clip, 0, 0, W, H, alpha, at);
if (hasFx) ctx.restore();
}
/**
* Draw the outgoing and incoming clips of a transition at progress `p`
* (0..1), in the geometry the style calls for: a blend, a dip through a
* colour, a push, a wipe, or a shape reveal.
*/
drawCrossJoin(style: TransitionStyle, p: number, d: CrossJoin, at: number) {
const ctx = this.ctx();
if (!ctx) return;
const W = this.canvas.width;
const H = this.canvas.height;
const drawMaster = (fx = d.masterFx, alpha = d.masterAlpha) =>
this.drawLayer(d.masterFrame, d.masterClip, false, alpha, at, d.masterZoom, fx);
// The blend styles ramp the incoming alpha; the geometric styles draw it
// opaque inside their own moving region.
const drawInc = (alpha = d.incAlpha, fx?: LayerFx) => {
this.drawLayer(d.incFrame, d.incClip, false, alpha, at, d.incZoom, fx);
};
const clipped = (path: () => void, draw: () => void) => {
ctx.save();
ctx.beginPath();
path();
ctx.clip();
draw();
ctx.restore();
};
// Only the absence of an incoming clip cancels the transition. One whose
// picture merely hasn't decoded yet still runs the layered styles: a dip's
// veil and a fade's ramp sit over the master, so the geometry is what the
// frame should look like mid-transition. The region styles are different —
// a push shoves the master aside and a circle carves it away, and with no
// incoming picture yet the vacated region renders black — so those hold
// the master in place until the frame decodes.
if (d.incFrame.kind === "missing" || p <= 0) {
drawMaster();
return;
}
const carves =
style.startsWith("push") ||
style.startsWith("wipe") ||
style.startsWith("circle") ||
style.startsWith("split") ||
style.startsWith("slice") ||
style === "clockwipe";
if (carves && d.incFrame.kind === "pending") {
drawMaster();
return;
}
// A softened edge: the incoming shot fades in across a ramp along the
// edge, the same profile the ffmpeg expression computes per pixel.
// A white band ahead of a wipe's edge draws a hard edge, so it wins
// over a feather.
const band = takesBand(style) ? clampBand(d.masterClip.transitionBand) : 0;
const feather = clampFeather(d.masterClip.transitionFeather);
if (feather > 0 && takesFeather(style) && band === 0) {
drawMaster();
this.drawFeathered(style, p, feather, () => drawInc(1));
return;
}
switch (style) {
case "dipblack":
case "dipwhite": {
// Matches xfade fadeblack/fadewhite's measured plateau: out by ~30%,
// solid to ~60%, in over the rest.
const veil = Math.max(0, Math.min(1, Math.min(p / 0.3, (1 - p) / 0.4)));
if (p < 0.45) drawMaster();
else drawInc(1);
this.fillVeil(style === "dipwhite" ? "255,255,255" : "0,0,0", veil);
break;
}
case "blur": {
// Defocus blend: blur peaks mid-transition on both sides (isotropic
// here vs ffmpeg's horizontal hblur — reads the same). Without
// ctx.filter this degrades to the plain crossfade below.
const supports = "filter" in ctx;
const r = Math.max(0.5, Math.min(W, H) * TRANSITION_BLUR * (1 - Math.abs(2 * p - 1)));
if (supports) ctx.filter = `blur(${r.toFixed(2)}px)`;
drawMaster();
drawInc();
if (supports) ctx.filter = "none";
break;
}
case "pushleft":
case "pushright":
case "pushup":
case "pushdown": {
// Both frames travel together; the incoming clip shoves the outgoing
// one off the named edge.
const [mx, my, ix, iy] =
style === "pushleft"
? [-p * W, 0, (1 - p) * W, 0]
: style === "pushright"
? [p * W, 0, -(1 - p) * W, 0]
: style === "pushup"
? [0, -p * H, 0, (1 - p) * H]
: [0, p * H, 0, -(1 - p) * H];
drawMaster({ ...d.masterFx, dx: d.masterFx.dx + mx, dy: d.masterFx.dy + my });
drawInc(1, { dx: ix, dy: iy });
break;
}
case "wipeleft":
case "wiperight":
case "wipeup":
case "wipedown": {
// Hard reveal edge traveling in the named direction; a band runs a
// solid white panel just ahead of it (`bandEdge`).
const e = bandEdge(p, band);
const [r, b] =
style === "wipeleft"
? [{ x: (1 - e) * W, y: 0, w: e * W, h: H }, { x: (1 - e - band) * W, y: 0, w: band * W, h: H }]
: style === "wiperight"
? [{ x: 0, y: 0, w: e * W, h: H }, { x: e * W, y: 0, w: band * W, h: H }]
: style === "wipeup"
? [{ x: 0, y: (1 - e) * H, w: W, h: e * H }, { x: 0, y: (1 - e - band) * H, w: W, h: band * H }]
: [{ x: 0, y: 0, w: W, h: e * H }, { x: 0, y: e * H, w: W, h: band * H }];
drawMaster();
if (e > 0) {
clipped(
() => ctx.rect(r.x, r.y, r.w, r.h),
() => drawInc(1)
);
}
if (band > 0) {
ctx.save();
ctx.globalAlpha = 1;
ctx.globalCompositeOperation = "source-over";
ctx.fillStyle = "#ffffff";
ctx.fillRect(b.x, b.y, b.w, b.h);
ctx.restore();
}
break;
}
case "circleopen": {
drawMaster();
clipped(
() => ctx.arc(W / 2, H / 2, (p * Math.hypot(W, H)) / 2, 0, Math.PI * 2),
() => drawInc(1)
);
break;
}
case "circleclose": {
// The outgoing picture collapses into a shrinking circle over the
// incoming one.
drawInc(1);
clipped(
() => ctx.arc(W / 2, H / 2, ((1 - p) * Math.hypot(W, H)) / 2, 0, Math.PI * 2),
() => drawMaster()
);
break;
}
case "splitopen": {
// Barn doors part from the center.
drawMaster();
clipped(
() => ctx.rect(((1 - p) * W) / 2, 0, p * W, H),
() => drawInc(1)
);
break;
}
case "splitclose": {
// The incoming picture closes in from both side edges.
drawMaster();
clipped(
() => {
ctx.rect(0, 0, (p * W) / 2, H);
ctx.rect(W - (p * W) / 2, 0, (p * W) / 2, H);
},
() => drawInc(1)
);
break;
}
case "clockwipe": {
// A hand pinned at the top-left corner sweeps from the top edge down
// to the left one, uncovering the incoming shot behind it.
const angle = shapeEase(style, p) * CLOCK_SWEEP;
drawMaster();
clipped(
() => {
ctx.moveTo(0, 0);
ctx.arc(0, 0, Math.hypot(W, H), 0, angle);
ctx.closePath();
},
() => drawInc(1)
);
break;
}
case "sliceleft":
case "sliceup": {
// The frame cut in two: across, the top half slides left and the
// bottom right; up, the left column slides up and the right down. In
// each half the incoming shot follows the outgoing one in. The halves
// streak along their slide: copies across the shutter's travel,
// folded into a running average (copy k at 1/(k+1)), each copy an
// opaque outgoing+incoming pair so the average needs no scratch.
const across = style === "sliceleft";
const e = shapeEase(style, p);
const len = across ? W : H;
const streak = sliceStreak(p, d.masterClip.transition ?? 0) * len;
const taps = streak >= 1 ? SLICE_TAPS : 1;
const halves: { rect: [number, number, number, number]; dir: number }[] = across
? [
{ rect: [0, 0, W, H / 2], dir: -1 },
{ rect: [0, H / 2, W, H - H / 2], dir: 1 },
]
: [
{ rect: [0, 0, W / 2, H], dir: -1 },
{ rect: [W / 2, 0, W - W / 2, H], dir: 1 },
];
for (const half of halves) {
clipped(
() => ctx.rect(...half.rect),
() => {
for (let k = 0; k < taps; k++) {
const off = taps > 1 ? streak * (k / (taps - 1) - 0.5) : 0;
const shift = half.dir * e * len + off;
const at = (v: number) => (across ? { dx: v, dy: 0 } : { dx: 0, dy: v });
const a = 1 / (k + 1);
const m = at(shift);
const i = at(shift - half.dir * len);
drawMaster({ ...d.masterFx, dx: d.masterFx.dx + m.dx, dy: d.masterFx.dy + m.dy }, a);
drawInc(a, i);
}
}
);
}
break;
}
default: {
// crossfade / crosszoom: the classic A·(1−α)+B·α blend (zooms already
// folded into the layer zoom factors).
drawMaster();
drawInc();
}
}
}
/**
* Lay the incoming shot over the frame through a soft edge: drawn into the
* scratch, faded by the style's profile (`featherAlpha`, sampled into a
* gradient along the edge's axis or out from the center), then composited.
*/
private drawFeathered(style: TransitionStyle, p: number, feather: number, drawInc: () => void) {
const ctx = this.ctx();
if (!ctx) return;
const W = this.canvas.width;
const H = this.canvas.height;
const { surface } = this.scratch("featherScratch", W, H);
const sctx = surface.getContext("2d") as Ctx | null;
if (!sctx) return;
sctx.clearRect(0, 0, W, H);
const main = this.canvas;
this.canvas = surface;
try {
drawInc();
} finally {
this.canvas = main;
}
// The profile, sampled along the axis it varies on: x for the sideways
// styles, y for the vertical wipes, the distance from the center for the
// circles.
const STOPS = 48;
const radial = style.startsWith("circle");
const vertical = style === "wipeup" || style === "wipedown";
const R = Math.hypot(W, H) / 2;
const grad = radial
? sctx.createRadialGradient(W / 2, H / 2, 0, W / 2, H / 2, R)
: vertical
? sctx.createLinearGradient(0, 0, 0, H)
: sctx.createLinearGradient(0, 0, W, 0);
for (let i = 0; i <= STOPS; i++) {
const u = i / STOPS;
const a = radial
? featherAlpha(style, p, W / 2 + u * R, H / 2, W, H, feather)
: vertical
? featherAlpha(style, p, 0, u * H, W, H, feather)
: featherAlpha(style, p, u * W, 0, W, H, feather);
grad.addColorStop(u, `rgba(0,0,0,${a.toFixed(4)})`);
}
sctx.save();
sctx.globalCompositeOperation = "destination-in";
sctx.fillStyle = grad;
sctx.fillRect(0, 0, W, H);
sctx.restore();
ctx.drawImage(surface as CanvasImageSource, 0, 0);
}
}