Open source CapCut alternative.
TypeScript
"use client";
/**
* Rendering the cut to a file, in the browser.
*
* The tab already knows how to draw this project — it does it sixty times a
* second in the preview. The only things it was missing were a clock it could
* step rather than follow, and somewhere to put the frames. This supplies both:
* frames are pulled at exact timestamps instead of played towards, and the
* finished picture goes into an MP4 muxer instead of onto the screen.
*
* Everything about what a frame looks like comes from the same two modules the
* preview uses — `framePlan` for what is on screen at a given moment, and
* `FrameCompositor` for turning that into pixels. Nothing here decides how the
* cut should look. That is the point: a render is the preview, evaluated at
* every frame instead of at whatever moment the display asked for.
*/
import {
CanvasSource,
getFirstEncodableAudioCodec,
getFirstEncodableVideoCodec,
MovOutputFormat,
Mp4OutputFormat,
Output,
AudioBufferSource,
Quality,
StreamTarget,
type AudioCodec,
type VideoCodec,
} from "mediabunny";
import { drawBlock } from "./blockSource";
import { audioFxSpans } from "./audioEffects";
import { duckWindows, renderMix, type MixClip, type MixItem, type MixSpec } from "./audioMix";
import { encoderLead } from "./encoderLead";
import { masterInPlace } from "./loudness";
import { STEM_CHANNELS, STEM_RATE, wavBytes, wavChunks, writeStoredZip, type StemDef } from "./stems";
import { spanSoundSpec, specSound } from "./soundSource";
import { FrameCompositor, MISSING_FRAME, type Frame } from "./composite";
import { ensureClipLuts, sourceLookup } from "./lutBuild";
import { colorRead, type ColorRead, type ReadFile } from "./sourceColor";
import type { OutputSpace } from "@donkeycut/effects-kit";
import { heldSpans, overlayPlan, trackZeroAt, trackZeroPlan } from "./framePlan";
import { heldFrames, holdOverlays, type HeldFrames } from "./frameGrid";
import { compatNeedOf } from "./codecSupport";
import { frameSink, keyframeTimeAt, openMedia, videoTrackOf, withAssetColors } from "./mediaRead";
import type { InputVideoTrack, WrappedCanvas } from "mediabunny";
import { allowance, canvasBytes, holdMemory } from "./memoryBudget";
import { getClipSpans, overlayLayers, projectDuration, spanSequence } from "./store";
import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
import { ElementFx, elementLook, paintElementCrop } from "@donkeycut/effects-kit";
import { applyEffectToCanvas, effectRecipe, diveView, divesAt, evalOverlayFrame, measureDiveFocus, retimeOf, grainTile, smoothsAt, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, planAnimatedLayers, type DiveFocus, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";
import { backdropStill, loadBackdropStill } from "./backdropStills";
import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";
import { overlayLiveAt } from "./overlayLive";
import { MatteAlpha, matteReadTime, matteStamp } from "./matteAlpha";
import { createRasterCanvas, snapshotRaster, type RasterSurface } from "./raster";
import { exportFrameSynth, SYNTH_EDGE, synthWeight, type FrameSynth } from "./frameSynth";
import { cutRenderEnv, renderElementCanvas, renderStillPng, requireStickers } from "./textRender";
import { cardMatteKey } from "./cameraCard";
import { assetIsSilent, behindSubjectOverlay, clipCovers, frameOf, frontSubjectOverlay, isEffectOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, removalActive } from "./types";
import type { ClipAnim, ClipSpan, EffectOverlay, MediaAsset, Overlay, StickerOverlay } from "./types";
import type { ExportDoc } from "./renderSnapshot";
import type { ExportSettings } from "./exportClient";
import { sourceExportPlan } from "@/cut/lib/sourceExportPlan";
import { prepareSourceRemux } from "@/cut/lib/sourceRemux";
import { deliverySpan, KEYFRAME_INTERVAL_S, videoBitrateFor } from "./exportDelivery";
import { UserError } from "./userError";
/** Audio is written at the rate and width a delivery file wants, rather than
* the 16 kHz mono a speech model reads. */
const AUDIO_RATE = 48000;
const AUDIO_CHANNELS = 2;
/**
* What a delivered file is made of: the codec the user chose, and nothing in
* its place.
*
* The container is only half of whether a file opens. An .mp4 carrying VP9,
* AV1, or Opus is a valid file that QuickTime, the Finder preview, and most
* phones refuse — the user gets something they cannot play and cannot convert.
* So there is no second codec to fall back to: a browser that cannot encode
* the chosen one at the chosen size fails `canRenderInBrowser`, and the export
* goes where that codec is available — the worker for a cloud project, the
* Mac app or the cloud for a browser-resident one. ProRes has no WebCodecs
* encoder anywhere, so it always takes that road.
*/
export function deliveryVideoCodec(settings: ExportSettings): VideoCodec | null {
return settings.codec === "h264" ? "avc" : settings.codec === "hevc" ? "hevc" : null;
}
export function deliveryAudioCodec(settings: ExportSettings): AudioCodec {
return settings.audioCodec === "pcm" ? "pcm-s16" : "aac";
}
/** Codecs a working clip may use — the removal bakes and the behind-speaker
* mask, which only this app's decoders and ffmpeg ever read, and both take
* any of these. A file that leaves the app uses `deliveryVideoCodec`. */
export const WORKING_VIDEO_CODECS: VideoCodec[] = ["avc", "hevc", "vp9", "av1"];
export interface RenderProgress {
/** 0..1 across the whole render. */
ratio: number;
stage: "audio" | "video" | "finishing";
}
/** A finished render: the file, and the scratch space it occupies. */
export interface RenderedExport {
/** The MP4, backed by origin-private disk rather than the tab's heap — it
* streams from there into whatever reads it. */
file: File;
/** Delete the backing scratch file. Call once the file has been consumed. */
discard(): Promise<void>;
}
/**
* Where an in-tab render writes its file while it is being made.
*
* Origin-private disk, not memory. The encoded output is the one thing in a
* render whose size grows with the cut — half a gigabyte for ten minutes of
* 1080p — and holding it in the heap is what crashes a tab on a project the
* gate otherwise allows. On disk, the tab's memory stays flat no matter how
* big the file gets.
*/
const SCRATCH_DIR = "cut-export-scratch";
/** After this long, a scratch file can only be the leaving of a tab that died
* mid-render — no render or upload runs for a day. */
const SCRATCH_ORPHAN_MS = 24 * 60 * 60 * 1000;
async function scratchDir(): Promise<FileSystemDirectoryHandle> {
const root = await navigator.storage.getDirectory();
return root.getDirectoryHandle(SCRATCH_DIR, { create: true });
}
/** Sweep scratch files no render can still own. Housekeeping — a failure here
* never stops the render that triggered it. */
async function sweepScratch(dir: FileSystemDirectoryHandle): Promise<void> {
try {
for await (const [name, handle] of dir) {
if (handle.kind !== "file") continue;
const file = await (handle as FileSystemFileHandle).getFile();
if (Date.now() - file.lastModified > SCRATCH_ORPHAN_MS) {
await dir.removeEntry(name).catch(() => {});
}
}
} catch {
// The sweep is best effort.
}
}
export interface RenderOptions {
/** Maps an asset's stored file name to a URL the reader can open. */
resolve: (asset: MediaAsset) => string;
onProgress?: (progress: RenderProgress) => void;
signal?: AbortSignal;
}
/** The bitrate the choice asks for — the model in `exportDelivery`, which
* the engine and the dialog's estimate read too. */
export function bitrateFor(settings: ExportSettings): number {
return videoBitrateFor(settings);
}
/** The stretch `[from, to]` of a rendered mix as a buffer of its own. */
function sliceAudio(mix: AudioBuffer, from: number, to: number): AudioBuffer {
const a = Math.max(0, Math.min(mix.length, Math.round(from * mix.sampleRate)));
const b = Math.max(a + 1, Math.min(mix.length, Math.round(to * mix.sampleRate)));
const out = new AudioBuffer({
length: b - a,
numberOfChannels: mix.numberOfChannels,
sampleRate: mix.sampleRate,
});
for (let ch = 0; ch < mix.numberOfChannels; ch++) {
out.copyToChannel(mix.getChannelData(ch).subarray(a, b), ch);
}
return out;
}
/** Standard cadences retain their fractional rates; unusual rates keep the measured value. */
const DELIVERY_RATES = [24000 / 1001, 24, 25, 30000 / 1001, 30, 48, 50, 60000 / 1001, 60];
/** Sources the probe reads for a rate: the ones with the most time on the
* timeline, so a cut of many files still answers quickly. */
const RATE_PROBE_SOURCES = 6;
/** How long the probe waits on a source before giving up on it. */
const RATE_PROBE_TIMEOUT_MS = 8_000;
/** The sources a rate probe would read, by footprint: the key a caller
* re-probes on, so a change elsewhere in the project leaves the answer. */
export function sourceFrameRateKey(doc: Pick<ExportDoc, "clips" | "assets">): string {
const byId = new Map(doc.assets.map((asset) => [asset.id, asset]));
return JSON.stringify(rateSources(doc).map(([id]) => [id, byId.get(id)?.fileName]));
}
function rateSources(doc: Pick<ExportDoc, "clips" | "assets">): [string, number][] {
const byId = new Map(doc.assets.map((a) => [a.id, a]));
const footprint = new Map<string, number>();
for (const c of doc.clips) {
const asset = byId.get(c.assetId);
if (!asset || asset.type !== "video" || c.hidden) continue;
footprint.set(asset.id, (footprint.get(asset.id) ?? 0) + retimeOf(c).len);
}
return [...footprint.entries()]
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
.slice(0, RATE_PROBE_SOURCES);
}
export type SourceExportProfile = { fps: number | null; videoBitrate?: number; audioBitrate?: number; codec?: "h264" | "hevc"; audioSampleRate?: number; audioChannels?: number };
/** Read cadence and compression budgets from the dominant video sources.
* The largest source bitrate preserves detail across the sampled footage;
* the frame rate is weighted by its time on the timeline. */
export async function sourceExportProfile(
doc: Pick<ExportDoc, "clips" | "assets">,
resolve: (asset: MediaAsset) => string,
opts: { signal?: AbortSignal } = {}
): Promise<SourceExportProfile> {
const byId = new Map(doc.assets.map((a) => [a.id, a]));
const seconds = new Map<number, number>();
let videoBitrate = 0;
let audioBitrate = 0;
const profiles = new Map<string, Partial<SourceExportProfile>>();
await Promise.all(
rateSources(doc).map(async ([id, len]) => {
if (opts.signal?.aborted) return;
const asset = byId.get(id)!;
const input = openMedia(resolve(asset));
// A cancel or the deadline closes the reader, which fails whatever it
// is waiting on; either way this source casts no vote.
const close = () => input.dispose();
const deadline = setTimeout(close, RATE_PROBE_TIMEOUT_MS);
opts.signal?.addEventListener("abort", close, { once: true });
try {
const track = await input.getPrimaryVideoTrack();
if (!track) return;
const stats = await track.computePacketStats(120);
const audio = await input.getPrimaryAudioTrack();
const audioStats = audio ? await audio.computePacketStats(120) : null;
const sizeBytes = asset.sizeBytes ?? await input.source.getSizeOrNull();
const sourceAudio = audioStats?.averageBitrate ?? 0;
const totalBitrate = sizeBytes && asset.duration > 0 ? sizeBytes * 8 / asset.duration : 0;
const sourceVideo = totalBitrate > sourceAudio ? totalBitrate - sourceAudio : stats.averageBitrate;
if (Number.isFinite(sourceVideo)) videoBitrate = Math.max(videoBitrate, sourceVideo);
if (Number.isFinite(sourceAudio)) audioBitrate = Math.max(audioBitrate, sourceAudio);
const codec = await track.getCodec();
profiles.set(id, {
...(codec === "avc" || codec === "hevc" ? { codec: codec === "avc" ? "h264" : "hevc" } : {}),
...(audio ? { audioSampleRate: await audio.getSampleRate(), audioChannels: await audio.getNumberOfChannels() } : {}),
});
const rate = stats.averagePacketRate;
if (!(rate > 0) || opts.signal?.aborted) return;
const snapped = DELIVERY_RATES.reduce((best, r) =>
Math.abs(r - rate) < Math.abs(best - rate) ? r : best
);
const cadence = Math.abs(snapped - rate) / rate < 0.0001 ? snapped : rate;
seconds.set(cadence, (seconds.get(cadence) ?? 0) + len);
} catch {
// A source the probe cannot read casts no vote.
} finally {
clearTimeout(deadline);
opts.signal?.removeEventListener("abort", close);
close();
}
})
);
if (opts.signal?.aborted) return { fps: null };
let pick: number | null = null;
for (const [rate, len] of seconds) {
if (pick === null || len > seconds.get(pick)! || (len === seconds.get(pick) && rate > pick)) pick = rate;
}
const primary = rateSources(doc).map(([id]) => profiles.get(id)).find(Boolean);
const audioChannels = Math.max(0, ...[...profiles.values()].map((p) => p.audioChannels ?? 0));
return { ...primary, ...(audioChannels ? { audioChannels } : {}), fps: pick, ...(videoBitrate > 0 ? { videoBitrate } : {}), ...(audioBitrate > 0 ? { audioBitrate } : {}) };
}
/** The frame reader for one open video source. */
type ClipSink = ReturnType<typeof frameSink>;
/** Canvases one clip reader cycles. Small: a render draws one frame at a time
* and never looks back, so the pool only has to keep the frame being drawn
* clear of the frame being decoded. */
const READER_POOL = 4;
/** Readers a render keeps open when memory is not the constraint. A cut drawn
* from more files than this reopens the ones it comes back to. */
const READERS_TUNED = 12;
/** Frames a reader may go unasked-for before it is a candidate to close. One
* frame's own readers are never candidates, so what a join or a stack of
* layers needs together is never taken away from the frame needing it. */
const READER_GRACE = 2;
/**
* Canvas bytes a reader may hold for a clip read backward.
*
* A reversed clip asks for its frames in falling order, and a decoder only
* runs forward: each ask on its own would decode from the keyframe below it
* every time, most of a group of pictures per frame. So a step down decodes
* the window of frames ending at the ask in one pass and holds it, and the
* asks below are answered from it until it runs out. The window is as many
* frames as this buys at the source's size, so a 4K clip holds a few and a
* small one holds a dozen.
*/
const BACK_WINDOW_BYTES = 64 << 20;
const BACK_WINDOW_MAX = 12;
/** Canvases the reader's pools cycle, in frames: the forward pool plus the
* backward window, the frame it keeps beyond the ask, and the two a fill
* needs around it. */
export function readerPoolFrames(r: { backWindow: number }): number {
return READER_POOL + (r.backWindow > 0 ? r.backWindow + 3 : 0);
}
const frameOfWrapped = (wrapped: WrappedCanvas): Frame => ({
kind: "ready",
image: wrapped.canvas,
width: wrapped.canvas.width,
height: wrapped.canvas.height,
});
/** The file a render reads: an asset's master, never the preview proxy. The
* readers open it and the compositors take their recipes from it through
* this one choice, so a frame is always drawn through the recipe of the read
* that produced it. */
export const renderFile = (): ReadFile => "master";
/** Settle the drawn matrix of every master the render reads, so the recipes
* built before its first frame are the final ones. */
export async function settleRenderColor(assets: readonly Pick<MediaAsset, "color" | "type">[]): Promise<void> {
await Promise.all(assets.filter((a) => a.type === "video").map((a) => colorRead(a, renderFile()).settled()));
}
/** A clip's source time at timeline time `t`. */
function sourceTimeAt(span: ClipSpan, t: number): number {
return retimeOf(span.clip).srcAt(Math.max(0, t - span.start));
}
/**
* Frames for one clip, pulled at exact timestamps.
*
* A video keeps its reader open across the whole render: the timestamps a
* render asks for only ever move forward, so the decoder walks the file once
* rather than seeking back to the start for every frame. A still decodes once
* and is handed back for as long as its clip is on screen.
*/
export class ClipReader {
private input: ReturnType<typeof openMedia> | null = null;
private sink: ClipSink | null = null;
private track: InputVideoTrack | null = null;
private still: Frame | null = null;
/** The source second last asked for; an ask below it is a step back. */
private lastAt = -Infinity;
/** The backward window: frames in rising order, on their own sink, and the
* frame length learned from them. */
private backSink: ClipSink | null = null;
private backFrames: WrappedCanvas[] = [];
private frameDt = 1 / 30;
private opened: Promise<ClipSink | null> | null = null;
/** Reads that failed in a row, each off a freshly rebuilt reader. Any
* successful read clears it. */
private failStreak = 0;
/** Set after the first decode failure: hardware decoder sessions are a
* finite machine resource, and a reader that lost one keeps losing while
* the preview holds the rest — the rebuilt reader decodes in software,
* which always opens. */
private software = false;
/** The frame number this reader was last asked for, for the painter's
* between-frames eviction. */
usedAt = 0;
/** How this reader reads its file, and the recipe its frames mean. */
readonly colorRead: ColorRead;
/**
* Pixels in a frame of the source.
*
* The sink is opened with no size, so its canvases are the file's own — a
* 4K clip in a 1080p render still cycles 4K canvases. Sizing the budget off
* the render's canvas would under-count that by the ratio between them.
*/
get sourcePixels(): number {
return (this.asset.width ?? 1920) * (this.asset.height ?? 1080);
}
/** Frames the backward window holds once it is open, 0 while it is not. */
get backWindow(): number {
return this.backSink ? this.windowFrames : 0;
}
/** Frames a backward window holds at this source's size. */
private get windowFrames(): number {
return Math.max(2, Math.min(BACK_WINDOW_MAX, Math.floor(BACK_WINDOW_BYTES / canvasBytes(this.sourcePixels))));
}
constructor(
private asset: MediaAsset,
/** Resolved on open, and again on a re-open, so a link re-minted while the
* render is running is picked up rather than the render dying on the
* snapshot's expired one. */
private url: () => string
) {
this.colorRead = colorRead(asset, renderFile());
}
/** The sink options every reader of this source opens with. */
private sinkOptions(poolSize: number) {
const colorSpace = this.colorRead.colorSpace;
return {
poolSize,
...(this.software ? { software: true } : {}),
...(colorSpace ? { colorSpace } : {}),
};
}
/** Open the source and hand back its frame reader, or null when it has no
* readable picture (a still, which is served from `this.still`, or a source
* with no decodable video). */
private open(): Promise<ClipSink | null> {
return (this.opened ??= (async () => {
if (this.asset.block) {
// Nothing to fetch: the block paints itself at the source size the
// render asks for, once, and that painting is the frame.
const canvas = drawBlock(this.asset, this.asset.width ?? 1080, this.asset.height ?? 1920);
this.still = { kind: "ready", image: canvas, width: canvas.width, height: canvas.height };
return null;
}
if (this.asset.type === "image") {
const blob = await (await fetch(this.url())).blob();
const bitmap = await createImageBitmap(blob);
this.still = {
kind: "ready",
image: bitmap,
width: bitmap.width,
height: bitmap.height,
};
return null;
}
const input = openMedia(this.url());
const track = await videoTrackOf(input);
if (!track) {
input.dispose();
return null;
}
this.input = input;
this.track = track;
// A small pool keeps the render's canvas allocation flat over thousands
// of frames.
this.sink = frameSink(track, undefined, this.sinkOptions(READER_POOL));
return this.sink;
})());
}
/** The clip's picture at source time `at`, or a missing frame when the
* source has none — a still-less video, or a time past its end. A render
* never reports `pending`: it waits for the decode instead of moving on. */
async frameAt(at: number): Promise<Frame> {
if (this.still) return this.still;
// A plain read cycles the pool past whatever pair was held.
this.pair = null;
const wrapped = await this.wrappedAt(at);
return wrapped ? frameOfWrapped(wrapped) : this.still ?? MISSING_FRAME;
}
/** The last pair handed out, kept while the asks stay between its two
* frames: an output frame between the same two sources costs no decode. */
private pair: { a: WrappedCanvas; b: WrappedCanvas | null } | null = null;
/**
* The source frames either side of `at`, for a clip smoothing its slow
* motion: `a` covers `at`, `b` is the next frame, or null at the end of the
* source. Asks walk forward like the plain read does — the pair advances by
* one decode when `at` passes `b`, and the frame that was `b` becomes `a`
* without being decoded again.
*/
async pairAt(at: number): Promise<{ a: WrappedCanvas; b: WrappedCanvas | null } | null> {
if (this.still) return null;
const held = this.pair;
// A pair with no `b` is the source's last frame; it covers every ask
// from there on, so a render finishing on it never reads again.
if (held && at >= held.a.timestamp - 1e-4 && (!held.b || at < held.b.timestamp - 1e-4)) return held;
let a: WrappedCanvas | null;
if (held?.b && at >= held.b.timestamp - 1e-4 && at < held.b.timestamp + Math.max(held.b.duration, 1e-3) - 1e-4) {
a = held.b;
this.lastAt = at;
} else {
this.pair = null;
a = await this.wrappedAt(at);
}
if (!a) return null;
const b = await this.wrappedAt(a.timestamp + Math.max(a.duration, 1e-3));
this.pair = { a, b: b && b.timestamp > a.timestamp + 1e-4 ? b : null };
return this.pair;
}
/** The decoded frame covering source time `at`, null when the source has
* none there. Readers are rebuilt through decoder failures that pass. */
private async wrappedAt(at: number): Promise<WrappedCanvas | null> {
let sink = await this.open();
if (this.still || !sink) return null;
const stepBack = at < this.lastAt - 1e-6;
this.lastAt = at;
for (;;) {
try {
const wrapped =
this.heldBack(at) ||
(stepBack && (await this.backFrame(at))) ||
(await sink.getCanvas(Math.max(0, at)));
this.failStreak = 0;
return wrapped ?? null;
} catch (err) {
// Decoders die mid-job for reasons that pass: a signed link that
// expired under a long render (reopening resolves the URL again), a
// hardware decoder wedged by contention ("Decoding error."). A fresh
// reader recovers both, so rebuild and re-read the same frame, with a
// breath between tries for the decoder to come back. Only a source
// that keeps failing straight off fresh decoders is truly
// unreadable, and that stops the job.
if (++this.failStreak > 3) throw err;
this.software = true;
await new Promise((r) => setTimeout(r, 250 * this.failStreak));
this.input?.dispose();
this.input = null;
this.sink = null;
this.track = null;
this.dropBack();
this.opened = null;
const reopened = await this.open();
if (!reopened) throw err;
sink = reopened;
}
}
}
/**
* The frame covering `at` for a reader stepping down its source, out of
* the backward window — filled first when the window has run out: the
* frames from the keyframe below `at` (or as many as the window holds)
* up to `at`, decoded in one pass on the window's own sink. Null when the
* source has no frame there, and the ordinary read answers.
*/
private async backFrame(at: number): Promise<WrappedCanvas | null> {
const held = this.heldBack(at);
if (held) return held;
if (!this.track) return null;
const kt = await keyframeTimeAt(this.track, Math.max(0, at));
if (kt === null) return null;
const window = this.windowFrames;
this.backSink ??= frameSink(this.track, undefined, this.sinkOptions(window + 3));
const start = Math.max(kt, at - (window - 1) * this.frameDt, 0);
const frames: WrappedCanvas[] = [];
const stream = this.backSink.canvases(start);
// The window runs one frame past the ask: a clip smoothing its slow
// motion reads the frame after the one covering `at` as well, and that
// one beyond keeps the read inside the window.
let covered = false;
try {
for (;;) {
const { value, done } = await stream.next();
if (done || !value) break;
if (value.duration > 0) this.frameDt = value.duration;
frames.push(value);
if (frames.length > window + 1) frames.shift();
if (covered || value.timestamp > at + 1e-4) break;
if (value.timestamp + value.duration > at + 1e-4) covered = true;
}
} finally {
void stream.return(undefined).catch(() => {});
}
this.backFrames = frames;
return (
this.heldBack(at) ??
frames.filter((f) => f.timestamp <= at + 1e-4).pop() ??
frames[frames.length - 1] ??
null
);
}
/** The frame in the backward window covering `at`, if it holds one. */
private heldBack(at: number): WrappedCanvas | null {
return (
this.backFrames.find(
(f) => f.timestamp <= at + 1e-4 && at < f.timestamp + Math.max(f.duration, 1e-4) + 1e-4
) ?? null
);
}
private dropBack(): void {
this.backFrames = [];
this.backSink = null;
this.pair = null;
}
dispose() {
this.input?.dispose();
this.input = null;
this.sink = null;
this.track = null;
this.dropBack();
if (this.still?.kind === "ready") (this.still.image as ImageBitmap).close?.();
this.still = null;
}
}
/** A text layer and the window it is on screen for. The picture is drawn when
* the render reaches it, not up front. */
/** The effect elements live at `t`, deepest lane first — the order the stack
* is walked in. */
function liveEffectsAt(overlays: Overlay[], t: number): EffectOverlay[] {
return overlays
.filter(isEffectOverlay)
.filter((o) => !isAudioEffect(o.effect))
.filter((o) => !o.hidden && overlayLiveAt(o, t))
.sort((a, b) => laneOf(b) - laneOf(a));
}
/** Captions sit above every element and every effect; lane 0 is the topmost
* element row, so they need a place above that. */
const CAPTION_LANE = -1;
interface StampedLayer {
overlay: Overlay;
start: number;
end: number;
/** Where this layer sits in the stack: the element's lane, or -1 for a
* caption, which rides above every element and every effect. */
stackLane: number;
/** Animated element: drawStamps evaluates this at (t − animStart) over
* animDur and applies the result as a canvas transform about the element
* center. Typewriter windows are pre-sliced into per-char layers, so their
* anim carries the remaining slots only. */
anim?: OverlayAnim;
animStart?: number;
animDur?: number;
/** The element this layer came from, before the picture was neutralized or
* a typewriter slice trimmed its text. The per-frame pose is evaluated from
* it, so keys and the element's own rotation/opacity survive. */
source?: Overlay;
/** Lottie sticker: seek this per frame instead of caching one bitmap. */
lottie?: LottieHandle;
/** Pixel-level animation baked into this window's picture (a wipe's
* uncovered share, a per-glyph ramp's position). */
phase?: PaintPhase;
}
/**
* Text overlays and captions, each as one layer with the window it belongs to.
*
* Every distinct look is a layer: a plain cue is one, and a word-highlight cue
* is one per word, since the highlight moves. What is *not* done here is
* turning them into pictures. A full-frame RGBA bitmap is about 8 MB at 1080p,
* and a four-minute subtitled cut has several hundred word windows — decoding
* them all before the first frame is drawn, and holding them for the whole
* render, exhausts the tab on projects the worker handles comfortably.
*
* The layout itself is the same rendering the ffmpeg path burns in, so the
* words land in the same place whichever renderer produced the file.
*/
async function stampText(doc: ExportDoc): Promise<StampedLayer[]> {
const duration = projectDuration(doc);
const layers: StampedLayer[] = [];
for (const o of doc.overlays) {
if (o.hidden || o.start >= duration) continue;
// A blank title has no pixels to draw; shapes and stickers always render.
if (isTextOverlay(o) && !o.text.trim()) continue;
// Behind-the-speaker elements composite in the subject pass, under the
// person; front subject-masked elements stamp normally and the draw
// trims them to the matte.
if (behindSubjectOverlay(o)) continue;
// Effect elements filter the video per frame; nothing to stamp.
if (o.kind === "effect") continue;
// A Lottie sticker seeks per frame; its handle rides the layer.
const lottie =
o.kind === "sticker" && o.lottie && o.assetId
? ((await import("./lottieAssets").then((m) =>
m.sharedLottieHandle(o.assetId!, doc.assets)
)) ?? undefined)
: undefined;
if (isOverlayAnimated(o) || lottie) {
const before = layers.length;
pushAnimatedLayers(layers, o, Math.min(o.end, duration));
for (let i = before; i < layers.length; i++) {
layers[i].stackLane = laneOf(o);
if (lottie) layers[i].lottie = lottie;
}
continue;
}
layers.push({
overlay: o,
start: o.start,
end: Math.min(o.end, duration),
stackLane: laneOf(o),
});
}
if (doc.subtitles.showOnVideo) {
const capStyle = captionStyle(doc.subtitles.style);
// Wrap in design space (1080 short side) from the project ratio — the same
// width the preview passes, whatever the render size.
const designWidth = frameOf(doc.aspect).w;
for (let lane = 0; lane < subtitleLaneCount(doc.subtitles); lane++) {
if (laneHidden(doc.subtitles, lane)) continue;
const cues = laneCues(doc.subtitles, lane);
const pos = trackPos(doc.subtitles, capStyle, lane);
for (let i = 0; i < cues.length; i++) {
const cue = cues[i];
if (cue.start >= duration || !cue.text.trim()) continue;
const windows = doc.subtitles.wordHighlight
? cueWordFrames(cue, capStyle, doc.subtitles)
: [{ start: cue.start, end: cue.end }];
for (let wi = 0; wi < windows.length; wi++) {
const win = windows[wi];
if (win.start >= duration) break;
layers.push({
overlay: cueOverlay(
cue,
capStyle,
i === 0,
pos,
doc.subtitles.wordHighlight ? (win.start + win.end) / 2 : undefined,
designWidth
),
start: win.start,
end: Math.min(win.end, duration),
stackLane: CAPTION_LANE,
});
}
}
}
}
return layers;
}
/**
* Text layer pictures, drawn on demand and released once their window passes.
*
* A render walks time forwards, so only the layers live at the current moment
* are needed — usually a caption and maybe a title. Holding those and dropping
* the rest keeps the memory a subtitled export uses flat in the length of the
* cut instead of growing with every word in it.
*/
/** A layer's picture and where its top-left sits in the frame: (0, 0) for a
* frame-sized picture, the ink box's corner for a cropped one. */
interface Stamp {
image: ImageBitmap;
x: number;
y: number;
}
class StampCache {
private drawn = new Map<StampedLayer, Stamp>();
private maskFrames = new Map<StampedLayer, { t: number; stamp: Stamp }>();
/** The mask pass's scratch for cropped pictures, made on first use. */
private maskScratch: HTMLCanvasElement | null = null;
/** Scratch for blurred, motion-blurred and darkening stamps, made on first
* use. */
readonly fx = new ElementFx(createRasterCanvas);
constructor(
private width: number,
private height: number,
private assets: MediaAsset[]
) {}
async bitmapFor(layer: StampedLayer): Promise<Stamp> {
let stamp = this.drawn.get(layer);
if (!stamp) {
// A layer with no animation holds still, its blur baked into the
// picture; an animated one is neutral, cropped to its own ink so a
// pose that shrinks an oversize element keeps what hung past the
// frame, and softens as it is drawn.
stamp = layer.anim
? await this.cropped(layer.overlay, 0, layer.phase)
: { image: await createImageBitmap(await renderStillPng(layer.overlay, this.width, this.height, this.assets)), x: 0, y: 0 };
this.drawn.set(layer, stamp);
}
return stamp;
}
/** The neutral element painted into a canvas cropped to its ink box,
* however far past the frame that reaches. */
private async cropped(el: Overlay, t: number, phase?: PaintPhase): Promise<Stamp> {
const canvas = createRasterCanvas(1, 1) as unknown as HTMLCanvasElement;
this.maskScratch ??= createRasterCanvas(1, 1) as unknown as HTMLCanvasElement;
const origin = await paintElementCrop(canvas, el, this.width, this.height, cutRenderEnv(this.assets), {
t,
phase,
maskScratch: this.maskScratch,
});
const image = await snapshotRaster(canvas);
canvas.width = canvas.height = 1;
return { image, ...origin };
}
/** The layer's picture drawn for this one moment, for the animations that
* change the pixels as time moves — a keyframed mask, a per-glyph loop.
* There is one live bitmap per layer, replaced as the render walks forward:
* the cached-stamp twin of the Lottie per-frame path. */
async bitmapAt(layer: StampedLayer, tLocal: number, phase?: PaintPhase): Promise<Stamp> {
const hit = this.maskFrames.get(layer);
if (hit && Math.abs(hit.t - tLocal) < 1e-6) return hit.stamp;
const m = layer.overlay.mask;
const el = isMaskAnimated(m)
? { ...layer.overlay, mask: { ...m!, ...maskFrameAt(m!, tLocal), kf: undefined } }
: layer.overlay;
// Drawn once and shown once, so it goes straight from the canvas. A
// dive's pose lives inside its frame-sized picture; everything else is
// cropped to its ink like the cached stamps.
const stamp = phase?.dive
? { image: await snapshotRaster(await renderElementCanvas(el, this.width, this.height, this.assets, phase)), x: 0, y: 0 }
: await this.cropped(el, tLocal, phase);
hit?.stamp.image.close();
this.maskFrames.set(layer, { t: tLocal, stamp });
return stamp;
}
/** Where a diving element flies into, measured once per look. */
diveFocus(o: Overlay): Promise<DiveFocus> {
return measureDiveFocus(o, this.width / this.height, cutRenderEnv(this.assets));
}
/** Release the pictures of layers that have finished by `t`. */
retire(t: number) {
for (const [layer, stamp] of this.drawn) {
if (layer.end <= t) {
stamp.image.close();
this.drawn.delete(layer);
}
}
for (const [layer, hit] of this.maskFrames) {
if (layer.end <= t) {
hit.stamp.image.close();
this.maskFrames.delete(layer);
}
}
}
dispose() {
for (const stamp of this.drawn.values()) stamp.image.close();
this.drawn.clear();
for (const hit of this.maskFrames.values()) hit.stamp.image.close();
this.maskFrames.clear();
if (this.maskScratch) {
this.maskScratch.width = this.maskScratch.height = 1;
this.maskScratch = null;
}
this.fx.dispose();
}
}
/** Head and tail audio ramps for one upper-track clip: its own entrance/exit
* animation, or the transition that owns that edge. Mirrors the ramps the
* export spec carries and the gain the preview applies, so a picture that
* fades up takes its sound with it. */
function overlayRamps(
spans: ClipSpan[]
): { head: number; tail: number; crossIn: number; crossOut: number }[] {
const ramps = spans.map(() => ({ head: 0, tail: 0, crossIn: 0, crossOut: 0 }));
spans.forEach((sp, i) => {
const apply = (anim: typeof sp.clip.animIn, side: "head" | "tail") => {
if (!anim) return;
// A zoom does not change how loud a clip is; every other style ramps.
if (overlayAnimStyle(anim.style) === "zoom") return;
ramps[i][side] = Math.max(ramps[i][side], Math.min(anim.seconds, sp.len));
};
// A transitioned joint owns its edge, so the animation there stands down.
if (!((spans[i - 1]?.transitionOut ?? 0) > 0)) apply(sp.clip.animIn, "head");
if (!(sp.transitionOut > 0)) apply(sp.clip.animOut, "tail");
if (sp.transitionOut > 0 && spans[i + 1]) {
ramps[i].tail = Math.max(ramps[i].tail, sp.transitionOut);
ramps[i + 1].head = Math.max(ramps[i + 1].head, sp.transitionOut);
}
// A cross dissolve is these ramps and nothing else: the picture cuts.
// They stay off `head`/`tail` — a fade ends at silence, a crossing ramps
// equal-power past the clip on the other side of the cut.
if (sp.soundOut > 0 && spans[i + 1]) {
ramps[i].crossOut = sp.soundOut;
ramps[i + 1].crossIn = sp.soundOut;
}
});
return ramps;
}
/**
* The audio ramps a track-0 clip's own entrance and exit carry — the gain
* the preview's frame plan applies: a fade takes the sound down with the
* picture, a zoom, a pop and a slide leave it, and an edge a transition owns
* stands down. The tail gets the room the head left.
*/
function trackZeroFades(spans: ClipSpan[], i: number): { fadeIn: number; fadeOut: number } {
const sp = spans[i];
const ramps = (anim: ClipAnim | undefined, room: number) =>
anim && !["zoom", "pop", "slideleft", "slideright", "slideup", "slidedown"].includes(anim.style)
? Math.min(anim.seconds, Math.max(0, room))
: 0;
const fadeIn = (spans[i - 1]?.transitionOut ?? 0) > 0 ? 0 : ramps(sp.clip.animIn, sp.len);
const fadeOut = sp.transitionOut > 0 ? 0 : ramps(sp.clip.animOut, sp.len - fadeIn);
return { fadeIn, fadeOut };
}
/**
* The mix spec for a doc: track-0 clip audio in sequence, plus everything
* placed at an absolute time — the soundtrack and every upper-track clip's own
* sound, which the preview plays and the file therefore has to carry.
*
* The track-0 fold is sequential, so gaps between free-placed clips have to
* ship as explicit silent spacers. Without them the audio closes up the gaps
* while the picture keeps them, and everything after the first gap plays early.
*/
export function mixSpecFor(doc: ExportDoc, resolve: (asset: MediaAsset) => string): MixSpec {
const duration = projectDuration(doc);
const spans = getClipSpans(doc.clips, doc.assets, 0);
const byId = new Map(doc.assets.map((a) => [a.id, a]));
const items: MixItem[] = [];
for (const track of new Set(overlayLayers(doc.clips).map((c) => c.track))) {
const trackSpans = getClipSpans(doc.clips, doc.assets, track);
const ramps = overlayRamps(trackSpans);
trackSpans.forEach((sp, i) => {
if (sp.clip.hidden || sp.start >= duration) return;
items.push({
file: resolve(sp.asset),
in: sp.clip.in,
out: sp.clip.out,
start: sp.start,
volume: sp.clip.volume ?? 1,
speed: sp.clip.speed,
speedCurve: sp.clip.speedCurve,
reverse: sp.clip.reverse,
sound: sp.clip.sound,
muted: sp.clip.muted || (!sp.sound && assetIsSilent(sp.asset)),
fadeIn: ramps[i].head,
fadeOut: ramps[i].tail,
crossIn: ramps[i].crossIn,
crossOut: ramps[i].crossOut,
soundBack: sp.soundBack,
soundAhead: sp.soundAhead,
...spanSoundSpec(sp, resolve),
});
});
}
for (const a of doc.audioClips) {
const asset = byId.get(a.assetId);
if (!asset || a.hidden || a.start >= duration) continue;
const soundFrom = specSound(asset, byId, resolve);
if (!soundFrom && assetIsSilent(asset)) continue;
items.push({
...(soundFrom ? { soundFrom } : {}),
file: resolve(asset),
in: a.in,
out: a.out,
start: a.start,
volume: a.volume,
speed: a.speed,
speedCurve: a.speedCurve,
reverse: a.reverse,
fadeIn: a.fadeIn,
fadeOut: a.fadeOut,
sound: a.sound,
duck: a.duck,
lane: a.lane ?? 0,
});
}
const spacer = (len: number): MixClip => ({ file: "", in: 0, out: len, muted: true });
const clips: MixClip[] =
spans.length === 0
? [spacer(duration)]
: spanSequence(spans).flatMap(({ gapBefore, span: sp }, i) => [
...(gapBefore > 0 ? [spacer(gapBefore)] : []),
{
file: resolve(sp.asset),
in: sp.clip.in,
out: sp.clip.out,
muted: sp.clip.muted || !!sp.clip.hidden || (!sp.sound && assetIsSilent(sp.asset)),
speed: sp.clip.speed,
speedCurve: sp.clip.speedCurve,
reverse: sp.clip.reverse,
volume: sp.clip.volume,
sound: sp.clip.sound,
...trackZeroFades(spans, i),
transition: sp.transitionOut,
soundCross: sp.soundOut,
soundBack: sp.soundBack,
soundAhead: sp.soundAhead,
...spanSoundSpec(sp, resolve),
},
]);
return {
duration,
clips,
items,
// Audio effect elements treat the finished mix over their own windows.
effects: audioFxSpans(doc.overlays, duration),
};
}
/**
* The mix spec of one stem: the video clips' sound, or one soundtrack lane.
* Everything else is left out, while the duck envelope still comes from the
* whole mix — a music lane printed alone still dips under the voiceover the
* way it does in the mix — and the effect elements treat every stem over
* their own windows.
*/
export function stemMixSpec(spec: MixSpec, stem: StemDef): MixSpec {
const ducks = duckWindows(spec.items);
if (stem.lane === null) {
return { ...spec, items: spec.items.filter((i) => i.lane === undefined), ducks };
}
return {
...spec,
clips: [{ file: "", in: 0, out: spec.duration, muted: true }],
items: spec.items.filter((i) => i.lane === stem.lane),
ducks,
};
}
/** Whether a mix spec has anything audible in it — the same test the mixer
* uses to decide which spans it decodes. */
export function mixHasSound(spec: MixSpec): boolean {
return spec.clips.some((c) => c.file && !c.muted) || spec.items.some((i) => !i.muted);
}
/** Slack, in frames, within which a render's start counts as on the grid. */
const GRID_SLACK = 1e-6;
/** The timeline time of output frame `i` of a render starting at `from`.
* A frame on the timeline grid is exactly k / fps, the time the kit cuts its
* windows at: `i * (1 / fps)` lands a hair early (111 * (1 / 30) is
* 3.6999…97) and would show the window before its own. */
export function exportFrameAt(from: number, i: number, fps: number): number {
const k = Math.round(from * fps);
// A range start off the grid keeps its own frame times.
if (Math.abs(from * fps - k) > GRID_SLACK) {
return from + i / fps;
}
return (k + i) / fps;
}
/**
* Render `doc` to an MP4 in scratch storage.
*
* Frames are produced in order and handed to the muxer as they are drawn, so
* the memory this holds is a handful of decoded frames rather than the render.
* The encoded output goes the same way: the muxer writes each chunk to the
* scratch file and lets it go, so the file on disk is the only whole copy that
* ever exists. The audio is mixed up front, because a single offline pass over
* the whole timeline is both cheaper and more accurate than trying to mix it
* in slices that have to agree at the seams.
*
* The file carries its index at the front, like the ffmpeg path's faststart
* files. That placement normally costs either memory (buffer everything) or a
* second pass (ffmpeg's), but a render knows its exact frame and sample counts
* before it starts, which is enough for the muxer to reserve the index's space
* up front and fill it in at the end.
*/
export async function renderProjectToMp4(
doc: ExportDoc,
settings: ExportSettings,
opts: RenderOptions
): Promise<RenderedExport> {
const { resolve, onProgress, signal } = opts;
const trim = sourceExportPlan(doc, settings);
if (trim && (settings.codec === "h264" || settings.codec === "hevc")) {
const inputs = new Map<string, ReturnType<typeof openMedia>>();
try {
for (const segment of trim) if (!inputs.has(segment.file)) inputs.set(segment.file, openMedia(resolve(doc.assets.find((a) => a.fileName === segment.file)!)));
const copy = await prepareSourceRemux(inputs, trim, settings.codec);
if (copy) {
const dir = await scratchDir();
void sweepScratch(dir);
const name = `source-${crypto.randomUUID()}.mp4`;
const handle = await dir.getFileHandle(name, { create: true });
const writable = await handle.createWritable();
const discard = async () => { await dir.removeEntry(name).catch(() => {}); };
try {
await copy({
write: (data, position) => writable.write({ type: "write", position, data: new Uint8Array(data) }),
truncate: (size) => writable.truncate(size),
}, (ratio) => onProgress?.({ ratio, stage: "video" }), () => signal?.throwIfAborted());
await writable.close();
return { file: await handle.getFile(), discard };
} catch (error) {
await writable.abort().catch(() => {});
await discard();
throw error;
}
}
} finally { for (const input of inputs.values()) input.dispose(); }
}
const duration = projectDuration(doc);
if (!(duration > 0)) {
throw new UserError("export_nothing_yet", "There is nothing to export yet.");
}
// The window of the timeline the file carries: a range when the settings
// name one, the whole cut otherwise. Frames are drawn at timeline time and
// written at file time, so the file starts at zero.
const from = Math.max(0, settings.range?.start ?? 0);
const span = deliverySpan(settings.range, duration);
if (!(span > 0)) {
throw new UserError("export_range_empty", "The export range is empty.");
}
const wanted = deliveryVideoCodec(settings);
if (!wanted) {
throw new UserError("browser_prores_unsupported", "This browser can't encode ProRes.");
}
const codec = await getFirstEncodableVideoCodec([wanted], {
width: settings.width,
height: settings.height,
});
if (!codec) {
const name = settings.codec === "hevc" ? "HEVC" : "H.264";
throw new UserError(
"browser_codec_unsupported",
`This browser can't encode ${name} at ${settings.width} × ${settings.height}.`,
{ codec: name, width: settings.width, height: settings.height },
);
}
const stop = () => {
if (signal?.aborted) throw new DOMException("Export canceled.", "AbortError");
};
onProgress?.({ ratio: 0, stage: "audio" });
const whole = await renderMix(mixSpecFor(doc, resolve), {
sampleRate: settings.audioSampleRate ?? AUDIO_RATE,
channels: settings.audioChannels ?? AUDIO_CHANNELS,
resolve: (file) => file, // mixSpecFor already resolved each asset to a URL
});
const mix = whole && settings.range ? sliceAudio(whole, from, from + span) : whole;
stop();
// A mastered delivery: the mix as it will be heard is measured, moved by
// one gain to its target, and limited only where that gain carries its
// true peak over the ceiling — the same code the engine runs on its mix.
if (mix && settings.loudness !== undefined) {
if (settings.truePeakCeiling === undefined) throw new Error("A mastered export needs its true-peak ceiling.");
await masterInPlace(
Array.from({ length: mix.numberOfChannels }, (_, c) => mix.getChannelData(c)),
{ sampleRate: mix.sampleRate, targetLufs: settings.loudness, ceilingDbtp: settings.truePeakCeiling }
);
stop();
}
const canvas = createRasterCanvas(settings.width, settings.height);
// Elements shorter than an output frame widen onto the one they hold.
const painter = new FramePainter({ ...doc, overlays: holdOverlays(doc.overlays, settings.fps) }, canvas, resolve, settings.fps);
const frameDur = 1 / settings.fps;
const frames = Math.max(1, Math.round(span * settings.fps));
const dir = await scratchDir();
void sweepScratch(dir);
const scratchName = `export-${crypto.randomUUID()}.mp4`;
const scratchHandle = await dir.getFileHandle(scratchName, { create: true });
const writable = await scratchHandle.createWritable();
const discard = async () => {
// An open writer holds a lock removeEntry respects; closing is a no-op
// once the muxer already has.
await writable.close().catch(() => {});
await dir.removeEntry(scratchName).catch(() => {});
};
const output = new Output({
format:
settings.container === "mov"
? new MovOutputFormat({ fastStart: "reserve" })
: new Mp4OutputFormat({ fastStart: "reserve" }),
target: new StreamTarget(writable, { chunked: true }),
});
try {
const video = new CanvasSource(canvas, {
codec,
quality: new Quality({ bitrate: bitrateFor(settings) }),
keyFrameInterval: KEYFRAME_INTERVAL_S,
});
// Reserving the index space needs a packet ceiling per track. The video's
// is exact — one packet per frame; the audio's is the mix's sample count
// over a floor below the samples a packet carries (AAC packs 1024, the
// muxer's own PCM 2048), plus slack for encoder priming. Over-reserving
// costs a few bytes of index; under-reserving fails the finalize.
output.addVideoTrack(video, { frameRate: settings.fps, maximumPacketCount: frames + 8 });
let audio: AudioBufferSource | null = null;
if (mix) {
// PCM is packed by the muxer itself; AAC needs the browser's encoder,
// and a cut with sound keeps it: an Opus stand-in plays nowhere the
// user is taking the file, so a browser without AAC hands the render on.
const audioCodec = await getFirstEncodableAudioCodec([deliveryAudioCodec(settings)], {
numberOfChannels: settings.audioChannels ?? AUDIO_CHANNELS,
sampleRate: settings.audioSampleRate ?? AUDIO_RATE,
});
if (!audioCodec) {
throw new UserError("browser_aac_unsupported", "This browser can't encode AAC audio.");
}
// The mix starts its encoder's priming early, so the edit list trims
// the priming and the sound stays on its picture.
const lead = await encoderLead(audioCodec, mix.sampleRate, mix.numberOfChannels);
audio = new AudioBufferSource(
{
codec: audioCodec,
...(audioCodec === "aac" ? { quality: new Quality({ bitrate: settings.audioBitrate ?? 192_000 }) } : {}),
},
{ startTimestamp: -lead }
);
output.addAudioTrack(audio, { maximumPacketCount: Math.ceil(mix.length / 960) + 32 });
}
await output.start();
// The audio is one buffer for the whole timeline, so it goes in before the
// frames rather than being interleaved with them.
if (audio && mix) await audio.add(mix);
await painter.prepare();
stop();
for (let i = 0; i < frames; i++) {
stop();
const t = i * frameDur;
await painter.drawAt(exportFrameAt(from, i, settings.fps));
await video.add(t, frameDur);
if (i % 15 === 0) {
onProgress?.({ ratio: i / frames, stage: "video" });
// Let the tab breathe: without a yield a long render starves every
// other task on this thread, including the one listening for a cancel.
await new Promise((r) => setTimeout(r, 0));
}
}
onProgress?.({ ratio: 1, stage: "finishing" });
// Finalize fills in the reserved index and closes the scratch file, so the
// handle reads back the finished MP4.
await output.finalize();
return { file: await scratchHandle.getFile(), discard };
} catch (err) {
await output.cancel().catch(() => {});
await discard();
throw err;
} finally {
painter.dispose();
}
}
/**
* Render the cut's stems into one stored zip in scratch storage: a 24-bit WAV
* per stem, each as long as the delivery and starting where it starts, from
* the same fold the export's mix comes from and before any master.
*
* One stem is rendered at a time, and its WAV streams into the archive a
* second at a time, so what this holds is one stem's mix.
*/
export async function renderStemsArchive(
doc: ExportDoc,
settings: ExportSettings,
stems: StemDef[],
opts: Pick<RenderOptions, "resolve" | "signal">
): Promise<RenderedExport> {
const duration = projectDuration(doc);
const from = Math.max(0, settings.range?.start ?? 0);
const span = deliverySpan(settings.range, duration);
const rate = STEM_RATE;
const count = STEM_CHANNELS;
const spec = mixSpecFor(doc, opts.resolve);
// The length the export's own mix comes out at, slice included.
const total = Math.max(1, Math.ceil(duration * rate));
const frames = settings.range
? Math.max(1, Math.min(total, Math.round((from + span) * rate)) - Math.min(total, Math.round(from * rate)))
: total;
const dir = await scratchDir();
void sweepScratch(dir);
const name = `stems-${crypto.randomUUID()}.zip`;
const handle = await dir.getFileHandle(name, { create: true });
const writable = await handle.createWritable();
const discard = async () => {
await writable.close().catch(() => {});
await dir.removeEntry(name).catch(() => {});
};
try {
// A stem's mix renders when its entry is first read; the entry is read
// twice (checksum, then bytes), so the last one rendered is kept.
let held: { stem: StemDef; channels: Float32Array[] } | null = null;
const channelsOf = async (stem: StemDef): Promise<Float32Array[]> => {
if (held?.stem === stem) return held.channels;
held = null;
opts.signal?.throwIfAborted();
const whole = await renderMix(stemMixSpec(spec, stem), { sampleRate: rate, channels: count, resolve: (f) => f });
const cut = whole && settings.range ? sliceAudio(whole, from, from + span) : whole;
const channels = cut ? Array.from({ length: cut.numberOfChannels }, (_, c) => cut.getChannelData(c)) : [];
held = { stem, channels };
return channels;
};
await writeStoredZip(
stems.map((stem) => ({
name: stem.file,
size: wavBytes(frames, count),
read: async function* () {
yield* wavChunks(await channelsOf(stem), frames, count, rate);
},
})),
async (bytes) => {
opts.signal?.throwIfAborted();
await writable.write(bytes as Uint8Array<ArrayBuffer>);
}
);
await writable.close();
return { file: await handle.getFile(), discard };
} catch (err) {
await writable.abort().catch(() => {});
await dir.removeEntry(name).catch(() => {});
throw err;
}
}
/** Draw the text layers live at `t` over the finished picture. */
/** Stamped layers for one animated element: the kit plans the windows (the
* same split its frame sequences use), and the render hangs what only it knows
* on them — the element they came from, and a Lottie handle to seek. */
function pushAnimatedLayers(layers: StampedLayer[], o: Overlay, end: number) {
const shared = {
animStart: o.start,
animDur: Math.max(0.1, o.end - o.start),
source: o,
stackLane: laneOf(o),
};
for (const layer of planAnimatedLayers(o, end)) layers.push({ ...layer, ...shared });
}
async function drawStamps(
canvas: HTMLCanvasElement | OffscreenCanvas,
layers: StampedLayer[],
stamps: StampCache,
t: number,
subject?: SubjectMaskCompositor | null
) {
const ctx = canvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!ctx) return;
for (const layer of layers) {
if (!overlayLiveAt(layer, t)) continue;
if (layer.lottie) {
// Nothing to cache: the animation's own canvas is sought per frame and
// drawn at the sticker's rect under the same delta transform.
const o = layer.overlay as StickerOverlay;
const ev = evalOverlayFrame(
{ ...(layer.source ?? o), anim: layer.anim },
t - (layer.animStart ?? layer.start),
canvas.width / canvas.height
);
if (ev.opacity <= 0.001) continue;
const scale = Math.min(canvas.width, canvas.height) / 1080;
const w = Math.max(1, o.w * canvas.width);
const aspect = layer.lottie.width > 0 ? layer.lottie.width / layer.lottie.height : 1;
const h = w / aspect;
// A blurred, streaking or darkening sticker poses into the scratch
// alone and lands softened or dimmed.
const at = { x: ev.x * canvas.width + ev.dx * scale, y: ev.y * canvas.height + ev.dy * scale };
const lc = stamps.fx.begin(ctx, canvas.width, canvas.height, elementLook(ev, scale, at));
lc.save();
lc.globalAlpha = ev.opacity;
lc.translate(at.x, at.y);
lc.rotate((ev.rotation * Math.PI) / 180);
lc.scale(ev.scale * (ev.sx ?? 1), ev.scale * (ev.sy ?? 1));
lc.drawImage(layer.lottie.seek(t - (layer.animStart ?? layer.start)), -w / 2, -h / 2, w, h);
lc.restore();
stamps.fx.end(ctx);
continue;
}
// Unkeyed, the bitmap bakes the element's own rotation/opacity and only
// the preset's delta composes on top. Keyed, the bitmap is neutral and the
// pose supplies position, scale, rotation and opacity outright — either
// way this samples the evaluator the ffmpeg path rasterized into frames.
const tLocal = t - (layer.animStart ?? layer.start);
const ev = layer.anim
? evalOverlayFrame(
{ ...(layer.source ?? layer.overlay), anim: layer.anim },
tLocal,
canvas.width / canvas.height
)
: null;
// A per-glyph loop moves the characters inside the picture, so its window
// is drawn afresh each frame with the loop folded into the layer's phase.
// So is a dive: a cached picture scaled up fifty times would be mush, so
// each frame draws the type at the size it is seen.
const dive =
ev?.dive && divesAt(layer.anim)
? diveView(ev.dive, await stamps.diveFocus(layer.source ?? layer.overlay), ev, layer.overlay, {
width: canvas.width,
height: canvas.height,
scale: Math.min(canvas.width, canvas.height) / 1080,
})
: undefined;
const livePhase =
ev?.glyphLoop || dive
? {
...layer.phase,
...(ev?.glyphLoop ? { glyphLoop: ev.glyphLoop } : {}),
...(dive && ev ? { dive, pose: ev } : {}),
}
: undefined;
const stamp =
livePhase || isMaskAnimated(layer.overlay.mask)
? await stamps.bitmapAt(layer, tLocal, livePhase ?? layer.phase)
: await stamps.bitmapFor(layer);
// A front subject-masked stamp trims to the pass's current matte — its
// pixels change with the video, so the trim happens at draw time, never
// in the cached raster.
const subjectFront = !!subject && frontSubjectOverlay(layer.overlay);
if (!ev) {
const picture = subjectFront
? subject!.mattedStamp(layer.overlay, canvas.width, canvas.height, (c) =>
c.drawImage(stamp.image, stamp.x, stamp.y)
)
: null;
if (picture) {
ctx.drawImage(picture, 0, 0, canvas.width, canvas.height);
} else {
ctx.drawImage(stamp.image, stamp.x, stamp.y);
}
continue;
}
if (ev.opacity <= 0.001) continue;
const scale = Math.min(canvas.width, canvas.height) / 1080;
const cx = layer.overlay.x * canvas.width;
const cy = layer.overlay.y * canvas.height;
const posed = (c: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) => {
// A dive's pose is already inside its frame-sized picture.
if (dive) {
c.drawImage(stamp.image, 0, 0, canvas.width, canvas.height);
return;
}
c.translate(ev.x * canvas.width + ev.dx * scale, ev.y * canvas.height + ev.dy * scale);
c.rotate((ev.rotation * Math.PI) / 180);
c.scale(ev.scale * (ev.sx ?? 1), ev.scale * (ev.sy ?? 1));
c.translate(-cx, -cy);
c.drawImage(stamp.image, stamp.x, stamp.y);
};
// A blurred, streaking, darkening or tilted element poses into the
// scratch alone and lands softened, dimmed or turned; a sharp one draws
// straight onto the frame.
const look = elementLook(ev, scale, {
x: ev.x * canvas.width + ev.dx * scale,
y: ev.y * canvas.height + ev.dy * scale,
});
const lay = (into: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) => {
const target = stamps.fx.begin(into, canvas.width, canvas.height, look);
target.save();
target.globalAlpha = ev.opacity;
posed(target);
target.restore();
stamps.fx.end(into);
};
if (subjectFront) {
// Pose the stamp inside the matte surface, so the matte stays put in
// frame space while the element travels beneath it. The softening
// lands first and the matte trims after, so the person's edge stays
// sharp over a moving element, as the preview shows it.
ctx.drawImage(subject!.mattedStamp(layer.overlay, canvas.width, canvas.height, lay), 0, 0);
} else {
lay(ctx);
}
}
}
/**
* The cut's picture, drawn at whatever moment is asked for.
*
* Everything a frame needs that outlives the frame lives here: the
* compositor, the open decoders, the stamped text layers, the subject pass.
* A render walks it forward frame by frame; a single capture opens it, draws
* once, and lets it go. Both get the same picture, because it is the same
* code drawing it.
*/
export class FramePainter {
private comp: FrameCompositor;
private stamps: StampCache;
private readers = new Map<string, ClipReader>();
/** What the open readers and the stamp scratch are standing on, so a
* render in the tab shows up beside the preview's own pictures in the
* memory report. */
private readonly releaseMemory = holdMemory("exportReaders", () => {
let n = 0;
for (const r of this.readers.values()) n += canvasBytes(readerPoolFrames(r) * r.sourcePixels);
return n + this.stamps.fx.bytes();
});
/** Frames drawn, which is the clock the readers' eviction order runs on. */
private frameNo = 0;
private behind: SubjectMaskCompositor | null = null;
private fxScratch: RasterSurface | null = null;
/** This frame's removal mattes, one alpha image per keyed clip — fetched
* with the layer frames (readers are async) and read synchronously by the
* compositor's provider mid-draw. */
private matteFrames = new Map<string, CanvasImageSource>();
/** Per-clip conversion of those mattes (luma frame → alpha), run once per
* matte frame. */
private matteAlpha = new Map<string, MatteAlpha>();
/** Text layers, deepest lane first — so a walk of them is a walk up the stack. */
private stacked: StampedLayer[] = [];
private spans: ClipSpan[] = [];
private overlayTracks: number[] = [];
private byTrack = new Map<number, ClipSpan[]>();
private spanOfClip = new Map<string, ClipSpan>();
/** The frames clips shorter than an output frame hold, on track 0 and
* above; empty for a painter drawing loose moments. */
private trackZeroHeld: (HeldFrames | null)[] = [];
private overlayHeld = new Map<ClipSpan, HeldFrames | null>();
private duration: number;
/** `fps` is the output grid of a render, absent for loose moments. */
constructor(
private doc: ExportDoc,
private canvas: RasterSurface,
private resolve: (asset: MediaAsset) => string,
private fps?: number
) {
this.comp = new FrameCompositor(canvas);
this.comp.background = projectBackground(doc.background);
// Every frame of a render goes through its clip's own LUT, built before
// the first frame (prepare) and on this thread if it ever falls out of
// the cache.
this.comp.colorMode = "exact";
this.comp.sourceProvider = sourceLookup(doc.assets, renderFile);
this.stamps = new StampCache(canvas.width, canvas.height, doc.assets);
this.duration = projectDuration(doc);
}
/** Make the text layers, the span geometry and the subject pass resident.
* Runs once, before the first frame. */
async prepare(): Promise<void> {
// Footage the lazy fill has not reached reads its color here, before any
// recipe is built from it.
this.doc = { ...this.doc, assets: await withAssetColors(this.doc.assets, this.doc, this.resolve) };
this.comp.sourceProvider = sourceLookup(this.doc.assets, renderFile);
await settleRenderColor(this.doc.assets);
await ensureClipLuts(this.doc.clips.map((c) => this.comp.recipeFor(c)));
// Every sticker the cut shows is read before the first frame: one that
// cannot be read stops the render by name.
await requireStickers(this.doc.overlays.filter((o) => o.start < this.duration), this.doc.assets);
this.stacked = [...(await stampText(this.doc))].sort((a, b) => b.stackLane - a.stackLane);
// The subject pass is created only when something reads the person matte;
// prepare() makes rasters and the segmenter resident before the first
// frame. Subject-masked clips pull their matte mid-stack through the
// compositor's provider, per frame (no throttling in an export).
if (
hasSubjectOverlays(this.doc.overlays) ||
this.doc.clips.some((c) => c.mask?.kind === "subject")
) {
const behind = new SubjectMaskCompositor();
await behind.prepare(
this.doc.overlays,
this.canvas.width,
this.canvas.height,
this.doc.assets
);
this.behind = behind;
this.comp.subjectMatteProvider = (at) =>
behind.clipMatteOf(this.canvas, at, { minMaskInterval: 0 });
}
// Removal: baked mattes are prefetched per frame into `matteFrames`; the
// provider is a synchronous read of that map. Backdrop stills decode once
// here, so mid-render reads are cache hits.
this.comp.removalMatteProvider = (clip) => this.matteFrames.get(clip.id) ?? null;
this.comp.cardMatteProvider = (clip) => this.matteFrames.get(cardMatteKey(clip.id)) ?? null;
this.comp.backdropImageProvider = (assetId) => backdropStill(this.doc.assets, assetId);
const backdropIds = new Set(
this.doc.clips
.map((c) => c.removal?.backdrop)
.filter((b) => b?.kind === "image" && b.assetId)
.map((b) => b!.assetId!)
);
await Promise.all(
[...backdropIds].map((id) => {
const asset = this.doc.assets.find((a) => a.id === id);
return asset ? loadBackdropStill({ ...asset, url: this.resolve(asset) }) : null;
})
);
// Span geometry is a property of the document, which does not change while
// rendering — so it is computed once rather than per overlay per frame.
this.spans = getClipSpans(this.doc.clips, this.doc.assets, 0);
this.overlayTracks = [...new Set(overlayLayers(this.doc.clips).map((c) => c.track))];
this.byTrack = new Map(
this.overlayTracks.map((track) => [track, getClipSpans(this.doc.clips, this.doc.assets, track)])
);
for (const list of this.byTrack.values())
for (const sp of list) this.spanOfClip.set(sp.clip.id, sp);
if (this.fps) {
const fps = this.fps;
this.trackZeroHeld = heldFrames(this.spans.map((sp) => ({ start: sp.start, end: sp.start + sp.len })), fps);
for (const list of this.byTrack.values())
for (const [sp, held] of heldSpans(list, fps)) this.overlayHeld.set(sp, held);
}
}
/**
* The reader for one asset, opening it if this frame is the first to ask.
*
* Readers are kept between frames because a render walks time forward and
* asks the same clip for thousands of consecutive frames; reopening one per
* frame would pay a keyframe seek for every one of them. They are not kept
* forever: each holds a decoder and a pool of canvases at the render size, so
* a cut drawing from a hundred files would otherwise finish the render
* holding a hundred decoders. Past the budget the least recently drawn are
* let go, which costs one reopen if the cut returns to them.
*/
/** The frame maker for smoothed slow motion, opened on the first frame
* that needs it. Its load — the runtime and the model — is part of the
* render, so a failure there fails the export like a decoder would. */
private synth: Promise<FrameSynth> | null = null;
/**
* A clip's picture at timeline time `t`: its source frame, or, on a clip
* smoothing its slow motion where the rate is under 1×, the frame made
* between the two source frames around that moment. At either edge of the
* pair the real frame shows.
*/
private async frameOf(span: ClipSpan, t: number): Promise<Frame> {
const reader = this.readerFor(span.asset);
const st = sourceTimeAt(span, t);
if (!smoothsAt(span.clip, retimeOf(span.clip), Math.max(0, t - span.start))) return reader.frameAt(st);
const pair = await reader.pairAt(st);
if (!pair) return reader.frameAt(st);
if (!pair.b) return frameOfWrapped(pair.a);
const w = synthWeight(st, pair.a.timestamp, pair.b.timestamp);
if (w <= SYNTH_EDGE) return frameOfWrapped(pair.a);
if (w >= 1 - SYNTH_EDGE) return frameOfWrapped(pair.b);
const synth = await (this.synth ??= exportFrameSynth());
const { width, height } = pair.a.canvas;
const key = `${span.asset.id}|${pair.a.timestamp.toFixed(4)}|${pair.b.timestamp.toFixed(4)}`;
const image = await synth.mid(key, pair.a.canvas, pair.b.canvas, width, height, w);
return { kind: "ready", image, width, height };
}
private readerFor(asset: MediaAsset): ClipReader {
let r = this.readers.get(asset.id);
if (!r) this.readers.set(asset.id, (r = new ClipReader(asset, () => this.resolve(asset))));
r.usedAt = this.frameNo;
return r;
}
/**
* Close the readers this cut has moved on from.
*
* Readers are kept between frames because a render walks time forward and
* asks the same clip for thousands of consecutive frames; reopening one per
* frame would pay a keyframe seek for every one. They are not kept forever:
* each holds a decoder and a pool of canvases at the *source's* own size, so
* a cut drawing from a hundred files would otherwise end the render holding a
* hundred decoders.
*
* This runs between frames, before anything has been asked for, and spares
* everything the last couple of frames used. A frame reaches for several
* readers at once — a join's two sides, a backdrop, a matte, every overlay
* layer — and closing one the frame is midway through would leave the
* compositor drawing from a disposed decoder. Closing them here, past the
* grace, cannot.
*/
private evictReaders(): void {
const cost = (r: ClipReader) => canvasBytes(readerPoolFrames(r) * r.sourcePixels);
const cap = allowance(
"exportReaders",
READERS_TUNED * canvasBytes(READER_POOL * 1920 * 1080)
);
let held = 0;
for (const r of this.readers.values()) held += cost(r);
if (held <= cap) return;
const old = [...this.readers.entries()]
.filter(([, r]) => this.frameNo - r.usedAt > READER_GRACE)
.sort((a, b) => a[1].usedAt - b[1].usedAt);
for (const [id, r] of old) {
if (held <= cap) break;
held -= cost(r);
this.readers.delete(id);
r.dispose();
}
}
/** Pull a clip's baked matte frames for timeline time `t` — its cutout's,
* and its camera card's when the card shows the head — and stage them,
* luma turned to alpha, where the compositor's providers read. A clip
* whose bake is still owed stages nothing. */
private async fetchRemovalMatte(span: ClipSpan, t: number): Promise<void> {
const clip = span.clip;
await this.stageMatte(clip.id, span, t, removalActive(clip.removal) ? clip.removal?.matte : undefined);
await this.stageMatte(cardMatteKey(clip.id), span, t, clip.card?.popOut ? clip.card.matte : undefined);
}
private async stageMatte(
key: string,
span: ClipSpan,
t: number,
m: { assetId: string; in: number } | undefined
): Promise<void> {
this.matteFrames.delete(key);
if (!m) return;
const asset = this.doc.assets.find((a) => a.id === m.assetId);
if (!asset) return;
const dur = Math.max(0.001, asset.duration);
const mt = Math.max(0, Math.min(sourceTimeAt(span, t) - m.in, dur - 0.001));
let alpha = this.matteAlpha.get(key);
if (!alpha) this.matteAlpha.set(key, (alpha = new MatteAlpha()));
// The matte advances at its own baked rate below the export's, so a
// converted frame is reused until the read crosses into the next one.
const stamp = matteStamp(m.assetId, mt);
const held = alpha.held(stamp);
const frame = held ? null : await this.readerFor(asset).frameAt(matteReadTime(mt, dur));
const img = held ?? (frame?.kind === "ready" ? alpha.of(frame, stamp) : null);
if (img) this.matteFrames.set(key, img);
}
/** Draw the whole cut at timeline time `t` onto the canvas. */
async drawAt(t: number): Promise<void> {
this.frameNo++;
this.evictReaders();
const { canvas, comp, doc, stamps } = this;
const W = canvas.width;
const H = canvas.height;
comp.clear();
// A clip holding a frame past its own end is drawn as at its start.
const shown = trackZeroAt(this.spans, this.trackZeroHeld, t);
if (shown) {
const { span: master, at } = shown;
const plan = trackZeroPlan(master, this.spans, at);
await this.fetchRemovalMatte(master, at);
if (plan.incoming) await this.fetchRemovalMatte(plan.incoming, at);
if (plan.backdrop) {
await this.fetchRemovalMatte(plan.backdrop.span, at);
const frame = await this.readerFor(plan.backdrop.span.asset).frameAt(plan.backdrop.at);
comp.drawLayer(frame, plan.backdrop.span.clip, false, 1, at);
}
const masterFrame = master.clip.hidden ? MISSING_FRAME : await this.frameOf(master, at);
const incFrame = plan.incoming ? await this.frameOf(plan.incoming, at) : MISSING_FRAME;
comp.drawCrossJoin(
plan.style,
plan.p,
{
masterFrame,
masterClip: master.clip,
masterAlpha: plan.masterAlpha,
masterZoom: plan.masterZoom,
masterFx: {
dx: plan.masterFxFrac.dx * W,
dy: plan.masterFxFrac.dy * H,
blur: plan.masterFxFrac.blur * Math.min(W, H),
},
incFrame,
incClip: plan.incoming?.clip,
incAlpha: plan.incAlpha,
incZoom: plan.incZoom,
},
at
);
if (plan.veil > 0) comp.fillBlackVeil(plan.veil, rectOf(master.clip));
}
const spansOf = (track: number) => this.byTrack.get(track) ?? [];
for (const layer of overlayPlan(this.overlayTracks, spansOf, t, this.overlayHeld)) {
const span = this.spanOfClip.get(layer.clip.id);
if (!span) continue;
await this.fetchRemovalMatte(span, layer.at);
const frame = await this.frameOf(span, layer.at);
comp.drawIntoRect(
frame,
rectOf(layer.clip),
clipCovers(layer.clip),
layer.alpha,
layer.at,
layer.zoom,
layer.clip
);
}
// The subject pass: video → behind elements → segmented person, exactly
// the preview's pass, sampled per drawn frame (no mask throttling).
// It also refreshes the matte the front subject-masked stamps read.
await this.behind?.ready(doc.overlays, canvas.width, canvas.height, doc.assets, t);
this.behind?.draw(canvas, doc.overlays, doc.assets, t, { minMaskInterval: 0 });
// The stack, bottom up: an effect grades what plays under it, so the
// elements below it burn in first, the effect runs over that much of the
// frame, and the elements above it land on the graded picture. Same
// order as the live preview and the ffmpeg graph.
let drawn = 0;
for (const o of liveEffectsAt(doc.overlays, t)) {
const upto = this.stacked.findIndex((l) => l.stackLane <= laneOf(o));
const end = upto < 0 ? this.stacked.length : upto;
if (end > drawn) await drawStamps(canvas, this.stacked.slice(drawn, end), stamps, t, this.behind);
drawn = Math.max(drawn, end);
this.fxScratch ??= createRasterCanvas(W, H);
applyEffectToCanvas(
canvas,
this.fxScratch,
effectRecipe(o),
o.amount,
t - o.start,
grainTile,
o.focus,
o.ramp,
o.end - o.start,
"opaque",
o
);
}
await drawStamps(canvas, this.stacked.slice(drawn), stamps, t, this.behind);
stamps.retire(t);
}
dispose(): void {
this.stamps.dispose();
this.behind?.dispose();
for (const r of this.readers.values()) r.dispose();
this.readers.clear();
void this.synth?.then((s) => s.dispose()).catch(() => {});
this.synth = null;
this.releaseMemory();
}
}
/**
* One composited frame of the cut, drawn the way a render of it would draw —
* clips, transitions, effects, elements, captions, the project fade. The
* canvas comes off the raster seam, so this answers on a page and in a job.
*/
/** Several frames of one unchanging document, from one prepared painter.
* Preparing stamps every title, decodes every backdrop still and may bring up
* the subject segmenter, so a call that wants four moments pays for that pass
* once. Each frame is handed to `each` before the next overwrites the canvas.
*/
export async function renderProjectFrames(
doc: ExportDoc,
times: number[],
size: { width: number; height: number },
resolve: (asset: MediaAsset) => string,
each: (canvas: RasterSurface, at: number) => Promise<void>
): Promise<void> {
const canvas = createRasterCanvas(size.width, size.height);
const painter = new FramePainter(doc, canvas, resolve);
try {
await painter.prepare();
for (const at of times) {
await painter.drawAt(at);
await each(canvas, at);
}
} finally {
painter.dispose();
}
}
export async function renderProjectFrame(
doc: ExportDoc,
at: number,
size: { width: number; height: number },
resolve: (asset: MediaAsset) => string
): Promise<RasterSurface> {
const canvas = createRasterCanvas(size.width, size.height);
const painter = new FramePainter(doc, canvas, resolve);
try {
await painter.prepare();
await painter.drawAt(at);
} finally {
painter.dispose();
}
return canvas;
}
/**
* Whether this browser can render the export at all. Two facts decide it,
* both about the browser: it has to offer scratch storage, because the render
* writes its file to origin-private disk, and WebCodecs has to offer a video
* encoder at these dimensions. A cut of any length renders in the tab — the
* pipeline streams to disk, so duration costs time, and the answer here has
* to be known up front because a cloud project whose browser can't carry the
* render quietly renders on the worker instead.
*/
export async function canRenderInBrowser(
doc: ExportDoc,
settings: ExportSettings,
/** The color space the file is delivered in: the project's for an export,
* SDR for the web pictures (hover preview, share card, stream). */
output: OutputSpace
): Promise<boolean> {
const duration = projectDuration(doc);
if (!(deliverySpan(settings.range, duration) > 0)) return false;
// The tab composites on 8-bit canvases; an HDR delivery is a 10-bit
// composite, which the engine and the worker build in ffmpeg.
if (output !== "sdr") return false;
if (
typeof navigator.storage?.getDirectory !== "function" ||
typeof FileSystemFileHandle === "undefined" ||
!("createWritable" in FileSystemFileHandle.prototype)
) {
return false;
}
// A master this browser cannot decode — H.264 where none is installed, AAC
// its decoder fails on — renders where ffmpeg reads the original.
for (const asset of doc.assets) {
if (await compatNeedOf(asset)) return false;
}
try {
const trim = sourceExportPlan(doc, settings);
if (trim && (settings.codec === "h264" || settings.codec === "hevc")) {
const inputs = new Map<string, ReturnType<typeof openMedia>>();
try {
for (const segment of trim) if (!inputs.has(segment.file)) inputs.set(segment.file, openMedia(doc.assets.find((a) => a.fileName === segment.file)!.url));
return !!await prepareSourceRemux(inputs, trim, settings.codec); }
finally { for (const input of inputs.values()) input.dispose(); }
}
const wanted = deliveryVideoCodec(settings);
if (!wanted) return false;
const video = await getFirstEncodableVideoCodec([wanted], {
width: settings.width,
height: settings.height,
});
if (!video) return false;
// A cut with sound needs the audio half of the file too; a browser with
// no AAC encoder would otherwise render the whole thing and fail at the
// muxer, or hand back a video whose sound went missing.
if (!mixHasSound(mixSpecFor(doc, (a) => a.url))) return true;
return !!(await getFirstEncodableAudioCodec([deliveryAudioCodec(settings)], {
numberOfChannels: settings.audioChannels ?? AUDIO_CHANNELS,
sampleRate: settings.audioSampleRate ?? AUDIO_RATE,
}));
} catch {
return false;
}
}