Open source CapCut alternative.
TypeScript
"use client";
/**
* Reading media in the browser.
*
* Containers are parsed directly and frames come off WebCodecs, rather than
* being coaxed out of `<video>`/`<audio>` elements. That difference is what
* this module exists for: an element answers questions about media by playing
* it, so every read used to cost a seek, a `seeked` event, and a timeout to
* cover the seek that never lands — and the answers were approximate anyway
* (`HTMLAudioElement.duration` overestimates MP3s, MediaRecorder WebM reports
* `Infinity`, and rotation is invisible). Reading the container gives exact
* answers for the cost of the bytes those answers live in.
*
* Everything here takes a URL or a `Blob`. A URL is read over ranged requests,
* which both backends already serve: the engine's file route answers 206, and
* the cloud's edge parses ranges. Requests go through `fetch`, whose default
* cross-origin mode sends an `Origin` and no credentials — the same request
* `crossOrigin="anonymous"` makes, so reads here share the cache entry the
* decoders use rather than racing them (see mediaCors.ts).
*
* Callers that read once open and dispose around the read. The one caller that
* reads the same file over and over — filmstrip edge frames on a trim drag —
* keeps its own pool of readers, because the decode cache inside a sink is the
* whole point of holding one open.
*/
import {
ALL_FORMATS,
AudioBufferSink,
BlobSource,
CanvasSink,
CustomSource,
EncodedPacketSink,
Input,
UnsupportedInputFormatError,
UrlSource,
type InputAudioTrack,
type InputVideoTrack,
type Rotation,
type WrappedAudioBuffer,
type WrappedCanvas,
} from "mediabunny";
import { confirmEngine, engineConnected, isEngineUrl } from "./api";
import { resolveRegisteredBlob } from "./backend/browser/registry";
import { CHUNK_SIZE, chunkSourceOptions } from "./chunkCache";
import { probeColor, probeMissingColors, rangeOfBlob, withFoundColors, type ColorProbe, type ReadRange } from "./colorProbe";
import { pictureAssetIds, type ItemLists } from "./itemKinds";
import { allowance, holdMemory } from "./memoryBudget";
import "./mediabunnyLog";
import { SpsColorRewriter, descriptionLengthSize, spsCodecOf, tagsOfColorSpace } from "./spsColor";
/** What a file turns out to be, read from its container. */
export interface MediaProbe {
/** Total bytes in the source file, when the source is sized. */
sizeBytes?: number;
/** Sample-exact, from the packets — not the container's rounded metadata. */
duration: number;
hasVideo: boolean;
hasAudio: boolean;
/** Display dimensions: rotation and pixel aspect already applied. */
width?: number;
height?: number;
rotation?: Rotation;
/** What the video's code values mean, from its header (video only). */
color?: ColorProbe;
}
/** Frames read back at a target size. Height alone preserves aspect. */
export interface FrameSize {
width?: number;
height?: number;
fit?: "fill" | "contain" | "cover";
}
export class UnreadableMediaError extends Error {
/** The file needs a demuxer or a decoder this runtime lacks; a converter elsewhere may read it. */
readonly undecodable: boolean;
constructor(message = "Cut can't read this media file.", undecodable = false) {
super(message);
this.name = "UnreadableMediaError";
this.undecodable = undecodable;
}
}
/** Ranged requests kept in the air for one URL.
*
* Widening only pays where a round trip is the cost. The local engine serves
* files off this Mac's disk over HTTP/1.1, where a browser allows six
* connections per origin in total — a handful of clips each asking for eight
* would put the frame being drawn behind readahead for clips nobody is
* looking at. So the local engine and any other plain-HTTP origin keep the
* library's pair, and so does a link that says it is slow or metered. The
* width is for multiplexed cloud media over a real network. */
export function urlParallelism(url: string, cache: number): number {
// Room decides first. Each worker holds the chunk it is reading in the
// source's one cache, so the width a source can afford is how many of those
// fit: past that the workers evict each other's reads and the width buys
// refetches. The pair is the floor — below that a walk stops overlapping its
// own reads at all — and the cache is sized by the memory budget, so a
// machine with less room reads narrower rather than thrashing.
const afforded = Math.max(2, Math.floor(cache / CHUNK_SIZE));
if (!/^https:/i.test(url)) return 2;
const conn =
typeof navigator === "undefined"
? undefined
: (navigator as { connection?: { saveData?: boolean; effectiveType?: string } }).connection;
if (conn?.saveData || /(^|-)2g$|^3g$/.test(conn?.effectiveType ?? "")) return 2;
return Math.min(8, afforded);
}
/** Backoff, in seconds, for a URL read whose request failed outright. The
* list ends, so a URL that is genuinely gone still fails — the preview's
* outage recovery (frameSource.ts) re-opens on its own cadence, and a read
* that hung on retries forever would never let it.
*
* The library's own policy reads a rejected cross-origin fetch as a CORS
* refusal and gives up without a single retry. Every URL Cut opens is
* cross-origin — signed cloud media on its own host, the engine on 127.0.0.1 —
* and each answers this page's requests by configuration, so a rejection is
* the connection dropping, not the browser refusing. Left to the default, one
* flaky minute turned into hard read failures with no retry behind them. */
const URL_RETRY_DELAYS_S = [0.5, 1.5, 4, 10];
/** A refused connection to this Mac is an answer rather than weather: the
* engine's port is listening or it is not. Two quick tries ride out the
* seconds an app update takes to put a new engine on the port, and nothing
* waits longer than that for a machine that is right here. */
const LOCAL_RETRY_DELAYS_S = [0.5, 1.5];
const urlRetryDelay = (
previousAttempts: number,
error: unknown,
src: string | URL | Request
): number | null => {
// A server's refusal is its answer: a gone file or a range past the end of
// an empty one reads the same on the next try.
if (error instanceof MediaFetchError && error.status !== undefined) return null;
// Headless has nothing to hand a giving-up read back to: the engine and the
// render worker open a file once, and a read that quits fails the job. They
// keep waiting, on a widening backoff, until the job's own cancellation
// ends it.
if (headless()) return Math.min(2 ** (previousAttempts - 2), 16);
const url = typeof src === "string" ? src : src instanceof URL ? src.href : src.url;
if (isEngineUrl(url)) {
// Nothing to wait for once the app is gone; the refusal stands until the
// gate connects an engine again.
return engineConnected() ? LOCAL_RETRY_DELAYS_S[previousAttempts - 1] ?? null : null;
}
return URL_RETRY_DELAYS_S[previousAttempts - 1] ?? null;
};
/** A read this module issued that never reached the file. The name is what
* error tracking matches on (instrumentation-client.ts) and what keeps a
* caller's own report of it worded as weather (report.ts). */
class MediaFetchError extends Error {
/** The server's answer, for a read it refused. */
readonly status?: number;
constructor(message: string, cause?: unknown, status?: number) {
super(message, { cause });
this.name = "MediaFetchError";
this.status = status;
}
}
const aborted = (err: unknown): boolean =>
err instanceof DOMException && err.name === "AbortError";
/**
* Wrap a source's own reads: a read of this Mac's media is refused outright
* once the app that serves it is gone, and any other failure comes back named.
*
* A closed app is a fact the page already holds — the ConnectGate has its
* connect modal up and no engine origin resolved — so a source that keeps
* asking is asking a port nothing is listening on. The first failure is what
* establishes it: it asks the engine's health endpoint, and an app that quit
* mid-session drops the connection there, which turns every later read into
* this refusal instead of another request.
*/
function trackedFetch(url: string): typeof fetch {
const engine = !headless() && isEngineUrl(url);
return (input, init) => {
if (engine && !engineConnected()) {
return Promise.reject(new MediaFetchError("The Donkey app isn't running."));
}
return fetch(input, init).then(
(res) => {
// A refusal is named here as well: the reader's readahead rejects with
// nothing awaiting it, and its own wording names neither the read nor
// this module.
if (!res.ok) throw new MediaFetchError(`Cut couldn't read this media (${res.status}).`, undefined, res.status);
return res;
},
(err: unknown) => {
if (aborted(err)) throw err;
if (engine) void confirmEngine();
throw new MediaFetchError("Cut couldn't read this media over the network.", err);
}
);
};
}
/** Open a file for reading. The caller owns it and must `dispose()` it. A URL
* the browser store minted resolves to its backing File and reads as a blob —
* ranged fetches of blob URLs are unreliable across browsers. Signed cloud
* media reads through the chunk cache (chunkCache.ts), so bytes touched once
* are on disk for the next read of the same object. */
/**
* Readers open at once, at the busiest the editor gets.
*
* The preview's decoder pool, the timeline's edge readers, and a render's
* per-clip readers all hold one each. The number is what a per-reader cache has
* to be divided by for the total to stay inside the tab's share: sizing each
* cache as if it were the only one is how a cache with a sensible-looking limit
* becomes a gigabyte.
*/
const READERS = 16;
/**
* Bytes of the file one reader keeps for itself: the reads its own walk just
* made, so a demuxer stepping back over a header it read a moment ago has it
* in hand. The bytes every reader on a file shares are held once, in the
* chunk cache's memory (chunkCache.ts), so this is only what a walk needs
* ahead of and behind its own position. Sixteen of these and the chunk memory
* together fill the reads share.
*/
const READ_CACHE = 8 * 2 ** 20;
/** Readers open right now, for the memory report. */
let openReaders = 0;
/** Decided on the first open and held: the ceiling does not move, so asking
* again would return the same number and re-record the same pressure. Reading
* what the readers hold must not itself count as a request for room. */
let readCache = 0;
const readCacheSize = (): number =>
(readCache ||= Math.round(
allowance("readerCaches", READ_CACHE * READERS) / READERS
));
holdMemory("readerCaches", () => openReaders * readCacheSize());
export function openMedia(src: string | Blob): Input {
const blob = typeof src === "string" ? resolveRegisteredBlob(src) ?? src : src;
let source;
// Every source keeps a cache of what it read, and every one of them is
// given the same size: a reader on this Mac's engine or on a store-minted
// blob holds file bytes in this tab the same as one on the cloud does. A
// cache this small is safe on a URL source because every server the page
// reads a file from answers a range with 206 — the engine's own file route
// and the media origin both do — so a chunk dropped is a chunk refetched;
// it is a server that ignores ranges that puts the source into the
// sequential mode where a read behind the cache throws.
const maxCacheSize = readCacheSize();
if (typeof blob === "string") {
const chunked = chunkSourceOptions(blob);
source = chunked
? new CustomSource({
...chunked,
prefetchProfile: "network",
maxCacheSize,
})
: new UrlSource(blob, {
parallelism: urlParallelism(blob, maxCacheSize),
getRetryDelay: urlRetryDelay,
fetchFn: trackedFetch(blob),
maxCacheSize,
});
} else {
source = new BlobSource(blob, { maxCacheSize });
}
const input = new Input({ formats: ALL_FORMATS, source });
// The cache lives inside the source, where nothing can ask it how big it has
// grown. Counting the readers that own one is what makes the total
// reportable, and the count is only right if it is taken where the reader is
// let go — so the dispose every caller already makes is where it is taken.
openReaders++;
const dispose = input.dispose.bind(input);
let closed = false;
input.dispose = () => {
if (!closed) {
closed = true;
openReaders--;
}
return dispose();
};
return input;
}
/**
* One reader per file, shared by everything reading it.
*
* A cut of forty segments from one recording asks for forty sources, and
* each would otherwise parse the same file and keep its own cache of the
* same bytes. Holders on a URL share one input — the parse, the reader's
* cache, the open file — and the last to let go closes it. The sinks and
* decoders stay per holder: a track serves any number of them. A store
* blob URL is a URL here too, since it names one file for as long as it is
* registered. A holder releases exactly once; a second release is nothing.
*/
export type MediaHandle = { input: Input; release: () => void };
const sharedInputs = new Map<string, { input: Input; refs: number }>();
export function openMediaShared(src: string): MediaHandle {
let entry = sharedInputs.get(src);
if (!entry) {
entry = { input: openMedia(src), refs: 0 };
sharedInputs.set(src, entry);
}
const held = entry;
held.refs++;
let released = false;
return {
input: held.input,
release: () => {
if (released) return;
released = true;
if (--held.refs > 0) return;
if (sharedInputs.get(src) === held) sharedInputs.delete(src);
held.input.dispose();
},
};
}
/** Run `fn` against an open input and dispose it however that ends. */
export async function withMedia<T>(src: string | Blob, fn: (input: Input) => Promise<T>): Promise<T> {
const input = openMedia(src);
try {
return await fn(input);
} finally {
input.dispose();
}
}
/** Read the source's byte length without decoding its contents. */
export function readMediaFileSize(src: string | Blob): Promise<number> {
return withMedia(src, (input) => input.source.getSize());
}
// A headless runtime (the turn runner) has no page, and there the
// decodability gate is not a gate: nothing plays in that process, rendering
// goes through ffmpeg, and the probe only reads container metadata — which
// parses fine. Tracks pass headless; every browser keeps the strict check,
// including one without WebCodecs, where canDecode answers false and the
// undecodable-import guard still refuses footage the page can't play.
const headless = () => typeof document === "undefined";
/** A file the container parser doesn't recognize reads back as this module's
* own error. mediabunny's own text names no file and reads like a bug report,
* and it reaches the user wherever a read runs — a caption run, an export, a
* chat tool. It is marked undecodable: ffmpeg reads far more containers, so an
* import hands the file to a converter. */
async function tracked<T>(read: Promise<T>): Promise<T> {
try {
return await read;
} catch (err) {
if (err instanceof UnsupportedInputFormatError) throw new UnreadableMediaError(undefined, true);
throw err;
}
}
/** The primary video track, or null when the file has no readable one — either
* no video at all, or video in a codec this browser can't decode. Callers that
* need to tell those apart ask `hasUndecodableVideo`. */
export async function videoTrackOf(input: Input): Promise<InputVideoTrack | null> {
const track = await tracked(input.getPrimaryVideoTrack());
if (!track) return null;
return (await decodable(track)) ? track : null;
}
/** Whether this runtime decodes the track. A ProRes track has no WebCodecs
* decoder anywhere; the WASM one is registered on the first ask and answers
* for every reader after — the preview of a master whose proxy has not
* landed, the in-tab export, which always reads the master. */
async function decodable(track: InputVideoTrack): Promise<boolean> {
if (headless()) return true;
// canDecode parses the stream's own header to build the decoder config, and
// a corrupt one throws from the parser ("Invalid exponential-Golomb code.").
// No decoder here opens that track, which is what false says.
if (await track.canDecode().catch(() => false)) return true;
if (track.codec !== "prores") return false;
await (await import("./proresDecoder")).ensureProresDecoder();
return track.canDecode();
}
/** True when the file carries video this browser cannot decode.
*
* The difference matters at import: a file with no video track is audio, and
* calling it audio is right. A camera file in 10-bit HEVC also yields no
* readable video track, and calling *that* audio drops the user's footage onto
* the timeline as a waveform with no explanation. */
export async function hasUndecodableVideo(input: Input): Promise<boolean> {
if (headless()) return false;
const track = await tracked(input.getPrimaryVideoTrack());
return !!track && !(await decodable(track));
}
/** The primary video track when it is H.264 in a browser with no H.264
* decoder at all (Firefox on Linux without the system's codecs). Its picture
* plays from a compat proxy (mediaProxy.ts), so the master imports as it is;
* a converter would only write H.264 again. A header the parser cannot read
* throws from canDecode, and that file goes to the converters. */
async function coverableVideo(input: Input): Promise<InputVideoTrack | null> {
if (headless()) return null;
const track = await tracked(input.getPrimaryVideoTrack());
if (!track || track.codec !== "avc") return null;
if (await track.canDecode().catch(() => true)) return null;
const { browserCodecs } = await import("./codecSupport");
return (await browserCodecs()).h264 ? null : track;
}
/** The primary audio track, or null — same decodability rule as video. */
export async function audioTrackOf(input: Input): Promise<InputAudioTrack | null> {
return tracked(input.getPrimaryAudioTrack({
filter: (track) => headless() ? track.codec !== null : track.canDecode().catch(() => false),
}));
}
/** Read what a file is: how long, whether it carries picture or sound, and at
* what size. Throws `UnreadableMediaError` for a container Cut can't parse, so
* a bad drop fails at the door with something to say instead of becoming an
* asset that plays black. */
export async function probeMediaFile(src: string | Blob): Promise<MediaProbe> {
return withMedia(src, async (input) => {
if (!(await input.canRead())) throw new UnreadableMediaError(undefined, true);
const [decoded, audio] = await Promise.all([videoTrackOf(input), audioTrackOf(input)]);
const video = decoded ?? (await coverableVideo(input));
// Footage this browser can't decode is refused outright. Treating it as
// audio would be a quieter failure, not a smaller one: the clip would look
// imported and simply have no picture.
if (!video && (await hasUndecodableVideo(input))) {
throw new UnreadableMediaError(
"This video is in a format Cut can't decode in this browser.",
true
);
}
if (!audio && (await tracked(input.getAudioTracks())).length) {
throw new UnreadableMediaError("This file's audio needs conversion.", true);
}
if (!video && !audio) throw new UnreadableMediaError();
// The color walk runs beside the rest of the probe: it is a few ranged
// reads of its own, and waiting on them in turn would hold every caller
// that writes what the probe found.
const color = video ? input.source.getSizeOrNull().then((bytes) => probeMediaColorOnce(src, bytes)) : null;
// Awaited below; a probe that fails before then fails on its own.
color?.catch(() => {});
const [duration, bytes] = await Promise.all([input.computeDuration(), input.source.getSizeOrNull()]);
const size = bytes === null ? {} : { sizeBytes: bytes };
if (!video || !color) return { ...size, duration, hasVideo: false, hasAudio: true };
const [width, height, rotation] = await Promise.all([
video.getDisplayWidth(),
video.getDisplayHeight(),
video.getRotation(),
]);
return { ...size, duration, hasVideo: true, hasAudio: !!audio, width, height, rotation, color: await color };
});
}
/** Color walks in flight, by source: an import's probe and the lazy fill of
* the same file share one walk, and so one write of what it found. */
const colorWalks = new Map<string | Blob, Promise<ColorProbe>>();
function probeMediaColorOnce(src: string | Blob, bytes?: number | null): Promise<ColorProbe> {
let walk = colorWalks.get(src);
if (!walk) {
const size = bytes ?? (typeof src === "string" ? Infinity : src.size);
walk = probeColor(rangeReader(src), size).finally(() => colorWalks.delete(src));
colorWalks.set(src, walk);
}
return walk;
}
/** Read a video's color from its header alone: the same walk the import
* probe makes, for media imported before the probe existed. */
export function probeMediaColor(src: string | Blob): Promise<ColorProbe> {
return probeMediaColorOnce(src);
}
/** `assets` with the color record of every video the timeline draws filled
* from its header: what a render or an export spec reads before its first
* frame, so footage the lazy fill has not reached yet is not drawn as
* Rec.709. A drawn file whose header cannot be read fails the call, naming
* it; files nothing on the timeline uses are not read. */
export async function withAssetColors<
A extends { id: string; type: string; fileName: string; url: string; color?: unknown; block?: unknown },
>(assets: A[], lists: Pick<ItemLists, "clips" | "overlays">, urlOf: (asset: A) => string = (a) => a.url): Promise<A[]> {
return withFoundColors(assets, await probeMissingColors(assets, pictureAssetIds(lists), (a) => probeMediaColor(urlOf(a))));
}
/** Ranged byte reads over what `openMedia` reads: a Blob (or the File behind
* a store-minted URL) by slicing, a URL by a Range request through the same
* fetch the readers use. A server that answers the whole file is sliced. */
function rangeReader(src: string | Blob): ReadRange {
const blob = typeof src === "string" ? (resolveRegisteredBlob(src) ?? src) : src;
if (typeof blob !== "string") return rangeOfBlob(blob);
const read = trackedFetch(blob);
return async (offset, length) => {
const res = await read(blob, {
mode: "cors",
headers: { Range: `bytes=${offset}-${offset + length - 1}` },
});
const bytes = new Uint8Array(await res.arrayBuffer());
return res.status === 206 ? bytes : bytes.subarray(offset, offset + length);
};
}
/** What every frame reader in Cut asks of a video track: one frame at a
* time, a span in order, or a set of moments in one pass. mediabunny's
* CanvasSink is this in a browser; a headless process installs its own,
* because CanvasSink draws through `VideoFrame`, which Node has no. */
export interface FrameCanvasSink {
getCanvas(timestamp: number): Promise<WrappedCanvas | null>;
canvases(start?: number, end?: number): AsyncGenerator<WrappedCanvas, void, unknown>;
canvasesAtTimestamps(
timestamps: Iterable<number> | AsyncIterable<number>
): AsyncGenerator<WrappedCanvas | null, void, unknown>;
}
export interface FrameSinkOptions {
poolSize?: number;
/** Ask the decoder to emit each frame as soon as it is decoded. Interactive
* readers — the preview, a trim drag — set this: Windows hardware decoders
* otherwise hold output until their input queue runs deep, and frames arrive
* in bursts with stalls between them. Batch readers leave it off and keep
* the deeper pipeline's throughput. */
lowLatency?: boolean;
/** Prefer the software decoder. Hardware decoder sessions are a finite
* machine resource; a reader rebuilt after losing one asks for software,
* which always opens. */
software?: boolean;
/** Tag the decoded frames with this color space in place of the file's own,
* so the browser draws the code values untouched: no tone mapping of an
* HDR file, no gamut conversion of a wide one. The grade pipeline then
* reads the meaning from the asset's `color`. Reaches WebCodecs as the
* decoder config's `colorSpace` and is stamped on the frames a WASM decoder
* produces. An H.264 or HEVC decoder reads its tags from the bitstream's
* own SPS, so the same tags are written into the SPS inside the codec
* description and into any SPS a sample carries in-band, the way ffmpeg's
* metadata filters do. What a route then draws is measured once per session
* by `decodePath` (decoderCheck.ts). */
colorSpace?: VideoColorSpaceInit;
}
/** Decoder hooks that write `colorSpace` into the bitstream's own SPS tags
* for H.264 and HEVC; empty for every other codec. */
export function spsHooks(colorSpace: VideoColorSpaceInit): {
transformConfig?: (config: VideoDecoderConfig) => VideoDecoderConfig;
transformPacket?: (data: Uint8Array, config: VideoDecoderConfig) => Uint8Array | null;
} {
const tags = tagsOfColorSpace(colorSpace);
let rewriter: SpsColorRewriter | null = null;
let lengthSize = 4;
return {
transformConfig: (config) => {
const codec = spsCodecOf(config.codec);
if (!codec || !config.description) return config;
rewriter = new SpsColorRewriter(codec, tags);
const desc = bytesOf(config.description);
lengthSize = descriptionLengthSize(codec, desc);
return { ...config, description: rewriter.description(desc) };
},
transformPacket: (data) => rewriter?.sample(data, lengthSize) ?? null,
};
}
const bytesOf = (buffer: AllowSharedBufferSource): Uint8Array =>
ArrayBuffer.isView(buffer)
? new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
: new Uint8Array(buffer);
export type FrameSinkFactory = (
track: InputVideoTrack,
size?: FrameSize,
opts?: FrameSinkOptions
) => FrameCanvasSink;
const canvasSinkFactory: FrameSinkFactory = (track, size, opts) => {
const decoderOptions = {
...(opts?.lowLatency ? { optimizeForLatency: true } : {}),
...(opts?.software ? { hardwareAcceleration: "prefer-software" as const } : {}),
...(opts?.colorSpace ? { colorSpace: opts.colorSpace, ...spsHooks(opts.colorSpace) } : {}),
};
return new CanvasSink(track, {
...size,
...(opts?.poolSize ? { poolSize: opts.poolSize } : {}),
...(Object.keys(decoderOptions).length ? { decoderOptions } : {}),
});
};
let sinkFactory: FrameSinkFactory = canvasSinkFactory;
/** Install a replacement frame reader, e.g. the skia-backed one a headless
* process uses; null puts the page's own back. Affects every later
* `frameSink` call. */
export function setFrameSinkFactory(f: FrameSinkFactory | null): void {
sinkFactory = f ?? canvasSinkFactory;
}
/** True once a replacement frame reader is installed: frames then come from
* the process's own decoder, drawn through the file's own matrix. */
export function frameSinkIsCustom(): boolean {
return sinkFactory !== canvasSinkFactory;
}
/** A sink that draws this track's frames at `size`. Rotation from the file's
* metadata is applied by default, so a phone clip comes back upright and no
* consumer has to know it was ever sideways. A requested size is capped at the
* track's own: a sampler asking 480 wide of a 360-wide file pays for the extra
* pixels and learns nothing from them. */
export function frameSink(
track: InputVideoTrack,
size?: FrameSize,
opts?: FrameSinkOptions
): FrameCanvasSink {
return fromFirstFrame(track, sinkFactory(track, capToTrack(track, size), opts));
}
/**
* A sink whose point reads never fall off the front of the track.
*
* A phone recording's first video frame often sits a few hundredths of a
* second after zero, and the underlying sink answers a timestamp before the
* first frame with null. A clip at the head of the timeline would then draw
* nothing for its first frames and pop in late, which the preview never
* shows because it streams from the nearest frame. Clamping the ask up to the
* first frame makes a read at the head land on the head.
*/
function fromFirstFrame(track: InputVideoTrack, sink: FrameCanvasSink): FrameCanvasSink {
let first: Promise<number> | null = null;
const floor = () => (first ??= track.getFirstTimestamp().catch(() => 0));
const clamp = async (t: number) => Math.max(await floor(), t);
return {
getCanvas: async (t) => sink.getCanvas(await clamp(t)),
canvases: (start, end) => sink.canvases(start, end),
canvasesAtTimestamps: (times) =>
sink.canvasesAtTimestamps(
(async function* () {
for await (const t of times) yield await clamp(t);
})()
),
};
}
function capToTrack(track: InputVideoTrack, size?: FrameSize): FrameSize | undefined {
if (!size) return size;
// A backing that resolves its metadata asynchronously — HLS, a segmented
// input — refuses the synchronous read. The sink is sync, so those sources
// decode at the size asked for.
let tw: number;
let th: number;
try {
tw = track.displayWidth;
th = track.displayHeight;
} catch {
return size;
}
return {
...size,
...(size.width !== undefined ? { width: Math.min(size.width, tw) } : {}),
...(size.height !== undefined ? { height: Math.min(size.height, th) } : {}),
};
}
/** Each stream a compat proxy covers in this page (mediaProxy.ts), by its
* master's address: the readers of that stream open the proxy. A render's
* clip reader opens the master itself, so an export reads the original. */
const streamCovers = { video: new Map<string, string>(), audio: new Map<string, string>() };
/** Replace the covers this page reads through: master address → proxy
* address, per stream. */
export function setStreamCovers(covers: { video: Map<string, string>; audio: Map<string, string> }): void {
streamCovers.video = covers.video;
streamCovers.audio = covers.audio;
}
/** Where this page reads `stream` of `src` from. */
export function coveredSrc<T extends string | Blob>(stream: "video" | "audio", src: T): T | string {
return typeof src === "string" ? (streamCovers[stream].get(src) ?? src) : src;
}
/**
* Frames at the given times, in one decode pass.
*
* Sorted timestamps decode each packet at most once, which is what makes a
* filmstrip or a contact sheet a single sweep of the file instead of N seeks
* into it. Yields null for a time the track has no frame for.
*/
export async function* framesAt(
src: string | Blob,
times: number[],
size?: FrameSize,
opts?: FrameSinkOptions
): AsyncGenerator<WrappedCanvas | null> {
const input = openMedia(coveredSrc("video", src));
try {
const track = await videoTrackOf(input);
if (!track) throw new UnreadableMediaError("This file has no readable video.");
yield* frameSink(track, size, opts).canvasesAtTimestamps(times);
} finally {
input.dispose();
}
}
/** When the keyframe at or before `t` starts. Read from the file's index —
* nothing decodes — so a scrub can find the one frame that answers in a
* single-packet decode. Null when the track has no keyframe there. */
export async function keyframeTimeAt(track: InputVideoTrack, t: number): Promise<number | null> {
const packet = await new EncodedPacketSink(track).getKeyPacket(t, { verifyKeyPackets: true });
return packet ? packet.timestamp : null;
}
/** A single frame at `time`, at native size unless one is given. */
export async function frameAt(
src: string | Blob,
time: number,
size?: FrameSize
): Promise<WrappedCanvas | null> {
return withMedia(coveredSrc("video", src), async (input) => {
const track = await videoTrackOf(input);
if (!track) throw new UnreadableMediaError("This file has no readable video.");
return frameSink(track, size).getCanvas(Math.max(0, time));
});
}
/**
* The file's audio, a buffer at a time, over `[from, to)`.
*
* Streaming is the point: a waveform or a silence scan reads a chunk, folds it
* into a running result, and drops it, so the peak memory is one buffer rather
* than a whole decoded file. Yields nothing when the file has no audio.
*/
export async function* audioChunks(
src: string | Blob,
from?: number,
to?: number
): AsyncGenerator<WrappedAudioBuffer> {
const input = openMedia(coveredSrc("audio", src));
try {
const track = await audioTrackOf(input);
if (!track) return;
yield* new AudioBufferSink(track).buffers(from, to);
} finally {
input.dispose();
}
}
/**
* One buffer holding `[from, to)` at the source's own rate and channel count,
* or null when the file has no audio.
*
* For callers that need random access to a span — resampling it, rendering it
* out — where streaming would only mean assembling this by hand. The span is
* what's decoded, so asking for thirty seconds of an hour-long file costs
* thirty seconds.
*/
export async function decodeAudioSpan(
src: string | Blob,
from = 0,
to?: number,
read: typeof audioChunks = audioChunks
): Promise<AudioBuffer | null> {
const chunks: WrappedAudioBuffer[] = [];
// A compressed frame leans on the one before it — an MP3's bit reservoir,
// AAC's overlap, Opus's pre-skip — so the first frame a decode opens on
// comes out near-silent. Opening a little early puts that frame before the
// span, and assembleAudio trims it away.
for await (const chunk of read(src, Math.max(0, from - AUDIO_PREROLL), to)) chunks.push(chunk);
return assembleAudio(chunks, from, to);
}
/** How far before a span its decode opens: past Opus's recommended 80 ms
* pre-roll, and several MP3 or AAC frames. */
const AUDIO_PREROLL = 0.12;
/**
* Lay decoded chunks into one buffer covering `[from, to)`, or null when there
* are none.
*
* The buffer begins exactly at `from`, which is what callers assume when they
* play it from 0. Decoding lands on packet boundaries, so the first chunk
* usually starts a little earlier; that head is trimmed rather than being
* allowed to shift the whole span early.
*/
export function assembleAudio(
chunks: WrappedAudioBuffer[],
from: number,
to?: number
): AudioBuffer | null {
if (chunks.length === 0) return null;
const sampleRate = chunks[0].buffer.sampleRate;
const numberOfChannels = Math.max(...chunks.map((c) => c.buffer.numberOfChannels));
const last = chunks[chunks.length - 1];
const start = from;
const end = Math.min(to ?? Infinity, last.timestamp + last.duration);
const length = Math.max(1, Math.round((end - start) * sampleRate));
const out = new AudioBuffer({ length, numberOfChannels, sampleRate });
for (const { buffer, timestamp } of chunks) {
// A chunk can start before the requested span, since decoding lands on
// packet boundaries. The negative offset trims its head rather than
// shifting everything after it late.
const offset = Math.round((timestamp - start) * sampleRate);
const head = Math.max(0, -offset);
const at = Math.max(0, offset);
for (let ch = 0; ch < numberOfChannels; ch++) {
const samples = buffer.getChannelData(Math.min(ch, buffer.numberOfChannels - 1));
const count = Math.min(samples.length - head, length - at);
if (count > 0) out.copyToChannel(samples.subarray(head, head + count), ch, at);
}
}
return out;
}
/** A reader part-way through one file's audio: `next` hands back the buffers
* in the order they play and answers null at the end of the span, and `close`
* gives up the file however far it got. */
export interface AudioWalk {
next(): Promise<WrappedAudioBuffer | null>;
/** The source time the next read comes from. A caller that has moved can ask
* whether this walk is standing where it wants to read. */
readonly position: number;
/** Re-aim the walk at `from` without giving up the file. Opening one is
* nearly all a walk costs — the container read out of a hundred megabytes
* over a link — and a caller that has to re-aim has every reason not to pay
* it twice: the clock ran past what this walk was reading, a seek landed
* somewhere else in the same file. The next read starts there. */
seek(from: number): void;
close(): void;
}
/**
* Walk a file's audio forward from `from`, a buffer at a time.
*
* Asking for a span at a time is what a caller reaches for first, and over a
* network it is the wrong shape: audio packets are interleaved with the
* picture through the whole file, so twenty seconds of sound is scattered
* across twenty seconds of *everything* — a hundred megabytes of a phone
* recording to fill a few hundred kilobytes of samples. A reader asking that
* far ahead is asking for bytes the link has not reached yet, and it goes on
* asking for them for as long as it plays, which is a sound that arrives after
* the moment it was for.
*
* A walk reads the file once, in order, at the pace the caller pulls, so the
* sound is exactly as far into the file as the picture is. It is the same
* shape the frame readers hold, and it is what lets one open file, one
* container parse and one decoder serve a whole clip.
*/
export function openAudioWalk(master: string | Blob, from = 0, to?: number): AudioWalk {
const src = coveredSrc("audio", master);
// A voice on a URL reads the file its picture's source already holds open.
const handle =
typeof src === "string"
? openMediaShared(src)
: { input: openMedia(src), release: () => void 0 };
const input = handle.input;
const close = typeof src === "string" ? handle.release : () => input.dispose();
let closed = false;
let finding: Promise<InputAudioTrack | null> | null = null;
let reader: AsyncGenerator<WrappedAudioBuffer, void, unknown> | null = null;
/** Where the next read comes from, and whether a reader stands at it. */
let at = from;
let aimed = false;
const stop = (gen: AsyncGenerator<WrappedAudioBuffer, void, unknown> | null) => {
// Returning a generator is where the decoder it holds is released.
if (gen) void gen.return().catch(() => {});
};
const open = async () => {
const track = await (finding ??= audioTrackOf(input));
if (!track || closed) return null;
if (!aimed) {
const old = reader;
reader = new AudioBufferSink(track).buffers(at, to);
aimed = true;
stop(old);
}
return reader;
};
return {
get position() {
return at;
},
async next() {
if (closed) return null;
const gen = await open();
if (!gen || closed) return null;
const step = await gen.next();
if (step.done) return null;
at = step.value.timestamp + step.value.duration;
return step.value;
},
seek(next) {
at = next;
aimed = false;
},
close() {
if (closed) return;
closed = true;
// The file goes at once behind the reader: disposing is what cancels a
// read still in the air, and a walk being closed because it stopped
// answering is waiting on exactly one of those.
stop(reader);
reader = null;
close();
},
};
}
/** How often a running peaks decode publishes what it has folded so far. */
const PEAKS_PUBLISH_MS = 250;
/**
* Normalized 0..1 waveform peaks across the whole file, `buckets` wide.
*
* Folded chunk by chunk, so the cost is the decode rather than the decode plus
* a copy of the file in memory — an hour of audio used to mean holding an hour
* of float samples to draw a strip a few hundred pixels wide.
*
* The decode walks the file front to back, and a file that has to come over
* the network walks at the link's pace, so `onPartial` hands out the buckets
* folded so far as the walk runs: the strip is the full width from the first
* chunk — the moments the walk has not reached yet sit flat — and fills in
* left to right until the returned array is the whole read.
*/
/** What a long read consults to know something more urgent — a play of this
* same file — wants the decoder and the link back. */
export interface IdleGate {
busy: () => boolean;
idle: () => Promise<void>;
}
export async function audioPeaks(
src: string | Blob,
buckets: number,
onPartial?: (peaks: number[]) => void,
gate?: IdleGate
): Promise<number[]> {
// Peaks are a real decode; a headless runtime skips them and the waveform
// enriches the next time a browser holds the asset.
if (typeof AudioDecoder === "undefined") return [];
const peaks = new Array<number>(buckets).fill(0);
let published = 0;
// Where the next pass picks up. A play ends the walk rather than suspending
// it — a suspended walk keeps the file, the decoder and the buffers it had
// in flight, which is what the preview was short of — so the read resumes
// from the second it had reached.
let from = 0;
for (;;) {
await gate?.idle();
const input = openMedia(coveredSrc("audio", src));
let read = 0;
let done = true;
try {
const track = await audioTrackOf(input);
if (!track) return [];
// Read the length first so each chunk can be folded into its bucket and
// dropped; without it the buckets aren't known until the last sample, and
// the whole file has to be held to place any of it.
const duration = await input.computeDuration();
if (!(duration > 0)) return [];
for await (const { buffer, timestamp } of new AudioBufferSink(track).buffers(from)) {
const data = buffer.getChannelData(0);
const rate = buffer.sampleRate;
// Every 8th sample, matching what the whole-file pass measured.
for (let i = 0; i < data.length; i += 8) {
const v = Math.abs(data[i]);
const at = (timestamp + i / rate) / duration;
const bucket = Math.min(buckets - 1, Math.max(0, Math.floor(at * buckets)));
if (v > peaks[bucket]) peaks[bucket] = v;
}
read++;
from = timestamp + buffer.duration;
if (gate?.busy()) {
done = false;
break;
}
if (!onPartial) continue;
const now = Date.now();
if (now - published < PEAKS_PUBLISH_MS) continue;
published = now;
onPartial(peaks.slice());
}
} finally {
input.dispose();
}
// Read to the end, or a pass that got nothing — an unreadable file, a walk
// that died — which another pass would only repeat.
if (done || read === 0) return peaks;
}
}