Skip to content

Code-first video & image, rendered natively on WebGPU.

19.2 KB
Raw Blame
import {
	AudioBufferSink,
	CanvasSink,
	type Input,
	type VideoSample,
	VideoSampleSink,
	type WrappedAudioBuffer,
	type WrappedCanvas,
} from "mediabunny";
import { premultiplyPixels } from "../color.js";
import { inputStore } from "../input-store.js";

/**
 * Base class for media decoders (Video/Audio) that manages shared Input resources
 * and provides robust, cached iterator access with seek-ahead invalidation.
 */
abstract class BaseMediaDecoder<T> {
	protected input: Input | null = null;
	protected initPromise: Promise<void> | null = null;
	protected iterator: AsyncGenerator<T, void, unknown> | null = null;
	protected iteratorStartTimestamp = -1;
	protected currentFetchId = 0;
	protected fetchLock: Promise<void> = Promise.resolve();
	protected destroyed = false;

	constructor(
		protected url: string,
		protected isHeadless: boolean,
	) {}

	async init(): Promise<void> {
		if (this.input) return;
		if (this.initPromise) return this.initPromise;

		this.destroyed = false;

		this.initPromise = (async () => {
			try {
				const input = await inputStore.acquire(this.url);
				if (this.destroyed) {
					inputStore.release(this.url);
				} else {
					this.input = input;
				}
			} catch (err) {
				this.initPromise = null;
				throw err;
			}
		})();

		return this.initPromise;
	}

	destroy(): void {
		this.destroyed = true;
		void this.iterator?.return();
		if (this.input) {
			inputStore.release(this.url);
			this.input = null;
		}
		this.initPromise = null;
		this.onDestroy();
	}

	protected abstract onDestroy(): void;
	protected abstract startIterator(
		timestampSec: number,
		fetchId: number,
	): Promise<void>;
}

export interface VideoFrameData {
	width: number;
	height: number;
	canvas?: HTMLCanvasElement | OffscreenCanvas;
	buffer?: Uint8Array;
}

export class VideoDecoder extends BaseMediaDecoder<
	WrappedCanvas | VideoSample
> {
	private canvasSink: CanvasSink | null = null;
	private videoSampleSink: VideoSampleSink | null = null;
	private currentFrame: WrappedCanvas | null = null;
	private nextFrame: WrappedCanvas | null = null;
	private currentSample: VideoSample | null = null;
	private nextSample: VideoSample | null = null;
	private headlessQueue: VideoSample[] = [];
	/** Recently decoded source frames for backwards access, bounded by RGBA size. */
	private backwardFrames: VideoSample[] = [];
	private backwardBytes = 0;
	private backwardEnd: number | undefined;
	private lastHeadlessTimestamp: number | undefined;
	private isDecoding = false;
	/**
	 * Reused RGBA staging array for headless decode. Only one decode is in
	 * flight per decoder (`isDecoding` serializes them) and callers consume the
	 * pixels synchronously (uploadFrameToTexture → writeTexture copies before
	 * the next decode can run), so the array is never read after it is
	 * overwritten. Avoids a full-frame allocation on every decoded frame.
	 */
	private decodeBuffer: Uint8Array | null = null;
	private nextRequest: {
		timestampSec: number;
		resolve: (value: VideoFrameData | null) => void;
		reject: (err: unknown) => void;
	} | null = null;

	/** Precise video-track duration, independent of a potentially longer audio track. */
	async durationSec(): Promise<number> {
		await this.init();
		const track = await this.input?.getPrimaryVideoTrack();
		if (!track) throw new Error(`No video track in ${this.url}`);
		return track.computeDuration();
	}

	private pushHeadlessQueue(sample: VideoSample) {
		const index = this.headlessQueue.findIndex(
			(s) => s.timestamp > sample.timestamp,
		);
		if (index === -1) {
			this.headlessQueue.push(sample);
		} else {
			this.headlessQueue.splice(index, 0, sample);
		}
	}

	private async fillHeadlessQueue(fetchId: number): Promise<void> {
		if (!this.iterator) return;
		while (this.headlessQueue.length < 2) {
			const result = await this.iterator.next();
			if (this.currentFetchId !== fetchId) {
				const sample = result.value as { close?: () => void } | null;
				sample?.close?.();
				return;
			}
			if (result.done || !result.value) break;
			this.pushHeadlessQueue(result.value as VideoSample);
		}
	}

	private async closeIterator(
		iter: AsyncGenerator<WrappedCanvas | VideoSample, void, unknown> | null,
	): Promise<void> {
		if (!iter) return;
		try {
			await iter.return();
		} catch (_) {}
	}

	resetSinks(): void {
		const iter = this.iterator;
		this.iterator = null;
		if (iter) {
			try {
				void iter.return();
			} catch (_) {}
		}
		this.canvasSink = null;
		this.videoSampleSink = null;
		this.currentFrame = null;
		this.nextFrame = null;
		this.currentSample?.close();
		this.currentSample = null;
		this.nextSample?.close();
		this.nextSample = null;
		for (const sample of this.headlessQueue) {
			try {
				sample.close();
			} catch (_) {}
		}
		this.headlessQueue = [];
		this.clearBackwardFrames();
		this.lastHeadlessTimestamp = undefined;
		this.decodeBuffer = null;
		this.iteratorStartTimestamp = -1;
	}

	private clearBackwardFrames(): void {
		for (const sample of this.backwardFrames) sample.close();
		this.backwardFrames = [];
		this.backwardBytes = 0;
		this.backwardEnd = undefined;
	}

	private async copyHeadlessSample(
		sample: VideoSample,
		fetchId: number,
	): Promise<VideoFrameData | null> {
		const width = sample.displayWidth;
		const height = sample.displayHeight;
		const byteLength = width * height * 4;
		let buffer = this.decodeBuffer;
		if (!buffer || buffer.length !== byteLength) {
			buffer = new Uint8Array(byteLength);
			this.decodeBuffer = buffer;
		}
		await sample.copyTo(buffer, { format: "RGBA" });
		if (this.currentFetchId !== fetchId || this.destroyed || !this.input)
			return null;
		premultiplyPixels(buffer);
		return { width, height, buffer };
	}

	private cachedBackwardFrame(timestamp: number): VideoSample | undefined {
		let selected: VideoSample | undefined;
		let index = -1;
		for (const [i, sample] of this.backwardFrames.entries()) {
			if (sample.timestamp > timestamp + 0.0001) break;
			selected = sample;
			index = i;
		}
		const next = this.backwardFrames[index + 1]?.timestamp ?? this.backwardEnd;
		if (
			!selected ||
			!(selected.duration > 0) ||
			timestamp + 0.0001 >=
				Math.min(selected.timestamp + selected.duration, next ?? Infinity)
		)
			return undefined;
		return selected;
	}

	/** Decode a small forward window once and retain its last frames for reverse playback. */
	private async fillBackwardFrames(
		timestamp: number,
		fetchId: number,
	): Promise<void> {
		if (!this.videoSampleSink) return;
		await this.closeIterator(this.iterator);
		this.iterator = null;
		for (const sample of this.headlessQueue) sample.close();
		this.headlessQueue = [];
		this.clearBackwardFrames();
		const iterator = this.videoSampleSink.samples(Math.max(0, timestamp - 0.5));
		try {
			for await (const sample of iterator) {
				if (sample.timestamp > timestamp + 0.0001)
					this.backwardEnd = sample.timestamp;
				if (
					this.destroyed ||
					this.currentFetchId !== fetchId ||
					sample.timestamp > timestamp + 0.0001
				) {
					sample.close();
					break;
				}
				const sampleBytes = sample.displayWidth * sample.displayHeight * 4;
				if (sampleBytes > 48 * 1024 * 1024) {
					sample.close();
					break;
				}
				this.backwardFrames.push(sample);
				this.backwardBytes += sampleBytes;
				while (
					this.backwardBytes > 48 * 1024 * 1024 &&
					this.backwardFrames.length > 1
				) {
					const oldest = this.backwardFrames.shift();
					if (oldest) {
						this.backwardBytes -=
							oldest.displayWidth * oldest.displayHeight * 4;
						oldest.close();
					}
				}
			}
		} finally {
			await this.closeIterator(iterator);
		}
	}

	protected onDestroy(): void {
		this.resetSinks();
	}

	protected async startIterator(
		timestampSec: number,
		fetchId: number,
	): Promise<void> {
		if (!this.canvasSink) return;

		const safeTimestamp = Math.max(
			0,
			Number.isFinite(timestampSec) ? timestampSec : 0,
		);
		const prevIterator = this.iterator;
		this.iterator = null;
		await this.closeIterator(prevIterator);
		if (this.currentFetchId !== fetchId || this.destroyed) return;

		this.iterator = this.canvasSink.canvases(safeTimestamp);

		const first = await this.iterator.next();
		if (this.currentFetchId !== fetchId || this.destroyed) return;
		this.currentFrame = (first.value as WrappedCanvas) ?? null;

		const second = await this.iterator.next();
		if (this.currentFetchId !== fetchId || this.destroyed) return;
		this.nextFrame = (second.value as WrappedCanvas) ?? null;

		this.iteratorStartTimestamp = safeTimestamp;
	}

	protected async startIteratorHeadless(
		timestampSec: number,
		fetchId: number,
	): Promise<void> {
		if (!this.videoSampleSink) return;

		const safeTimestamp = Math.max(
			0,
			Number.isFinite(timestampSec) ? timestampSec : 0,
		);
		const prevIterator = this.iterator;
		this.iterator = null;
		await this.closeIterator(prevIterator);
		if (this.currentFetchId !== fetchId || this.destroyed) return;

		for (const sample of this.headlessQueue) {
			sample.close();
		}
		this.headlessQueue = [];

		const seekTime = Math.max(0, safeTimestamp - 0.1);
		this.iterator = this.videoSampleSink.samples(seekTime);

		await this.fillHeadlessQueue(fetchId);
		if (this.currentFetchId !== fetchId || this.destroyed) return;

		this.iteratorStartTimestamp = safeTimestamp;
	}

	async getFrame(timestampSec: number): Promise<VideoFrameData | null> {
		await this.init();
		if (this.destroyed || !this.input) return null;

		const safeTimestamp = Math.max(
			0,
			Number.isFinite(timestampSec) ? timestampSec : 0,
		);

		if (this.isDecoding) {
			if (this.nextRequest) {
				this.nextRequest.resolve(null);
			}
			return new Promise<VideoFrameData | null>((resolve, reject) => {
				this.nextRequest = { timestampSec: safeTimestamp, resolve, reject };
			});
		}

		return this.executeDecode(safeTimestamp);
	}

	private async executeDecode(
		timestampSec: number,
	): Promise<VideoFrameData | null> {
		this.isDecoding = true;
		try {
			return await this.performDecode(timestampSec);
		} finally {
			this.isDecoding = false;
			if (this.nextRequest) {
				const req = this.nextRequest;
				this.nextRequest = null;
				this.executeDecode(req.timestampSec)
					.then(req.resolve)
					.catch(req.reject);
			}
		}
	}

	private async performDecodeHeadless(
		safeTimestamp: number,
		fetchId: number,
	): Promise<VideoFrameData | null> {
		if (!this.videoSampleSink) {
			if (this.destroyed || !this.input) return null;
			const track = await this.input.getPrimaryVideoTrack();
			if (!track) return null;
			this.videoSampleSink = new VideoSampleSink(track);
		}

		const eps = 0.0001;
		const forwardSeekThreshold = 1.0;
		const backwardSeekThreshold = 0.01;
		const backwards =
			this.lastHeadlessTimestamp !== undefined &&
			safeTimestamp < this.lastHeadlessTimestamp - 0.0001;
		this.lastHeadlessTimestamp = safeTimestamp;
		let cached = this.cachedBackwardFrame(safeTimestamp);
		if (!cached && backwards) {
			await this.fillBackwardFrames(safeTimestamp, fetchId);
			cached = this.cachedBackwardFrame(safeTimestamp);
		}
		if (cached) return this.copyHeadlessSample(cached, fetchId);

		if (
			!this.iterator ||
			safeTimestamp < this.iteratorStartTimestamp - eps ||
			(this.headlessQueue.length > 0 &&
				safeTimestamp <
					this.headlessQueue[0].timestamp - backwardSeekThreshold) ||
			(this.headlessQueue.length > 0 &&
				safeTimestamp - this.headlessQueue[0].timestamp > forwardSeekThreshold)
		) {
			await this.startIteratorHeadless(safeTimestamp, fetchId);
			if (this.currentFetchId !== fetchId || this.destroyed || !this.input)
				return null;
		}

		while (true) {
			await this.fillHeadlessQueue(fetchId);
			if (this.currentFetchId !== fetchId || this.destroyed || !this.input)
				return null;

			if (this.headlessQueue.length >= 2) {
				const next = this.headlessQueue[1];
				if (next.timestamp <= safeTimestamp + eps) {
					const old = this.headlessQueue.shift();
					old?.close();
					continue;
				}
			}
			break;
		}

		const currentSample = this.headlessQueue[0];
		if (!currentSample) return null;

		return this.copyHeadlessSample(currentSample, fetchId);
	}

	private async performDecodeInternal(
		safeTimestamp: number,
	): Promise<VideoFrameData | null> {
		this.currentFetchId++;
		const fetchId = this.currentFetchId;

		if (this.isHeadless) {
			return this.performDecodeHeadless(safeTimestamp, fetchId);
		}

		if (!this.canvasSink) {
			if (this.destroyed || !this.input) return null;
			const track = await this.input.getPrimaryVideoTrack();
			if (!track) return null;
			this.canvasSink = new CanvasSink(track, { poolSize: 3, alpha: true });
		}

		const eps = 0.0001;
		const forwardSeekThreshold = 1.0;
		const backwardSeekThreshold = 0.01;

		if (
			!this.iterator ||
			safeTimestamp < this.iteratorStartTimestamp - eps ||
			(this.currentFrame &&
				safeTimestamp < this.currentFrame.timestamp - backwardSeekThreshold) ||
			(this.currentFrame &&
				safeTimestamp - this.currentFrame.timestamp > forwardSeekThreshold)
		) {
			await this.startIterator(safeTimestamp, fetchId);
			if (this.currentFetchId !== fetchId || this.destroyed || !this.input)
				return null;
		}

		while (this.nextFrame && this.nextFrame.timestamp <= safeTimestamp + eps) {
			this.currentFrame = this.nextFrame;
			this.nextFrame = null;
			if (this.iterator) {
				const result = await this.iterator.next();
				if (this.currentFetchId !== fetchId || this.destroyed || !this.input)
					return null;
				this.nextFrame = (result.value as WrappedCanvas) ?? null;
			}
		}

		if (!this.currentFrame) return null;
		const canvas = this.currentFrame.canvas;
		return { width: canvas.width, height: canvas.height, canvas };
	}

	private async performDecode(
		timestampSec: number,
	): Promise<VideoFrameData | null> {
		const safeTimestamp = Math.max(
			0,
			Number.isFinite(timestampSec) ? timestampSec : 0,
		);
		try {
			return await this.performDecodeInternal(safeTimestamp);
		} catch (err: unknown) {
			const errObj = err as { name?: string; message?: string } | null;
			if (
				errObj?.name === "InputDisposedError" ||
				errObj?.message?.includes("disposed") ||
				this.destroyed ||
				!this.input
			) {
				return null;
			}

			// If WebCodecs decode failed (e.g. EncodingError), reset dead sinks and retry once
			// with software decoding fallback preference to avoid permanent poisoning.
			this.resetSinks();

			try {
				if (!this.isHeadless && this.input && !this.destroyed) {
					const track = await this.input.getPrimaryVideoTrack();
					if (track && !this.destroyed) {
						this.canvasSink = new CanvasSink(track, {
							poolSize: 3,
							alpha: true,
							decoderOptions: { hardwareAcceleration: "prefer-software" },
						});
						return await this.performDecodeInternal(safeTimestamp);
					}
				}
			} catch (_) {}

			throw err;
		}
	}
}

export class AudioDecoder extends BaseMediaDecoder<WrappedAudioBuffer> {
	private audioSink: AudioBufferSink | null = null;

	protected onDestroy(): void {
		this.audioSink = null;
	}

	protected async startIterator(
		timestampSec: number,
		_fetchId: number,
	): Promise<void> {
		if (!this.audioSink) return;
		void this.iterator?.return();
		this.iterator = this.audioSink.buffers(timestampSec);
		this.iteratorStartTimestamp = timestampSec;
	}

	async *getBuffers(
		timestampSec: number,
	): AsyncGenerator<WrappedAudioBuffer, void, unknown> {
		try {
			await this.init();
			if (this.destroyed || !this.input) return;

			if (!this.audioSink) {
				const track = await this.input.getPrimaryAudioTrack();
				if (!track) return;
				this.audioSink = new AudioBufferSink(track);
			}

			const eps = 0.05; // 50ms tolerance for audio sync jumps
			if (
				!this.iterator ||
				Math.abs(timestampSec - this.iteratorStartTimestamp) > eps
			) {
				this.currentFetchId++;
				await this.startIterator(timestampSec, this.currentFetchId);
			}

			const fetchId = this.currentFetchId;
			if (!this.iterator) return;

			for await (const wrapped of this.iterator) {
				if (this.currentFetchId !== fetchId || this.destroyed || !this.input)
					break;
				this.iteratorStartTimestamp = wrapped.timestamp + wrapped.duration;
				yield wrapped;
			}
		} catch (err: unknown) {
			const errObj = err as { name?: string; message?: string } | null;
			if (
				errObj?.name === "InputDisposedError" ||
				errObj?.message?.includes("disposed") ||
				this.destroyed ||
				!this.input
			) {
				return;
			}
			throw err;
		} finally {
			const fetchId = this.currentFetchId;
			if (this.currentFetchId === fetchId) {
				this.iterator = null;
			}
		}
	}
}

class MediaDecoderCache {
	private videoDecoders = new Map<string, VideoDecoder>();
	private audioDecoders = new Map<string, AudioDecoder>();
	private lastAccessed = new Map<string, number>();
	/** Rendered frames so far (see `tick`), and the frame each decoder was last used on. */
	private ticks = 0;
	private lastTick = new Map<string, number>();

	private touchKey(key: string) {
		this.lastAccessed.set(key, Date.now());
		this.lastTick.set(key, this.ticks);
	}

	private dropKey(key: string): void {
		const videoDecoder = this.videoDecoders.get(key);
		if (videoDecoder) {
			try {
				videoDecoder.destroy();
			} catch (_) {}
			this.videoDecoders.delete(key);
		}
		const audioDecoder = this.audioDecoders.get(key);
		if (audioDecoder) {
			try {
				audioDecoder.destroy();
			} catch (_) {}
			this.audioDecoders.delete(key);
		}
		this.lastAccessed.delete(key);
		this.lastTick.delete(key);
	}

	/**
	 * Called once per rendered frame by headless renders. A decoder holds a
	 * full-frame RGBA buffer and native decoder state, so one that no layer has
	 * drawn for `idleFrames` frames (its clip ended, or a still render used a
	 * one-off id) is released instead of waiting for the wall-clock prune.
	 */
	tick(idleFrames = 120): void {
		this.ticks++;
		for (const [key, last] of this.lastTick) {
			if (this.ticks - last > idleFrames) this.dropKey(key);
		}
	}

	destroy(): void {
		for (const decoder of this.videoDecoders.values()) {
			try {
				decoder.destroy();
			} catch (_) {}
		}
		this.videoDecoders.clear();

		for (const decoder of this.audioDecoders.values()) {
			try {
				decoder.destroy();
			} catch (_) {}
		}
		this.audioDecoders.clear();
		this.lastAccessed.clear();
		this.lastTick.clear();
	}

	/** Decoders currently open (for tests and diagnostics). */
	get size(): number {
		return this.videoDecoders.size + this.audioDecoders.size;
	}

	private prune(maxAgeMs = 300000) {
		const now = Date.now();
		for (const [key, lastTime] of this.lastAccessed.entries()) {
			if (now - lastTime > maxAgeMs) this.dropKey(key);
		}
	}

	getVideo(url: string, isHeadless: boolean, nodeId?: string): VideoDecoder {
		const key = `${url}-${isHeadless}-${nodeId ?? "default"}`;
		let decoder = this.videoDecoders.get(key);
		if (!decoder) {
			decoder = new VideoDecoder(url, isHeadless);
			this.videoDecoders.set(key, decoder);
		}
		this.touchKey(key);
		this.prune();
		return decoder;
	}

	getAudio(url: string, nodeId?: string): AudioDecoder {
		const key = `${url}-${nodeId ?? "default"}`;
		let decoder = this.audioDecoders.get(key);
		if (!decoder) {
			decoder = new AudioDecoder(url, false);
			this.audioDecoders.set(key, decoder);
		}
		this.touchKey(key);
		this.prune();
		return decoder;
	}

	clearNode(nodeId: string) {
		const keys = [...this.videoDecoders.keys(), ...this.audioDecoders.keys()];
		for (const key of keys) {
			if (key.includes(nodeId)) this.dropKey(key);
		}
	}
}

export const mediaDecoderCache = new MediaDecoderCache();