Skip to content

The VS Code of video editing, built for agents.

TypeScript
4.3 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { VideoSampleSink } from 'mediabunny';

import type { InputVideoTrack, VideoSample } from 'mediabunny';
import type { Sam2Mask } from './mask';
import type { Sam2Correction, Sam2Point, Sam2Video } from './tracker';

export type FrameRequest = {
	track: InputVideoTrack;
	timestamp: number;
};

export type TrackRequest = {
	track: InputVideoTrack;
	/** Where each frame to segment is in the file, in seconds, ascending. */
	timestamps: number[];
	/** Which of `timestamps` the points were placed on. */
	seedIndex: number;
	points: Sam2Point[];
	/** The seed's mask as the user corrected it: what the object is tracked from. */
	correct?: Sam2Correction;
	signal?: AbortSignal;
	/** Called as each frame's mask is ready, in tracking order: the seed, then forward, then backward. */
	onMask: (index: number, mask: Sam2Mask) => void;
};

/**
 * Frames decoded in reverse are read a batch at a time, in file order, and
 * walked backward: seeking to every frame on its own would decode from its
 * keyframe each time.
 */
const REVERSE_BATCH = 12;

/**
 * Decodes one frame and holds it in the model, encoded, so prompts on it
 * (`model.preview`, `model.seedHeld`) cost the mask decoder alone.
 */
export async function holdFrame(model: Sam2Video, request: FrameRequest): Promise<void> {
	const { track, timestamp } = request;
	const sample = await new VideoSampleSink(track).getSample(timestamp);
	if (!sample) throw new Error('The frame could not be decoded');
	await withVideoFrame(sample, (frame) => model.hold(frame, track.rotation));
}

/**
 * Segments the object at the seed and follows it through every frame of the
 * request, away from the seed in both directions. Frames that decode to the
 * same picture as the one before (a source slower than the timestamps) share
 * its mask instead of being run again.
 */
export async function trackObject(model: Sam2Video, request: TrackRequest): Promise<void> {
	const { track, timestamps, seedIndex, points, correct, signal, onMask } = request;
	const sink = new VideoSampleSink(track);
	const rotation = track.rotation;
	const total = timestamps.length;

	const seed = await sink.getSample(timestamps[seedIndex]!);
	if (!seed) throw new Error('The prompted frame could not be decoded');

	const seedTimestamp = seed.timestamp;
	const seedMask = await withVideoFrame(seed, (frame) => model.seed(frame, rotation, points, seedIndex, correct));
	onMask(seedIndex, seedMask);

	let previous = { timestamp: seedTimestamp, mask: seedMask };

	const process = async (sample: VideoSample, index: number) => {
		if (sample.timestamp === previous.timestamp) {
			sample.close();
			onMask(index, previous.mask);
			return;
		}
		const timestamp = sample.timestamp;
		const mask = await withVideoFrame(sample, (frame) => model.track(frame, rotation, index, total));
		previous = { timestamp, mask };
		onMask(index, mask);
	};

	const forward = sink.samplesAtTimestamps(timestamps.slice(seedIndex + 1));
	let index = seedIndex + 1;
	// The next decode is requested before the current frame is processed, so
	// decoding and inference overlap.
	let pending = forward.next();
	try {
		for (;;) {
			const { done, value: sample } = await pending;
			if (done) break;
			pending = forward.next();

			if (signal?.aborted) {
				sample?.close();
				return;
			}
			if (sample) await process(sample, index);
			index++;
		}
	} finally {
		await forward.return();
	}

	model.rewind();
	previous = { timestamp: seedTimestamp, mask: seedMask };

	for (let end = seedIndex; end > 0; end -= REVERSE_BATCH) {
		const start = Math.max(0, end - REVERSE_BATCH);
		const samples: (VideoSample | null)[] = [];
		for await (const sample of sink.samplesAtTimestamps(timestamps.slice(start, end))) samples.push(sample);

		for (let i = samples.length - 1; i >= 0; i--) {
			const sample = samples[i];
			if (!sample) continue;
			if (signal?.aborted) {
				sample.close();
				continue;
			}
			await process(sample, start + i);
		}
		if (signal?.aborted) return;
	}
}

async function withVideoFrame<T>(sample: VideoSample, run: (frame: VideoFrame) => Promise<T>): Promise<T> {
	const frame = sample.toVideoFrame();
	try {
		return await run(frame);
	} finally {
		frame.close();
		sample.close();
	}
}