Skip to content

The VS Code of video editing, built for agents.

TypeScript
4.1 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { MASK_EXTENSION, assetName, encodeMaskFile } from '@diffusionstudio/assets';
import { Effect as EffectElement, Mask } from '@diffusionstudio/reconciler';
import { Cache, Effect, EffectType, FrameRate, Hidden, Library, getParentNode } from '@diffusionstudio/runtime';
import { geometryOf } from '@diffusionstudio/sam2/models';

import { getDocumentEditor } from '../editor';
import { getVideoRect } from './media';
import { getTargetEffect } from './store';

import type { Entity, World } from 'koota';
import type { AssetLibrary, MaskFrame, MaskRecipe, VideoAsset } from '@diffusionstudio/assets';
import type { ObjectTrack } from './store';

/** Where masks go in the library: a folder per video under this one. */
const MASKS_FOLDER = 'masks';

/**
 * Turns a tracked session into the document: its frames go into the
 * library as one mask file, `masks/<video>/Tracking <n>.mask`, with the
 * recipe that made them,
 * and the clip gets a `<mask>` naming it — `src` the file, `sourceIn` the
 * clip's source time its first frame belongs to — under the effect the tool
 * was started for, or else under an `opacity` effect: the cut-out. A clip
 * that already has an opacity effect gets the mask under that one. Returns
 * the mask, or null when nothing was written (the session was cancelled or
 * the clip is not a video).
 */
export async function commitObjectMask(world: World, track: ObjectTrack): Promise<Entity | null> {
	const library = world.get(Library);
	const rect = getVideoRect(world, track.clip);
	const model = track.model;
	if (!library || !rect || !model || track.masks.length === 0) return null;

	const { signal } = track.controller;
	const fps = world.get(FrameRate)?.value ?? 30;
	const folder = objectMaskFolder(rect.asset);

	const blob = encodeObjectMask(rect.asset, fps, geometryOf(model.imageSize).maskSize, track.masks, {
		model: model.repo,
		source: rect.asset.id,
		first: track.first,
		seedFrame: track.seedFrame,
		points: track.points,
		strokes: track.strokes,
	});
	if (signal.aborted) return null;

	const asset = await library.store(blob, { folder, name: nextTrackingName(library, folder) });
	if (signal.aborted || !track.clip.isAlive()) return null;

	const editor = getDocumentEditor(world);
	const mask = () => <Mask src={asset.path} sourceIn={track.first / fps} />;

	const target = getTargetEffect();
	const holder = target && getParentNode(target) === track.clip ? target : opacityEffect(track.clip);
	if (holder) {
		const [inserted] = editor.insertElement(holder, mask);
		return inserted ?? null;
	}
	const [effect] = editor.insertElement(track.clip, () => (
		<EffectElement type="opacity" value={1}>
			{mask()}
		</EffectElement>
	));
	return effect?.get(Cache)?.masks[0] ?? null;
}

/** A mask file of tracked frames, on a `grid` square, over footage of `size`, one a frame at `frameRate`. */
export function encodeObjectMask(
	size: { width: number; height: number },
	frameRate: number,
	grid: number,
	masks: readonly (MaskFrame | null)[],
	recipe: MaskRecipe,
): Blob {
	return encodeMaskFile(
		{ gridWidth: grid, gridHeight: grid, width: size.width, height: size.height, frameRate, recipe },
		masks,
	);
}

/** Where masks tracked on `video` go in the library: `masks/<video>`. */
export function objectMaskFolder(video: VideoAsset): string {
	return `${MASKS_FOLDER}/${assetName(video).replace(/\.[^.]+$/, '')}`;
}

/** `Tracking <n>.mask`, `n` the first count not yet taken in `folder`. */
export function nextTrackingName(library: AssetLibrary, folder: string): string {
	for (let n = 1; ; n++) {
		const name = `Tracking ${n}.${MASK_EXTENSION}`;
		if (!library.get(`${folder}/${name}`)) return name;
	}
}

/** The clip's opacity effect, if it has one that is switched on: where another mask joins. */
function opacityEffect(clip: Entity): Entity | null {
	for (const effect of clip.get(Cache)?.effects ?? []) {
		if (effect.get(Effect)?.type === EffectType.OPACITY && !effect.has(Hidden)) return effect;
	}
	return null;
}