Skip to content

The VS Code of video editing, built for agents.

TypeScript
9.6 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { Culled, Hidden, HitRegions, Tool, ToolType, entityQuad, isPointerInEntity } from '@diffusionstudio/runtime';
import { traceMask } from '@diffusionstudio/assets';
import { geometryOf } from '@diffusionstudio/sam2/models';

import { ObjectMaskTool, Pointer } from '../traits';
import { endMaskStroke, paintableObjectTrack } from './brush';
import { maskFrame } from './frame';
import { handleObjectMaskInteraction, heldObjectMaskLabel } from './interaction';
import { currentSourceFrame, getVideoRect, pointOnVideo, videoPointAt, videoPointToDevice } from './media';
import { clearTargetEffect, getObjectHover, getObjectMask, getObjectTrack, getTargetClip } from './store';
import { clearObjectHover, hoverObjectMask } from './tracking';

import type { Entity, World } from 'koota';
import type { MaskFrame } from '@diffusionstudio/assets';
import type { Sam2Mask } from '@diffusionstudio/sam2/mask';
import type { MaskPoint } from './store';
import type { VideoRect } from './media';

type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;

const ACCENT = '#008CFF';
const BACKGROUND_POINT = '#F43535';
const MASK_ALPHA = 0.45;
const HOVER_ALPHA = 0.28;
/** The mask's edge: an accent line over a dark halo, legible on any footage; the hovered segment's is fainter. */
const OUTLINE_WIDTH = 1.5;
const OUTLINE_HALO = 3.5;
const OUTLINE_ALPHA = 1;
const HOVER_OUTLINE_ALPHA = 0.6;
const POINT_RADIUS = 5;
/** How far the pointer moves, in 0..1 of the frame, before a new segment is asked for. */
const HOVER_STEP = 0.004;

/** Traced masks, kept for scrubbing back and forth; the oldest goes first. */
const MASK_OUTLINES = 48;
const outlines = new Map<Sam2Mask | MaskFrame, Path2D>();

/** Whether the tool was up last frame, so putting it down cleans up once. */
let active = false;
let lastHover: { frame: number; x: number; y: number; label: 0 | 1 } | null = null;

/**
 * Everything the tool puts on the canvas while it is up: the session over its
 * clip, the segment under the pointer or the brush, the points that prompted
 * it, and the region that claims clicks on the video. Drawn last in the HUD so
 * that region sits over the selection's.
 */
export function drawObjectMasks(world: World, ctx: Ctx2D, resolution: number): void {
	if (world.get(Tool)?.value !== ToolType.OBJECT_MASK) {
		if (active) {
			active = false;
			lastHover = null;
			clearObjectHover();
			clearTargetEffect();
			endMaskStroke();
			world.set(ObjectMaskTool, { mode: 'points', op: 'add', brushShown: false });
		}
		return;
	}
	active = true;

	const target = toolTarget(world);
	const brush = world.get(ObjectMaskTool)!.mode === 'brush';
	updateHover(world, brush ? null : target);

	const track = getObjectTrack();
	if (track && track.clip.isAlive() && !track.clip.has(Culled) && !track.clip.has(Hidden)) {
		const rect = getVideoRect(world, track.clip);
		if (rect) {
			const frame = currentSourceFrame(world, track.clip);
			const shown = getObjectMask(track, frame);
			// A tracked frame is on the grid of the model that made it; the prompted one says its own.
			const size = shown && ('size' in shown ? shown.size : track.model && geometryOf(track.model.imageSize).maskSize);
			if (shown && size) drawMask(ctx, rect, shown, size, MASK_ALPHA, OUTLINE_ALPHA, resolution);
			if (frame === track.seedFrame) drawPoints(ctx, rect, track.points, resolution);
		}
	}

	const hover = getObjectHover();
	if (hover && hover.clip.isAlive()) {
		const rect = getVideoRect(world, hover.clip);
		if (rect && hover.frame === currentSourceFrame(world, hover.clip)) drawMask(ctx, rect, hover.mask, hover.size, HOVER_ALPHA, HOVER_OUTLINE_ALPHA, resolution);
	}

	const shown = !!(brush && target && drawBrush(world, ctx, target, resolution));
	if (world.get(ObjectMaskTool)!.brushShown !== shown) {
		world.set(ObjectMaskTool, { brushShown: shown });
	}

	if (target) {
		world.get(HitRegions)!.list.push({
			target: { kind: 'hud', id: 'object-mask', entity: target.clip, quad: entityQuad(world, target.clip) },
			callback: handleObjectMaskInteraction,
		});
	}
}

/**
 * Asks the model what is under the pointer while it rests over the tool's
 * video, so the segment a click would pick shows before the click.
 */
function updateHover(world: World, target: { clip: Entity; rect: VideoRect } | null): void {
	const pointer = world.get(Pointer);
	const point = target && pointer?.over && pointer.phase === 'lifted'
		? pointOnVideo(target.rect, pointer.clientX, pointer.clientY)
		: null;

	if (!target || !point) {
		if (lastHover) clearObjectHover();
		lastHover = null;
		return;
	}

	const frame = currentSourceFrame(world, target.clip);
	const label = heldObjectMaskLabel(world);
	if (
		lastHover && lastHover.frame === frame && lastHover.label === label
		&& Math.hypot(point.x - lastHover.x, point.y - lastHover.y) < HOVER_STEP
	) return;
	lastHover = { frame, x: point.x, y: point.y, label };
	hoverObjectMask(world, target.clip, { ...point, label });
}

/**
 * What the tool works on: the clip of the effect the tool was started for,
 * else the video clip under the pointer, where a click prompts. Null when
 * there is no video to work on.
 */
function toolTarget(world: World): { clip: Entity; rect: VideoRect } | null {
	const clip = getTargetClip() ?? videoUnderPointer(world);
	if (!clip || !clip.isAlive() || clip.has(Culled) || clip.has(Hidden)) return null;
	const rect = getVideoRect(world, clip);
	return rect ? { clip, rect } : null;
}

/** The topmost video clip under the pointer, from the regions the last frame drew. */
function videoUnderPointer(world: World): Entity | null {
	const pointer = world.get(Pointer);
	if (!pointer?.over) return null;

	const regions = world.get(HitRegions)?.list ?? [];
	const point = { x: pointer.clientX, y: pointer.clientY };
	for (let i = regions.length - 1; i >= 0; i--) {
		const { target } = regions[i]!;
		if (target.kind !== 'entity' || !target.id.isAlive()) continue;
		if (!isPointerInEntity(world, target.id, point)) continue;
		return getVideoRect(world, target.id) ? target.id : null;
	}
	return null;
}

/** Draws `mask`, on a `size` square grid, over the video. */
function drawMask(ctx: Ctx2D, rect: VideoRect, mask: Sam2Mask | MaskFrame, size: number, alpha: number, outlineAlpha: number, resolution: number): void {
	const { mat } = rect;

	// The outline is traced in grid pixels and placed on the device, then
	// filled and stroked there, so the edge is smooth and the line keeps its
	// width at any zoom. It is the edge the mask is cut along.
	const outline = new Path2D();
	const grid = new DOMMatrix([mat.a, mat.b, mat.c, mat.d, mat.e, mat.f])
		.translate(rect.x, rect.y)
		.scale(rect.w / size, rect.h / size);
	outline.addPath(maskOutline(mask, size), grid);

	ctx.save();
	ctx.setTransform(mat.a, mat.b, mat.c, mat.d, mat.e, mat.f);
	ctx.beginPath();
	ctx.rect(0, 0, rect.width, rect.height);
	ctx.clip();
	ctx.resetTransform();
	ctx.globalAlpha = alpha;
	ctx.fillStyle = ACCENT;
	ctx.fill(outline);

	ctx.globalAlpha = outlineAlpha;
	ctx.lineCap = 'round';
	ctx.lineJoin = 'round';
	ctx.lineWidth = OUTLINE_HALO * resolution;
	ctx.strokeStyle = 'rgba(0, 0, 0, 0.45)';
	ctx.stroke(outline);
	ctx.lineWidth = OUTLINE_WIDTH * resolution;
	ctx.strokeStyle = ACCENT;
	ctx.stroke(outline);
	ctx.restore();
}

/**
 * The brush's outline under the pointer, round on the footage, while there is
 * a mask on the clip to correct. Says whether it drew.
 */
function drawBrush(world: World, ctx: Ctx2D, target: { clip: Entity; rect: VideoRect }, resolution: number): boolean {
	const pointer = world.get(Pointer);
	if (!pointer?.over || !paintableObjectTrack(world, target.clip)) return false;

	const { rect } = target;
	const pressed = pointer.phase === 'pressed';
	const point = pressed ? videoPointAt(rect, pointer.clientX, pointer.clientY) : pointOnVideo(rect, pointer.clientX, pointer.clientY);
	if (!point) return false;

	const { mat } = rect;
	const radius = world.get(ObjectMaskTool)!.brushRadius * rect.h;
	ctx.save();
	// The path is laid down in box space and stroked in device pixels, so the line keeps its width at any zoom.
	ctx.setTransform(mat.a, mat.b, mat.c, mat.d, mat.e, mat.f);
	ctx.beginPath();
	ctx.ellipse(rect.x + point.x * rect.w, rect.y + point.y * rect.h, radius, radius, 0, 0, Math.PI * 2);
	ctx.resetTransform();
	ctx.lineWidth = 3 * resolution;
	ctx.strokeStyle = 'rgba(0, 0, 0, 0.4)';
	ctx.stroke();
	ctx.lineWidth = 1.5 * resolution;
	ctx.strokeStyle = '#FFFFFF';
	ctx.stroke();
	ctx.restore();

	return true;
}

function drawPoints(ctx: Ctx2D, rect: VideoRect, points: MaskPoint[], resolution: number): void {
	ctx.resetTransform();
	ctx.lineWidth = 1.5 * resolution;
	ctx.strokeStyle = '#FFFFFF';

	for (const point of points) {
		const { x, y } = videoPointToDevice(rect, point);
		ctx.beginPath();
		ctx.arc(x, y, POINT_RADIUS * resolution, 0, Math.PI * 2);
		ctx.closePath();
		ctx.fillStyle = point.label === 1 ? ACCENT : BACKGROUND_POINT;
		ctx.fill();
		ctx.stroke();
	}
}

/**
 * The mask's edge in grid cells, traced once and kept while it is shown: a
 * tracked frame as its file will hold it, the prompted one, which the brush
 * paints on, from the model's logits.
 */
function maskOutline(mask: Sam2Mask | MaskFrame, size: number): Path2D {
	let outline = outlines.get(mask);
	if (outline) return outline;

	outline = new Path2D();
	traceMask('field' in mask ? mask.field : maskFrame(mask).field, size, size, outline);
	outlines.set(mask, outline);

	if (outlines.size > MASK_OUTLINES) outlines.delete(outlines.keys().next().value!);
	return outline;
}