Skip to content

The VS Code of video editing, built for agents.

TypeScript
9.8 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { parseSource } from '@diffusionstudio/jsx';

import { Cache, Computed, Delay, FrameRate, FramePromises, Host, Library, LoadRequest, PendingSource, PendingSync, PlaybackRate, Source, SourceError, SyncRequest, Trim } from '../traits';
import { bindAsset, getAssetFile } from '../actions/assets';
import { getParentNode } from '../queries/hierarchy';
import { findAssetDuration, findGeometryAsset } from '../utils/time';
import { computeAudioSyncOffsetCached } from '../media/audio-sync';
import { store } from '../world/store';
import { assert } from '../utils/assert';

import type { Entity, World } from 'koota';
import type { Asset } from '@diffusionstudio/assets';

export function assetSystem(world: World): void {
	for (const entity of world.query(LoadRequest, Host)) {
		const source = entity.get(LoadRequest)!.value;
		if (!isDomImage(entity)) continue;
		pointDomImageAt(entity, null);
		if (source !== '' && !/^(?:data|blob):/i.test(source)) continue;
		entity.remove(LoadRequest);
		pointDomImageAt(entity, source || null);
	}

	const library = world.get(Library);
	if (library) {
		for (const entity of world.query(LoadRequest)) {
			const source = entity.get(LoadRequest)!.value;
			entity.remove(LoadRequest);
			resolve(world, entity, library.resolve(source));
		}
	}

	// Handle syncTo requests
	for (const entity of world.query(SyncRequest)) {
		if (waitingOnSources(entity)) continue;

		const id = entity.get(SyncRequest)!.value;
		const target = findElementById(world, id, entity);

		if (target === undefined) {
			fail(entity, `syncTo: no other element carries id "${id}"`);
			continue;
		}

		if (inCycle(world, entity)) {
			fail(entity, `syncTo: "${id}" leads back around to this element; a sync cannot go in a circle`);
			continue;
		}

		if (target.has(SyncRequest) || target.has(PendingSync) || waitingOnSources(target)) continue;

		const source = findGeometryAsset(world, entity);
		if (source === null || (source.type !== 'AUDIO' && source.type !== 'VIDEO')) {
			fail(entity, 'syncTo: the element has no audio or video source to align');
			continue;
		}

		const reference = findGeometryAsset(world, target);
		if (reference === null || (reference.type !== 'AUDIO' && reference.type !== 'VIDEO')) {
			fail(entity, `syncTo: "${id}" has no audio or video source to align against`);
			continue;
		}

		entity.remove(SyncRequest);

		const token = {};
		entity.add(PendingSync);
		entity.set(PendingSync, { value: token });

		const done = computeAudioSyncOffsetCached(source, reference)
			.then(({ offsetSeconds }) => {
				if (!currentSync(entity, token)) return;
				if (target.isAlive()) placeSynced(world, entity, target, id, offsetSeconds);
				entity.remove(PendingSync);
			})
			.catch((error: unknown) => {
				if (!currentSync(entity, token)) return;
				fail(entity, error instanceof Error ? error.message : String(error));
			});

		world.get(FramePromises)?.list?.push(done);
	}
}

/**
 * Places `entity` so its recording coincides with `target`'s: the measured
 * offset says how long after the target's recording this one started, so the
 * element's source frame 0 goes that far after the target's origin,
 * re-expressed against the element's own parent (Delay is parent-relative
 * where Computed is absolute). The origin keeps its fraction so the audio
 * scheduled against it doesn't inherit a half-frame of slip.
 *
 * An authored trim bound survives (`sourceIn`/`sourceOut` remain yours to
 * set); a missing one defaults to the intersection of the element's natural
 * extent with the target's window. A window that closes before it opens means
 * the recordings do not overlap, which is reported rather than guessed at.
 */
function placeSynced(world: World, entity: Entity, target: Entity, id: string, offsetSeconds: number): void {
	const computed = store(world, Computed);
	const frameRate = world.get(FrameRate)?.value ?? 30;

	const parent = getParentNode(entity);
	const parentOrigin = parent !== null ? computed.origin[parent.id()] ?? 0 : 0;
	const origin = (computed.origin[target.id()] ?? 0) + offsetSeconds * frameRate;

	entity.add(Delay);
	entity.set(Delay, { value: origin - parentOrigin });

	const rate = entity.get(PlaybackRate)?.value || 1;
	const trim = entity.get(Trim);
	const targetStart = computed.start[target.id()] ?? 0;
	const targetEnd = computed.end[target.id()] ?? 0;

	const start = trim?.start ?? Math.max(0, (targetStart - origin) * rate);
	let end = trim?.end ?? null;
	if (end === null) {
		end = (targetEnd - origin) * rate;
		const duration = findAssetDuration(world, entity);
		if (duration !== null) end = Math.min(end, duration);
	}

	assert(end > start, `syncTo: the aligned clip does not overlap the window of "${id}"`);

	entity.add(Trim);
	entity.set(Trim, { start, end });
}

/**
 * The element whose authored id is `id`: ids ride into the runtime as the
 * locator of the source stamp (see SOURCE_ATTR in @diffusionstudio/jsx), so
 * only stamped elements can be found — and a positional locator, being a
 * number, never matches. `except` keeps an element from finding itself.
 * Iterations of a loop share one stamp; the first stands for them all.
 */
function findElementById(world: World, id: string, except: Entity): Entity | undefined {
	for (const entity of world.query(Source)) {
		if (entity === except) continue;
		if (parseSource(entity.get(Source)!.value)?.locator === id) return entity;
	}
	return undefined;
}

/**
 * Whether following `syncTo` from `entity` comes back around to it. Only
 * requests still standing are links: a placed element is a fixed point, not a
 * hop. Of a cycle's members, each fails on its own turn through the request
 * loop; whoever is checked once the others have failed no longer sits on a
 * cycle and aligns against their default placement.
 */
function inCycle(world: World, entity: Entity): boolean {
	const seen = new Set<Entity>([entity]);
	let current = entity;
	for (; ;) {
		const id = current.get(SyncRequest)?.value;
		if (id === undefined) return false;
		const next = findElementById(world, id, current);
		if (next === undefined) return false;
		if (next === entity) return true;
		if (seen.has(next)) return false;
		seen.add(next);
		current = next;
	}
}

/**
 * Whether the element's media is still being resolved — a request this system
 * has not consumed, or a resolution inflight — on the element itself or on a
 * fill (a `<rect><videoPaint/>` carries its src there).
 */
function waitingOnSources(entity: Entity): boolean {
	if (pendingSource(entity)) return true;
	for (const fill of entity.get(Cache)?.fills ?? []) {
		if (pendingSource(fill)) return true;
	}
	return false;
}

function pendingSource(entity: Entity): boolean {
	return entity.has(LoadRequest) || entity.has(PendingSource);
}

/** Takes the sync off and leaves why on the element, `resolve`-style. */
function fail(entity: Entity, message: string): void {
	entity.remove(SyncRequest, PendingSync);
	entity.add(SourceError);
	entity.set(SourceError, { value: message });
	console.error('[runtime] could not sync:', message);
}

function currentSync(entity: Entity, token: object): boolean {
	return entity.isAlive() && entity.get(PendingSync)?.value === token;
}

/**
 * Tracks one started resolution: the entity remembers which one it is waiting
 * on (PendingSource), so of overlapping resolutions only the latest binds, and
 * one that outlives its element (or its src) is dropped. The wait itself is
 * the identity rather than what was asked for — the later question is always
 * the one being asked.
 */
function resolve(world: World, entity: Entity, promise: Promise<Asset>): void {
	const token = {};
	entity.remove(SourceError);
	entity.add(PendingSource);
	entity.set(PendingSource, { value: token });

	const done = promise.then(async (asset) => {
		if (!current(entity, token)) return;
		bindAsset(entity, asset);
		await bindDomImage(entity, asset, token);
		if (current(entity, token)) entity.remove(PendingSource);
	}).catch((error: unknown) => {
		if (!current(entity, token)) return;
		entity.remove(PendingSource);
		entity.add(SourceError);
		entity.set(SourceError, { value: errorMessage(error) });
		console.error('[runtime] could not resolve src:', error);
	});

	world.get(FramePromises)?.list?.push(done);
}

function isDomImage(entity: Entity): boolean {
	return typeof HTMLImageElement !== 'undefined'
		&& entity.get(Host)?.element instanceof HTMLImageElement;
}

function pointDomImageAt(entity: Entity, source: string | null): void {
	const node = entity.get(Host);
	const image = node?.element;
	if (typeof HTMLImageElement === 'undefined' || !(image instanceof HTMLImageElement) || !node) return;

	releaseDomImageSource(image);
	if (source === null) image.removeAttribute('src');
	else image.src = source;
}

export function releaseDomImageSource(image: HTMLImageElement): void {
	if (image.src.startsWith('blob:')) URL.revokeObjectURL(image.src);
}

async function bindDomImage(entity: Entity, asset: Asset, token: object): Promise<void> {
	if (!isDomImage(entity)) return;
	const file = await getAssetFile(asset);
	if (!current(entity, token)) return;

	const url = URL.createObjectURL(file);
	if (!current(entity, token)) {
		URL.revokeObjectURL(url);
		return;
	}

	pointDomImageAt(entity, url);
	const image = entity.get(Host)?.element;
	if (typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement) {
		await image.decode().catch(() => undefined);
	}
}

/** What a rejection says, for an entity to carry and the host to show. */
function errorMessage(error: unknown): string {
	const message = error instanceof Error ? error.message : String(error);
	return message.trim() || 'Something went wrong';
}

function current(entity: Entity, token: object): boolean {
	return entity.isAlive() && entity.get(PendingSource)?.value === token;
}