The VS Code of video editing, built for agents.
TypeScript
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// Playback system (was systems/playback.ts): advances root playheads, derives
// per-entity local time + visibility, forwards decoders/hosts, keeps the
// audio bus tree in sync, and steps live mounts. In offline modes the encoder
// sets the playhead explicitly and awaits FramePromises.
import { Not, Or } from 'koota';
import { store } from '../world/store';
import { PaintType } from '../constants';
import {
ChildOf, Hidden, Culled, Dragging, AssetId, Mask,
Geometry, Group, AdjustmentLayer, Paint, Audio, Caption, Muted, Soloed,
Sequential, Transition, Playback, Workarea,
AudioPlayback, Computed,
AudioDecoderHandle, AudioBusHandle, Host,
Mode, Silent, FrameRate, Time, AudioEngine, FramePromises, Tickers,
Root,
} from '../traits';
import { getMaskContainers, getParentNode } from '../queries/hierarchy';
import { getIntrinsicPaint, getSourceWindow } from '../utils/time';
import { clamp } from '../math/common';
import { getTransitionWindow } from '../utils/transition';
import {
resolveAudioDecoder, resolveCaptionDecoder, resolveImageDecoder,
resolveShaderHost, resolveVideoDecoder,
} from '../media';
import { whenHtmlReady } from '../media/html';
import { AudioBus } from '../media/audio-bus';
import type { Entity, World } from 'koota';
const WARMUP_FRAMES = 15;
function framePromises(world: World) {
return world.get(FramePromises)?.list ?? null;
}
function advancePlayhead(world: World, entity: Entity): void {
// In offline mode the encoder sets the playhead explicitly per frame, so
// skip the delta-based advance — running it would either double-step or
// drift depending on how the caller set `Time.delta`.
if (world.get(Mode)?.value !== 'realtime') return;
const playback = store(world, Playback);
const computed = store(world, Computed);
const eid = entity.id();
const fps = world.get(FrameRate)?.value ?? 30;
const speed = playback.speed[eid] || 1;
let looped = false;
// Only advance the playhead while playing. The audio play/stop bookkeeping
// below must still run when stopped, otherwise a manual stop (playing
// flipped off externally) is never observed — `wasPlaying` stays stuck and
// the next play isn't re-anchored, so scheduled buffers land in the past
// and play silently.
if (playback.playing[eid] === true) {
const dt = (world.get(Time)?.delta ?? 0) / 1000;
const previousTime = computed.localTimeInSeconds[eid]!;
let time = previousTime + dt * speed;
const hasWorkarea = entity.has(Workarea);
const workarea = store(world, Workarea);
const duration = computed.duration[eid]!;
const durationSeconds = duration / fps;
const maxSeconds = hasWorkarea
? (workarea.end[eid] ?? duration) / fps
: durationSeconds;
const minSeconds = hasWorkarea
? (workarea.start[eid] ?? 0) / fps
: 0;
if (time >= maxSeconds) {
if (playback.loop[eid] === true) {
time = minSeconds;
looped = true;
} else {
playback.playing[eid] = false;
}
} else if (time <= minSeconds) {
if (playback.loop[eid] === true) {
time = maxSeconds;
looped = true;
} else {
playback.playing[eid] = false;
time = minSeconds;
}
}
computed.localTimeInSeconds[eid] = clamp(time, minSeconds, maxSeconds);
computed.localTime[eid] = Math.round(computed.localTimeInSeconds[eid]! * fps);
}
// Handle audio play/stop transitions
if (!entity.has(AudioPlayback)) entity.add(AudioPlayback);
const audioPlayback = store(world, AudioPlayback);
const ctx = world.get(AudioEngine)?.context;
const playing = playback.playing[eid] ?? false;
const wasPlaying = audioPlayback.wasPlaying[eid] ?? false;
// Shuttling (J/L at anything but 1x) plays silent
if ((playing && !wasPlaying) || looped || (playing && speed !== 1)) {
audioPlayback.contextOffsetInSeconds[eid] = ctx?.currentTime ?? 0;
audioPlayback.timelineOffsetInSeconds[eid] = computed.localTimeInSeconds[eid]!;
}
if (!playing && wasPlaying) {
resetDecoders(world, entity);
}
audioPlayback.wasPlaying[eid] = playing;
}
function forwardVideoDecoder(world: World, scene: Entity, entity: Entity, fill: Entity): void {
const computed = store(world, Computed);
const eid = entity.id();
const fps = world.get(FrameRate)?.value ?? 30;
const globalFrame = computed.localTime[scene.id()]!;
const localFrame = computed.localTime[eid]!;
const start = computed.start[eid]!;
const end = computed.end[eid]!;
const hasCache = world.get(Mode)?.value === 'realtime';
const warmupDecoder = globalFrame >= start - WARMUP_FRAMES && globalFrame < end + WARMUP_FRAMES && hasCache;
const source = getSourceWindow(entity);
const decoder = resolveVideoDecoder(world, fill);
if (!decoder) return;
if (computed.visibility[eid] === 1 || warmupDecoder) {
const seekFrame = clamp(localFrame, source.in, source.out);
const seekPromise = decoder.seekTo(seekFrame, fps);
framePromises(world)?.push(seekPromise ?? null);
}
}
function forwardCaptionDecoder(world: World, _scene: Entity, entity: Entity): void {
const localFrame = store(world, Computed).localTime[entity.id()]!;
const fps = world.get(FrameRate)?.value ?? 30;
const decoder = resolveCaptionDecoder(world, entity);
decoder?.seekTo(world, entity, localFrame / fps);
}
/**
* Forward the audio decoder for a child entity. `audioSource` optionally
* points at the sub-entity carrying the audio (a video fill), while the
* timing still comes from the clip entity itself.
*/
function forwardAudioDecoder(world: World, scene: Entity, entity: Entity, audioSource?: Entity): void {
if (world.has(Silent) || entity.has(Muted)) return;
const resolvedDecoder = resolveAudioDecoder(world, audioSource ?? entity);
if (!resolvedDecoder) return;
const { decoder, initPromise } = resolvedDecoder;
const computed = store(world, Computed);
const audioPlayback = store(world, AudioPlayback);
const playback = store(world, Playback);
const eid = entity.id();
const sid = scene.id();
const fps = world.get(FrameRate)?.value ?? 30;
const currentTime = computed.localTime[sid]!;
const localFrame = computed.localTime[eid]!;
const source = getSourceWindow(entity);
const playbackRate = computed.playbackRate[eid] || 1;
const origin = computed.origin[eid]!;
const audioOffset = audioPlayback.contextOffsetInSeconds[sid] ?? 0;
const playbackOffset = audioPlayback.timelineOffsetInSeconds[sid] ?? 0;
// In offline rendering the encoder pins contextOffset to 0 and playbackOffset
// to the workarea start, so this term shifts scheduled audio back into the
// encoded window (and resolves to 0 when there is no workarea).
const audioDelay = audioOffset - playbackOffset;
const bus = resolveAudioBus(world, entity);
if (!decoder.ready) {
framePromises(world)?.push(initPromise);
} else if (computed.visibility[eid] === 1 && playback.playing[sid] === true && (playback.speed[sid] || 1) === 1) {
const playPromise = decoder.playTo(bus, {
relativeFrom: localFrame / fps,
relativeTo: (localFrame + 15) / fps,
trimStart: source.in / fps,
trimEnd: source.out / fps,
playbackRate,
currentTime: currentTime / fps,
relativeDelay: (origin / fps) + audioDelay,
});
framePromises(world)?.push(playPromise);
} else {
decoder.reset();
}
}
function forwardHtmlHost(world: World, scene: Entity, entity: Entity, fill: Entity): void {
const computed = store(world, Computed);
const root = fill.get(Host)?.element;
if (world.get(Mode)?.value === 'realtime'
|| computed.visibility[entity.id()] !== 1
|| !(root instanceof HTMLElement)) return;
framePromises(world)?.push(whenHtmlReady(root, computed.localTimeInSeconds[scene.id()] ?? 0));
}
function forwardImageDecoder(world: World, _scene: Entity, _entity: Entity, fill: Entity): void {
const resolvedDecoder = resolveImageDecoder(world, fill);
if (!resolvedDecoder) return;
const { decoder, initPromise } = resolvedDecoder;
if (!decoder.ready) {
framePromises(world)?.push(initPromise);
}
}
/**
* Seeks the frame sequences of the masks cutting the entity or its effects: a
* mask's frame `i` belongs to source frame `offset + i` of the clip, so the
* seek is the clip's own source frame less the offset, at the rate the
* sequence was written at (the composition's, authored as its frameRate).
*/
function forwardMaskDecoders(world: World, scene: Entity, entity: Entity): void {
const containers = getMaskContainers(world, entity);
if (!containers.length) return;
const computed = store(world, Computed);
const maskStore = store(world, Mask);
const eid = entity.id();
const fps = world.get(FrameRate)?.value ?? 30;
const globalFrame = computed.localTime[scene.id()]!;
const localFrame = computed.localTime[eid]!;
const start = computed.start[eid]!;
const end = computed.end[eid]!;
const hasCache = world.get(Mode)?.value === 'realtime';
const warmupDecoder = globalFrame >= start - WARMUP_FRAMES && globalFrame < end + WARMUP_FRAMES && hasCache;
if (computed.visibility[eid] !== 1 && !warmupDecoder) return;
const source = getSourceWindow(entity);
const sourceFrame = clamp(localFrame, source.in, source.out);
for (const mask of containers) {
if (!mask.has(Mask) || mask.has(Hidden) || !mask.has(AssetId)) continue;
if (getParentNode(mask)?.has(Hidden)) continue;
const decoder = resolveVideoDecoder(world, mask);
if (!decoder) continue;
const frame = Math.max(0, sourceFrame - (maskStore.offset[mask.id()] ?? 0));
// Seek first: the editor keeps no promise list, and an optional call
// skips its arguments along with itself.
const seek = decoder.seekTo(frame, fps);
framePromises(world)?.push(seek ?? null);
}
}
/**
* Forward decoders for a child entity and its node descendants.
*/
function forwardDecoders(world: World, scene: Entity, entity: Entity): void {
const paintStore = store(world, Paint);
const hidden = entity.has(Hidden) || entity.has(Culled);
const visualsEnabled = !hidden && world.get(Mode)?.value !== 'offline-audio';
let intrinsicVideo = false;
let paintAudioSource: Entity | undefined;
if (!hidden) {
const intrinsic = getIntrinsicPaint(entity);
if (intrinsic === PaintType.VIDEO) {
if (visualsEnabled) {
forwardVideoDecoder(world, scene, entity, entity);
}
intrinsicVideo = true;
} else if (intrinsic === PaintType.IMAGE && visualsEnabled) {
forwardImageDecoder(world, scene, entity, entity);
} else if (intrinsic === PaintType.HTML && visualsEnabled) {
forwardHtmlHost(world, scene, entity, entity);
}
for (const fill of world.query(ChildOf(entity), Paint, Not(Geometry), Not(Hidden))) {
const paint = paintStore.value[fill.id()];
if (paint === PaintType.VIDEO) {
if (visualsEnabled) {
forwardVideoDecoder(world, scene, entity, fill);
}
paintAudioSource = fill;
}
if (paint === PaintType.IMAGE && visualsEnabled) {
forwardImageDecoder(world, scene, entity, fill);
}
if (paint === PaintType.HTML && visualsEnabled) {
forwardHtmlHost(world, scene, entity, fill);
}
if (paint === PaintType.SHADER && visualsEnabled) {
const ready = resolveShaderHost(world, fill)?.whenReady();
if (ready) framePromises(world)?.push(ready);
}
}
if (visualsEnabled) {
forwardMaskDecoders(world, scene, entity);
}
}
if (entity.has(Caption) && visualsEnabled) {
forwardCaptionDecoder(world, scene, entity);
}
if (intrinsicVideo || entity.has(Audio) || paintAudioSource) {
forwardAudioDecoder(world, scene, entity, paintAudioSource);
}
for (const child of world.query(ChildOf(entity), Or(Geometry, Group, AdjustmentLayer), Not(Hidden))) {
forwardDecoders(world, scene, child);
}
for (const container of getMaskContainers(world, entity)) {
if (container.has(Hidden)) continue;
for (const child of world.query(ChildOf(container), Or(Geometry, Group, AdjustmentLayer), Not(Hidden))) {
forwardDecoders(world, scene, child);
}
}
}
function updateVisibility(world: World, scene: Entity, entity: Entity): void {
const computed = store(world, Computed);
const eid = entity.id();
// Root or dragging entity is always visible
if (entity === scene || entity.has(Dragging)) {
computed.visibility[eid] = 1;
} else {
const globalFrame = computed.localTime[scene.id()]!;
const origin = computed.origin[eid] ?? 0;
const playbackRate = computed.playbackRate[eid] || 1;
const start = computed.start[eid]!;
const end = computed.end[eid]!;
computed.localTime[eid] = Math.round((globalFrame - origin) * playbackRate);
computed.visibility[eid] = globalFrame >= start && globalFrame < end ? 1 : 0;
}
for (const child of world.query(Or(Geometry, Group, AdjustmentLayer), ChildOf(entity))) {
updateVisibility(world, scene, child);
}
// A `<clipPath>` or `<mask>` runs on the clock of the node it cuts.
for (const container of getMaskContainers(world, entity)) {
const cid = container.id();
computed.localTime[cid] = computed.localTime[eid]!;
computed.visibility[cid] = computed.visibility[eid]!;
for (const child of world.query(Or(Geometry, Group, AdjustmentLayer), ChildOf(container))) {
updateVisibility(world, scene, child);
}
}
}
function resetDecoders(world: World, entity: Entity): void {
if (entity.has(AudioDecoderHandle)) {
entity.get(AudioDecoderHandle)?.reset();
}
for (const child of world.query(ChildOf(entity), AudioDecoderHandle)) {
resetDecoders(world, child);
}
}
function collectAncestors(entity: Entity): Entity[] {
const path: Entity[] = [];
let current: Entity | null = entity;
while (current) {
path.push(current);
current = getParentNode(current);
}
return path.toReversed();
}
/**
* The entity's audio bus, building (and wiring) the ancestor bus chain on
* first use. Null when the world has no audio context.
*/
export function resolveAudioBus(world: World, entity: Entity): AudioBus | null {
const existing = entity.has(AudioBusHandle) ? entity.get(AudioBusHandle) : null;
if (existing) return existing;
const context = world.get(AudioEngine)?.context;
if (!context) return null;
const ancestors = collectAncestors(entity);
// Walk root -> leaf, connecting each bus's gain into its parent's input
// (root connects into the context destination). Reuse buses that already
// exist; only newly created ones need wiring.
let currentInput: AudioNode = context.destination;
for (const ancestor of ancestors) {
let bus = ancestor.has(AudioBusHandle) ? ancestor.get(AudioBusHandle) : null;
if (!bus) {
bus = new AudioBus(world, ancestor);
ancestor.add(AudioBusHandle);
ancestor.set(AudioBusHandle, bus);
bus.connect(currentInput);
}
currentInput = bus.input;
}
return entity.get(AudioBusHandle) ?? null;
}
function getGlobalFrame(world: World, entity: Entity): number {
const computed = store(world, Computed);
let current: Entity | null = entity;
while (current) {
if (current.has(Playback)) {
return computed.localTime[current.id()]!;
}
current = getParentNode(current);
}
return 0;
}
export function playbackSystem(world: World): void {
const computed = store(world, Computed);
// handle root playback
for (const entity of world.query(Playback)) {
advancePlayhead(world, entity);
}
for (const entity of world.query(Or(Geometry, Group, AdjustmentLayer), ChildOf(world.get(Root)!))) {
updateVisibility(world, entity, entity);
forwardDecoders(world, entity, entity);
}
// handle transition visibility
for (const clip of world.query(Or(Geometry, Group, AdjustmentLayer), Transition)) {
const parent = getParentNode(clip);
if (parent == null || !parent.has(Sequential)) {
console.info('Removing transition due to non-sequential parent', clip);
// System-driven cleanup of orphaned state — not a user edit.
clip.remove(Transition);
continue;
}
const children = world.query(ChildOf(parent), Or(Geometry, Group, AdjustmentLayer));
const partner = children.find(sibling => computed.start[sibling.id()] === computed.end[clip.id()]);
if (!partner) continue;
const window = getTransitionWindow(world, clip, partner);
const globalFrame = getGlobalFrame(world, clip);
if (globalFrame >= window.start && globalFrame < window.end) {
computed.visibility[clip.id()] = 1;
computed.visibility[partner.id()] = 1;
}
}
// Sync audio buses
let soloed: Set<Entity> | null = null;
for (const entity of world.query(AudioBusHandle)) {
entity.get(AudioBusHandle)?.sync();
if (!entity.has(Soloed)) continue;
if (soloed === null) {
soloed = new Set();
}
for (const ancestor of collectAncestors(entity)) {
soloed.add(ancestor);
}
}
if (soloed) {
for (const entity of world.query(AudioBusHandle)) {
if (!soloed.has(entity)) {
entity.get(AudioBusHandle)?.mute();
}
}
}
for (const tick of world.get(Tickers) ?? []) tick();
}