Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor playback handling and enhance media synchronization

konstantin-paulusaf64295ed983
diff --git a/apps/web/src/engine/input/interactions.ts b/apps/web/src/engine/input/interactions.ts
index f0ca5b62..1c54577f 100644
--- a/apps/web/src/engine/input/interactions.ts
+++ b/apps/web/src/engine/input/interactions.ts
@@ -17,8 +17,8 @@
*/
import {
- AudioEngine, ChildOf, Computed, Culled, Geometry, Group, Hovering,
- Interactive, KeepAspectRatio, Playback, RenderSurface, Root, Scene,
+ ChildOf, Computed, Culled, Geometry, Group, Hovering,
+ Interactive, KeepAspectRatio, RenderSurface, Root, Scene,
Selected, Skew, Time,
computeGroupBounds, computeLocalMatrix, decompose2D, entityAnchor,
entityOffset, entityQuad, entityWorldMat, enterEntity,
@@ -26,7 +26,7 @@ import {
getSelection, getSelectionMask, identity2D, invert2D, isPointerInEntity,
multiply2D, quadCenter, quadContainsQuad, quadsIntersect, rectToQuad,
rotate2D, scale2D,
- store, syncInteractiveState, transformPoint, translate2D,
+ store, syncInteractiveState, togglePlayback, transformPoint, translate2D,
} from '@diffusionstudio/runtime';
import { Not, Or } from 'koota';
@@ -248,20 +248,15 @@ export function handleCanvasInteraction(world: World, event: DispatchedPointerEv
}
}
-/** The play/pause button on a scene header. */
+/** The play/pause button on a header: a scene's, or a stage-level video/audio's. */
export function handlePlayInteraction(world: World, event: DispatchedPointerEvent): void {
if (event.type !== 'click' || event.target.kind !== 'hud') return;
// The region is a frame old, so what it points at may be gone.
const entity = event.target.entity;
- const playback = entity?.isAlive() ? entity.get(Playback) : undefined;
- if (!entity || !playback) return;
-
- entity.set(Playback, { playing: !playback.playing, speed: 1 });
+ if (!entity?.isAlive()) return;
- // Audio is gated until the context has seen a gesture; this is one.
- const context = world.get(AudioEngine)?.context;
- if (context instanceof AudioContext) void context.resume();
+ togglePlayback(world, entity);
}
/** The name on a scene header: selects, renames on double-click, drags the node. */
diff --git a/packages/runtime/src/actions/playback.ts b/packages/runtime/src/actions/playback.ts
index 191a2622..d1be771f 100644
--- a/packages/runtime/src/actions/playback.ts
+++ b/packages/runtime/src/actions/playback.ts
@@ -9,10 +9,50 @@
// editor. `playbackSystem` advances the playhead from here while playing.
import { store } from '../world/store';
-import { AudioEngine, Computed, FrameRate, Playback } from '../traits';
+import {
+ Audio, AudioEngine, Cache, Computed, FrameRate, Geometry, Paint,
+ Playback, Scene, Stage, Workarea,
+} from '../traits';
+import { PaintType } from '../constants';
+import { getIntrinsicPaint } from '../utils/time';
+import { getParentEntity } from '../queries/hierarchy';
import type { Entity, World } from 'koota';
+/**
+ * Whether a node plays media of its own: audio, or video through its
+ * intrinsic paint or one of its fills.
+ */
+function hasPlayableMedia(entity: Entity): boolean {
+ if (entity.has(Audio) || getIntrinsicPaint(entity) === PaintType.VIDEO) return true;
+
+ for (const fill of entity.get(Cache)?.fills ?? []) {
+ if (fill.get(Paint)?.value === PaintType.VIDEO) return true;
+ }
+
+ return false;
+}
+
+/**
+ * Keeps Playback in step with where `entity` sits: a video or audio directly
+ * on the stage plays like a scene — the playhead its header's play toggle
+ * drives is its own — while nested under a scene it derives its time from it.
+ * The hierarchy observers call this after every attach and detach, for both
+ * the node itself and the parent a media fill just made (or stopped making)
+ * playable, once the caches it reads are rebuilt. Scenes own their Playback
+ * from creation and are left alone, as is anything without a geometry (the
+ * encoder's synthetic clock is a bare group).
+ */
+export function syncStagePlayback(entity: Entity): void {
+ if (!entity.has(Geometry) || entity.has(Scene)) return;
+
+ if (getParentEntity(entity)?.has(Stage) && hasPlayableMedia(entity)) {
+ if (!entity.has(Playback)) entity.add(Playback);
+ } else if (entity.has(Playback)) {
+ entity.remove(Playback);
+ }
+}
+
/**
* Moves `scene`'s playhead to `frame` — a seek, which is what a scrub of the
* ruler or a click in the timeline comes to. Both mirrors of the playhead are
@@ -42,6 +82,19 @@ export function togglePlayback(world: World, scene: Entity): void {
const playback = scene.get(Playback);
if (!playback) return;
+ // Starting parked at the end would stop on the first advance, so a
+ // finished scene plays again from the top (of its workarea, if any).
+ if (!playback.playing) {
+ const computed = store(world, Computed);
+ const eid = scene.id();
+ const workarea = scene.has(Workarea) ? scene.get(Workarea) : undefined;
+ const end = workarea?.end ?? computed.duration[eid] ?? 0;
+
+ if (end > 0 && (computed.localTime[eid] ?? 0) >= end) {
+ setPlayhead(world, scene, workarea?.start ?? 0);
+ }
+ }
+
scene.set(Playback, { playing: !playback.playing, speed: 1 });
const context = world.get(AudioEngine)?.context;
diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.ts
index f3268c60..2b16805d 100644
--- a/packages/runtime/src/world/observers.ts
+++ b/packages/runtime/src/world/observers.ts
@@ -16,6 +16,7 @@ import {
} from '../traits';
import { getParentEntity } from '../queries/hierarchy';
import { evictFromCaches, rebuildCaches, refileMask } from '../actions/cache';
+import { syncStagePlayback } from '../actions/playback';
import { disposeDecoders, disconnectAudioBus } from '../media/dispose';
import {
reactToChildAttached, reactToChildDetached, reactToAssetChange,
@@ -43,6 +44,8 @@ export function observeWorld(world: World): () => void {
resolveConstraintOffsets(world, child);
disconnectAudioBus(world, child);
reactToChildAttached(world, child);
+ syncStagePlayback(child);
+ syncStagePlayback(parent);
}));
// On re-target (and destroy) the remove event fires while the child is
@@ -58,6 +61,7 @@ export function observeWorld(world: World): () => void {
rebuildCaches(world, child, parent, child);
evictFromCaches(world, child, parent);
reactToChildDetached(world, child);
+ syncStagePlayback(parent);
}));
// Live media handles are released with their trait. Destroy removes every