Skip to content

The VS Code of video editing, built for agents.

TypeScript

Implement syncTo feature for audio/video alignment

konstantin-pauluse0e9b761bb7e
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 9b921beb..5ff12ce2 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Stroke, StrokeCap, StrokeJoin, StrokeStyle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';
+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';
import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
import { createSignal } from 'solid-js';
import { SVGElements } from 'solid-js/web';
@@ -1151,6 +1151,18 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
}
return;
}
+ case 'syncTo': {
+ entity.remove(PendingSync);
+
+ if (typeof value !== 'string' || value === '') {
+ entity.remove(SyncRequest);
+ return;
+ }
+
+ entity.add(SyncRequest);
+ entity.set(SyncRequest, { value });
+ return;
+ }
case 'fill':
case 'color': {
const color = parseColor(value);
diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.ts
index 6298798f..05f0bf26 100644
--- a/packages/runtime/src/systems/assets.ts
+++ b/packages/runtime/src/systems/assets.ts
@@ -2,9 +2,15 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Ai, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, SourceError, TranscriptionRequest } from '../traits';
+import { parseSource } from '@diffusionstudio/jsx';
+
+import { Ai, Cache, Computed, Delay, FrameRate, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, PendingSync, PlaybackRate, Source, SourceError, SyncRequest, TranscriptionRequest, Trim } from '../traits';
import { bindAsset, getAssetFile, getModifiers } from '../actions/assets';
-import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';
+import { getEntityTree, getParentNode, getSceneAncestor } from '../queries/hierarchy';
+import { findAssetDuration, findGeometryAsset } from '../utils/time';
+import { computeAudioSyncOffsetCached } from '../media/audio-sync';
+import { store } from '../world/store';
+import { assert } from '../utils/assert';
import type { Entity, World } from 'koota';
import type { Asset } from '@diffusionstudio/assets';
@@ -48,6 +54,166 @@ export function assetSystem(world: World): void {
resolve(world, entity, ai.transcribe(world, scene, seed), true);
}
}
+
+ // Handle syncTo requests
+ for (const entity of world.query(SyncRequest)) {
+ if (waitingOnSources(entity)) continue;
+
+ const id = entity.get(SyncRequest)!.value;
+ const target = findElementById(world, id, entity);
+
+ if (target === undefined) {
+ fail(entity, `syncTo: no other element carries id "${id}"`);
+ continue;
+ }
+
+ if (inCycle(world, entity)) {
+ fail(entity, `syncTo: "${id}" leads back around to this element; a sync cannot go in a circle`);
+ continue;
+ }
+
+ if (target.has(SyncRequest) || target.has(PendingSync) || waitingOnSources(target)) continue;
+
+ const source = findGeometryAsset(world, entity);
+ if (source === null || (source.type !== 'AUDIO' && source.type !== 'VIDEO')) {
+ fail(entity, 'syncTo: the element has no audio or video source to align');
+ continue;
+ }
+
+ const reference = findGeometryAsset(world, target);
+ if (reference === null || (reference.type !== 'AUDIO' && reference.type !== 'VIDEO')) {
+ fail(entity, `syncTo: "${id}" has no audio or video source to align against`);
+ continue;
+ }
+
+ entity.remove(SyncRequest);
+
+ const token = {};
+ entity.add(PendingSync);
+ entity.set(PendingSync, { value: token });
+
+ const done = computeAudioSyncOffsetCached(source, reference)
+ .then(({ offsetSeconds }) => {
+ if (!currentSync(entity, token)) return;
+ if (target.isAlive()) placeSynced(world, entity, target, id, offsetSeconds);
+ entity.remove(PendingSync);
+ })
+ .catch((error: unknown) => {
+ if (!currentSync(entity, token)) return;
+ fail(entity, error instanceof Error ? error.message : String(error));
+ });
+
+ world.get(FramePromises)?.list?.push(done);
+ }
+}
+
+/**
+ * Places `entity` so its recording coincides with `target`'s: the measured
+ * offset says how long after the target's recording this one started, so the
+ * element's source frame 0 goes that far after the target's origin,
+ * re-expressed against the element's own parent (Delay is parent-relative
+ * where Computed is absolute). The origin keeps its fraction so the audio
+ * scheduled against it doesn't inherit a half-frame of slip.
+ *
+ * An authored trim bound survives (`sourceIn`/`sourceOut` remain yours to
+ * set); a missing one defaults to the intersection of the element's natural
+ * extent with the target's window. A window that closes before it opens means
+ * the recordings do not overlap, which is reported rather than guessed at.
+ */
+function placeSynced(world: World, entity: Entity, target: Entity, id: string, offsetSeconds: number): void {
+ const computed = store(world, Computed);
+ const frameRate = world.get(FrameRate)?.value ?? 30;
+
+ const parent = getParentNode(entity);
+ const parentOrigin = parent !== null ? computed.origin[parent.id()] ?? 0 : 0;
+ const origin = (computed.origin[target.id()] ?? 0) + offsetSeconds * frameRate;
+
+ entity.add(Delay);
+ entity.set(Delay, { value: origin - parentOrigin });
+
+ const rate = entity.get(PlaybackRate)?.value || 1;
+ const trim = entity.get(Trim);
+ const targetStart = computed.start[target.id()] ?? 0;
+ const targetEnd = computed.end[target.id()] ?? 0;
+
+ const start = trim?.start ?? Math.max(0, (targetStart - origin) * rate);
+ let end = trim?.end ?? null;
+ if (end === null) {
+ end = (targetEnd - origin) * rate;
+ const duration = findAssetDuration(world, entity);
+ if (duration !== null) end = Math.min(end, duration);
+ }
+
+ assert(end > start, `syncTo: the aligned clip does not overlap the window of "${id}"`);
+
+ entity.add(Trim);
+ entity.set(Trim, { start, end });
+}
+
+/**
+ * The element whose authored id is `id`: ids ride into the runtime as the
+ * locator of the source stamp (see SOURCE_ATTR in @diffusionstudio/jsx), so
+ * only stamped elements can be found — and a positional locator, being a
+ * number, never matches. `except` keeps an element from finding itself.
+ * Iterations of a loop share one stamp; the first stands for them all.
+ */
+function findElementById(world: World, id: string, except: Entity): Entity | undefined {
+ for (const entity of world.query(Source)) {
+ if (entity === except) continue;
+ if (parseSource(entity.get(Source)!.value)?.locator === id) return entity;
+ }
+ return undefined;
+}
+
+/**
+ * Whether following `syncTo` from `entity` comes back around to it. Only
+ * requests still standing are links: a placed element is a fixed point, not a
+ * hop. Of a cycle's members, each fails on its own turn through the request
+ * loop; whoever is checked once the others have failed no longer sits on a
+ * cycle and aligns against their default placement.
+ */
+function inCycle(world: World, entity: Entity): boolean {
+ const seen = new Set<Entity>([entity]);
+ let current = entity;
+ for (; ;) {
+ const id = current.get(SyncRequest)?.value;
+ if (id === undefined) return false;
+ const next = findElementById(world, id, current);
+ if (next === undefined) return false;
+ if (next === entity) return true;
+ if (seen.has(next)) return false;
+ seen.add(next);
+ current = next;
+ }
+}
+
+/**
+ * Whether the element's media is still being resolved — a request this system
+ * has not consumed, or a resolution inflight — on the element itself or on a
+ * fill (a `<rect><videoPaint/>` carries its src there).
+ */
+function waitingOnSources(entity: Entity): boolean {
+ if (pendingSource(entity)) return true;
+ for (const fill of entity.get(Cache)?.fills ?? []) {
+ if (pendingSource(fill)) return true;
+ }
+ return false;
+}
+
+function pendingSource(entity: Entity): boolean {
+ return entity.has(LoadRequest) || entity.has(GenerationRequest) || entity.has(PendingSource);
+}
+
+/** Takes the sync off and leaves why on the element, `resolve`-style. */
+function fail(entity: Entity, message: string): void {
+ entity.remove(SyncRequest, PendingSync);
+ entity.add(SourceError);
+ entity.set(SourceError, { value: message, generated: false });
+ console.error('[runtime] could not sync:', message);
+}
+
+function currentSync(entity: Entity, token: object): boolean {
+ return entity.isAlive() && entity.get(PendingSync)?.value === token;
}
/**
@@ -71,12 +237,14 @@ function start(world: World, entity: Entity, base: Promise<Asset>, generating =
/**
* Whether anything in `scene`'s subtree (besides `except`, the requesting
* element itself) is still waiting on a source: a request this system has
- * not consumed, or a resolution it started that has not landed.
+ * not consumed, or a resolution it started that has not landed. A pending
+ * `syncTo` counts — the clip has a source but not yet the placement a
+ * transcription would read it at.
*/
function hasPendingSources(world: World, scene: Entity, except: Entity): boolean {
for (const entity of getEntityTree(world, scene)) {
if (entity === except) continue;
- if (entity.has(LoadRequest) || entity.has(GenerationRequest) || entity.has(PendingSource)) {
+ if (pendingSource(entity) || entity.has(SyncRequest) || entity.has(PendingSync)) {
return true;
}
}
diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.ts
index ad77b4dd..a7960d92 100644
--- a/packages/runtime/src/traits/node.ts
+++ b/packages/runtime/src/traits/node.ts
@@ -101,6 +101,20 @@ export const SourceModifiers = trait({ removeBackground: false, upscale: 1, addA
// bumping the seed transcribes the scene again. Seed 0 is the default take.
export const TranscriptionRequest = trait({ seed: 0 });
+// On an element authored with `syncTo`: the id of the element to align
+// against by listening instead of arithmetic. The asset system waits for both
+// sides' assets to land, cross-correlates the two recordings, and derives
+// Delay/Trim so they coincide on the timeline. Stands (with PendingSync) as a
+// pending source, so a transcription of the scene waits for the clip to land
+// where it will play. Never serialized, for the same reason the requests
+// above are not.
+export const SyncRequest = trait({ value: '' });
+
+// The syncTo measurement inflight, kept while the asset system correlates, so
+// a result that arrives after the element was pointed at another target (or
+// none) is dropped. The document clears it whenever a new syncTo arrives.
+export const PendingSync = trait(() => ({ value: undefined as unknown }));
+
// The src whose resolution is inflight, kept while the asset system works,
// so a resolution that arrives after the element was given another source
// (or none) is dropped. The document clears it whenever a new src arrives.
diff --git a/reference/jsx/audio-sync.md b/reference/jsx/audio-sync.md
index 24a42137..8605fc22 100644
--- a/reference/jsx/audio-sync.md
+++ b/reference/jsx/audio-sync.md
@@ -9,15 +9,13 @@
</scene>
```
-> **Status:** `syncTo` is part of the authoring surface and typechecks, and the correlation itself is implemented in the runtime, but the mount does not apply it yet — a node carrying `syncTo` is placed by its own `start` (0 by default) as if the prop were absent. Set `start` explicitly until it lands.
-
## Semantics
- `syncTo` names the [`id`](./module.md#ids) of another element in the same render. Both sides must carry an audio track; any pairing works (audio-to-video, audio-to-audio, video-to-video).
- The computed placement is `start = target.start + offset`, where `offset` is the measured source-time offset between the two recordings (positive when this node's recording started after the target's; possibly negative). `syncTo` and `start` are mutually exclusive.
- `sourceIn`/`sourceOut` keep their normal meaning and remain yours to set. When omitted on a synced node, the window defaults to the intersection of the node's natural extent with the target's window (instead of the usual natural-duration fit), so a lav track simply covers its take.
- Alignment reads source content: `muted` and `volume` on either side do not affect the measurement. Use `muted` to keep only one side audible, as on the camera track above.
-- Chains resolve in dependency order (A may sync to B while B syncs to C). Unknown ids and cycles are mount errors.
+- Chains resolve in dependency order (A may sync to B while B syncs to C). An unknown id or a cycle is reported on the element like a failed source (see [errors.md](./errors.md)), and the node keeps its default placement.
## Execution