The VS Code of video editing, built for agents.
TypeScript
Implement syncTo feature for audio/video alignment
konstantin-pauluse0e9b761bb7e
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 9b921beb..5ff12ce2 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Stroke, StrokeCap, StrokeJoin, StrokeStyle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import { createSignal } from 'solid-js';import { SVGElements } from 'solid-js/web';@@ -1151,6 +1151,18 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {}return;}+ case 'syncTo': {+ entity.remove(PendingSync);++ if (typeof value !== 'string' || value === '') {+ entity.remove(SyncRequest);+ return;+ }++ entity.add(SyncRequest);+ entity.set(SyncRequest, { value });+ return;+ }case 'fill':case 'color': {const color = parseColor(value);diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsindex 6298798f..05f0bf26 100644--- a/packages/runtime/src/systems/assets.ts+++ b/packages/runtime/src/systems/assets.ts@@ -2,9 +2,15 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Ai, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, SourceError, TranscriptionRequest } from '../traits';+import { parseSource } from '@diffusionstudio/jsx';++import { Ai, Cache, Computed, Delay, FrameRate, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, PendingSync, PlaybackRate, Source, SourceError, SyncRequest, TranscriptionRequest, Trim } from '../traits';import { bindAsset, getAssetFile, getModifiers } from '../actions/assets';-import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';+import { getEntityTree, getParentNode, getSceneAncestor } from '../queries/hierarchy';+import { findAssetDuration, findGeometryAsset } from '../utils/time';+import { computeAudioSyncOffsetCached } from '../media/audio-sync';+import { store } from '../world/store';+import { assert } from '../utils/assert';import type { Entity, World } from 'koota';import type { Asset } from '@diffusionstudio/assets';@@ -48,6 +54,166 @@ export function assetSystem(world: World): void {resolve(world, entity, ai.transcribe(world, scene, seed), true);}}++ // Handle syncTo requests+ for (const entity of world.query(SyncRequest)) {+ if (waitingOnSources(entity)) continue;++ const id = entity.get(SyncRequest)!.value;+ const target = findElementById(world, id, entity);++ if (target === undefined) {+ fail(entity, `syncTo: no other element carries id "${id}"`);+ continue;+ }++ if (inCycle(world, entity)) {+ fail(entity, `syncTo: "${id}" leads back around to this element; a sync cannot go in a circle`);+ continue;+ }++ if (target.has(SyncRequest) || target.has(PendingSync) || waitingOnSources(target)) continue;++ const source = findGeometryAsset(world, entity);+ if (source === null || (source.type !== 'AUDIO' && source.type !== 'VIDEO')) {+ fail(entity, 'syncTo: the element has no audio or video source to align');+ continue;+ }++ const reference = findGeometryAsset(world, target);+ if (reference === null || (reference.type !== 'AUDIO' && reference.type !== 'VIDEO')) {+ fail(entity, `syncTo: "${id}" has no audio or video source to align against`);+ continue;+ }++ entity.remove(SyncRequest);++ const token = {};+ entity.add(PendingSync);+ entity.set(PendingSync, { value: token });++ const done = computeAudioSyncOffsetCached(source, reference)+ .then(({ offsetSeconds }) => {+ if (!currentSync(entity, token)) return;+ if (target.isAlive()) placeSynced(world, entity, target, id, offsetSeconds);+ entity.remove(PendingSync);+ })+ .catch((error: unknown) => {+ if (!currentSync(entity, token)) return;+ fail(entity, error instanceof Error ? error.message : String(error));+ });++ world.get(FramePromises)?.list?.push(done);+ }+}++/**+ * Places `entity` so its recording coincides with `target`'s: the measured+ * offset says how long after the target's recording this one started, so the+ * element's source frame 0 goes that far after the target's origin,+ * re-expressed against the element's own parent (Delay is parent-relative+ * where Computed is absolute). The origin keeps its fraction so the audio+ * scheduled against it doesn't inherit a half-frame of slip.+ *+ * An authored trim bound survives (`sourceIn`/`sourceOut` remain yours to+ * set); a missing one defaults to the intersection of the element's natural+ * extent with the target's window. A window that closes before it opens means+ * the recordings do not overlap, which is reported rather than guessed at.+ */+function placeSynced(world: World, entity: Entity, target: Entity, id: string, offsetSeconds: number): void {+ const computed = store(world, Computed);+ const frameRate = world.get(FrameRate)?.value ?? 30;++ const parent = getParentNode(entity);+ const parentOrigin = parent !== null ? computed.origin[parent.id()] ?? 0 : 0;+ const origin = (computed.origin[target.id()] ?? 0) + offsetSeconds * frameRate;++ entity.add(Delay);+ entity.set(Delay, { value: origin - parentOrigin });++ const rate = entity.get(PlaybackRate)?.value || 1;+ const trim = entity.get(Trim);+ const targetStart = computed.start[target.id()] ?? 0;+ const targetEnd = computed.end[target.id()] ?? 0;++ const start = trim?.start ?? Math.max(0, (targetStart - origin) * rate);+ let end = trim?.end ?? null;+ if (end === null) {+ end = (targetEnd - origin) * rate;+ const duration = findAssetDuration(world, entity);+ if (duration !== null) end = Math.min(end, duration);+ }++ assert(end > start, `syncTo: the aligned clip does not overlap the window of "${id}"`);++ entity.add(Trim);+ entity.set(Trim, { start, end });+}++/**+ * The element whose authored id is `id`: ids ride into the runtime as the+ * locator of the source stamp (see SOURCE_ATTR in @diffusionstudio/jsx), so+ * only stamped elements can be found — and a positional locator, being a+ * number, never matches. `except` keeps an element from finding itself.+ * Iterations of a loop share one stamp; the first stands for them all.+ */+function findElementById(world: World, id: string, except: Entity): Entity | undefined {+ for (const entity of world.query(Source)) {+ if (entity === except) continue;+ if (parseSource(entity.get(Source)!.value)?.locator === id) return entity;+ }+ return undefined;+}++/**+ * Whether following `syncTo` from `entity` comes back around to it. Only+ * requests still standing are links: a placed element is a fixed point, not a+ * hop. Of a cycle's members, each fails on its own turn through the request+ * loop; whoever is checked once the others have failed no longer sits on a+ * cycle and aligns against their default placement.+ */+function inCycle(world: World, entity: Entity): boolean {+ const seen = new Set<Entity>([entity]);+ let current = entity;+ for (; ;) {+ const id = current.get(SyncRequest)?.value;+ if (id === undefined) return false;+ const next = findElementById(world, id, current);+ if (next === undefined) return false;+ if (next === entity) return true;+ if (seen.has(next)) return false;+ seen.add(next);+ current = next;+ }+}++/**+ * Whether the element's media is still being resolved — a request this system+ * has not consumed, or a resolution inflight — on the element itself or on a+ * fill (a `<rect><videoPaint/>` carries its src there).+ */+function waitingOnSources(entity: Entity): boolean {+ if (pendingSource(entity)) return true;+ for (const fill of entity.get(Cache)?.fills ?? []) {+ if (pendingSource(fill)) return true;+ }+ return false;+}++function pendingSource(entity: Entity): boolean {+ return entity.has(LoadRequest) || entity.has(GenerationRequest) || entity.has(PendingSource);+}++/** Takes the sync off and leaves why on the element, `resolve`-style. */+function fail(entity: Entity, message: string): void {+ entity.remove(SyncRequest, PendingSync);+ entity.add(SourceError);+ entity.set(SourceError, { value: message, generated: false });+ console.error('[runtime] could not sync:', message);+}++function currentSync(entity: Entity, token: object): boolean {+ return entity.isAlive() && entity.get(PendingSync)?.value === token;}/**@@ -71,12 +237,14 @@ function start(world: World, entity: Entity, base: Promise<Asset>, generating =/*** Whether anything in `scene`'s subtree (besides `except`, the requesting* element itself) is still waiting on a source: a request this system has- * not consumed, or a resolution it started that has not landed.+ * not consumed, or a resolution it started that has not landed. A pending+ * `syncTo` counts — the clip has a source but not yet the placement a+ * transcription would read it at.*/function hasPendingSources(world: World, scene: Entity, except: Entity): boolean {for (const entity of getEntityTree(world, scene)) {if (entity === except) continue;- if (entity.has(LoadRequest) || entity.has(GenerationRequest) || entity.has(PendingSource)) {+ if (pendingSource(entity) || entity.has(SyncRequest) || entity.has(PendingSync)) {return true;}}diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex ad77b4dd..a7960d92 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -101,6 +101,20 @@ export const SourceModifiers = trait({ removeBackground: false, upscale: 1, addA// bumping the seed transcribes the scene again. Seed 0 is the default take.export const TranscriptionRequest = trait({ seed: 0 });+// On an element authored with `syncTo`: the id of the element to align+// against by listening instead of arithmetic. The asset system waits for both+// sides' assets to land, cross-correlates the two recordings, and derives+// Delay/Trim so they coincide on the timeline. Stands (with PendingSync) as a+// pending source, so a transcription of the scene waits for the clip to land+// where it will play. Never serialized, for the same reason the requests+// above are not.+export const SyncRequest = trait({ value: '' });++// The syncTo measurement inflight, kept while the asset system correlates, so+// a result that arrives after the element was pointed at another target (or+// none) is dropped. The document clears it whenever a new syncTo arrives.+export const PendingSync = trait(() => ({ value: undefined as unknown }));+// The src whose resolution is inflight, kept while the asset system works,// so a resolution that arrives after the element was given another source// (or none) is dropped. The document clears it whenever a new src arrives.diff --git a/reference/jsx/audio-sync.md b/reference/jsx/audio-sync.mdindex 24a42137..8605fc22 100644--- a/reference/jsx/audio-sync.md+++ b/reference/jsx/audio-sync.md@@ -9,15 +9,13 @@</scene>```-> **Status:** `syncTo` is part of the authoring surface and typechecks, and the correlation itself is implemented in the runtime, but the mount does not apply it yet — a node carrying `syncTo` is placed by its own `start` (0 by default) as if the prop were absent. Set `start` explicitly until it lands.-## Semantics- `syncTo` names the [`id`](./module.md#ids) of another element in the same render. Both sides must carry an audio track; any pairing works (audio-to-video, audio-to-audio, video-to-video).- The computed placement is `start = target.start + offset`, where `offset` is the measured source-time offset between the two recordings (positive when this node's recording started after the target's; possibly negative). `syncTo` and `start` are mutually exclusive.- `sourceIn`/`sourceOut` keep their normal meaning and remain yours to set. When omitted on a synced node, the window defaults to the intersection of the node's natural extent with the target's window (instead of the usual natural-duration fit), so a lav track simply covers its take.- Alignment reads source content: `muted` and `volume` on either side do not affect the measurement. Use `muted` to keep only one side audible, as on the camera track above.-- Chains resolve in dependency order (A may sync to B while B syncs to C). Unknown ids and cycles are mount errors.+- Chains resolve in dependency order (A may sync to B while B syncs to C). An unknown id or a cycle is reported on the element like a failed source (see [errors.md](./errors.md)), and the node keeps its default placement.## Execution