The VS Code of video editing, built for agents.
TypeScript
Refactor caption handling and enhance transcript duration management
konstantin-paulusf33d5570ccdd
diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsxindex 5a368411..385cdfa5 100644--- a/apps/web/src/components/genai/use-auto-captions.tsx+++ b/apps/web/src/components/genai/use-auto-captions.tsx@@ -4,14 +4,7 @@import { createMemo } from "solid-js";import { Captions, authoredElement } from "@diffusionstudio/reconciler";-import {- Caption,- Computed,- FrameRate,- framesToSeconds,- getEntityTree,- isScene,-} from "@diffusionstudio/runtime";+import { Caption, getEntityTree, isScene } from "@diffusionstudio/runtime";import { useWorld } from "@diffusionstudio/koota-solid";import { useEditor, useSelection } from "@/engine/hooks";import { toast } from "somoto";@@ -26,21 +19,9 @@ export function useAutoCaptions() {const scenes = createMemo(() => nodes().filter(isScene));const hasScene = () => scenes().length > 0;- /**- * The scene's own length in seconds, for the caption element's `end`. A- * fresh <Captions> has no source to take a length from, so without one it- * runs for the 16s default and stretches the very scene it captions.- */- const sceneSpan = (scene: Entity) => {- const computed = scene.get(Computed);- const frames = computed?.duration ?? 0;- if (frames <= 0) return undefined;- return framesToSeconds(frames, world.get(FrameRate)?.value ?? 30);- };-const insertFresh = (scene: Entity) => {const [entity] = editor.insertElement(scene, () => (- <Captions end={sceneSpan(scene)} seed={Date.now()} />+ <Captions seed={Date.now()} />));if (!entity) {toast.error("Could not add captions", {diff --git a/packages/runtime/src/actions/timing.ts b/packages/runtime/src/actions/timing.tsindex 86fe184c..5138e5b4 100644--- a/packages/runtime/src/actions/timing.ts+++ b/packages/runtime/src/actions/timing.ts@@ -10,11 +10,14 @@ import { store } from '../world/store';import { DEFAULT_DURATION_FRAMES } from '../constants';import {Geometry, Group, AdjustmentLayer, Paint, Caption, Cache, Computed,- Delay, Trim, PlaybackRate, CaptionDecoderHandle,+ Delay, Trim, PlaybackRate, CaptionDecoderHandle, AssetId, FrameRate,+ FramePromises,} from '../traits';import { isGroupLike } from '../queries/predicates';import { getParentNode } from '../queries/hierarchy';-import { findAssetDuration, getSourceFrameAt, isPaintEntity } from '../utils/time';+import { findAssetDuration, getSourceFrameAt, isPaintEntity, secondsToFrames } from '../utils/time';+import { getAsset } from './assets';+import { getTranscriptDuration, primeTranscriptDuration } from '../media/caption/utils';import type { Entity, World } from 'koota';import type { Ignorable, TimeTrait } from '../utils/time';@@ -60,6 +63,59 @@ function resolveSourceOut(return Math.max(trimStart, out);}+/**+ * The caption's transcript length in project frames, or null while the+ * transcript hasn't been read (see `primeTranscriptDuration`) or has no words.+ */+function findTranscriptDuration(world: World, entity: Entity, ignore?: Ignorable): number | null {+ if (ignore === AssetId) return null;++ const asset = getAsset(world, entity.get(AssetId)?.value ?? '');+ if (!asset) return null;++ const duration = getTranscriptDuration(asset);+ if (duration === null) return null;++ return secondsToFrames(duration, world.get(FrameRate)?.value ?? 30);+}++/**+ * Where a caption's source window closes, in source frames. Unlike media, the+ * transcript is a fallback length rather than a cap — an authored end wins+ * outright, then the transcript's last word. Null when the caption has no+ * length of its own at all: it follows its parent (see `resolveParentSpan`).+ */+function resolveCaptionSourceOut(+ world: World,+ entity: Entity,+ trimStart: number,+ trimEnd: number | null,+ ignore?: Ignorable,+): number | null {+ if (trimEnd !== null) return Math.max(trimStart, trimEnd);++ const duration = findTranscriptDuration(world, entity, ignore);+ if (duration !== null) return Math.max(trimStart, duration);++ return null;+}++/**+ * How far a length-less caption's source runs, in source frames: the span its+ * parent last computed. This is an initial value, not a live derivation — it+ * is read when the caption's range happens to be recomputed and stands until+ * then, which is all it needs to be: the length a fresh caption has while its+ * transcript is still being made. The parent's fit already contains the+ * caption's previous span, so the read is a fixed point, never a runaway.+ */+function resolveParentSpan(world: World, entity: Entity, origin: number, playbackRate: number): number {+ const parent = getParentNode(entity);+ const end = parent === null ? -Infinity : store(world, Computed).end[parent.id()] ?? -Infinity;++ const span = (end - origin) * playbackRate;+ return Number.isFinite(span) && span > 0 ? span : DEFAULT_DURATION_FRAMES;+}+/*** Resolve one node's time traits into its absolute bounds, origin, and rate.*@@ -114,7 +170,13 @@ export function recomputeEntityTimeRange(world: World, entity: Entity, ignore?:return;}- const sourceOut = resolveSourceOut(world, entity, trimStart, trim?.end ?? null, ignore);+ let sourceOut = resolveSourceOut(world, entity, trimStart, trim?.end ?? null, ignore);++ if (entity.has(Caption)) {+ sourceOut = resolveCaptionSourceOut(world, entity, trimStart, trim?.end ?? null, ignore)+ ?? resolveParentSpan(world, entity, origin, playbackRate)+ }+const end = Math.max(start, Math.round(origin + sourceOut / playbackRate));computed.start[eid] = start;@@ -214,6 +276,22 @@ export function reactToAssetChange(world: World, entity: Entity, ignore?: Ignoraentity.get(CaptionDecoderHandle)?.dispose();entity.set(CaptionDecoderHandle, null);}++ // The transcript is the caption's fallback length: recompute against+ // what is known now, and once the new transcript's words are read+ // (a file read, so async) recompute again with its actual end.+ recomputeEntityTimeRange(world, entity, ignore);+ bubbleTimeRangeUp(world, entity);++ const asset = ignore === AssetId ? undefined : getAsset(world, entity.get(AssetId)?.value ?? '');+ if (asset && getTranscriptDuration(asset) === null) {+ const done = primeTranscriptDuration(asset).then(() => {+ if (!entity.isAlive() || entity.get(AssetId)?.value !== asset.id) return;+ recomputeEntityTimeRange(world, entity);+ bubbleTimeRangeUp(world, entity);+ }).catch(() => undefined);+ world.get(FramePromises)?.list?.push(done);+ }} else if (entity.has(Geometry)) {// A geometry's own asset backs its intrinsic paint (a video's footage)// or, on an audio clip, its recording: a new one is a new source length.@@ -276,6 +354,10 @@ export function reactToChildDetached(world: World, child: Entity) {if (child.has(Paint)) {const parent = getParentNode(child);if (parent === null || !parent.has(Geometry)) return;+ // A caption's length comes from its transcript or parent, never from+ // its fills; pinning here would freeze that derived length into a+ // trim whenever a preset's styling children are cleared.+ if (parent.has(Caption)) return;// Pin the current duration before recomputing; otherwise the geometry// would silently fall back to the 16s default after losing its paint.pinTrimToCurrentBounds(world, parent);diff --git a/packages/runtime/src/media/caption/utils.ts b/packages/runtime/src/media/caption/utils.tsindex fe1bf7c9..23a7f09b 100644--- a/packages/runtime/src/media/caption/utils.ts+++ b/packages/runtime/src/media/caption/utils.ts@@ -112,6 +112,40 @@ export function splitSequence(sequence: WordGroup): [WordGroup, WordGroup] {}+// Transcript lengths already read, by asset id — a content hash, so an entry+// can never go stale. Timing recomputes are synchronous while reading a+// transcript is not; this is the seam between the two (see the caption+// branches in actions/timing.ts).+const transcriptDurations = new Map<string, number>();++/**+ * How long the transcript runs, in seconds: its last word's end. Null until+ * `primeTranscriptDuration` has read the asset, and null for a transcript+ * with no words — a caption without one takes its span from its parent+ * instead.+ */+export function getTranscriptDuration(asset: Asset): number | null {+ const duration = transcriptDurations.get(asset.id);+ return duration === undefined || duration <= 0 ? null : duration;+}++/**+ * Reads the transcript and caches its length, so `getTranscriptDuration` can+ * answer synchronously from then on.+ */+export async function primeTranscriptDuration(asset: Asset): Promise<void> {+ if (transcriptDurations.has(asset.id)) return;++ const transcript = await resolveTranscript(asset);+ let end = 0;+ for (const segment of transcript) {+ for (const word of segment.words) {+ if (word.end > end) end = word.end;+ }+ }+ transcriptDurations.set(asset.id, end);+}+/*** Resolve a transcript from any asset type that can carry one.*/