Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor caption handling and enhance transcript duration management

konstantin-paulusf33d5570ccdd
diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsx
index 5a368411..385cdfa5 100644
--- a/apps/web/src/components/genai/use-auto-captions.tsx
+++ b/apps/web/src/components/genai/use-auto-captions.tsx
@@ -4,14 +4,7 @@
import { createMemo } from "solid-js";
import { Captions, authoredElement } from "@diffusionstudio/reconciler";
-import {
- Caption,
- Computed,
- FrameRate,
- framesToSeconds,
- getEntityTree,
- isScene,
-} from "@diffusionstudio/runtime";
+import { Caption, getEntityTree, isScene } from "@diffusionstudio/runtime";
import { useWorld } from "@diffusionstudio/koota-solid";
import { useEditor, useSelection } from "@/engine/hooks";
import { toast } from "somoto";
@@ -26,21 +19,9 @@ export function useAutoCaptions() {
const scenes = createMemo(() => nodes().filter(isScene));
const hasScene = () => scenes().length > 0;
- /**
- * The scene's own length in seconds, for the caption element's `end`. A
- * fresh <Captions> has no source to take a length from, so without one it
- * runs for the 16s default and stretches the very scene it captions.
- */
- const sceneSpan = (scene: Entity) => {
- const computed = scene.get(Computed);
- const frames = computed?.duration ?? 0;
- if (frames <= 0) return undefined;
- return framesToSeconds(frames, world.get(FrameRate)?.value ?? 30);
- };
-
const insertFresh = (scene: Entity) => {
const [entity] = editor.insertElement(scene, () => (
- <Captions end={sceneSpan(scene)} seed={Date.now()} />
+ <Captions seed={Date.now()} />
));
if (!entity) {
toast.error("Could not add captions", {
diff --git a/packages/runtime/src/actions/timing.ts b/packages/runtime/src/actions/timing.ts
index 86fe184c..5138e5b4 100644
--- a/packages/runtime/src/actions/timing.ts
+++ b/packages/runtime/src/actions/timing.ts
@@ -10,11 +10,14 @@ import { store } from '../world/store';
import { DEFAULT_DURATION_FRAMES } from '../constants';
import {
Geometry, Group, AdjustmentLayer, Paint, Caption, Cache, Computed,
- Delay, Trim, PlaybackRate, CaptionDecoderHandle,
+ Delay, Trim, PlaybackRate, CaptionDecoderHandle, AssetId, FrameRate,
+ FramePromises,
} from '../traits';
import { isGroupLike } from '../queries/predicates';
import { getParentNode } from '../queries/hierarchy';
-import { findAssetDuration, getSourceFrameAt, isPaintEntity } from '../utils/time';
+import { findAssetDuration, getSourceFrameAt, isPaintEntity, secondsToFrames } from '../utils/time';
+import { getAsset } from './assets';
+import { getTranscriptDuration, primeTranscriptDuration } from '../media/caption/utils';
import type { Entity, World } from 'koota';
import type { Ignorable, TimeTrait } from '../utils/time';
@@ -60,6 +63,59 @@ function resolveSourceOut(
return Math.max(trimStart, out);
}
+/**
+ * The caption's transcript length in project frames, or null while the
+ * transcript hasn't been read (see `primeTranscriptDuration`) or has no words.
+ */
+function findTranscriptDuration(world: World, entity: Entity, ignore?: Ignorable): number | null {
+ if (ignore === AssetId) return null;
+
+ const asset = getAsset(world, entity.get(AssetId)?.value ?? '');
+ if (!asset) return null;
+
+ const duration = getTranscriptDuration(asset);
+ if (duration === null) return null;
+
+ return secondsToFrames(duration, world.get(FrameRate)?.value ?? 30);
+}
+
+/**
+ * Where a caption's source window closes, in source frames. Unlike media, the
+ * transcript is a fallback length rather than a cap — an authored end wins
+ * outright, then the transcript's last word. Null when the caption has no
+ * length of its own at all: it follows its parent (see `resolveParentSpan`).
+ */
+function resolveCaptionSourceOut(
+ world: World,
+ entity: Entity,
+ trimStart: number,
+ trimEnd: number | null,
+ ignore?: Ignorable,
+): number | null {
+ if (trimEnd !== null) return Math.max(trimStart, trimEnd);
+
+ const duration = findTranscriptDuration(world, entity, ignore);
+ if (duration !== null) return Math.max(trimStart, duration);
+
+ return null;
+}
+
+/**
+ * How far a length-less caption's source runs, in source frames: the span its
+ * parent last computed. This is an initial value, not a live derivation — it
+ * is read when the caption's range happens to be recomputed and stands until
+ * then, which is all it needs to be: the length a fresh caption has while its
+ * transcript is still being made. The parent's fit already contains the
+ * caption's previous span, so the read is a fixed point, never a runaway.
+ */
+function resolveParentSpan(world: World, entity: Entity, origin: number, playbackRate: number): number {
+ const parent = getParentNode(entity);
+ const end = parent === null ? -Infinity : store(world, Computed).end[parent.id()] ?? -Infinity;
+
+ const span = (end - origin) * playbackRate;
+ return Number.isFinite(span) && span > 0 ? span : DEFAULT_DURATION_FRAMES;
+}
+
/**
* Resolve one node's time traits into its absolute bounds, origin, and rate.
*
@@ -114,7 +170,13 @@ export function recomputeEntityTimeRange(world: World, entity: Entity, ignore?:
return;
}
- const sourceOut = resolveSourceOut(world, entity, trimStart, trim?.end ?? null, ignore);
+ let sourceOut = resolveSourceOut(world, entity, trimStart, trim?.end ?? null, ignore);
+
+ if (entity.has(Caption)) {
+ sourceOut = resolveCaptionSourceOut(world, entity, trimStart, trim?.end ?? null, ignore)
+ ?? resolveParentSpan(world, entity, origin, playbackRate)
+ }
+
const end = Math.max(start, Math.round(origin + sourceOut / playbackRate));
computed.start[eid] = start;
@@ -214,6 +276,22 @@ export function reactToAssetChange(world: World, entity: Entity, ignore?: Ignora
entity.get(CaptionDecoderHandle)?.dispose();
entity.set(CaptionDecoderHandle, null);
}
+
+ // The transcript is the caption's fallback length: recompute against
+ // what is known now, and once the new transcript's words are read
+ // (a file read, so async) recompute again with its actual end.
+ recomputeEntityTimeRange(world, entity, ignore);
+ bubbleTimeRangeUp(world, entity);
+
+ const asset = ignore === AssetId ? undefined : getAsset(world, entity.get(AssetId)?.value ?? '');
+ if (asset && getTranscriptDuration(asset) === null) {
+ const done = primeTranscriptDuration(asset).then(() => {
+ if (!entity.isAlive() || entity.get(AssetId)?.value !== asset.id) return;
+ recomputeEntityTimeRange(world, entity);
+ bubbleTimeRangeUp(world, entity);
+ }).catch(() => undefined);
+ world.get(FramePromises)?.list?.push(done);
+ }
} else if (entity.has(Geometry)) {
// A geometry's own asset backs its intrinsic paint (a video's footage)
// or, on an audio clip, its recording: a new one is a new source length.
@@ -276,6 +354,10 @@ export function reactToChildDetached(world: World, child: Entity) {
if (child.has(Paint)) {
const parent = getParentNode(child);
if (parent === null || !parent.has(Geometry)) return;
+ // A caption's length comes from its transcript or parent, never from
+ // its fills; pinning here would freeze that derived length into a
+ // trim whenever a preset's styling children are cleared.
+ if (parent.has(Caption)) return;
// Pin the current duration before recomputing; otherwise the geometry
// would silently fall back to the 16s default after losing its paint.
pinTrimToCurrentBounds(world, parent);
diff --git a/packages/runtime/src/media/caption/utils.ts b/packages/runtime/src/media/caption/utils.ts
index fe1bf7c9..23a7f09b 100644
--- a/packages/runtime/src/media/caption/utils.ts
+++ b/packages/runtime/src/media/caption/utils.ts
@@ -112,6 +112,40 @@ export function splitSequence(sequence: WordGroup): [WordGroup, WordGroup] {
}
+// Transcript lengths already read, by asset id — a content hash, so an entry
+// can never go stale. Timing recomputes are synchronous while reading a
+// transcript is not; this is the seam between the two (see the caption
+// branches in actions/timing.ts).
+const transcriptDurations = new Map<string, number>();
+
+/**
+ * How long the transcript runs, in seconds: its last word's end. Null until
+ * `primeTranscriptDuration` has read the asset, and null for a transcript
+ * with no words — a caption without one takes its span from its parent
+ * instead.
+ */
+export function getTranscriptDuration(asset: Asset): number | null {
+ const duration = transcriptDurations.get(asset.id);
+ return duration === undefined || duration <= 0 ? null : duration;
+}
+
+/**
+ * Reads the transcript and caches its length, so `getTranscriptDuration` can
+ * answer synchronously from then on.
+ */
+export async function primeTranscriptDuration(asset: Asset): Promise<void> {
+ if (transcriptDurations.has(asset.id)) return;
+
+ const transcript = await resolveTranscript(asset);
+ let end = 0;
+ for (const segment of transcript) {
+ for (const word of segment.words) {
+ if (word.end > end) end = word.end;
+ }
+ }
+ transcriptDurations.set(asset.id, end);
+}
+
/**
* Resolve a transcript from any asset type that can carry one.
*/