The VS Code of video editing, built for agents.
TypeScript
Enhance audio fingerprinting and transcription process with optional seed parameter
konstantin-paulus5c30d9161380
diff --git a/apps/web/src/components/engine/utils/captions.ts b/apps/web/src/components/engine/utils/captions.tsindex 8e6c30b6..5b29e5b3 100644--- a/apps/web/src/components/engine/utils/captions.ts+++ b/apps/web/src/components/engine/utils/captions.ts@@ -41,35 +41,39 @@ export function hasAudioSources(world: EngineWorld, sceneEid: number): boolean {* content, placement, source offset, rate, and gain. Volume fades and audio* transitions are not captured; they don't change the spoken words.*/-export function sceneAudioFingerprint(world: EngineWorld, sceneEid: number): string {+export function sceneAudioFingerprint(world: EngineWorld, sceneEid: number, seed?: number): string {const c = world.components;const entries: string[] = [];for (const eid of getEntityTree(world, sceneEid)) {if (hasComponent(world, eid, c.Muted)) continue;- // Video paints carry the asset; their timing lives on the geometry parent.- let timingEid = eid;- if (hasComponent(world, eid, c.Paint) && c.Paint[eid] === PaintType.VIDEO) {- timingEid = getParentEntity(world, eid) ?? eid;- } else if (!hasComponent(world, eid, c.Audio)) {- continue;+ let audioEid: number | null = null;++ if (c.Paint[eid] === PaintType.VIDEO) {+ audioEid = getParentEntity(world, eid) ?? null;+ }++ if (!hasComponent(world, eid, c.Audio)) {+ audioEid = eid;}- const asset = c.AssetId[eid] ? world.assets.get(c.AssetId[eid]) : undefined;- if (!asset) continue;++ const asset = world.assets.get(c.AssetId[eid] ?? "");+ if (audioEid === null || !asset) continue;entries.push([asset.hash,- c.Computed.start[timingEid] ?? 0,- c.Computed.end[timingEid] ?? 0,- c.Computed.delay[timingEid] ?? 0,- c.PlaybackRate[timingEid] ?? 1,- c.Volume[timingEid] ?? 0,+ c.Computed.start[audioEid] ?? 0,+ c.Computed.end[audioEid] ?? 0,+ c.Computed.delay[audioEid] ?? 0,+ c.PlaybackRate[audioEid] ?? 1,+ c.Volume[audioEid] ?? 0,].join(":"));}- return `v1:${fnv1a(entries.sort().join("|"))}`;+ const mix = entries.sort().join("|");+ return `v1:${fnv1a(seed === undefined ? mix : `${mix}#seed=${seed}`)}`;}function fnv1a(input: string): string {@@ -96,6 +100,7 @@ export async function transcribeScene(engine: Engine,sceneEid: number,onStatus?: (status: TranscribeStatus) => void,+ seed?: number,) {const world = engine.world;const c = world.components;@@ -107,7 +112,7 @@ export async function transcribeScene(throw new Error("No audio found. Add an audio or video clip to the scene to generate captions.");}- const generationKey = sceneAudioFingerprint(world, sceneEid);+ const generationKey = sceneAudioFingerprint(world, sceneEid, seed);const cached = Array.from(world.assets.values()).find(a => a.type === "TRANSCRIPT" && a.generationKey === generationKey);diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.tsindex c3f0c5b7..09715b6e 100644--- a/apps/web/src/utils/jsx.ts+++ b/apps/web/src/utils/jsx.ts@@ -536,15 +536,19 @@ export class WorldDocument implements ProjectDocument<HostNode> {// Handle captioning queue{for (const { eid: nodeEid, type } of this.queue) {- if (hasComponent(world, nodeEid, c.Deleted) || type !== "caption" || c.AssetId[nodeEid]) continue;+ if (hasComponent(world, nodeEid, c.Deleted) || type !== "caption") continue;++ const props = this.data(nodeEid).props;+ const seed = props.seed as number | undefined;+ if (c.AssetId[nodeEid] && (seed === undefined || props.src !== undefined)) continue;+const sceneEid = getSceneAncestor(world, nodeEid);if (!sceneEid || !hasAudioSources(world, sceneEid)) continue;assert(this.engine, "captioning requires an engine (author-mode mount)");- const { asset, trim } = await transcribeScene(this.engine, sceneEid);+ const { asset, trim } = await transcribeScene(this.engine, sceneEid, undefined, seed);setComponent(world, nodeEid, c.AssetId, asset.id);- const props = this.data(nodeEid).props;const hasIn = timingProp(props, "sourceIn") !== undefined;const hasOut = timingProp(props, "sourceOut") !== undefined || timingProp(props, "end") !== undefined;setComponent(world, nodeEid, c.Trim, {@@ -1360,6 +1364,13 @@ export class WorldDocument implements ProjectDocument<HostNode> {// A live decoder has already placed the box from the old alignment.c.CaptionDecoder[eid]?.reposition(world, eid);break;+ } case "seed": {+ assert(hasComponent(world, eid, c.Caption), "`seed` only applies to <Captions>");+ assert(typeof value === "number", "`seed` must be a number" + `, value: ${value}`);+ if (!this.committed && !this.queue.some((item) => item.type === "caption" && item.eid === eid)) {+ this.queue.push({ eid, type: "caption" });+ }+ break;}}}diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 1b618902..345da255 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -230,6 +230,12 @@ export type PatchProps = {* horizontal placement. Defaults to the preset's own alignment.*/verticalAlign?: "top" | "center" | "bottom";+ /**+ * Transcription seed — `<Captions>` only. Part of the transcript cache key,+ * so a new value bypasses the cached transcript and transcribes the scene+ * again; reusing a value replays that take from cache.+ */+ seed?: number;};/** Every PatchProps key, at runtime — the `dapi node patch` allowlist. */@@ -271,6 +277,7 @@ export const PATCH_PROP_KEYS = Object.keys({preset: true,colors: true,verticalAlign: true,+ seed: true,} satisfies Record<keyof PatchProps, true>) as ReadonlyArray<keyof PatchProps>;type TimingProps = Pick<PatchProps, "start" | "end" | "sourceIn" | "sourceOut">;@@ -372,5 +379,5 @@ export type SequenceProps = Pick<PatchProps, "name"> & {export type CaptionsProps = Pick<PatchProps,| "preset" | "colors" | "verticalAlign" | "offsetX" | "offsetY" | "src"- | "start" | "end" | "sourceIn" | "sourceOut" | "animations"+ | "start" | "end" | "sourceIn" | "sourceOut" | "animations" | "seed">;diff --git a/reference/jsx/captions.md b/reference/jsx/captions.mdindex e5c07b6d..ae9f3f53 100644--- a/reference/jsx/captions.md+++ b/reference/jsx/captions.md@@ -8,7 +8,7 @@Transcription is **asynchronous and non-blocking**: the caption node is inserted at commit and its transcript attaches once ready. Because it reads the scene's audio, it runs **after** any generated assets in the scene have landed and after [audio sync](./audio-sync.md) has resolved; captioning a generated `voice`/`audio` track transcribes the finished audio at its final placement. The scene must contain an unmuted audio or video source; otherwise the caption node is left empty.-Transcripts are **cached**: every transcript asset records a fingerprint of the scene's audible mix (source content, placement, source offset, playback rate, gain). When a scene's fingerprint matches an existing transcript, that asset is reused instead of transcribing again; re-mounting a project with unchanged audio consumes no credits. Any change to the scene's audio invalidates the fingerprint.+Transcripts are **cached**: every transcript asset records a fingerprint of the scene's audible mix (source content, placement, source offset, playback rate, gain). When a scene's fingerprint matches an existing transcript, that asset is reused instead of transcribing again; re-mounting a project with unchanged audio consumes no credits. Any change to the scene's audio invalidates the fingerprint, and so does a new `seed`.## Props@@ -17,6 +17,7 @@ Transcripts are **cached**: every transcript asset records a fingerprint of the| `preset` | see below | `"classic"` | Caption style preset. || `colors` | `string[]` | preset defaults | Fills the preset's color slots in order; any CSS color, alpha ignored. Ignored by presets without slots. || `verticalAlign` | `"top" \| "center" \| "bottom"` | preset default | Vertical placement of the caption block: anchored to the top or bottom safe margin (100 px), or centered. Horizontal placement stays with the preset. |+| `seed` | `number` | none | Joins the transcript cache key: a value unused for this audio transcribes the scene again instead of reusing the cached transcript (it costs credits). Reusing a value replays that take from cache. || `offsetX`, `offsetY` | `Animatable<number>` | `0` | Render-time nudge in px on top of the preset placement; subpixel values are kept. A slide animation drives the same channel and wins while it plays. || `start`, `end`, `sourceIn`, `sourceOut` | `Time` | full transcript | Trim which stretch of the transcript is captioned, using the same [timing](./timing.md) semantics as media nodes. Set the source window (`sourceIn`/`sourceOut`) and timeline placement (`start`/`end`) together — see [Trimming](#trimming). |