Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance audio fingerprinting and transcription process with optional seed parameter

konstantin-paulus5c30d9161380
diff --git a/apps/web/src/components/engine/utils/captions.ts b/apps/web/src/components/engine/utils/captions.ts
index 8e6c30b6..5b29e5b3 100644
--- a/apps/web/src/components/engine/utils/captions.ts
+++ b/apps/web/src/components/engine/utils/captions.ts
@@ -41,35 +41,39 @@ export function hasAudioSources(world: EngineWorld, sceneEid: number): boolean {
* content, placement, source offset, rate, and gain. Volume fades and audio
* transitions are not captured; they don't change the spoken words.
*/
-export function sceneAudioFingerprint(world: EngineWorld, sceneEid: number): string {
+export function sceneAudioFingerprint(world: EngineWorld, sceneEid: number, seed?: number): string {
const c = world.components;
const entries: string[] = [];
for (const eid of getEntityTree(world, sceneEid)) {
if (hasComponent(world, eid, c.Muted)) continue;
- // Video paints carry the asset; their timing lives on the geometry parent.
- let timingEid = eid;
- if (hasComponent(world, eid, c.Paint) && c.Paint[eid] === PaintType.VIDEO) {
- timingEid = getParentEntity(world, eid) ?? eid;
- } else if (!hasComponent(world, eid, c.Audio)) {
- continue;
+ let audioEid: number | null = null;
+
+ if (c.Paint[eid] === PaintType.VIDEO) {
+ audioEid = getParentEntity(world, eid) ?? null;
+ }
+
+ if (!hasComponent(world, eid, c.Audio)) {
+ audioEid = eid;
}
- const asset = c.AssetId[eid] ? world.assets.get(c.AssetId[eid]) : undefined;
- if (!asset) continue;
+
+ const asset = world.assets.get(c.AssetId[eid] ?? "");
+ if (audioEid === null || !asset) continue;
entries.push([
asset.hash,
- c.Computed.start[timingEid] ?? 0,
- c.Computed.end[timingEid] ?? 0,
- c.Computed.delay[timingEid] ?? 0,
- c.PlaybackRate[timingEid] ?? 1,
- c.Volume[timingEid] ?? 0,
+ c.Computed.start[audioEid] ?? 0,
+ c.Computed.end[audioEid] ?? 0,
+ c.Computed.delay[audioEid] ?? 0,
+ c.PlaybackRate[audioEid] ?? 1,
+ c.Volume[audioEid] ?? 0,
].join(":"));
}
- return `v1:${fnv1a(entries.sort().join("|"))}`;
+ const mix = entries.sort().join("|");
+ return `v1:${fnv1a(seed === undefined ? mix : `${mix}#seed=${seed}`)}`;
}
function fnv1a(input: string): string {
@@ -96,6 +100,7 @@ export async function transcribeScene(
engine: Engine,
sceneEid: number,
onStatus?: (status: TranscribeStatus) => void,
+ seed?: number,
) {
const world = engine.world;
const c = world.components;
@@ -107,7 +112,7 @@ export async function transcribeScene(
throw new Error("No audio found. Add an audio or video clip to the scene to generate captions.");
}
- const generationKey = sceneAudioFingerprint(world, sceneEid);
+ const generationKey = sceneAudioFingerprint(world, sceneEid, seed);
const cached = Array.from(world.assets.values())
.find(a => a.type === "TRANSCRIPT" && a.generationKey === generationKey);
diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.ts
index c3f0c5b7..09715b6e 100644
--- a/apps/web/src/utils/jsx.ts
+++ b/apps/web/src/utils/jsx.ts
@@ -536,15 +536,19 @@ export class WorldDocument implements ProjectDocument<HostNode> {
// Handle captioning queue
{
for (const { eid: nodeEid, type } of this.queue) {
- if (hasComponent(world, nodeEid, c.Deleted) || type !== "caption" || c.AssetId[nodeEid]) continue;
+ if (hasComponent(world, nodeEid, c.Deleted) || type !== "caption") continue;
+
+ const props = this.data(nodeEid).props;
+ const seed = props.seed as number | undefined;
+ if (c.AssetId[nodeEid] && (seed === undefined || props.src !== undefined)) continue;
+
const sceneEid = getSceneAncestor(world, nodeEid);
if (!sceneEid || !hasAudioSources(world, sceneEid)) continue;
assert(this.engine, "captioning requires an engine (author-mode mount)");
- const { asset, trim } = await transcribeScene(this.engine, sceneEid);
+ const { asset, trim } = await transcribeScene(this.engine, sceneEid, undefined, seed);
setComponent(world, nodeEid, c.AssetId, asset.id);
- const props = this.data(nodeEid).props;
const hasIn = timingProp(props, "sourceIn") !== undefined;
const hasOut = timingProp(props, "sourceOut") !== undefined || timingProp(props, "end") !== undefined;
setComponent(world, nodeEid, c.Trim, {
@@ -1360,6 +1364,13 @@ export class WorldDocument implements ProjectDocument<HostNode> {
// A live decoder has already placed the box from the old alignment.
c.CaptionDecoder[eid]?.reposition(world, eid);
break;
+ } case "seed": {
+ assert(hasComponent(world, eid, c.Caption), "`seed` only applies to <Captions>");
+ assert(typeof value === "number", "`seed` must be a number" + `, value: ${value}`);
+ if (!this.committed && !this.queue.some((item) => item.type === "caption" && item.eid === eid)) {
+ this.queue.push({ eid, type: "caption" });
+ }
+ break;
}
}
}
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index 1b618902..345da255 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -230,6 +230,12 @@ export type PatchProps = {
* horizontal placement. Defaults to the preset's own alignment.
*/
verticalAlign?: "top" | "center" | "bottom";
+ /**
+ * Transcription seed — `<Captions>` only. Part of the transcript cache key,
+ * so a new value bypasses the cached transcript and transcribes the scene
+ * again; reusing a value replays that take from cache.
+ */
+ seed?: number;
};
/** Every PatchProps key, at runtime — the `dapi node patch` allowlist. */
@@ -271,6 +277,7 @@ export const PATCH_PROP_KEYS = Object.keys({
preset: true,
colors: true,
verticalAlign: true,
+ seed: true,
} satisfies Record<keyof PatchProps, true>) as ReadonlyArray<keyof PatchProps>;
type TimingProps = Pick<PatchProps, "start" | "end" | "sourceIn" | "sourceOut">;
@@ -372,5 +379,5 @@ export type SequenceProps = Pick<PatchProps, "name"> & {
export type CaptionsProps = Pick<
PatchProps,
| "preset" | "colors" | "verticalAlign" | "offsetX" | "offsetY" | "src"
- | "start" | "end" | "sourceIn" | "sourceOut" | "animations"
+ | "start" | "end" | "sourceIn" | "sourceOut" | "animations" | "seed"
>;
diff --git a/reference/jsx/captions.md b/reference/jsx/captions.md
index e5c07b6d..ae9f3f53 100644
--- a/reference/jsx/captions.md
+++ b/reference/jsx/captions.md
@@ -8,7 +8,7 @@
Transcription is **asynchronous and non-blocking**: the caption node is inserted at commit and its transcript attaches once ready. Because it reads the scene's audio, it runs **after** any generated assets in the scene have landed and after [audio sync](./audio-sync.md) has resolved; captioning a generated `voice`/`audio` track transcribes the finished audio at its final placement. The scene must contain an unmuted audio or video source; otherwise the caption node is left empty.
-Transcripts are **cached**: every transcript asset records a fingerprint of the scene's audible mix (source content, placement, source offset, playback rate, gain). When a scene's fingerprint matches an existing transcript, that asset is reused instead of transcribing again; re-mounting a project with unchanged audio consumes no credits. Any change to the scene's audio invalidates the fingerprint.
+Transcripts are **cached**: every transcript asset records a fingerprint of the scene's audible mix (source content, placement, source offset, playback rate, gain). When a scene's fingerprint matches an existing transcript, that asset is reused instead of transcribing again; re-mounting a project with unchanged audio consumes no credits. Any change to the scene's audio invalidates the fingerprint, and so does a new `seed`.
## Props
@@ -17,6 +17,7 @@ Transcripts are **cached**: every transcript asset records a fingerprint of the
| `preset` | see below | `"classic"` | Caption style preset. |
| `colors` | `string[]` | preset defaults | Fills the preset's color slots in order; any CSS color, alpha ignored. Ignored by presets without slots. |
| `verticalAlign` | `"top" \| "center" \| "bottom"` | preset default | Vertical placement of the caption block: anchored to the top or bottom safe margin (100 px), or centered. Horizontal placement stays with the preset. |
+| `seed` | `number` | none | Joins the transcript cache key: a value unused for this audio transcribes the scene again instead of reusing the cached transcript (it costs credits). Reusing a value replays that take from cache. |
| `offsetX`, `offsetY` | `Animatable<number>` | `0` | Render-time nudge in px on top of the preset placement; subpixel values are kept. A slide animation drives the same channel and wins while it plays. |
| `start`, `end`, `sourceIn`, `sourceOut` | `Time` | full transcript | Trim which stretch of the transcript is captioned, using the same [timing](./timing.md) semantics as media nodes. Set the source window (`sourceIn`/`sourceOut`) and timeline placement (`start`/`end`) together — see [Trimming](#trimming). |