Skip to content

The VS Code of video editing, built for agents.

TypeScript

Integrate transcription functionality and enhance audio processing

konstantin-paulus907a56ef5960
diff --git a/apps/web/package.json b/apps/web/package.json
index 73a9da2c..88c073eb 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -16,6 +16,7 @@
"@diffusionstudio/api-contract": "^0.1.0",
"@diffusionstudio/assets": "*",
"@diffusionstudio/cli": "*",
+ "@diffusionstudio/encoder": "*",
"@diffusionstudio/jsx": "*",
"@diffusionstudio/koota-solid": "*",
"@diffusionstudio/reconciler": "*",
diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.ts
index 6196f7bb..880e95d6 100644
--- a/apps/web/src/utils/gen-ai.ts
+++ b/apps/web/src/utils/gen-ai.ts
@@ -10,9 +10,13 @@
// finished file lands in the library under `generated/` with the spec's key
// in the manifest.
-import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";
-import { Ai, GenAi, getAssetFile, Project } from "@diffusionstudio/runtime";
-import { GENERATED_DIR } from "@diffusionstudio/assets";
+import { getAssetSpec, isAssetRef, parseSource } from "@diffusionstudio/jsx";
+import {
+ Ai, AssetId, Audio, GenAi, getAssetFile, getEntityTree, Hidden, Muted,
+ Paint, PaintType, Project, Source,
+} from "@diffusionstudio/runtime";
+import { createEncoder } from "@diffusionstudio/encoder";
+import { assetName, GENERATED_DIR } from "@diffusionstudio/assets";
import {
PROMPT_INPUT_AUDIO_MODEL_OPTIONS,
PROMPT_INPUT_IMAGE_MODEL_OPTIONS,
@@ -25,7 +29,7 @@ import { trpc } from "@/lib/trpc";
import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";
import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
-import type { World } from "koota";
+import type { Entity, World } from "koota";
// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).
const VOICE_MODEL = "elevenlabs-v3";
@@ -58,7 +62,7 @@ export class EditorGenAi extends GenAi {
* forgotten, so a remount tries again.
*/
private readonly memo = new Map<AssetRef, Promise<Asset>>();
- /** In-flight generations keyed by `generationKey`. */
+ /** In-flight generations and transcriptions keyed by `generationKey`. */
private readonly inflight = new Map<string, Promise<Asset>>();
public constructor(library: AssetLibrary, projectId: string) {
@@ -80,6 +84,72 @@ export class EditorGenAi extends GenAi {
return promise;
}
+ /**
+ * Transcribes the scene's audible mix for a `<captions>` element (see the
+ * runtime's asset system). Cached by scene id + seed: the same pair is the
+ * same transcript asset across sessions (the transcript lands in the
+ * library under `generated/` with that key in the manifest), and a new
+ * seed transcribes the scene again.
+ */
+ public async transcribe(world: World, scene: Entity, seed: number): Promise<Asset> {
+ const key = transcriptKey(scene, seed);
+
+ const cached = this.library.list().find((asset) => asset.generation?.key === key);
+ if (cached) return cached;
+
+ const running = this.inflight.get(key);
+ if (running) return await running;
+
+ const promise = this.runTranscription(world, scene, key);
+ this.inflight.set(key, promise);
+ try {
+ return await promise;
+ } finally {
+ this.inflight.delete(key);
+ }
+ }
+
+ /** Encodes the scene's audio, transcribes it, and stores the transcript. */
+ private async runTranscription(world: World, scene: Entity, key: string): Promise<Asset> {
+ assert(sceneHasAudio(world, scene), "No audio found. Add an audio or video clip to the scene to generate captions.");
+
+ const encoder = await createEncoder(world, {
+ scene,
+ format: "ogg",
+ video: { enabled: false },
+ audio: { enabled: true, codec: "opus", sampleRate: 24000 },
+ });
+ const result = await encoder.render();
+ assert(result.type === "success" && result.data !== undefined, "Failed to encode the scene audio");
+
+ const uploadId = crypto.randomUUID();
+ const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });
+ const fileRef = await uploadBlob(audioFile, uploadId);
+ assert(fileRef, "Failed to upload the scene audio for transcription");
+
+ const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });
+ assert(
+ transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),
+ "No speech detected. The audio does not appear to contain recognizable speech.",
+ );
+
+ const blob = new Blob([JSON.stringify(transcript)], { type: "application/json" });
+ return this.library.store(blob, {
+ name: `${this.nextCaptionsName()}.json`,
+ folder: GENERATED_DIR,
+ generation: { key },
+ });
+ }
+
+ private nextCaptionsName(): string {
+ let max = 0;
+ for (const asset of this.library.list()) {
+ const match = assetName(asset).match(/^Captions (\d+)\.json$/);
+ if (match) max = Math.max(max, Number(match[1]));
+ }
+ return `Captions ${max + 1}`;
+ }
+
private resolveInput(input: AssetInput): Promise<Asset> {
return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);
}
@@ -233,6 +303,31 @@ export class EditorGenAi extends GenAi {
}
}
+/**
+ * The transcript cache key: scene id + seed. The scene's durable name is the
+ * id in its source stamp (`<file>:<id>`, stamped once by the compiler — the
+ * same identity the project config keys by); a scene without one falls back
+ * to its entity id, which only holds within the session.
+ */
+function transcriptKey(scene: Entity, seed: number): string {
+ const source = scene.get(Source)?.value;
+ const locator = source ? parseSource(source)?.locator : undefined;
+ const sceneId = typeof locator === "string" ? locator : source ?? String(scene.id());
+ return `transcript:v1:${sceneId}:${seed}`;
+}
+
+/**
+ * Whether anything in the scene contributes to its audible mix: an unmuted,
+ * unhidden audio clip, video, or video paint with its asset bound.
+ */
+function sceneHasAudio(world: World, scene: Entity): boolean {
+ for (const entity of getEntityTree(world, scene)) {
+ if (entity.has(Hidden) || entity.has(Muted) || !entity.has(AssetId)) continue;
+ if (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO) return true;
+ }
+ return false;
+}
+
/**
* Per-model constraints (`dapi models video`); unknown models are left to the server.
*/
diff --git a/package-lock.json b/package-lock.json
index 42ff1dbe..0e27a40e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -94,6 +94,7 @@
"@diffusionstudio/api-contract": "^0.1.0",
"@diffusionstudio/assets": "*",
"@diffusionstudio/cli": "*",
+ "@diffusionstudio/encoder": "*",
"@diffusionstudio/jsx": "*",
"@diffusionstudio/koota-solid": "*",
"@diffusionstudio/reconciler": "*",
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index 1a30f51d..695bbeba 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -686,9 +686,9 @@ export type CaptionsProps = IdentityProps & TimingProps & OffsetProps & Partial<
*/
verticalAlign?: "top" | "center" | "bottom";
/**
- * Transcription seed. Part of the transcript cache key, so a new value
- * bypasses the cached transcript and transcribes the scene again; reusing
- * a value replays that take from cache.
+ * Transcription seed. Part of the transcript cache key (scene id + seed),
+ * so a new value bypasses the cached transcript and transcribes the scene
+ * again; reusing a value replays that take from cache. Default 0.
*/
seed?: number;
/** `<Animation>` children. */
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index a82011d9..623d5deb 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';
@@ -434,6 +434,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
entity.set(Position, { x: 0, y: 0 });
entity.add(Chars);
entity.add(Caption);
+ entity.add(TranscriptionRequest);
break;
}
case 'adjustmentLayer': {
@@ -988,9 +989,16 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
if (value === undefined || value === null || value === '') {
entity.remove(AssetId);
+ // A `<captions>` without a src transcribes its scene instead.
+ if (entity.has(Caption)) {
+ entity.add(TranscriptionRequest);
+ entity.set(TranscriptionRequest, { seed: toNumber(entity.get(Authored)?.props.seed) ?? 0 });
+ }
return;
}
+ entity.remove(TranscriptionRequest);
+
if (typeof value !== 'string') {
entity.remove(AssetId);
entity.add(GenerationRequest);
@@ -1148,6 +1156,20 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
});
return;
}
+ case 'seed': {
+ if (!entity.has(Caption)) return;
+ // An authored src mounts a transcript directly; there is no
+ // transcription for the seed to key. (An AssetRef src is not a
+ // PropValue, so its request stands in for the authored record.)
+ if (entity.get(Authored)?.props.src !== undefined) return;
+ if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;
+
+ // A resolution running for another seed must not bind late.
+ entity.remove(PendingSource);
+ entity.add(TranscriptionRequest);
+ entity.set(TranscriptionRequest, { seed: toNumber(value) ?? 0 });
+ return;
+ }
case 'background': {
const color = parseColor(value);
this.stage.entity.set(Background, { value: color ?? DEFAULT_BACKGROUND });
diff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.ts
index 6dc653db..18bc7910 100644
--- a/packages/runtime/src/ai.ts
+++ b/packages/runtime/src/ai.ts
@@ -2,15 +2,9 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// The generation service a host may attach to the world (the `Ai` trait):
-// what turns a `generate.*` declaration into an asset of the project's
-// library. The runtime itself cannot generate — it only knows the contract —
-// so a world without an Ai never resolves such sources (see the asset
-// system). The asset library knows nothing of generation either; an
-// implementation *uses* a library to look up and store what it makes.
-
import { generate } from '@diffusionstudio/jsx';
+import type { Entity, World } from 'koota';
import type { Asset } from '@diffusionstudio/assets';
import type {
AssetRef,
@@ -29,6 +23,14 @@ export abstract class GenAi {
*/
public abstract resolve(ref: AssetRef): Promise<Asset>;
+ /**
+ * The transcript of `scene`'s audible mix, as a TRANSCRIPT asset of the
+ * project's library. Cached by scene id + seed: the same pair resolves to
+ * the same asset, in this session and the next, and a new seed transcribes
+ * the scene again. Identical concurrent requests share one run.
+ */
+ public abstract transcribe(world: World, scene: Entity, seed: number): Promise<Asset>;
+
/**
* The imperative surface: `ai.generate.image({...})` declares and resolves
* in one call, returning the finished asset.
diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.ts
index f6a5a84a..8080d0cc 100644
--- a/packages/runtime/src/systems/assets.ts
+++ b/packages/runtime/src/systems/assets.ts
@@ -2,20 +2,9 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Binds entities to the assets their `src` names. The document is
-// synchronous and only leaves requests behind (LoadRequest for a source
-// outside the library, GenerationRequest for a `generate.*` declaration);
-// this system consumes them each frame, starts the async work, and stamps
-// AssetId when the asset lands. Loads go through the world's library;
-// generations through the world's Ai — deduplication (a spec whose hash
-// already produced a library asset binds to it, identical inflight specs
-// share one run) is the Ai implementation's contract (see GenAi). A request
-// the world cannot serve yet — no library, no Ai — is left in place for a
-// frame that can. In offline mode the started work joins FramePromises, so
-// the encoder's barrier waits for the binding before it samples a frame.
-
-import { Ai, FramePromises, GenerationRequest, Library, LoadRequest, PendingSource } from '../traits';
+import { Ai, FramePromises, GenerationRequest, Library, LoadRequest, PendingSource, TranscriptionRequest } from '../traits';
import { bindAsset } from '../actions/assets';
+import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';
import type { Entity, World } from 'koota';
import type { Asset } from '@diffusionstudio/assets';
@@ -38,7 +27,31 @@ export function assetSystem(world: World): void {
if (ref === null) continue;
resolve(world, entity, ref, ai.resolve(ref));
}
+
+ for (const entity of world.query(TranscriptionRequest)) {
+ const scene = getSceneAncestor(entity);
+ if (!scene || hasPendingSources(world, scene, entity)) continue;
+
+ const seed = entity.get(TranscriptionRequest)!.seed;
+ entity.remove(TranscriptionRequest);
+ resolve(world, entity, `transcript:${scene.id()}:${seed}`, ai.transcribe(world, scene, seed));
+ }
+ }
+}
+
+/**
+ * Whether anything in `scene`'s subtree (besides `except`, the requesting
+ * element itself) is still waiting on a source: a request this system has
+ * not consumed, or a resolution it started that has not landed.
+ */
+function hasPendingSources(world: World, scene: Entity, except: Entity): boolean {
+ for (const entity of getEntityTree(world, scene)) {
+ if (entity === except) continue;
+ if (entity.has(LoadRequest) || entity.has(GenerationRequest) || entity.has(PendingSource)) {
+ return true;
+ }
}
+ return false;
}
/**
diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.ts
index e7dee2f7..a40395b3 100644
--- a/packages/runtime/src/traits/node.ts
+++ b/packages/runtime/src/traits/node.ts
@@ -85,6 +85,12 @@ export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }))
// through the library.
export const LoadRequest = trait({ value: '' });
+// On a `<captions>` element without a src: transcribe the scene it sits
+// under, through the world's Ai. The seed keys the take — the transcript is
+// cached by scene id + seed, so re-running with a seed replays that take and
+// bumping the seed transcribes the scene again. Seed 0 is the default take.
+export const TranscriptionRequest = trait({ seed: 0 });
+
// The src whose resolution is inflight, kept while the asset system works,
// so a resolution that arrives after the element was given another source
// (or none) is dropped. The document clears it whenever a new src arrives.