The VS Code of video editing, built for agents.
TypeScript
Integrate transcription functionality and enhance audio processing
konstantin-paulus907a56ef5960
diff --git a/apps/web/package.json b/apps/web/package.jsonindex 73a9da2c..88c073eb 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -16,6 +16,7 @@"@diffusionstudio/api-contract": "^0.1.0","@diffusionstudio/assets": "*","@diffusionstudio/cli": "*",+ "@diffusionstudio/encoder": "*","@diffusionstudio/jsx": "*","@diffusionstudio/koota-solid": "*","@diffusionstudio/reconciler": "*",diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.tsindex 6196f7bb..880e95d6 100644--- a/apps/web/src/utils/gen-ai.ts+++ b/apps/web/src/utils/gen-ai.ts@@ -10,9 +10,13 @@// finished file lands in the library under `generated/` with the spec's key// in the manifest.-import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";-import { Ai, GenAi, getAssetFile, Project } from "@diffusionstudio/runtime";-import { GENERATED_DIR } from "@diffusionstudio/assets";+import { getAssetSpec, isAssetRef, parseSource } from "@diffusionstudio/jsx";+import {+ Ai, AssetId, Audio, GenAi, getAssetFile, getEntityTree, Hidden, Muted,+ Paint, PaintType, Project, Source,+} from "@diffusionstudio/runtime";+import { createEncoder } from "@diffusionstudio/encoder";+import { assetName, GENERATED_DIR } from "@diffusionstudio/assets";import {PROMPT_INPUT_AUDIO_MODEL_OPTIONS,PROMPT_INPUT_IMAGE_MODEL_OPTIONS,@@ -25,7 +29,7 @@ import { trpc } from "@/lib/trpc";import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";import type { Asset, AssetLibrary } from "@diffusionstudio/assets";-import type { World } from "koota";+import type { Entity, World } from "koota";// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).const VOICE_MODEL = "elevenlabs-v3";@@ -58,7 +62,7 @@ export class EditorGenAi extends GenAi {* forgotten, so a remount tries again.*/private readonly memo = new Map<AssetRef, Promise<Asset>>();- /** In-flight generations keyed by `generationKey`. */+ /** In-flight generations and transcriptions keyed by `generationKey`. */private readonly inflight = new Map<string, Promise<Asset>>();public constructor(library: AssetLibrary, projectId: string) {@@ -80,6 +84,72 @@ export class EditorGenAi extends GenAi {return promise;}+ /**+ * Transcribes the scene's audible mix for a `<captions>` element (see the+ * runtime's asset system). Cached by scene id + seed: the same pair is the+ * same transcript asset across sessions (the transcript lands in the+ * library under `generated/` with that key in the manifest), and a new+ * seed transcribes the scene again.+ */+ public async transcribe(world: World, scene: Entity, seed: number): Promise<Asset> {+ const key = transcriptKey(scene, seed);++ const cached = this.library.list().find((asset) => asset.generation?.key === key);+ if (cached) return cached;++ const running = this.inflight.get(key);+ if (running) return await running;++ const promise = this.runTranscription(world, scene, key);+ this.inflight.set(key, promise);+ try {+ return await promise;+ } finally {+ this.inflight.delete(key);+ }+ }++ /** Encodes the scene's audio, transcribes it, and stores the transcript. */+ private async runTranscription(world: World, scene: Entity, key: string): Promise<Asset> {+ assert(sceneHasAudio(world, scene), "No audio found. Add an audio or video clip to the scene to generate captions.");++ const encoder = await createEncoder(world, {+ scene,+ format: "ogg",+ video: { enabled: false },+ audio: { enabled: true, codec: "opus", sampleRate: 24000 },+ });+ const result = await encoder.render();+ assert(result.type === "success" && result.data !== undefined, "Failed to encode the scene audio");++ const uploadId = crypto.randomUUID();+ const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });+ const fileRef = await uploadBlob(audioFile, uploadId);+ assert(fileRef, "Failed to upload the scene audio for transcription");++ const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });+ assert(+ transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),+ "No speech detected. The audio does not appear to contain recognizable speech.",+ );++ const blob = new Blob([JSON.stringify(transcript)], { type: "application/json" });+ return this.library.store(blob, {+ name: `${this.nextCaptionsName()}.json`,+ folder: GENERATED_DIR,+ generation: { key },+ });+ }++ private nextCaptionsName(): string {+ let max = 0;+ for (const asset of this.library.list()) {+ const match = assetName(asset).match(/^Captions (\d+)\.json$/);+ if (match) max = Math.max(max, Number(match[1]));+ }+ return `Captions ${max + 1}`;+ }+private resolveInput(input: AssetInput): Promise<Asset> {return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);}@@ -233,6 +303,31 @@ export class EditorGenAi extends GenAi {}}+/**+ * The transcript cache key: scene id + seed. The scene's durable name is the+ * id in its source stamp (`<file>:<id>`, stamped once by the compiler — the+ * same identity the project config keys by); a scene without one falls back+ * to its entity id, which only holds within the session.+ */+function transcriptKey(scene: Entity, seed: number): string {+ const source = scene.get(Source)?.value;+ const locator = source ? parseSource(source)?.locator : undefined;+ const sceneId = typeof locator === "string" ? locator : source ?? String(scene.id());+ return `transcript:v1:${sceneId}:${seed}`;+}++/**+ * Whether anything in the scene contributes to its audible mix: an unmuted,+ * unhidden audio clip, video, or video paint with its asset bound.+ */+function sceneHasAudio(world: World, scene: Entity): boolean {+ for (const entity of getEntityTree(world, scene)) {+ if (entity.has(Hidden) || entity.has(Muted) || !entity.has(AssetId)) continue;+ if (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO) return true;+ }+ return false;+}+/*** Per-model constraints (`dapi models video`); unknown models are left to the server.*/diff --git a/package-lock.json b/package-lock.jsonindex 42ff1dbe..0e27a40e 100644--- a/package-lock.json+++ b/package-lock.json@@ -94,6 +94,7 @@"@diffusionstudio/api-contract": "^0.1.0","@diffusionstudio/assets": "*","@diffusionstudio/cli": "*",+ "@diffusionstudio/encoder": "*","@diffusionstudio/jsx": "*","@diffusionstudio/koota-solid": "*","@diffusionstudio/reconciler": "*",diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 1a30f51d..695bbeba 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -686,9 +686,9 @@ export type CaptionsProps = IdentityProps & TimingProps & OffsetProps & Partial<*/verticalAlign?: "top" | "center" | "bottom";/**- * Transcription seed. Part of the transcript cache key, so a new value- * bypasses the cached transcript and transcribes the scene again; reusing- * a value replays that take from cache.+ * Transcription seed. Part of the transcript cache key (scene id + seed),+ * so a new value bypasses the cached transcript and transcribes the scene+ * again; reusing a value replays that take from cache. Default 0.*/seed?: number;/** `<Animation>` children. */diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex a82011d9..623d5deb 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';@@ -434,6 +434,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.set(Position, { x: 0, y: 0 });entity.add(Chars);entity.add(Caption);+ entity.add(TranscriptionRequest);break;}case 'adjustmentLayer': {@@ -988,9 +989,16 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {if (value === undefined || value === null || value === '') {entity.remove(AssetId);+ // A `<captions>` without a src transcribes its scene instead.+ if (entity.has(Caption)) {+ entity.add(TranscriptionRequest);+ entity.set(TranscriptionRequest, { seed: toNumber(entity.get(Authored)?.props.seed) ?? 0 });+ }return;}+ entity.remove(TranscriptionRequest);+if (typeof value !== 'string') {entity.remove(AssetId);entity.add(GenerationRequest);@@ -1148,6 +1156,20 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {});return;}+ case 'seed': {+ if (!entity.has(Caption)) return;+ // An authored src mounts a transcript directly; there is no+ // transcription for the seed to key. (An AssetRef src is not a+ // PropValue, so its request stands in for the authored record.)+ if (entity.get(Authored)?.props.src !== undefined) return;+ if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;++ // A resolution running for another seed must not bind late.+ entity.remove(PendingSource);+ entity.add(TranscriptionRequest);+ entity.set(TranscriptionRequest, { seed: toNumber(value) ?? 0 });+ return;+ }case 'background': {const color = parseColor(value);this.stage.entity.set(Background, { value: color ?? DEFAULT_BACKGROUND });diff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.tsindex 6dc653db..18bc7910 100644--- a/packages/runtime/src/ai.ts+++ b/packages/runtime/src/ai.ts@@ -2,15 +2,9 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// The generation service a host may attach to the world (the `Ai` trait):-// what turns a `generate.*` declaration into an asset of the project's-// library. The runtime itself cannot generate — it only knows the contract —-// so a world without an Ai never resolves such sources (see the asset-// system). The asset library knows nothing of generation either; an-// implementation *uses* a library to look up and store what it makes.-import { generate } from '@diffusionstudio/jsx';+import type { Entity, World } from 'koota';import type { Asset } from '@diffusionstudio/assets';import type {AssetRef,@@ -29,6 +23,14 @@ export abstract class GenAi {*/public abstract resolve(ref: AssetRef): Promise<Asset>;+ /**+ * The transcript of `scene`'s audible mix, as a TRANSCRIPT asset of the+ * project's library. Cached by scene id + seed: the same pair resolves to+ * the same asset, in this session and the next, and a new seed transcribes+ * the scene again. Identical concurrent requests share one run.+ */+ public abstract transcribe(world: World, scene: Entity, seed: number): Promise<Asset>;+/*** The imperative surface: `ai.generate.image({...})` declares and resolves* in one call, returning the finished asset.diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsindex f6a5a84a..8080d0cc 100644--- a/packages/runtime/src/systems/assets.ts+++ b/packages/runtime/src/systems/assets.ts@@ -2,20 +2,9 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Binds entities to the assets their `src` names. The document is-// synchronous and only leaves requests behind (LoadRequest for a source-// outside the library, GenerationRequest for a `generate.*` declaration);-// this system consumes them each frame, starts the async work, and stamps-// AssetId when the asset lands. Loads go through the world's library;-// generations through the world's Ai — deduplication (a spec whose hash-// already produced a library asset binds to it, identical inflight specs-// share one run) is the Ai implementation's contract (see GenAi). A request-// the world cannot serve yet — no library, no Ai — is left in place for a-// frame that can. In offline mode the started work joins FramePromises, so-// the encoder's barrier waits for the binding before it samples a frame.--import { Ai, FramePromises, GenerationRequest, Library, LoadRequest, PendingSource } from '../traits';+import { Ai, FramePromises, GenerationRequest, Library, LoadRequest, PendingSource, TranscriptionRequest } from '../traits';import { bindAsset } from '../actions/assets';+import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';import type { Entity, World } from 'koota';import type { Asset } from '@diffusionstudio/assets';@@ -38,7 +27,31 @@ export function assetSystem(world: World): void {if (ref === null) continue;resolve(world, entity, ref, ai.resolve(ref));}++ for (const entity of world.query(TranscriptionRequest)) {+ const scene = getSceneAncestor(entity);+ if (!scene || hasPendingSources(world, scene, entity)) continue;++ const seed = entity.get(TranscriptionRequest)!.seed;+ entity.remove(TranscriptionRequest);+ resolve(world, entity, `transcript:${scene.id()}:${seed}`, ai.transcribe(world, scene, seed));+ }+ }+}++/**+ * Whether anything in `scene`'s subtree (besides `except`, the requesting+ * element itself) is still waiting on a source: a request this system has+ * not consumed, or a resolution it started that has not landed.+ */+function hasPendingSources(world: World, scene: Entity, except: Entity): boolean {+ for (const entity of getEntityTree(world, scene)) {+ if (entity === except) continue;+ if (entity.has(LoadRequest) || entity.has(GenerationRequest) || entity.has(PendingSource)) {+ return true;+ }}+ return false;}/**diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex e7dee2f7..a40395b3 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -85,6 +85,12 @@ export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }))// through the library.export const LoadRequest = trait({ value: '' });+// On a `<captions>` element without a src: transcribe the scene it sits+// under, through the world's Ai. The seed keys the take — the transcript is+// cached by scene id + seed, so re-running with a seed replays that take and+// bumping the seed transcribes the scene again. Seed 0 is the default take.+export const TranscriptionRequest = trait({ seed: 0 });+// The src whose resolution is inflight, kept while the asset system works,// so a resolution that arrives after the element was given another source// (or none) is dropped. The document clears it whenever a new src arrives.