Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance audio handling in media generation and update documentation

konstantin-paulus21589f4843a2
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 1d256b7b..fb33dabd 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -278,7 +278,7 @@ program
.addArgument(new Argument("<model>", field("generate", "model")).choices(toolByName("generate").input.shape.model.options))
.argument(
"[fields]",
- `the model's other fields as one JSON object, e.g. '{"aspectRatio":"16:9","images":[{"path":"./ref.png"}]}'; a { "path" } that exists relative to the working directory is sent as its absolute path`,
+ `the model's other fields as one JSON object, e.g. '{"aspectRatio":"16:9","images":[{"path":"./ref.png"}]}'; a { "path" } that exists relative to the working directory is sent as its absolute path; an "audio" field also takes { "scene": "<id>" }, the scene's mix`,
)
.option("-p, --prompt <text>", "the request's `prompt`, without JSON quoting; wins over one in the fields")
.option(
diff --git a/apps/web/src/dapi/handlers/generate.ts b/apps/web/src/dapi/handlers/generate.ts
index 8a12b37e..365122ba 100644
--- a/apps/web/src/dapi/handlers/generate.ts
+++ b/apps/web/src/dapi/handlers/generate.ts
@@ -2,14 +2,6 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-/**
- * `generate` and `job`: the API's generation endpoints with a local layer
- * and nothing more. A request goes out as given, except that `{ path }` file
- * references are uploaded and swapped for the API's asset refs; the API
- * validates the rest. A job comes back as the API has it, except that once it
- * has succeeded its files are saved, here into the library, or by the server
- * to a directory on disk (see `present.ts` in the desktop app).
- */
import { getAssetFile, getLibrary } from "@diffusionstudio/runtime";
import { isAbsoluteSource, isProjectSource, isUrlSource } from "@diffusionstudio/assets";
@@ -19,7 +11,9 @@ import { TRPCClientError } from "@trpc/client";
import { api } from "@/lib/api";
import { supabase } from "@/lib/supabase";
import { uploadFile } from "@/lib/uploads";
+import { encodeSceneAudio, hasAudio } from "@/engine/scene-audio";
import { resolveAsset } from "../lib/assets";
+import { requireScene } from "../lib/scene";
import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
import type { AssetRef, GenerateRequestInput, Job } from "@diffusionstudio/api-contract";
@@ -75,6 +69,7 @@ export const job: ToolHandler<"job"> = async ({ id, cancel }, ctx) => {
// ── Requests ─────────────────────────────────────────────────
type FileRef = { path: string };
+type SceneRef = { scene: string };
const isPlainObject = (value: unknown): value is Record<string, unknown> => {
return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -85,12 +80,22 @@ const isFileRef = (value: unknown): value is FileRef => {
return isPlainObject(value) && typeof value.path === "string" && Object.keys(value).length === 1;
}
-/** The request with every `{ path }` uploaded and replaced by the ref the API names it by; the rest untouched. */
-async function withUploads(value: unknown, ctx: ToolContext): Promise<unknown> {
+/** `{ scene }` and nothing else: a scene whose audio the caller means. */
+const isSceneRef = (value: unknown): value is SceneRef => {
+ return isPlainObject(value) && typeof value.scene === "string" && Object.keys(value).length === 1;
+}
+
+/**
+ * The request with every `{ path }` and `{ scene }` uploaded and replaced by
+ * the ref the API names it by; the rest untouched. `field` is the key the
+ * value sits under (an array's items sit under the array's).
+ */
+async function withUploads(value: unknown, ctx: ToolContext, field?: string): Promise<unknown> {
if (isFileRef(value)) return upload(value.path, ctx);
- if (Array.isArray(value)) return Promise.all(value.map((item) => withUploads(item, ctx)));
+ if (isSceneRef(value)) return uploadScene(value.scene, field, ctx);
+ if (Array.isArray(value)) return Promise.all(value.map((item) => withUploads(item, ctx, field)));
if (!isPlainObject(value)) return value;
- const entries = await Promise.all(Object.entries(value).map(async ([key, item]) => [key, await withUploads(item, ctx)] as const));
+ const entries = await Promise.all(Object.entries(value).map(async ([key, item]) => [key, await withUploads(item, ctx, key)] as const));
return Object.fromEntries(entries);
}
@@ -99,6 +104,30 @@ async function upload(path: string, ctx: ToolContext): Promise<AssetRef> {
return uploadFile(await getAssetFile(asset));
}
+/**
+ * The scene's audible mix, rendered from its start to its end and uploaded,
+ * so the times of what comes back (a transcript's words) are scene times. Audio
+ * only: what a scene would be as a picture or a video is a frame to pick or a
+ * full export to run, which `capture` and `export` are for. Checked before
+ * anything renders, and the render runs before the job starts, so a scene
+ * with nothing to hear never costs credits.
+ */
+async function uploadScene(id: string, field: string | undefined, ctx: ToolContext): Promise<AssetRef> {
+ if (field !== "audio") {
+ throw new DapiError(
+ "invalid-input",
+ `A { "scene" } reference sends the scene's audio, so it goes in an \`audio\` field${field ? ` (got \`${field}\`)` : ""}.`,
+ );
+ }
+ const { world, project } = ctx.requireSession();
+ const scene = requireScene(world, id, "send", `{ "scene" } sends a scene's audio`);
+ if (!hasAudio(world, scene)) {
+ throw new DapiError("invalid-input", `Scene "${id}" has nothing to hear: no unmuted, visible audio or video clip in it.`);
+ }
+ const audio = await encodeSceneAudio(world, scene, project.dir());
+ return uploadFile(audio);
+}
+
// ── Where the files go ───────────────────────────────────────
/**
diff --git a/apps/web/src/dapi/lib/scene.ts b/apps/web/src/dapi/lib/scene.ts
index d62c79fc..25679456 100644
--- a/apps/web/src/dapi/lib/scene.ts
+++ b/apps/web/src/dapi/lib/scene.ts
@@ -15,9 +15,10 @@ import type { Entity, World } from "koota";
* requested positions first, and that pre-roll runs the project's code ahead
* of the frames being drawn, which is exactly what an export never does.
*
- * `verb` names the caller in the error ("capture", "export").
+ * `verb` names the caller in the error ("capture", "export"), and `reason`
+ * says why it takes a scene.
*/
-export function requireScene(world: World, id: string, verb: string): Entity {
+export function requireScene(world: World, id: string, verb: string, reason = `${verb} renders what an export renders`): Entity {
const node = resolveNode(world, id);
if (isScene(node)) return node;
@@ -27,8 +28,8 @@ export function requireScene(world: World, id: string, verb: string): Entity {
throw new DapiError(
"wrong-kind",
stamp
- ? `"${id}" is not a scene — ${verb} renders what an export renders. ${capitalize(verb)} its scene "${stamp}" instead.`
- : `"${id}" is not a scene — ${verb} renders what an export renders, so it takes a scene id.`,
+ ? `"${id}" is not a scene — ${reason}. ${capitalize(verb)} its scene "${stamp}" instead.`
+ : `"${id}" is not a scene — ${reason}, so it takes a scene id.`,
);
}
diff --git a/apps/web/src/engine/generate/captions.tsx b/apps/web/src/engine/generate/captions.tsx
index 0206c237..14131f29 100644
--- a/apps/web/src/engine/generate/captions.tsx
+++ b/apps/web/src/engine/generate/captions.tsx
@@ -10,16 +10,15 @@
*/
import { Captions } from '@diffusionstudio/reconciler';
-import { AssetId, Audio, Caption, getEntityTree, Hidden, Library, Muted, Paint, PaintType } from '@diffusionstudio/runtime';
+import { Caption, getEntityTree, Library } from '@diffusionstudio/runtime';
import { assetName } from '@diffusionstudio/assets';
-import { createEncoder } from '@diffusionstudio/encoder';
import { toast } from 'somoto';
import { jobFailure, runJob } from '@/lib/jobs';
import { uploadFile } from '@/lib/uploads';
-import { createCapture } from '../capture';
import { getDocumentEditor } from '../editor';
+import { encodeSceneAudio, hasAudio } from '../scene-audio';
import type { AssetLibrary } from '@diffusionstudio/assets';
import type { Job } from '@diffusionstudio/api-contract';
@@ -27,32 +26,6 @@ import type { Entity, World } from 'koota';
const TITLE = 'Caption generation failed';
-/** Whether anything in the scene is heard: an unmuted, visible clip or video paint with its asset bound. */
-function hasAudio(world: World, scene: Entity): boolean {
- return getEntityTree(world, scene).some((entity) =>
- !entity.has(Hidden) && !entity.has(Muted) && entity.has(AssetId)
- && (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO));
-}
-
-/** The scene's audible mix as an Ogg/Opus file; `dir` is the project's folder, compiled fresh. */
-async function encodeAudio(world: World, scene: Entity, dir?: string): Promise<File> {
- const capture = await createCapture(world, scene, { mode: 'offline-audio', dir });
- try {
- const encoder = await createEncoder(capture.world, {
- format: 'ogg',
- video: { enabled: false },
- audio: { enabled: true, codec: 'opus', sampleRate: 24000 },
- });
- const result = await encoder.render();
- if (result.type !== 'success' || !result.data) {
- throw new Error('Could not encode the scene audio.');
- }
- return new File([result.data], 'scene.ogg', { type: 'audio/ogg' });
- } finally {
- capture.dispose();
- }
-}
-
/** The next free `Captions N.json`. */
function captionsName(library: AssetLibrary): string {
let max = 0;
@@ -87,7 +60,7 @@ export async function generateCaptions(world: World, scene: Entity, dir?: string
toast.loading('Generating captions', { id, description: status(undefined) });
try {
- const audio = await uploadFile(await encodeAudio(world, scene, dir));
+ const audio = await uploadFile(await encodeSceneAudio(world, scene, dir));
const job = await runJob({ model: 'transcribe', audio }, (update) => {
toast.loading('Generating captions', { id, description: status(update) });
});
diff --git a/apps/web/src/engine/scene-audio.ts b/apps/web/src/engine/scene-audio.ts
new file mode 100644
index 00000000..eb2a9912
--- /dev/null
+++ b/apps/web/src/engine/scene-audio.ts
@@ -0,0 +1,46 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * A scene's audible mix as a file of its own, for what listens to a scene
+ * rather than to one of its assets: transcription, above all. The mix is the
+ * export's, from the scene's start to its end — the workarea is an export
+ * setting, not part of the composition — so a time in the file is a time in
+ * the scene, and a transcript of it lines up with `<captions>` untrimmed.
+ */
+
+import { AssetId, Audio, getEntityTree, Hidden, Muted, Paint, PaintType, Workarea } from '@diffusionstudio/runtime';
+import { createEncoder } from '@diffusionstudio/encoder';
+
+import { createCapture } from './capture';
+
+import type { Entity, World } from 'koota';
+
+/** Whether anything in the scene is heard: an unmuted, visible clip or video paint with its asset bound. */
+export function hasAudio(world: World, scene: Entity): boolean {
+ return getEntityTree(world, scene).some((entity) =>
+ !entity.has(Hidden) && !entity.has(Muted) && entity.has(AssetId)
+ && (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO));
+}
+
+/** The scene's audible mix as an Ogg/Opus file; `dir` is the project's folder, compiled fresh. */
+export async function encodeSceneAudio(world: World, scene: Entity, dir?: string): Promise<File> {
+ const capture = await createCapture(world, scene, { mode: 'offline-audio', dir });
+ try {
+ // The whole scene, not the workarea the encoder would otherwise keep to.
+ capture.node.remove(Workarea);
+ const encoder = await createEncoder(capture.world, {
+ format: 'ogg',
+ video: { enabled: false },
+ audio: { enabled: true, codec: 'opus', sampleRate: 24000 },
+ });
+ const result = await encoder.render();
+ if (result.type !== 'success' || !result.data) {
+ throw new Error('Could not encode the scene audio.');
+ }
+ return new File([result.data], 'scene.ogg', { type: 'audio/ogg' });
+ } finally {
+ capture.dispose();
+ }
+}
diff --git a/docs/reference/jsx/captions.md b/docs/reference/jsx/captions.md
index 7cefd2a8..8a579070 100644
--- a/docs/reference/jsx/captions.md
+++ b/docs/reference/jsx/captions.md
@@ -9,7 +9,25 @@
</scene>
```
-`src` is **required**: a transcript file — `.srt`, `.vtt`, or a transcript `.json` — or an audio or video asset that carries a transcript. It resolves like any other [`src`](./media.md) (library path, asset id, path, URL) and is mounted as it is; `<captions>` does not transcribe its scene. Loading is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once loaded. A source that fails to load leaves the node carrying the reason (see [errors.md](./errors.md#failed-sources)).
+`src` is **required**: a transcript file — `.srt`, `.vtt`, or a transcript `.json` — or an audio or video asset that carries a transcript. It resolves like any other [`src`](./media.md) (library path, asset id, path, URL) and is mounted as it is; `<captions>` does not transcribe its scene — make the transcript first (see [Captioning a scene](#captioning-a-scene)). Loading is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once loaded. A source that fails to load leaves the node carrying the reason (see [errors.md](./errors.md#failed-sources)).
+
+## Captioning a scene
+
+To caption what a scene plays — every voice in it, as mixed — transcribe the scene itself: pass `{ "scene": "<scene id>" }` as the `transcribe` model's `audio` to [`generate`](../tools/generate.md), save the transcript into the library, and mount it.
+
+```bash
+diffusion generate transcribe '{"audio":{"scene":"talk"}}' -o transcripts/talk.json
+diffusion job <id> # until its status is succeeded
+```
+
+```tsx
+<scene id="talk" name="Talk" width={1920} height={1080}>
+ <video src="a-roll/take-3.mp4" width={1920} height={1080} />
+ <captions src="transcripts/talk.json" preset="whisper" />
+</scene>
+```
+
+The scene's audible mix is rendered from its start to its end (the workarea is ignored), so the transcript's times are scene times and the `<captions>` lines up untrimmed — no `start` or `sourceIn` needed. Transcribe again after the edit changes what is heard or when (a cut, a moved clip, a muted track): the transcript is a snapshot of the mix, not a live view of it. A single recording rather than the whole scene takes `{ "path": … }` instead, and a transcript of a clip that starts later in the scene needs the clip's timing (see [Trimming](#trimming)).
## Props
diff --git a/docs/reference/models.md b/docs/reference/models.md
index 6a7f44fb..c3e431df 100644
--- a/docs/reference/models.md
+++ b/docs/reference/models.md
@@ -91,5 +91,5 @@ Each takes one file and returns one.
| `remove-background` | `image`: `{ path }`, required | a PNG of the image with a transparent background |
| `upscale-image` | `image`: `{ path }`, required | a PNG of the image upscaled 2× |
| `upscale-video` | `video`: `{ path }`, required | an MP4 of the video upscaled to 2K at 30 fps |
-| `transcribe` | `audio`: `{ path }`, required (audio or video); `languageCode`: e.g. `en`, `en_us`, `de` (detected when left out) | `transcript.json`: `[{ text, speaker?, words: [{ text, start, end }] }]`, times in seconds, speakers labelled `A`, `B`, … when told apart |
+| `transcribe` | `audio`: `{ path }` (audio or video) or `{ scene }` (the scene's mix, see [Scenes](./tools/generate.md#scenes)), required; `languageCode`: e.g. `en`, `en_us`, `de` (detected when left out) | `transcript.json`: `[{ text, speaker?, words: [{ text, start, end }] }]`, times in seconds, speakers labelled `A`, `B`, … when told apart |
| `analyze` | `media`: `{ path }`, required (image, video or audio); `prompt`: what to look for, up to 10,000 characters (a general description when left out) | the analysis, as a file |
diff --git a/docs/skills/editor.md b/docs/skills/editor.md
index a19426f4..816fe949 100644
--- a/docs/skills/editor.md
+++ b/docs/skills/editor.md
@@ -49,7 +49,6 @@ How to confirm a change actually produced what you intended. A clean save does n
- To cut an object out of footage, or confine an effect to it, track it with `media_segment` (preview the prompt first) and name the mask file in a `<mask>` under the clip's `<effect>`. A track into the library runs in the background: keep working, and poll `context` until its row in `masks` is done before naming it.
- For motion graphics, overlays and UI-heavy graphics, the `<html>` tag driven by a paused [anime.js](https://animejs.com) timeline
- Before animating anything, read the [easings reference](../guides/motion/easings.md) and choose easings deliberately — default or linear easing is what makes motion read as a slideshow.
-- Add captions last, after everything else is assembled. `<captions>` mounts the transcript its `src` names; it does not transcribe the scene.
- For tasks that don't need an editing UI, keep the app in the background: from a shell, `diffusion open -b <dir>` launches it that way. Over MCP the app is already running, and `open` only opens the folder.
- Only render (export) the result when prompted.
- Start with a fresh project.