The VS Code of video editing, built for agents.
TypeScript
Enhance audio handling in media generation and update documentation
konstantin-paulus21589f4843a2
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 1d256b7b..fb33dabd 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -278,7 +278,7 @@ program.addArgument(new Argument("<model>", field("generate", "model")).choices(toolByName("generate").input.shape.model.options)).argument("[fields]",- `the model's other fields as one JSON object, e.g. '{"aspectRatio":"16:9","images":[{"path":"./ref.png"}]}'; a { "path" } that exists relative to the working directory is sent as its absolute path`,+ `the model's other fields as one JSON object, e.g. '{"aspectRatio":"16:9","images":[{"path":"./ref.png"}]}'; a { "path" } that exists relative to the working directory is sent as its absolute path; an "audio" field also takes { "scene": "<id>" }, the scene's mix`,).option("-p, --prompt <text>", "the request's `prompt`, without JSON quoting; wins over one in the fields").option(diff --git a/apps/web/src/dapi/handlers/generate.ts b/apps/web/src/dapi/handlers/generate.tsindex 8a12b37e..365122ba 100644--- a/apps/web/src/dapi/handlers/generate.ts+++ b/apps/web/src/dapi/handlers/generate.ts@@ -2,14 +2,6 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-/**- * `generate` and `job`: the API's generation endpoints with a local layer- * and nothing more. A request goes out as given, except that `{ path }` file- * references are uploaded and swapped for the API's asset refs; the API- * validates the rest. A job comes back as the API has it, except that once it- * has succeeded its files are saved, here into the library, or by the server- * to a directory on disk (see `present.ts` in the desktop app).- */import { getAssetFile, getLibrary } from "@diffusionstudio/runtime";import { isAbsoluteSource, isProjectSource, isUrlSource } from "@diffusionstudio/assets";@@ -19,7 +11,9 @@ import { TRPCClientError } from "@trpc/client";import { api } from "@/lib/api";import { supabase } from "@/lib/supabase";import { uploadFile } from "@/lib/uploads";+import { encodeSceneAudio, hasAudio } from "@/engine/scene-audio";import { resolveAsset } from "../lib/assets";+import { requireScene } from "../lib/scene";import type { Asset, AssetLibrary } from "@diffusionstudio/assets";import type { AssetRef, GenerateRequestInput, Job } from "@diffusionstudio/api-contract";@@ -75,6 +69,7 @@ export const job: ToolHandler<"job"> = async ({ id, cancel }, ctx) => {// ── Requests ─────────────────────────────────────────────────type FileRef = { path: string };+type SceneRef = { scene: string };const isPlainObject = (value: unknown): value is Record<string, unknown> => {return typeof value === "object" && value !== null && !Array.isArray(value);@@ -85,12 +80,22 @@ const isFileRef = (value: unknown): value is FileRef => {return isPlainObject(value) && typeof value.path === "string" && Object.keys(value).length === 1;}-/** The request with every `{ path }` uploaded and replaced by the ref the API names it by; the rest untouched. */-async function withUploads(value: unknown, ctx: ToolContext): Promise<unknown> {+/** `{ scene }` and nothing else: a scene whose audio the caller means. */+const isSceneRef = (value: unknown): value is SceneRef => {+ return isPlainObject(value) && typeof value.scene === "string" && Object.keys(value).length === 1;+}++/**+ * The request with every `{ path }` and `{ scene }` uploaded and replaced by+ * the ref the API names it by; the rest untouched. `field` is the key the+ * value sits under (an array's items sit under the array's).+ */+async function withUploads(value: unknown, ctx: ToolContext, field?: string): Promise<unknown> {if (isFileRef(value)) return upload(value.path, ctx);- if (Array.isArray(value)) return Promise.all(value.map((item) => withUploads(item, ctx)));+ if (isSceneRef(value)) return uploadScene(value.scene, field, ctx);+ if (Array.isArray(value)) return Promise.all(value.map((item) => withUploads(item, ctx, field)));if (!isPlainObject(value)) return value;- const entries = await Promise.all(Object.entries(value).map(async ([key, item]) => [key, await withUploads(item, ctx)] as const));+ const entries = await Promise.all(Object.entries(value).map(async ([key, item]) => [key, await withUploads(item, ctx, key)] as const));return Object.fromEntries(entries);}@@ -99,6 +104,30 @@ async function upload(path: string, ctx: ToolContext): Promise<AssetRef> {return uploadFile(await getAssetFile(asset));}+/**+ * The scene's audible mix, rendered from its start to its end and uploaded,+ * so the times of what comes back (a transcript's words) are scene times. Audio+ * only: what a scene would be as a picture or a video is a frame to pick or a+ * full export to run, which `capture` and `export` are for. Checked before+ * anything renders, and the render runs before the job starts, so a scene+ * with nothing to hear never costs credits.+ */+async function uploadScene(id: string, field: string | undefined, ctx: ToolContext): Promise<AssetRef> {+ if (field !== "audio") {+ throw new DapiError(+ "invalid-input",+ `A { "scene" } reference sends the scene's audio, so it goes in an \`audio\` field${field ? ` (got \`${field}\`)` : ""}.`,+ );+ }+ const { world, project } = ctx.requireSession();+ const scene = requireScene(world, id, "send", `{ "scene" } sends a scene's audio`);+ if (!hasAudio(world, scene)) {+ throw new DapiError("invalid-input", `Scene "${id}" has nothing to hear: no unmuted, visible audio or video clip in it.`);+ }+ const audio = await encodeSceneAudio(world, scene, project.dir());+ return uploadFile(audio);+}+// ── Where the files go ───────────────────────────────────────/**diff --git a/apps/web/src/dapi/lib/scene.ts b/apps/web/src/dapi/lib/scene.tsindex d62c79fc..25679456 100644--- a/apps/web/src/dapi/lib/scene.ts+++ b/apps/web/src/dapi/lib/scene.ts@@ -15,9 +15,10 @@ import type { Entity, World } from "koota";* requested positions first, and that pre-roll runs the project's code ahead* of the frames being drawn, which is exactly what an export never does.*- * `verb` names the caller in the error ("capture", "export").+ * `verb` names the caller in the error ("capture", "export"), and `reason`+ * says why it takes a scene.*/-export function requireScene(world: World, id: string, verb: string): Entity {+export function requireScene(world: World, id: string, verb: string, reason = `${verb} renders what an export renders`): Entity {const node = resolveNode(world, id);if (isScene(node)) return node;@@ -27,8 +28,8 @@ export function requireScene(world: World, id: string, verb: string): Entity {throw new DapiError("wrong-kind",stamp- ? `"${id}" is not a scene — ${verb} renders what an export renders. ${capitalize(verb)} its scene "${stamp}" instead.`- : `"${id}" is not a scene — ${verb} renders what an export renders, so it takes a scene id.`,+ ? `"${id}" is not a scene — ${reason}. ${capitalize(verb)} its scene "${stamp}" instead.`+ : `"${id}" is not a scene — ${reason}, so it takes a scene id.`,);}diff --git a/apps/web/src/engine/generate/captions.tsx b/apps/web/src/engine/generate/captions.tsxindex 0206c237..14131f29 100644--- a/apps/web/src/engine/generate/captions.tsx+++ b/apps/web/src/engine/generate/captions.tsx@@ -10,16 +10,15 @@*/import { Captions } from '@diffusionstudio/reconciler';-import { AssetId, Audio, Caption, getEntityTree, Hidden, Library, Muted, Paint, PaintType } from '@diffusionstudio/runtime';+import { Caption, getEntityTree, Library } from '@diffusionstudio/runtime';import { assetName } from '@diffusionstudio/assets';-import { createEncoder } from '@diffusionstudio/encoder';import { toast } from 'somoto';import { jobFailure, runJob } from '@/lib/jobs';import { uploadFile } from '@/lib/uploads';-import { createCapture } from '../capture';import { getDocumentEditor } from '../editor';+import { encodeSceneAudio, hasAudio } from '../scene-audio';import type { AssetLibrary } from '@diffusionstudio/assets';import type { Job } from '@diffusionstudio/api-contract';@@ -27,32 +26,6 @@ import type { Entity, World } from 'koota';const TITLE = 'Caption generation failed';-/** Whether anything in the scene is heard: an unmuted, visible clip or video paint with its asset bound. */-function hasAudio(world: World, scene: Entity): boolean {- return getEntityTree(world, scene).some((entity) =>- !entity.has(Hidden) && !entity.has(Muted) && entity.has(AssetId)- && (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO));-}--/** The scene's audible mix as an Ogg/Opus file; `dir` is the project's folder, compiled fresh. */-async function encodeAudio(world: World, scene: Entity, dir?: string): Promise<File> {- const capture = await createCapture(world, scene, { mode: 'offline-audio', dir });- try {- const encoder = await createEncoder(capture.world, {- format: 'ogg',- video: { enabled: false },- audio: { enabled: true, codec: 'opus', sampleRate: 24000 },- });- const result = await encoder.render();- if (result.type !== 'success' || !result.data) {- throw new Error('Could not encode the scene audio.');- }- return new File([result.data], 'scene.ogg', { type: 'audio/ogg' });- } finally {- capture.dispose();- }-}-/** The next free `Captions N.json`. */function captionsName(library: AssetLibrary): string {let max = 0;@@ -87,7 +60,7 @@ export async function generateCaptions(world: World, scene: Entity, dir?: stringtoast.loading('Generating captions', { id, description: status(undefined) });try {- const audio = await uploadFile(await encodeAudio(world, scene, dir));+ const audio = await uploadFile(await encodeSceneAudio(world, scene, dir));const job = await runJob({ model: 'transcribe', audio }, (update) => {toast.loading('Generating captions', { id, description: status(update) });});diff --git a/apps/web/src/engine/scene-audio.ts b/apps/web/src/engine/scene-audio.tsnew file mode 100644index 00000000..eb2a9912--- /dev/null+++ b/apps/web/src/engine/scene-audio.ts@@ -0,0 +1,46 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * A scene's audible mix as a file of its own, for what listens to a scene+ * rather than to one of its assets: transcription, above all. The mix is the+ * export's, from the scene's start to its end — the workarea is an export+ * setting, not part of the composition — so a time in the file is a time in+ * the scene, and a transcript of it lines up with `<captions>` untrimmed.+ */++import { AssetId, Audio, getEntityTree, Hidden, Muted, Paint, PaintType, Workarea } from '@diffusionstudio/runtime';+import { createEncoder } from '@diffusionstudio/encoder';++import { createCapture } from './capture';++import type { Entity, World } from 'koota';++/** Whether anything in the scene is heard: an unmuted, visible clip or video paint with its asset bound. */+export function hasAudio(world: World, scene: Entity): boolean {+ return getEntityTree(world, scene).some((entity) =>+ !entity.has(Hidden) && !entity.has(Muted) && entity.has(AssetId)+ && (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO));+}++/** The scene's audible mix as an Ogg/Opus file; `dir` is the project's folder, compiled fresh. */+export async function encodeSceneAudio(world: World, scene: Entity, dir?: string): Promise<File> {+ const capture = await createCapture(world, scene, { mode: 'offline-audio', dir });+ try {+ // The whole scene, not the workarea the encoder would otherwise keep to.+ capture.node.remove(Workarea);+ const encoder = await createEncoder(capture.world, {+ format: 'ogg',+ video: { enabled: false },+ audio: { enabled: true, codec: 'opus', sampleRate: 24000 },+ });+ const result = await encoder.render();+ if (result.type !== 'success' || !result.data) {+ throw new Error('Could not encode the scene audio.');+ }+ return new File([result.data], 'scene.ogg', { type: 'audio/ogg' });+ } finally {+ capture.dispose();+ }+}diff --git a/docs/reference/jsx/captions.md b/docs/reference/jsx/captions.mdindex 7cefd2a8..8a579070 100644--- a/docs/reference/jsx/captions.md+++ b/docs/reference/jsx/captions.md@@ -9,7 +9,25 @@</scene>```-`src` is **required**: a transcript file — `.srt`, `.vtt`, or a transcript `.json` — or an audio or video asset that carries a transcript. It resolves like any other [`src`](./media.md) (library path, asset id, path, URL) and is mounted as it is; `<captions>` does not transcribe its scene. Loading is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once loaded. A source that fails to load leaves the node carrying the reason (see [errors.md](./errors.md#failed-sources)).+`src` is **required**: a transcript file — `.srt`, `.vtt`, or a transcript `.json` — or an audio or video asset that carries a transcript. It resolves like any other [`src`](./media.md) (library path, asset id, path, URL) and is mounted as it is; `<captions>` does not transcribe its scene — make the transcript first (see [Captioning a scene](#captioning-a-scene)). Loading is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once loaded. A source that fails to load leaves the node carrying the reason (see [errors.md](./errors.md#failed-sources)).++## Captioning a scene++To caption what a scene plays — every voice in it, as mixed — transcribe the scene itself: pass `{ "scene": "<scene id>" }` as the `transcribe` model's `audio` to [`generate`](../tools/generate.md), save the transcript into the library, and mount it.++```bash+diffusion generate transcribe '{"audio":{"scene":"talk"}}' -o transcripts/talk.json+diffusion job <id> # until its status is succeeded+```++```tsx+<scene id="talk" name="Talk" width={1920} height={1080}>+ <video src="a-roll/take-3.mp4" width={1920} height={1080} />+ <captions src="transcripts/talk.json" preset="whisper" />+</scene>+```++The scene's audible mix is rendered from its start to its end (the workarea is ignored), so the transcript's times are scene times and the `<captions>` lines up untrimmed — no `start` or `sourceIn` needed. Transcribe again after the edit changes what is heard or when (a cut, a moved clip, a muted track): the transcript is a snapshot of the mix, not a live view of it. A single recording rather than the whole scene takes `{ "path": … }` instead, and a transcript of a clip that starts later in the scene needs the clip's timing (see [Trimming](#trimming)).## Propsdiff --git a/docs/reference/models.md b/docs/reference/models.mdindex 6a7f44fb..c3e431df 100644--- a/docs/reference/models.md+++ b/docs/reference/models.md@@ -91,5 +91,5 @@ Each takes one file and returns one.| `remove-background` | `image`: `{ path }`, required | a PNG of the image with a transparent background || `upscale-image` | `image`: `{ path }`, required | a PNG of the image upscaled 2× || `upscale-video` | `video`: `{ path }`, required | an MP4 of the video upscaled to 2K at 30 fps |-| `transcribe` | `audio`: `{ path }`, required (audio or video); `languageCode`: e.g. `en`, `en_us`, `de` (detected when left out) | `transcript.json`: `[{ text, speaker?, words: [{ text, start, end }] }]`, times in seconds, speakers labelled `A`, `B`, … when told apart |+| `transcribe` | `audio`: `{ path }` (audio or video) or `{ scene }` (the scene's mix, see [Scenes](./tools/generate.md#scenes)), required; `languageCode`: e.g. `en`, `en_us`, `de` (detected when left out) | `transcript.json`: `[{ text, speaker?, words: [{ text, start, end }] }]`, times in seconds, speakers labelled `A`, `B`, … when told apart || `analyze` | `media`: `{ path }`, required (image, video or audio); `prompt`: what to look for, up to 10,000 characters (a general description when left out) | the analysis, as a file |diff --git a/docs/skills/editor.md b/docs/skills/editor.mdindex a19426f4..816fe949 100644--- a/docs/skills/editor.md+++ b/docs/skills/editor.md@@ -49,7 +49,6 @@ How to confirm a change actually produced what you intended. A clean save does n- To cut an object out of footage, or confine an effect to it, track it with `media_segment` (preview the prompt first) and name the mask file in a `<mask>` under the clip's `<effect>`. A track into the library runs in the background: keep working, and poll `context` until its row in `masks` is done before naming it.- For motion graphics, overlays and UI-heavy graphics, the `<html>` tag driven by a paused [anime.js](https://animejs.com) timeline- Before animating anything, read the [easings reference](../guides/motion/easings.md) and choose easings deliberately — default or linear easing is what makes motion read as a slideshow.-- Add captions last, after everything else is assembled. `<captions>` mounts the transcript its `src` names; it does not transcribe the scene.- For tasks that don't need an editing UI, keep the app in the background: from a shell, `diffusion open -b <dir>` launches it that way. Over MCP the app is already running, and `open` only opens the folder.- Only render (export) the result when prompted.- Start with a fresh project.