Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor audio handling and enhance media processing tools

konstantin-paulus6e251239417f
diff --git a/apps/web/src/dapi/handlers/generate.ts b/apps/web/src/dapi/handlers/generate.ts
index 282d4f64..ea15f466 100644
--- a/apps/web/src/dapi/handlers/generate.ts
+++ b/apps/web/src/dapi/handlers/generate.ts
@@ -5,20 +5,20 @@
import { getAssetFile, getLibrary } from "@diffusionstudio/runtime";
import { DapiError, generatedPath, isLocalJobId, isLocalModel } from "@diffusionstudio/dapi";
-import { TRPCClientError } from "@trpc/client";
import { api } from "@/lib/api";
-import { supabase } from "@/lib/supabase";
import { uploadFile } from "@/lib/uploads";
import { encodeSceneAudio, hasAudio } from "@/engine/scene-audio";
import { resolveAsset } from "../lib/assets";
+import { audioFile } from "../lib/audio";
import { destination, storeAt } from "../lib/outputs";
+import { apiError, requireSignIn } from "../lib/remote";
import { requireScene } from "../lib/scene";
import { localJob, startLocalJob } from "../local/jobs";
import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
import type { AssetRef, GenerateRequestInput, Job } from "@diffusionstudio/api-contract";
-import type { DapiErrorCode, GenerationResult } from "@diffusionstudio/dapi";
+import type { GenerationResult } from "@diffusionstudio/dapi";
import type { ToolContext, ToolHandler } from "../handler";
import type { Destination } from "../lib/outputs";
@@ -105,7 +105,7 @@ const isSceneRef = (value: unknown): value is SceneRef => {
* value sits under (an array's items sit under the array's).
*/
async function withUploads(value: unknown, ctx: ToolContext, field?: string): Promise<unknown> {
- if (isFileRef(value)) return upload(value.path, ctx);
+ if (isFileRef(value)) return upload(value.path, field, ctx);
if (isSceneRef(value)) return uploadScene(value.scene, field, ctx);
if (Array.isArray(value)) return Promise.all(value.map((item) => withUploads(item, ctx, field)));
if (!isPlainObject(value)) return value;
@@ -113,9 +113,10 @@ async function withUploads(value: unknown, ctx: ToolContext, field?: string): Pr
return Object.fromEntries(entries);
}
-async function upload(path: string, ctx: ToolContext): Promise<AssetRef> {
+/** A file uploaded; in an `audio` field only its sound, so a video's picture is not sent to be ignored. */
+async function upload(path: string, field: string | undefined, ctx: ToolContext): Promise<AssetRef> {
const asset = await resolveAsset(ctx, path);
- return uploadFile(await getAssetFile(asset));
+ return uploadFile(field === "audio" ? await audioFile(asset) : await getAssetFile(asset));
}
/**
@@ -201,30 +202,3 @@ async function saveToLibrary(library: AssetLibrary, job: Job, output: string | n
}
return assets;
}
-
-// ── The API ──────────────────────────────────────────────────
-
-async function requireSignIn(): Promise<void> {
- const session = await supabase?.auth.getSession();
- if (!session?.data.session) {
- throw new DapiError("sign-in-required", "Generating needs a Diffusion Studio account — sign in to the app first.");
- }
-}
-
-const API_ERRORS: Record<string, DapiErrorCode> = {
- BAD_REQUEST: "invalid-input",
- UNAUTHORIZED: "sign-in-required",
- NOT_FOUND: "not-found",
- CONFLICT: "busy",
-};
-
-/** An API call's failure as the tools report it: a tool error where one fits, a sentence for credits, else as it came. */
-function apiError(error: unknown): unknown {
- if (!(error instanceof TRPCClientError)) return error;
- const code: string | undefined = error.data?.code;
- const mapped = code && API_ERRORS[code];
- if (mapped) return new DapiError(mapped, error.message, { cause: error });
- if (code === "PAYMENT_REQUIRED") return new Error(`Not enough credits: ${error.message}`, { cause: error });
- return error;
-}
-
diff --git a/apps/web/src/dapi/handlers/media-listen.ts b/apps/web/src/dapi/handlers/media-listen.ts
index 733d5bf3..fdf1e39c 100644
--- a/apps/web/src/dapi/handlers/media-listen.ts
+++ b/apps/web/src/dapi/handlers/media-listen.ts
@@ -3,14 +3,40 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { DapiError } from "@diffusionstudio/dapi";
+
+import { jobFailure, runJob } from "@/lib/jobs";
+import { uploadFile } from "@/lib/uploads";
import { requireAssetType, resolveAsset } from "../lib/assets";
+import { audioFile } from "../lib/audio";
+import { apiError, requireSignIn } from "../lib/remote";
import type { ToolHandler } from "../handler";
-// Placeholder: the analysis ran on the backend, which is being rebuilt. The
-// tool keeps its place in the catalog so callers get a clear answer.
-export const mediaListen: ToolHandler<"media_listen"> = async ({ path }, ctx) => {
+/** The asset's audio, or a window of it, analyzed on the API; times in the answer count from `start`, echoed with `end` when a window was given. */
+export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end }, ctx) => {
const asset = await resolveAsset(ctx, path);
requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");
- throw new DapiError("unsupported", "Audio analysis is not available in this version of the app.");
+
+ const windowed = start !== undefined || end !== undefined;
+ const from = start ?? 0;
+ const to = Math.min(end ?? asset.duration, asset.duration);
+ if (windowed && from >= to) {
+ throw new DapiError("invalid-input", `The window is empty: start (${from.toFixed(2)}s) is at or past the end (${to.toFixed(2)}s).`);
+ }
+ await requireSignIn();
+
+ try {
+ const media = await uploadFile(windowed ? await audioFile(asset, from, to) : await audioFile(asset));
+ const job = await runJob({ model: "gemini-3.5-flash", media, prompt }, () => {}, ctx.signal);
+ if (job.status !== "succeeded") throw new Error(jobFailure(job));
+
+ const [file] = job.assets;
+ if (!file) throw new Error("The analysis returned nothing.");
+ const response = await fetch(file.url, { signal: ctx.signal });
+ if (!response.ok) throw new Error(`Could not download the analysis (${response.status})`);
+ const result = (await response.text()).trim();
+ return windowed ? { result, start: from, end: to } : { result };
+ } catch (error) {
+ throw apiError(error);
+ }
};
diff --git a/apps/web/src/dapi/handlers/media-transcribe.ts b/apps/web/src/dapi/handlers/media-transcribe.ts
index 0b9e33a3..a47dd344 100644
--- a/apps/web/src/dapi/handlers/media-transcribe.ts
+++ b/apps/web/src/dapi/handlers/media-transcribe.ts
@@ -2,15 +2,52 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { DapiError } from "@diffusionstudio/dapi";
+import { jobFailure, runJob } from "@/lib/jobs";
+import { sha256, uploadFile } from "@/lib/uploads";
import { requireAssetType, resolveAsset } from "../lib/assets";
+import { audioFile } from "../lib/audio";
+import { apiError, requireSignIn } from "../lib/remote";
+import type { Transcript } from "@diffusionstudio/api-contract";
import type { ToolHandler } from "../handler";
-// Placeholder: transcription ran on the backend, which is being rebuilt. The
-// tool keeps its place in the catalog so callers get a clear answer.
+/** Transcripts by the sha256 of the audio sent, for this app session: asking again costs nothing. */
+const transcripts = new Map<string, Promise<Transcript>>();
+
+/** The asset's audio, transcribed on the API; the main process writes the transcript to its file. */
export const mediaTranscribe: ToolHandler<"media_transcribe"> = async ({ path }, ctx) => {
const asset = await resolveAsset(ctx, path);
requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");
- throw new DapiError("unsupported", "Transcription is not available in this version of the app.");
+ await requireSignIn();
+
+ const audio = await audioFile(asset);
+ const key = await sha256(audio);
+ let transcript = transcripts.get(key);
+ if (!transcript) {
+ transcript = transcribe(audio, ctx.signal);
+ transcripts.set(key, transcript);
+ transcript.catch(() => transcripts.delete(key));
+ }
+
+ const segments = await transcript;
+ if (!segments.some((segment) => segment.words.length > 0)) {
+ throw new Error("No speech detected: the audio does not appear to contain recognizable speech.");
+ }
+ return { segments };
};
+
+/** `audio` uploaded and transcribed; aborting `signal` cancels the job. */
+async function transcribe(audio: Blob, signal: AbortSignal): Promise<Transcript> {
+ try {
+ const job = await runJob({ model: "universal-3.5-pro", audio: await uploadFile(audio) }, () => {}, signal);
+ if (job.status !== "succeeded") throw new Error(jobFailure(job));
+
+ const [file] = job.assets;
+ if (!file) throw new Error("The transcription returned nothing.");
+ const response = await fetch(file.url, { signal });
+ if (!response.ok) throw new Error(`Could not download the transcript (${response.status})`);
+ return await response.json() as Transcript;
+ } catch (error) {
+ throw apiError(error);
+ }
+}
diff --git a/apps/web/src/dapi/lib/audio.ts b/apps/web/src/dapi/lib/audio.ts
new file mode 100644
index 00000000..32f1743d
--- /dev/null
+++ b/apps/web/src/dapi/lib/audio.ts
@@ -0,0 +1,81 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * An asset's sound as a file of its own, for a model that only listens
+ * (transcription, audio analysis): a video's picture never leaves the
+ * machine, which for footage is nearly all of its bytes. The audio track is
+ * copied as it is when a container of its own holds it, which takes no
+ * longer than reading the file; anything else is encoded to Opus.
+ */
+
+import { ALL_FORMATS, BlobSource, BufferTarget, Conversion, Input, Output } from "mediabunny";
+import { getAssetFile } from "@diffusionstudio/runtime";
+import { DapiError } from "@diffusionstudio/dapi";
+
+import type { Asset } from "@diffusionstudio/assets";
+import type { AudioCodec, OutputFormat } from "mediabunny";
+
+/** The audio-only container each compressed codec is copied into. */
+const CONTAINERS: Partial<Record<AudioCodec, () => Promise<OutputFormat>>> = {
+ aac: async () => {
+ const { AdtsOutputFormat } = await import("mediabunny");
+ return new AdtsOutputFormat();
+ },
+ mp3: async () => {
+ const { Mp3OutputFormat } = await import("mediabunny");
+ return new Mp3OutputFormat();
+ },
+ opus: ogg,
+ vorbis: ogg,
+};
+
+/** Opus and Vorbis's own container, and what everything encoded to Opus goes in. */
+async function ogg(): Promise<OutputFormat> {
+ const { OggOutputFormat } = await import("mediabunny");
+ return new OggOutputFormat();
+}
+
+/** What the rest (PCM, FLAC, AC-3, …) is encoded to: plenty for speech, a fraction of the size. */
+const OPUS = { codec: "opus", numberOfChannels: 1, sampleRate: 24_000, bitrate: 48_000 } as const;
+
+/**
+ * `asset`'s audio, from `start` to `end` seconds when given. An audio file
+ * that is already compressed and wanted whole is sent as it is; an asset
+ * that is neither audio nor video is too, for the API to judge.
+ */
+export async function audioFile(asset: Asset, start?: number, end?: number): Promise<Blob> {
+ const file = await getAssetFile(asset);
+ if (asset.type !== "AUDIO" && asset.type !== "VIDEO") return file;
+
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
+ try {
+ const track = await input.getPrimaryAudioTrack();
+ if (!track) throw new DapiError("wrong-kind", `Asset ${asset.id} has no audio track.`);
+
+ const container = track.codec ? CONTAINERS[track.codec] : undefined;
+ const whole = start === undefined && end === undefined;
+ if (asset.type === "AUDIO" && container && whole) return file;
+
+ const format = await (container ?? ogg)();
+ const target = new BufferTarget();
+ const conversion = await Conversion.init({
+ input,
+ output: new Output({ format, target }),
+ tracks: "primary",
+ video: { discard: true },
+ audio: container ? {} : OPUS,
+ trim: whole ? undefined : { start, end },
+ showWarnings: false,
+ });
+ if (!conversion.isValid) {
+ throw new Error(`Could not extract the audio of ${asset.id} (${conversion.discardedTracks.map((t) => t.reason).join(", ")}).`);
+ }
+ await conversion.execute();
+ // Ogg says `application/ogg`; the models take audio by its `audio/` type.
+ return new Blob([target.buffer!], { type: format.mimeType.replace(/^application\//, "audio/") });
+ } finally {
+ input.dispose();
+ }
+}
diff --git a/apps/web/src/dapi/lib/remote.ts b/apps/web/src/dapi/lib/remote.ts
new file mode 100644
index 00000000..90a4f4b6
--- /dev/null
+++ b/apps/web/src/dapi/lib/remote.ts
@@ -0,0 +1,36 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/** The API as the tools call it: signed in, its failures as tool errors. */
+
+import { DapiError } from "@diffusionstudio/dapi";
+import { TRPCClientError } from "@trpc/client";
+
+import { supabase } from "@/lib/supabase";
+
+import type { DapiErrorCode } from "@diffusionstudio/dapi";
+
+export async function requireSignIn(): Promise<void> {
+ const session = await supabase?.auth.getSession();
+ if (!session?.data.session) {
+ throw new DapiError("sign-in-required", "Generating needs a Diffusion Studio account — sign in to the app first.");
+ }
+}
+
+const API_ERRORS: Record<string, DapiErrorCode> = {
+ BAD_REQUEST: "invalid-input",
+ UNAUTHORIZED: "sign-in-required",
+ NOT_FOUND: "not-found",
+ CONFLICT: "busy",
+};
+
+/** An API call's failure as the tools report it: a tool error where one fits, a sentence for credits, else as it came. */
+export function apiError(error: unknown): unknown {
+ if (!(error instanceof TRPCClientError)) return error;
+ const code: string | undefined = error.data?.code;
+ const mapped = code && API_ERRORS[code];
+ if (mapped) return new DapiError(mapped, error.message, { cause: error });
+ if (code === "PAYMENT_REQUIRED") return new Error(`Not enough credits: ${error.message}`, { cause: error });
+ return error;
+}
diff --git a/apps/web/src/lib/jobs.ts b/apps/web/src/lib/jobs.ts
index 083083c2..c79d016e 100644
--- a/apps/web/src/lib/jobs.ts
+++ b/apps/web/src/lib/jobs.ts
@@ -27,15 +27,20 @@ const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve,
* Starts a job for `request` and follows it to its end. `onUpdate` hears
* every state the job is seen in, the first one included. Resolves with the
* job as it ended — failed and canceled jobs included, which say why in
- * `error`; rejects only when the job could not be started or followed.
+ * `error`; rejects only when the job could not be started or followed. When
+ * `signal` aborts, the job is canceled and the abort reason thrown.
*/
-export async function runJob(request: GenerateRequestInput, onUpdate: (job: Job) => void): Promise<Job> {
+export async function runJob(request: GenerateRequestInput, onUpdate: (job: Job) => void, signal?: AbortSignal): Promise<Job> {
let job = await api.generate.mutate({ request, idempotencyKey: crypto.randomUUID() });
onUpdate(job);
let failures = 0;
while (!isTerminal(job)) {
await sleep(POLL_INTERVAL);
+ if (signal?.aborted) {
+ await api.jobs.cancel.mutate({ id: job.id }).catch(() => {});
+ signal.throwIfAborted();
+ }
try {
job = await api.jobs.get.query({ id: job.id });
failures = 0;
diff --git a/apps/web/src/lib/uploads.ts b/apps/web/src/lib/uploads.ts
index 378a9083..e5299df1 100644
--- a/apps/web/src/lib/uploads.ts
+++ b/apps/web/src/lib/uploads.ts
@@ -14,7 +14,7 @@ import { api } from "./api";
import type { AssetRef } from "@diffusionstudio/api-contract";
/** Lowercase hex sha256 of `blob`'s bytes: what the API names an upload by. */
-async function sha256(blob: Blob): Promise<string> {
+export async function sha256(blob: Blob): Promise<string> {
const digest = await crypto.subtle.digest("SHA-256", await blob.arrayBuffer());
return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
}
diff --git a/docs/reference/tools/generate.md b/docs/reference/tools/generate.md
index 78207481..42c0b7c9 100644
--- a/docs/reference/tools/generate.md
+++ b/docs/reference/tools/generate.md
@@ -24,7 +24,7 @@ An API model's fields are not checked here: they go to the API as given, and the
## Files
-Where a field takes a file (`images`, `startFrame`, `endFrame`, `image`, `video`, `audio`, `media`), put a `{ "path": … }` object: an absolute path or URL (with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project). The file is uploaded before the job starts — once per file, however many jobs name it — and the reference becomes the API's own; a local model reads it in place. The API's references (`{ "kind": "asset", "id" }`, `{ "kind": "url", "url" }`) pass through unchanged. On the CLI a `path` that exists relative to the working directory is sent as its absolute path.
+Where a field takes a file (`images`, `startFrame`, `endFrame`, `image`, `video`, `audio`, `media`), put a `{ "path": … }` object: an absolute path or URL (with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project). The file is uploaded before the job starts — once per file, however many jobs name it — and the reference becomes the API's own; in an `audio` field only its sound is sent, so a video's picture stays on the machine; a local model reads it in place. The API's references (`{ "kind": "asset", "id" }`, `{ "kind": "url", "url" }`) pass through unchanged. On the CLI a `path` that exists relative to the working directory is sent as its absolute path.
```json
{
diff --git a/docs/reference/tools/media/listen.md b/docs/reference/tools/media/listen.md
index 4799a1f4..6277a883 100644
--- a/docs/reference/tools/media/listen.md
+++ b/docs/reference/tools/media/listen.md
@@ -2,8 +2,6 @@
Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; of a video only the audio track is analyzed.
-**Temporarily unavailable in this version:** every call fails with `unsupported`.
-
| | |
| --- | --- |
| MCP tool | `media_listen` |
@@ -30,4 +28,4 @@ One JSON object, the model's answer. `start`/`end` echo the analyzed window (in
## Errors
-Fails when the path can't be resolved, the asset isn't a video or audio asset, or `start`/`end` cross (`start` >= `end`). In this version every call fails with `unsupported`.
+Fails when the path can't be resolved, the asset isn't a video or audio asset, or `start`/`end` cross (`start` >= `end`).
diff --git a/docs/reference/tools/media/transcribe.md b/docs/reference/tools/media/transcribe.md
index 816bcbf8..bf82478f 100644
--- a/docs/reference/tools/media/transcribe.md
+++ b/docs/reference/tools/media/transcribe.md
@@ -1,8 +1,6 @@
# media_transcribe
-Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, with word-level start/end times in seconds; returns the file's path and its segment and word counts. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).
-
-**Temporarily unavailable in this version:** every call fails with `unsupported`.
+Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, with word-level start/end times in seconds and speaker labels; returns the file's path and its segment and word counts. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).
| | |
| --- | --- |
@@ -16,7 +14,7 @@ Transcribe the speech in a video or audio file and write the timed transcript to
| `path` | `string`, required | `<path>` | absolute file path or URL (works with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project) |
| `output` | `string` | `-o, --output <path>` | absolute path to write the transcript JSON to (default: a fresh file under the system temp dir) |
-Times are in **seconds** of source/content time. An `output` naming an existing directory gets a fresh `dapi-transcript-*.json` inside it. The whole asset is transcribed once per app session (cached in memory, keyed by file content; an app restart or an edited file re-transcribes). Every call writes the file again, so a cached transcript returns at once.
+Times are in **seconds** of source/content time. An `output` naming an existing directory gets a fresh `dapi-transcript-*.json` inside it. The whole asset is transcribed once per app session (cached in memory, keyed by file content; an app restart or an edited file re-transcribes). Every call writes the file again, so a cached transcript returns at once. Only the audio is uploaded: a video's audio track is copied out (or encoded to Opus when its codec has no audio-only container), so the picture never leaves the machine.
## Output
@@ -35,7 +33,8 @@ The file holds the transcript as indented JSON, so it searches line by line: a g
```ts
{
segments: Array<{
- text: string; // spoken words only (no silence markers)
+ text: string; // spoken words only (no silence markers)
+ speaker?: string; // "A", "B", … when speakers are told apart
words: Array<{ text: string; start: number; end: number }>; // seconds
}>;
}
@@ -43,4 +42,4 @@ The file holds the transcript as indented JSON, so it searches line by line: a g
## Errors
-Fails when the path can't be resolved, the asset is not a video/audio asset, or no speech is detected in the audio at all (`No speech detected`). In this version every call fails with `unsupported`.
+Fails when the path can't be resolved, the asset is not a video/audio asset, or no speech is detected in the audio at all (`No speech detected`).
diff --git a/docs/skills/editor.md b/docs/skills/editor.md
index 57a0ab61..372f4d6f 100644
--- a/docs/skills/editor.md
+++ b/docs/skills/editor.md
@@ -4,8 +4,8 @@ How to understand source material before editing it. Inspect only the modalities
- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.
- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.
-- **Listen to the audio.** (`media_listen` is temporarily unavailable in this version.) Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.
-- **Transcribe speech.** (`media_transcribe` is temporarily unavailable in this version.) For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.
+- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.
+- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.
- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.
# The editing loop
diff --git a/docs/skills/watch.md b/docs/skills/watch.md
index 8769a870..c83f6ab3 100644
--- a/docs/skills/watch.md
+++ b/docs/skills/watch.md
@@ -4,8 +4,8 @@ How to understand source material. Inspect only the modalities the question turn
- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.
- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.
-- **Listen to the audio.** (`media_listen` is temporarily unavailable in this version.) Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.
-- **Transcribe speech.** (`media_transcribe` is temporarily unavailable in this version.) For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.
+- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.
+- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.
- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.
The `media_*` tools take a path or a direct media URL. Footage behind a page URL (YouTube, TikTok, Instagram, …) is downloaded first with `yt-dlp` from a shell — audio only (`yt-dlp -x --audio-format m4a -o talk.m4a <url>`) unless the question needs frames; see the [tool reference](https://github.com/yt-dlp/yt-dlp).
@@ -16,8 +16,8 @@ Read only as much of the footage as the answer requires — each pass costs time
- A duration or format question ends at `media_probe`.
- "Where is the quiet part" or "how is it paced" is usually answered by the waveform and filmstrip alone.
-- A question about what was said resolves fastest through `media_transcribe` (temporarily unavailable in this version); search the transcript file for the passage and quote it with its times.
-- Questions about non-speech audio (music, tone, sound events, speaker identity) are what `media_listen` is for (temporarily unavailable in this version).
+- A question about what was said resolves fastest through `media_transcribe`; search the transcript file for the passage and quote it with its times.
+- Questions about non-speech audio (music, tone, sound events, speaker identity) are what `media_listen` is for.
- Only questions about what is *seen* need frames — and the audio pass usually tells you which moments to grab, so grab those instead of scanning blind.
- For open-ended questions ("summarize this", "what happens here"), combine passes: structure from waveform + filmstrip, content from transcript or listen, then frames at the salient moments to confirm what the picture shows.
diff --git a/packages/dapi/src/tools/media-transcribe.ts b/packages/dapi/src/tools/media-transcribe.ts
index 4e4a31cc..1b6ec1ac 100644
--- a/packages/dapi/src/tools/media-transcribe.ts
+++ b/packages/dapi/src/tools/media-transcribe.ts
@@ -14,6 +14,7 @@ export const TranscriptWord = z.object({
export const TranscriptSegment = z.object({
text: z.string(),
+ speaker: z.string().optional().describe("speaker label (A, B, …) when speakers are told apart"),
words: z.array(TranscriptWord),
});
@@ -21,7 +22,7 @@ export const mediaTranscribe = defineTool({
name: "media_transcribe",
title: "Transcribe speech",
description:
- "Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, `{ segments: [{ text, words: [{ text, start, end }] }] }` with word-level times in seconds; returns the file's path and its segment and word counts. Search the file for the passage you need (grep, jq) rather than reading it whole: a long recording's word timings run to tens of thousands of tokens. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).",
+ "Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, `{ segments: [{ text, speaker?, words: [{ text, start, end }] }] }` with word-level times in seconds and speakers labelled A, B, … when told apart; returns the file's path and its segment and word counts. Search the file for the passage you need (grep, jq) rather than reading it whole: a long recording's word timings run to tens of thousands of tokens. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).",
input: z.object({
path: AssetPath,
output: z.string().optional().describe("absolute path to write the transcript JSON to (default: a fresh file under the system temp dir)"),