The VS Code of video editing, built for agents.
TypeScript
Refactor audio handling and enhance media processing tools
konstantin-paulus6e251239417f
diff --git a/apps/web/src/dapi/handlers/generate.ts b/apps/web/src/dapi/handlers/generate.tsindex 282d4f64..ea15f466 100644--- a/apps/web/src/dapi/handlers/generate.ts+++ b/apps/web/src/dapi/handlers/generate.ts@@ -5,20 +5,20 @@import { getAssetFile, getLibrary } from "@diffusionstudio/runtime";import { DapiError, generatedPath, isLocalJobId, isLocalModel } from "@diffusionstudio/dapi";-import { TRPCClientError } from "@trpc/client";import { api } from "@/lib/api";-import { supabase } from "@/lib/supabase";import { uploadFile } from "@/lib/uploads";import { encodeSceneAudio, hasAudio } from "@/engine/scene-audio";import { resolveAsset } from "../lib/assets";+import { audioFile } from "../lib/audio";import { destination, storeAt } from "../lib/outputs";+import { apiError, requireSignIn } from "../lib/remote";import { requireScene } from "../lib/scene";import { localJob, startLocalJob } from "../local/jobs";import type { Asset, AssetLibrary } from "@diffusionstudio/assets";import type { AssetRef, GenerateRequestInput, Job } from "@diffusionstudio/api-contract";-import type { DapiErrorCode, GenerationResult } from "@diffusionstudio/dapi";+import type { GenerationResult } from "@diffusionstudio/dapi";import type { ToolContext, ToolHandler } from "../handler";import type { Destination } from "../lib/outputs";@@ -105,7 +105,7 @@ const isSceneRef = (value: unknown): value is SceneRef => {* value sits under (an array's items sit under the array's).*/async function withUploads(value: unknown, ctx: ToolContext, field?: string): Promise<unknown> {- if (isFileRef(value)) return upload(value.path, ctx);+ if (isFileRef(value)) return upload(value.path, field, ctx);if (isSceneRef(value)) return uploadScene(value.scene, field, ctx);if (Array.isArray(value)) return Promise.all(value.map((item) => withUploads(item, ctx, field)));if (!isPlainObject(value)) return value;@@ -113,9 +113,10 @@ async function withUploads(value: unknown, ctx: ToolContext, field?: string): Prreturn Object.fromEntries(entries);}-async function upload(path: string, ctx: ToolContext): Promise<AssetRef> {+/** A file uploaded; in an `audio` field only its sound, so a video's picture is not sent to be ignored. */+async function upload(path: string, field: string | undefined, ctx: ToolContext): Promise<AssetRef> {const asset = await resolveAsset(ctx, path);- return uploadFile(await getAssetFile(asset));+ return uploadFile(field === "audio" ? await audioFile(asset) : await getAssetFile(asset));}/**@@ -201,30 +202,3 @@ async function saveToLibrary(library: AssetLibrary, job: Job, output: string | n}return assets;}--// ── The API ──────────────────────────────────────────────────--async function requireSignIn(): Promise<void> {- const session = await supabase?.auth.getSession();- if (!session?.data.session) {- throw new DapiError("sign-in-required", "Generating needs a Diffusion Studio account — sign in to the app first.");- }-}--const API_ERRORS: Record<string, DapiErrorCode> = {- BAD_REQUEST: "invalid-input",- UNAUTHORIZED: "sign-in-required",- NOT_FOUND: "not-found",- CONFLICT: "busy",-};--/** An API call's failure as the tools report it: a tool error where one fits, a sentence for credits, else as it came. */-function apiError(error: unknown): unknown {- if (!(error instanceof TRPCClientError)) return error;- const code: string | undefined = error.data?.code;- const mapped = code && API_ERRORS[code];- if (mapped) return new DapiError(mapped, error.message, { cause: error });- if (code === "PAYMENT_REQUIRED") return new Error(`Not enough credits: ${error.message}`, { cause: error });- return error;-}-diff --git a/apps/web/src/dapi/handlers/media-listen.ts b/apps/web/src/dapi/handlers/media-listen.tsindex 733d5bf3..fdf1e39c 100644--- a/apps/web/src/dapi/handlers/media-listen.ts+++ b/apps/web/src/dapi/handlers/media-listen.ts@@ -3,14 +3,40 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { DapiError } from "@diffusionstudio/dapi";++import { jobFailure, runJob } from "@/lib/jobs";+import { uploadFile } from "@/lib/uploads";import { requireAssetType, resolveAsset } from "../lib/assets";+import { audioFile } from "../lib/audio";+import { apiError, requireSignIn } from "../lib/remote";import type { ToolHandler } from "../handler";-// Placeholder: the analysis ran on the backend, which is being rebuilt. The-// tool keeps its place in the catalog so callers get a clear answer.-export const mediaListen: ToolHandler<"media_listen"> = async ({ path }, ctx) => {+/** The asset's audio, or a window of it, analyzed on the API; times in the answer count from `start`, echoed with `end` when a window was given. */+export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end }, ctx) => {const asset = await resolveAsset(ctx, path);requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");- throw new DapiError("unsupported", "Audio analysis is not available in this version of the app.");++ const windowed = start !== undefined || end !== undefined;+ const from = start ?? 0;+ const to = Math.min(end ?? asset.duration, asset.duration);+ if (windowed && from >= to) {+ throw new DapiError("invalid-input", `The window is empty: start (${from.toFixed(2)}s) is at or past the end (${to.toFixed(2)}s).`);+ }+ await requireSignIn();++ try {+ const media = await uploadFile(windowed ? await audioFile(asset, from, to) : await audioFile(asset));+ const job = await runJob({ model: "gemini-3.5-flash", media, prompt }, () => {}, ctx.signal);+ if (job.status !== "succeeded") throw new Error(jobFailure(job));++ const [file] = job.assets;+ if (!file) throw new Error("The analysis returned nothing.");+ const response = await fetch(file.url, { signal: ctx.signal });+ if (!response.ok) throw new Error(`Could not download the analysis (${response.status})`);+ const result = (await response.text()).trim();+ return windowed ? { result, start: from, end: to } : { result };+ } catch (error) {+ throw apiError(error);+ }};diff --git a/apps/web/src/dapi/handlers/media-transcribe.ts b/apps/web/src/dapi/handlers/media-transcribe.tsindex 0b9e33a3..a47dd344 100644--- a/apps/web/src/dapi/handlers/media-transcribe.ts+++ b/apps/web/src/dapi/handlers/media-transcribe.ts@@ -2,15 +2,52 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { DapiError } from "@diffusionstudio/dapi";+import { jobFailure, runJob } from "@/lib/jobs";+import { sha256, uploadFile } from "@/lib/uploads";import { requireAssetType, resolveAsset } from "../lib/assets";+import { audioFile } from "../lib/audio";+import { apiError, requireSignIn } from "../lib/remote";+import type { Transcript } from "@diffusionstudio/api-contract";import type { ToolHandler } from "../handler";-// Placeholder: transcription ran on the backend, which is being rebuilt. The-// tool keeps its place in the catalog so callers get a clear answer.+/** Transcripts by the sha256 of the audio sent, for this app session: asking again costs nothing. */+const transcripts = new Map<string, Promise<Transcript>>();++/** The asset's audio, transcribed on the API; the main process writes the transcript to its file. */export const mediaTranscribe: ToolHandler<"media_transcribe"> = async ({ path }, ctx) => {const asset = await resolveAsset(ctx, path);requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");- throw new DapiError("unsupported", "Transcription is not available in this version of the app.");+ await requireSignIn();++ const audio = await audioFile(asset);+ const key = await sha256(audio);+ let transcript = transcripts.get(key);+ if (!transcript) {+ transcript = transcribe(audio, ctx.signal);+ transcripts.set(key, transcript);+ transcript.catch(() => transcripts.delete(key));+ }++ const segments = await transcript;+ if (!segments.some((segment) => segment.words.length > 0)) {+ throw new Error("No speech detected: the audio does not appear to contain recognizable speech.");+ }+ return { segments };};++/** `audio` uploaded and transcribed; aborting `signal` cancels the job. */+async function transcribe(audio: Blob, signal: AbortSignal): Promise<Transcript> {+ try {+ const job = await runJob({ model: "universal-3.5-pro", audio: await uploadFile(audio) }, () => {}, signal);+ if (job.status !== "succeeded") throw new Error(jobFailure(job));++ const [file] = job.assets;+ if (!file) throw new Error("The transcription returned nothing.");+ const response = await fetch(file.url, { signal });+ if (!response.ok) throw new Error(`Could not download the transcript (${response.status})`);+ return await response.json() as Transcript;+ } catch (error) {+ throw apiError(error);+ }+}diff --git a/apps/web/src/dapi/lib/audio.ts b/apps/web/src/dapi/lib/audio.tsnew file mode 100644index 00000000..32f1743d--- /dev/null+++ b/apps/web/src/dapi/lib/audio.ts@@ -0,0 +1,81 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * An asset's sound as a file of its own, for a model that only listens+ * (transcription, audio analysis): a video's picture never leaves the+ * machine, which for footage is nearly all of its bytes. The audio track is+ * copied as it is when a container of its own holds it, which takes no+ * longer than reading the file; anything else is encoded to Opus.+ */++import { ALL_FORMATS, BlobSource, BufferTarget, Conversion, Input, Output } from "mediabunny";+import { getAssetFile } from "@diffusionstudio/runtime";+import { DapiError } from "@diffusionstudio/dapi";++import type { Asset } from "@diffusionstudio/assets";+import type { AudioCodec, OutputFormat } from "mediabunny";++/** The audio-only container each compressed codec is copied into. */+const CONTAINERS: Partial<Record<AudioCodec, () => Promise<OutputFormat>>> = {+ aac: async () => {+ const { AdtsOutputFormat } = await import("mediabunny");+ return new AdtsOutputFormat();+ },+ mp3: async () => {+ const { Mp3OutputFormat } = await import("mediabunny");+ return new Mp3OutputFormat();+ },+ opus: ogg,+ vorbis: ogg,+};++/** Opus and Vorbis's own container, and what everything encoded to Opus goes in. */+async function ogg(): Promise<OutputFormat> {+ const { OggOutputFormat } = await import("mediabunny");+ return new OggOutputFormat();+}++/** What the rest (PCM, FLAC, AC-3, …) is encoded to: plenty for speech, a fraction of the size. */+const OPUS = { codec: "opus", numberOfChannels: 1, sampleRate: 24_000, bitrate: 48_000 } as const;++/**+ * `asset`'s audio, from `start` to `end` seconds when given. An audio file+ * that is already compressed and wanted whole is sent as it is; an asset+ * that is neither audio nor video is too, for the API to judge.+ */+export async function audioFile(asset: Asset, start?: number, end?: number): Promise<Blob> {+ const file = await getAssetFile(asset);+ if (asset.type !== "AUDIO" && asset.type !== "VIDEO") return file;++ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });+ try {+ const track = await input.getPrimaryAudioTrack();+ if (!track) throw new DapiError("wrong-kind", `Asset ${asset.id} has no audio track.`);++ const container = track.codec ? CONTAINERS[track.codec] : undefined;+ const whole = start === undefined && end === undefined;+ if (asset.type === "AUDIO" && container && whole) return file;++ const format = await (container ?? ogg)();+ const target = new BufferTarget();+ const conversion = await Conversion.init({+ input,+ output: new Output({ format, target }),+ tracks: "primary",+ video: { discard: true },+ audio: container ? {} : OPUS,+ trim: whole ? undefined : { start, end },+ showWarnings: false,+ });+ if (!conversion.isValid) {+ throw new Error(`Could not extract the audio of ${asset.id} (${conversion.discardedTracks.map((t) => t.reason).join(", ")}).`);+ }+ await conversion.execute();+ // Ogg says `application/ogg`; the models take audio by its `audio/` type.+ return new Blob([target.buffer!], { type: format.mimeType.replace(/^application\//, "audio/") });+ } finally {+ input.dispose();+ }+}diff --git a/apps/web/src/dapi/lib/remote.ts b/apps/web/src/dapi/lib/remote.tsnew file mode 100644index 00000000..90a4f4b6--- /dev/null+++ b/apps/web/src/dapi/lib/remote.ts@@ -0,0 +1,36 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/** The API as the tools call it: signed in, its failures as tool errors. */++import { DapiError } from "@diffusionstudio/dapi";+import { TRPCClientError } from "@trpc/client";++import { supabase } from "@/lib/supabase";++import type { DapiErrorCode } from "@diffusionstudio/dapi";++export async function requireSignIn(): Promise<void> {+ const session = await supabase?.auth.getSession();+ if (!session?.data.session) {+ throw new DapiError("sign-in-required", "Generating needs a Diffusion Studio account — sign in to the app first.");+ }+}++const API_ERRORS: Record<string, DapiErrorCode> = {+ BAD_REQUEST: "invalid-input",+ UNAUTHORIZED: "sign-in-required",+ NOT_FOUND: "not-found",+ CONFLICT: "busy",+};++/** An API call's failure as the tools report it: a tool error where one fits, a sentence for credits, else as it came. */+export function apiError(error: unknown): unknown {+ if (!(error instanceof TRPCClientError)) return error;+ const code: string | undefined = error.data?.code;+ const mapped = code && API_ERRORS[code];+ if (mapped) return new DapiError(mapped, error.message, { cause: error });+ if (code === "PAYMENT_REQUIRED") return new Error(`Not enough credits: ${error.message}`, { cause: error });+ return error;+}diff --git a/apps/web/src/lib/jobs.ts b/apps/web/src/lib/jobs.tsindex 083083c2..c79d016e 100644--- a/apps/web/src/lib/jobs.ts+++ b/apps/web/src/lib/jobs.ts@@ -27,15 +27,20 @@ const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve,* Starts a job for `request` and follows it to its end. `onUpdate` hears* every state the job is seen in, the first one included. Resolves with the* job as it ended — failed and canceled jobs included, which say why in- * `error`; rejects only when the job could not be started or followed.+ * `error`; rejects only when the job could not be started or followed. When+ * `signal` aborts, the job is canceled and the abort reason thrown.*/-export async function runJob(request: GenerateRequestInput, onUpdate: (job: Job) => void): Promise<Job> {+export async function runJob(request: GenerateRequestInput, onUpdate: (job: Job) => void, signal?: AbortSignal): Promise<Job> {let job = await api.generate.mutate({ request, idempotencyKey: crypto.randomUUID() });onUpdate(job);let failures = 0;while (!isTerminal(job)) {await sleep(POLL_INTERVAL);+ if (signal?.aborted) {+ await api.jobs.cancel.mutate({ id: job.id }).catch(() => {});+ signal.throwIfAborted();+ }try {job = await api.jobs.get.query({ id: job.id });failures = 0;diff --git a/apps/web/src/lib/uploads.ts b/apps/web/src/lib/uploads.tsindex 378a9083..e5299df1 100644--- a/apps/web/src/lib/uploads.ts+++ b/apps/web/src/lib/uploads.ts@@ -14,7 +14,7 @@ import { api } from "./api";import type { AssetRef } from "@diffusionstudio/api-contract";/** Lowercase hex sha256 of `blob`'s bytes: what the API names an upload by. */-async function sha256(blob: Blob): Promise<string> {+export async function sha256(blob: Blob): Promise<string> {const digest = await crypto.subtle.digest("SHA-256", await blob.arrayBuffer());return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");}diff --git a/docs/reference/tools/generate.md b/docs/reference/tools/generate.mdindex 78207481..42c0b7c9 100644--- a/docs/reference/tools/generate.md+++ b/docs/reference/tools/generate.md@@ -24,7 +24,7 @@ An API model's fields are not checked here: they go to the API as given, and the## Files-Where a field takes a file (`images`, `startFrame`, `endFrame`, `image`, `video`, `audio`, `media`), put a `{ "path": … }` object: an absolute path or URL (with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project). The file is uploaded before the job starts — once per file, however many jobs name it — and the reference becomes the API's own; a local model reads it in place. The API's references (`{ "kind": "asset", "id" }`, `{ "kind": "url", "url" }`) pass through unchanged. On the CLI a `path` that exists relative to the working directory is sent as its absolute path.+Where a field takes a file (`images`, `startFrame`, `endFrame`, `image`, `video`, `audio`, `media`), put a `{ "path": … }` object: an absolute path or URL (with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project). The file is uploaded before the job starts — once per file, however many jobs name it — and the reference becomes the API's own; in an `audio` field only its sound is sent, so a video's picture stays on the machine; a local model reads it in place. The API's references (`{ "kind": "asset", "id" }`, `{ "kind": "url", "url" }`) pass through unchanged. On the CLI a `path` that exists relative to the working directory is sent as its absolute path.```json{diff --git a/docs/reference/tools/media/listen.md b/docs/reference/tools/media/listen.mdindex 4799a1f4..6277a883 100644--- a/docs/reference/tools/media/listen.md+++ b/docs/reference/tools/media/listen.md@@ -2,8 +2,6 @@Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; of a video only the audio track is analyzed.-**Temporarily unavailable in this version:** every call fails with `unsupported`.-| | || --- | --- || MCP tool | `media_listen` |@@ -30,4 +28,4 @@ One JSON object, the model's answer. `start`/`end` echo the analyzed window (in## Errors-Fails when the path can't be resolved, the asset isn't a video or audio asset, or `start`/`end` cross (`start` >= `end`). In this version every call fails with `unsupported`.+Fails when the path can't be resolved, the asset isn't a video or audio asset, or `start`/`end` cross (`start` >= `end`).diff --git a/docs/reference/tools/media/transcribe.md b/docs/reference/tools/media/transcribe.mdindex 816bcbf8..bf82478f 100644--- a/docs/reference/tools/media/transcribe.md+++ b/docs/reference/tools/media/transcribe.md@@ -1,8 +1,6 @@# media_transcribe-Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, with word-level start/end times in seconds; returns the file's path and its segment and word counts. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).--**Temporarily unavailable in this version:** every call fails with `unsupported`.+Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, with word-level start/end times in seconds and speaker labels; returns the file's path and its segment and word counts. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).| | || --- | --- |@@ -16,7 +14,7 @@ Transcribe the speech in a video or audio file and write the timed transcript to| `path` | `string`, required | `<path>` | absolute file path or URL (works with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project) || `output` | `string` | `-o, --output <path>` | absolute path to write the transcript JSON to (default: a fresh file under the system temp dir) |-Times are in **seconds** of source/content time. An `output` naming an existing directory gets a fresh `dapi-transcript-*.json` inside it. The whole asset is transcribed once per app session (cached in memory, keyed by file content; an app restart or an edited file re-transcribes). Every call writes the file again, so a cached transcript returns at once.+Times are in **seconds** of source/content time. An `output` naming an existing directory gets a fresh `dapi-transcript-*.json` inside it. The whole asset is transcribed once per app session (cached in memory, keyed by file content; an app restart or an edited file re-transcribes). Every call writes the file again, so a cached transcript returns at once. Only the audio is uploaded: a video's audio track is copied out (or encoded to Opus when its codec has no audio-only container), so the picture never leaves the machine.## Output@@ -35,7 +33,8 @@ The file holds the transcript as indented JSON, so it searches line by line: a g```ts{segments: Array<{- text: string; // spoken words only (no silence markers)+ text: string; // spoken words only (no silence markers)+ speaker?: string; // "A", "B", … when speakers are told apartwords: Array<{ text: string; start: number; end: number }>; // seconds}>;}@@ -43,4 +42,4 @@ The file holds the transcript as indented JSON, so it searches line by line: a g## Errors-Fails when the path can't be resolved, the asset is not a video/audio asset, or no speech is detected in the audio at all (`No speech detected`). In this version every call fails with `unsupported`.+Fails when the path can't be resolved, the asset is not a video/audio asset, or no speech is detected in the audio at all (`No speech detected`).diff --git a/docs/skills/editor.md b/docs/skills/editor.mdindex 57a0ab61..372f4d6f 100644--- a/docs/skills/editor.md+++ b/docs/skills/editor.md@@ -4,8 +4,8 @@ How to understand source material before editing it. Inspect only the modalities- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.-- **Listen to the audio.** (`media_listen` is temporarily unavailable in this version.) Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.-- **Transcribe speech.** (`media_transcribe` is temporarily unavailable in this version.) For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.+- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.+- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.# The editing loopdiff --git a/docs/skills/watch.md b/docs/skills/watch.mdindex 8769a870..c83f6ab3 100644--- a/docs/skills/watch.md+++ b/docs/skills/watch.md@@ -4,8 +4,8 @@ How to understand source material. Inspect only the modalities the question turn- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.-- **Listen to the audio.** (`media_listen` is temporarily unavailable in this version.) Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.-- **Transcribe speech.** (`media_transcribe` is temporarily unavailable in this version.) For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.+- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.+- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.The `media_*` tools take a path or a direct media URL. Footage behind a page URL (YouTube, TikTok, Instagram, …) is downloaded first with `yt-dlp` from a shell — audio only (`yt-dlp -x --audio-format m4a -o talk.m4a <url>`) unless the question needs frames; see the [tool reference](https://github.com/yt-dlp/yt-dlp).@@ -16,8 +16,8 @@ Read only as much of the footage as the answer requires — each pass costs time- A duration or format question ends at `media_probe`.- "Where is the quiet part" or "how is it paced" is usually answered by the waveform and filmstrip alone.-- A question about what was said resolves fastest through `media_transcribe` (temporarily unavailable in this version); search the transcript file for the passage and quote it with its times.-- Questions about non-speech audio (music, tone, sound events, speaker identity) are what `media_listen` is for (temporarily unavailable in this version).+- A question about what was said resolves fastest through `media_transcribe`; search the transcript file for the passage and quote it with its times.+- Questions about non-speech audio (music, tone, sound events, speaker identity) are what `media_listen` is for.- Only questions about what is *seen* need frames — and the audio pass usually tells you which moments to grab, so grab those instead of scanning blind.- For open-ended questions ("summarize this", "what happens here"), combine passes: structure from waveform + filmstrip, content from transcript or listen, then frames at the salient moments to confirm what the picture shows.diff --git a/packages/dapi/src/tools/media-transcribe.ts b/packages/dapi/src/tools/media-transcribe.tsindex 4e4a31cc..1b6ec1ac 100644--- a/packages/dapi/src/tools/media-transcribe.ts+++ b/packages/dapi/src/tools/media-transcribe.ts@@ -14,6 +14,7 @@ export const TranscriptWord = z.object({export const TranscriptSegment = z.object({text: z.string(),+ speaker: z.string().optional().describe("speaker label (A, B, …) when speakers are told apart"),words: z.array(TranscriptWord),});@@ -21,7 +22,7 @@ export const mediaTranscribe = defineTool({name: "media_transcribe",title: "Transcribe speech",description:- "Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, `{ segments: [{ text, words: [{ text, start, end }] }] }` with word-level times in seconds; returns the file's path and its segment and word counts. Search the file for the passage you need (grep, jq) rather than reading it whole: a long recording's word timings run to tens of thousands of tokens. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).",+ "Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, `{ segments: [{ text, speaker?, words: [{ text, start, end }] }] }` with word-level times in seconds and speakers labelled A, B, … when told apart; returns the file's path and its segment and word counts. Search the file for the passage you need (grep, jq) rather than reading it whole: a long recording's word timings run to tens of thousands of tokens. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).",input: z.object({path: AssetPath,output: z.string().optional().describe("absolute path to write the transcript JSON to (default: a fresh file under the system temp dir)"),