The VS Code of video editing, built for agents.
TypeScript
Enhance media_transcribe tool and logging functionality
konstantin-paulus093aa671ac49
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 167c6cf9..7eb5e5f7 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -152,7 +152,10 @@ media.command("transcribe").description(describe("media_transcribe")).argument("<path>", field("media_transcribe", "path"))- .action((ref: string) => run("media_transcribe", { path: assetPath(ref) }));+ .option("-o, --output <path>", field("media_transcribe", "output"))+ .action((ref: string, opts: Omit<ToolInput<"media_transcribe">, "path">) =>+ run("media_transcribe", { path: assetPath(ref), ...opts, output: opts.output && resolve(opts.output) }),+ );media.command("grab")@@ -229,6 +232,8 @@ program.description(describe("logs")).option("-n, --tail <n>", field("logs", "tail"), numeric).option("-l, --level <level>", field("logs", "level"))+ .option("--since <ms>", field("logs", "since"), numeric)+ .option("-c, --contains <text>", field("logs", "contains")).action((opts: ToolInput<"logs">) => run("logs", opts));programdiff --git a/apps/desktop/src/dapi/handlers/fonts.ts b/apps/desktop/src/dapi/handlers/fonts.tsindex 56f54b5f..25961210 100644--- a/apps/desktop/src/dapi/handlers/fonts.ts+++ b/apps/desktop/src/dapi/handlers/fonts.ts@@ -4,7 +4,7 @@import { spawnSync } from "node:child_process";import { platform } from "node:os";-import { DapiError } from "@diffusionstudio/dapi";+import { DapiError, FONT_LIMIT } from "@diffusionstudio/dapi";import type { FontFamily } from "@diffusionstudio/dapi";import type { MainHandler } from "../handler";@@ -70,7 +70,7 @@ function listLocalFonts(): FontFamily[] {return JSON.parse(result.stdout.trim()) as FontFamily[];}-export const fonts: MainHandler<"fonts"> = async ({ family, weights, style, limit }) => {+export const fonts: MainHandler<"fonts"> = async ({ family, weights, style, limit = FONT_LIMIT }) => {const pattern = family?.toLowerCase();const wanted = weights && weights.length > 0 ? new Set(weights) : null;@@ -84,7 +84,6 @@ export const fonts: MainHandler<"fonts"> = async ({ family, weights, style, limi});if (variants.length === 0) continue;families.push({ family: entry.family, variants });- if (limit !== undefined && families.length >= limit) break;}- return { families };+ return { families: families.slice(0, limit), total: families.length };};diff --git a/apps/desktop/src/dapi/handlers/logs.test.ts b/apps/desktop/src/dapi/handlers/logs.test.tsnew file mode 100644index 00000000..d8496feb--- /dev/null+++ b/apps/desktop/src/dapi/handlers/logs.test.ts@@ -0,0 +1,42 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { describe, expect, it } from "vitest";+import { LOG_MESSAGE_MAX, LOG_TAIL } from "@diffusionstudio/dapi";+import { logs } from "./logs";++import type { LogEntry } from "@diffusionstudio/dapi";+import type { MainContext } from "../handler";++const entry = (ts: number, message: string, level: LogEntry["level"] = "info"): LogEntry => ({ ts, level, message, source: "" });++const ctx = (entries: LogEntry[]): MainContext => ({ signal: new AbortController().signal, logs: () => entries, version: "0" });++describe("logs", () => {+ it("returns the last LOG_TAIL entries when no tail is given", async () => {+ const buffer = Array.from({ length: LOG_TAIL + 5 }, (_, i) => entry(i, `m${i}`));+ const { entries } = await logs({}, ctx(buffer));+ expect(entries).toHaveLength(LOG_TAIL);+ expect(entries[0]!.ts).toBe(5);+ });++ it("filters by level, since, and contains before taking the tail", async () => {+ const buffer = [+ entry(1, "Export 10%"),+ entry(2, "export 50%"),+ entry(3, "render done"),+ entry(4, "EXPORT failed", "error"),+ entry(5, "export 90%", "debug"),+ ];+ const { entries } = await logs({ since: 1, contains: "export", level: "info", tail: 1 }, ctx(buffer));+ expect(entries.map((e) => e.ts)).toEqual([4]);+ const all = await logs({ since: 1, contains: "export" }, ctx(buffer));+ expect(all.entries.map((e) => e.ts)).toEqual([2, 4, 5]);+ });++ it("cuts long messages and says how much was dropped", async () => {+ const { entries } = await logs({}, ctx([entry(1, "x".repeat(LOG_MESSAGE_MAX + 10))]));+ expect(entries[0]!.message).toBe(`${"x".repeat(LOG_MESSAGE_MAX)}… (10 more chars)`);+ });+});diff --git a/apps/desktop/src/dapi/handlers/logs.ts b/apps/desktop/src/dapi/handlers/logs.tsindex 7b04e6af..638b0de9 100644--- a/apps/desktop/src/dapi/handlers/logs.ts+++ b/apps/desktop/src/dapi/handlers/logs.ts@@ -2,21 +2,32 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */+import { LOG_MESSAGE_MAX, LOG_TAIL } from "@diffusionstudio/dapi";+import type { LogEntry, LogLevel } from "@diffusionstudio/dapi";import type { MainHandler } from "../handler";const LEVEL_RANK: Record<LogLevel, number> = { debug: 0, info: 1, warning: 2, error: 3 };-export const logs: MainHandler<"logs"> = async ({ tail, level }, ctx) => {- let entries = ctx.logs();- if (level !== undefined) {- const min = LEVEL_RANK[level];- entries = entries.filter((e) => LEVEL_RANK[e.level] >= min);- }- if (tail !== undefined) entries = entries.slice(-tail);- return { entries };+export const logs: MainHandler<"logs"> = async ({ tail = LOG_TAIL, level, since, contains }, ctx) => {+ const min = level === undefined ? 0 : LEVEL_RANK[level];+ const needle = contains?.toLowerCase();+ const entries = ctx+ .logs()+ .filter(+ (e) =>+ LEVEL_RANK[e.level] >= min &&+ (since === undefined || e.ts > since) &&+ (needle === undefined || e.message.toLowerCase().includes(needle)),+ );+ return { entries: entries.slice(-tail).map(clip) };};+function clip(entry: LogEntry): LogEntry {+ const over = entry.message.length - LOG_MESSAGE_MAX;+ return over > 0 ? { ...entry, message: `${entry.message.slice(0, LOG_MESSAGE_MAX)}… (${over} more chars)` } : entry;+}+/** One log entry as a line: local time, level, message, source. */export function formatLogEntry(entry: LogEntry): string {const pad = (n: number, w = 2) => String(n).padStart(w, "0");diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.tsindex 5f5115c8..a090d642 100644--- a/apps/desktop/src/dapi/present.test.ts+++ b/apps/desktop/src/dapi/present.test.ts@@ -46,6 +46,14 @@ describe("present", () => {expect(b).not.toBe(a);});+ it("writes a transcript to a file, unchanged, and returns its path and size", async () => {+ const file = join(dir, "talk.json");+ const segments = [{ text: "Hi there", words: [{ text: "Hi", start: 0, end: 0.2 }, { text: "there", start: 0.3, end: 0.6 }] }];+ const presented = await present("media_transcribe", { path: "/c.mp4", output: file }, { segments });+ expect(presented).toEqual({ output: { path: file, segments: 1, words: 2 }, images: [] });+ expect(JSON.parse(readFileSync(file, "utf8"))).toEqual({ segments });+ });+it("passes other results through untouched", async () => {expect(await present("check", { id: "x" }, { stats: {}, issues: [] })).toEqual({ output: { stats: {}, issues: [] }, images: [] });});diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.tsindex 67d90d1f..0bdfc957 100644--- a/apps/desktop/src/dapi/present.ts+++ b/apps/desktop/src/dapi/present.ts@@ -2,12 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Turns what a handler returned into what a caller receives. Tools that-// render images hand back bytes; here they become files on disk, paths in-// the structured result, and — when the result is small — inline images the-// agent sees without opening anything.--import { existsSync } from "node:fs";+ import { existsSync } from "node:fs";import { mkdir, mkdtemp, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { dirname, join } from "node:path";@@ -39,6 +34,8 @@ export async function present(name: ToolName, args: unknown, result: unknown): Preturn presentPreview(result as ToolResult<"media_waveform">, (args as ToolArgs<"media_waveform">).output, "waveform");case "screenshot":return presentScreenshot(result as ToolResult<"screenshot">, (args as ToolArgs<"screenshot">).output);+ case "media_transcribe":+ return presentTranscript(result as ToolResult<"media_transcribe">, (args as ToolArgs<"media_transcribe">).output);default:return { output: result, images: [] };}@@ -85,6 +82,15 @@ async function presentScreenshot(result: ToolResult<"screenshot">, output: strinreturn { output: presented, images: [{ path, png: result.png }] };}+async function presentTranscript(transcript: ToolResult<"media_transcribe">, output: string | undefined): Promise<Presented> {+ const path = output ?? join(tmpdir(), `dapi-transcript-${randomUUID()}.json`);+ await mkdir(dirname(path), { recursive: true });+ await writeFile(path, JSON.stringify(transcript, null, 2));+ const words = transcript.segments.reduce((sum, segment) => sum + segment.words.length, 0);+ const presented: ToolOutput<"media_transcribe"> = { path, segments: transcript.segments.length, words };+ return { output: presented, images: [] };+}+function screenshotFilename(taken: Date, attempt: number): string {const pad = (value: number) => String(value).padStart(2, "0");const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-");diff --git a/docs/guides/walkthroughs/podcast-clip.md b/docs/guides/walkthroughs/podcast-clip.mdindex 8b9268a9..0bc66aee 100644--- a/docs/guides/walkthroughs/podcast-clip.md+++ b/docs/guides/walkthroughs/podcast-clip.md@@ -36,7 +36,7 @@ Asking for a rating and a one-line summary is what makes the candidates comparabThe analysis gives you seconds; a clip needs the frame. Tighten both ends against the real audio:-- `media_transcribe` returns word-level start/end times (use ffmpeg to shorten). Put the in-point on the first word of the hook line and the out-point after the last word of the button.+- `media_transcribe` writes word-level start/end times to a JSON file (use ffmpeg to shorten); search it for the hook line. Put the in-point on the first word of the hook line and the out-point after the last word of the button.- `media_waveform` with `start`/`end` around the cut shows the breaths around those words, so you can open the in-point a beat early and let the out-point land on the silence after the line instead of clipping its tail.## 6. Download the segment and lay it outdiff --git a/docs/guides/walkthroughs/talking-head.md b/docs/guides/walkthroughs/talking-head.mdindex 3d384225..436c5e7b 100644--- a/docs/guides/walkthroughs/talking-head.md+++ b/docs/guides/walkthroughs/talking-head.md@@ -10,7 +10,7 @@ Write the brief first. It records the source files, the target duration (or "keeThe transcript is the spine, so transcribe everything with speech.-- **Transcribe every video and any external audio.** `media_transcribe` returns word-level start/end times — the times you cut on. Run it on each camera take and on any separate recording (a lav or interface track you will sync to).+- **Transcribe every video and any external audio.** `media_transcribe` writes word-level start/end times to a JSON file — the times you cut on. Run it on each camera take and on any separate recording (a lav or interface track you will sync to).- **Render the waveform to read the gaps.** `media_waveform` marks silence in red and returns it as second ranges. A transcript marks only speech, so a gap in it can be silence *or* something to keep — a laugh, a breath. The waveform tells them apart: red is dead air to cut; a gap that still shows signal is performance to keep.- **Resolve double takes visually.** The transcript shows repeated lines; usually the **last** take is the keeper. When takes read equally well, `media_grab` the first word of each and keep the sharpest — motion blur means a bad take.diff --git a/docs/reference/tools/fonts.md b/docs/reference/tools/fonts.mdindex d7aa766c..180d2dd8 100644--- a/docs/reference/tools/fonts.md+++ b/docs/reference/tools/fonts.md@@ -14,10 +14,12 @@ List the local fonts available on this machine (macOS only). These family names| `family` | `string` | `-f, --family <pattern>` | filter to families whose name contains this (case-insensitive) || `weights` | `string[]` | `-w, --weights <weights...>` | filter to variants with the given CSS weights, e.g. ["400", "700"] || `style` | `"normal" \| "italic"` | `-s, --style <style>` | filter to variants with the given style, normal or italic |-| `limit` | `integer` | `-l, --limit <n>` | return at most this many families |+| `limit` | `integer` | `-l, --limit <n>` | return at most this many families (default: 50) |Font families listed here are valid `fontFamily` values on [`<text>`](../jsx/text.md); see [jsx/fonts.md](../jsx/fonts.md) for how a family and variant are named in a composition, and [`context`](./context.md) for the families the open project has actually registered. Runs in the app's main process, so no project needs to be open.+A machine has hundreds of families, so filter by `family` when looking for one. `total` counts every family the filters match; when it is larger than `families.length`, the limit cut the list.+## OutputOne JSON object:@@ -32,5 +34,6 @@ One JSON object:source: string; // CSS local() source}>;}>;+ total: number; // families matching the filters, before the limit}```diff --git a/docs/reference/tools/logs.md b/docs/reference/tools/logs.mdindex 9e4e7bc2..273b1f05 100644--- a/docs/reference/tools/logs.md+++ b/docs/reference/tools/logs.md@@ -11,10 +11,14 @@ Recent console output from the running app (what the devtools console shows: pag| Field | Type | CLI | Description || --- | --- | --- | --- |-| `tail` | `integer` | `-n, --tail <n>` | return only the last n entries |+| `tail` | `integer` | `-n, --tail <n>` | return only the last n entries (default: 100) || `level` | `"debug" \| "info" \| "warning" \| "error"` | `-l, --level <level>` | minimum level to include: debug, info, warning, or error |+| `since` | `number` | `--since <ms>` | only entries logged after this unix time in milliseconds; pass the last entry's `ts` to get only new ones |+| `contains` | `string` | `-c, --contains <text>` | only entries whose message contains this (case-insensitive) |-The buffer lives in the app's main process, so the log survives page reloads and project switches. Progress of long operations — an export's percentage, a generation landing — shows up here, so polling `logs` is how a caller follows work it started.+The buffer lives in the app's main process, so the log survives page reloads and project switches. Progress of long operations — an export's percentage, a generation landing — shows up here, so polling `logs` is how a caller follows work it started; pass `since` with the last `ts` seen so each poll returns only what is new.++`level`, `since`, and `contains` filter first, then `tail` keeps the last entries of what is left. A message longer than 4000 characters is cut, ending in `… (N more chars)`.## Output@@ -33,4 +37,4 @@ One JSON object, the entries oldest first:## Errors-Fails when `tail` is not a positive integer or `level` is not one of the four levels.+Fails when `tail` is not a positive integer, `level` is not one of the four levels, or `contains` is empty.diff --git a/docs/reference/tools/media/transcribe.md b/docs/reference/tools/media/transcribe.mdindex 653eedca..61d244d4 100644--- a/docs/reference/tools/media/transcribe.md+++ b/docs/reference/tools/media/transcribe.md@@ -1,6 +1,6 @@# media_transcribe-Transcribe the speech in a video or audio file and return the timed transcript, with word-level start/end times in seconds. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).+Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, with word-level start/end times in seconds; returns the file's path and its segment and word counts. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).| | || --- | --- |@@ -12,12 +12,23 @@ Transcribe the speech in a video or audio file and return the timed transcript,| Field | Type | CLI | Description || --- | --- | --- | --- || `path` | `string`, required | `<path>` | absolute file path or URL (works with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project) |+| `output` | `string` | `-o, --output <path>` | absolute path to write the transcript JSON to (default: a fresh file under the system temp dir) |-Times are in **seconds** of source/content time. The whole asset is transcribed once per app session (cached in memory, keyed by file content; an app restart or an edited file re-transcribes).+Times are in **seconds** of source/content time. The whole asset is transcribed once per app session (cached in memory, keyed by file content; an app restart or an edited file re-transcribes). Every call writes the file again, so a cached transcript returns at once.## Output-One JSON object, the transcript:+One JSON object, where the transcript went and how large it is:++```ts+{+ path: string; // absolute path of the transcript JSON+ segments: number; // segments in the transcript+ words: number; // words across all segments+}+```++The file holds the transcript as indented JSON, so it searches line by line: a grep for a phrase lands on its segment's `text`, with the word times in the lines after it. Search it (grep, jq) or read the part you need rather than loading it whole — a long recording's word timings run to tens of thousands of tokens.```ts{diff --git a/docs/skills/editor.md b/docs/skills/editor.mdindex e7266e6b..c556409b 100644--- a/docs/skills/editor.md+++ b/docs/skills/editor.md@@ -5,7 +5,7 @@ How to understand source material before editing it. Inspect only the modalities- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.-- **Transcribe speech.** For speech, `media_transcribe` returns the full transcript with word-level start/end times directly — read any segment straight from it.+- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.# The editing loopdiff --git a/docs/skills/watch.md b/docs/skills/watch.mdindex c44de813..0e1d3d1f 100644--- a/docs/skills/watch.md+++ b/docs/skills/watch.md@@ -5,7 +5,7 @@ How to understand source material. Inspect only the modalities the question turn- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.-- **Transcribe speech.** For speech, `media_transcribe` returns the full transcript with word-level start/end times directly — read any segment straight from it.+- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.# Matching depth to the question@@ -14,7 +14,7 @@ Read only as much of the footage as the answer requires — each pass costs time- A duration or format question ends at `media_probe`.- "Where is the quiet part" or "how is it paced" is usually answered by the waveform and filmstrip alone.-- A question about what was said resolves fastest through `media_transcribe`; quote the transcript and its times directly.+- A question about what was said resolves fastest through `media_transcribe`; search the transcript file for the passage and quote it with its times.- Questions about non-speech audio (music, tone, sound events, speaker identity) are what `media_listen` is for.- Only questions about what is *seen* need frames — and the audio pass usually tells you which moments to grab, so grab those instead of scanning blind.- For open-ended questions ("summarize this", "what happens here"), combine passes: structure from waveform + filmstrip, content from transcript or listen, then frames at the salient moments to confirm what the picture shows.diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.tsindex 83b45d5c..ae9bdb2c 100644--- a/packages/dapi/src/index.ts+++ b/packages/dapi/src/index.ts@@ -25,6 +25,8 @@ export { DAPI_WIRE } from "./ipc";export type { DapiCall, DapiCancel, DapiReply } from "./ipc";export { FRAME_CAP } from "./tools/media-grab";export { ISSUE_LOG_TAIL } from "./tools/report";+export { LOG_TAIL, LOG_MESSAGE_MAX } from "./tools/logs";+export { FONT_LIMIT } from "./tools/fonts";// Named request and result types, for handlers that spell out their// signature. Each is the parsed (output) side of the tool's schema.diff --git a/packages/dapi/src/tool.ts b/packages/dapi/src/tool.tsindex 02466d9c..ec2fabf1 100644--- a/packages/dapi/src/tool.ts+++ b/packages/dapi/src/tool.ts@@ -29,8 +29,9 @@ export interface Tool<readonly output: Output;/*** What the handler returns, when that is not the output: image tools hand- * back bytes, and the server presents them as files and inline images. Same- * as `output` when omitted.+ * back bytes, and the server presents them as files and inline images;+ * transcribe hands back the transcript, and the server writes it to a file.+ * Same as `output` when omitted.*/readonly result?: Result;readonly environment: Environment;diff --git a/packages/dapi/src/tools/fonts.ts b/packages/dapi/src/tools/fonts.tsindex 0ba24884..3d0d0cdf 100644--- a/packages/dapi/src/tools/fonts.ts+++ b/packages/dapi/src/tools/fonts.ts@@ -18,17 +18,23 @@ export const FontFamily = z.object({),});+/** Families returned when no limit is given. */+export const FONT_LIMIT = 50;+export const fonts = defineTool({name: "fonts",title: "Local fonts",description:- "List the local fonts available on this machine (macOS only). These family names are valid `fontFamily` values on <text>; each family lists its variants.",+ "List the local fonts available on this machine (macOS only). These family names are valid `fontFamily` values on <text>; each family lists its variants. A machine has hundreds of families, so filter by family name when looking for one; total tells whether the limit cut the list.",input: z.object({family: z.string().optional().describe("filter to families whose name contains this (case-insensitive)"),weights: z.array(z.string()).optional().describe('filter to variants with the given CSS weights, e.g. ["400", "700"]'),style: FontStyle.optional().describe("filter to variants with the given style, normal or italic"),- limit: z.int().min(1).optional().describe("return at most this many families"),+ limit: z.int().min(1).optional().describe(`return at most this many families (default: ${FONT_LIMIT})`),+ }),+ output: z.object({+ families: z.array(FontFamily),+ total: z.int().describe("families matching the filters before the limit; more than were returned means the list was cut"),}),- output: z.object({ families: z.array(FontFamily) }),environment: "main",});diff --git a/packages/dapi/src/tools/logs.ts b/packages/dapi/src/tools/logs.tsindex 25c509d5..ec9f37e2 100644--- a/packages/dapi/src/tools/logs.ts+++ b/packages/dapi/src/tools/logs.ts@@ -6,14 +6,21 @@ import { z } from "zod";import { defineTool } from "../tool";import { LogEntry, LogLevel } from "../schemas";+/** Entries returned when no tail is given. */+export const LOG_TAIL = 100;++/** Characters a message is cut at, so one dumped object or stack cannot crowd out the rest. */+export const LOG_MESSAGE_MAX = 4000;+export const logs = defineTool({name: "logs",title: "App logs",- description:- "Recent console output from the running app (what the devtools console shows: page logs, worker logs, uncaught errors), oldest first. The app buffers the last 2000 entries across reloads and project switches, so this replaces relaunching with ELECTRON_ENABLE_LOGGING=1 when debugging renderer-side behavior.",+ description: `Recent console output from the running app (what the devtools console shows: page logs, worker logs, uncaught errors), oldest first. The app buffers the last 2000 entries across reloads and project switches, so this replaces relaunching with ELECTRON_ENABLE_LOGGING=1 when debugging renderer-side behavior. Filters apply before tail; messages longer than ${LOG_MESSAGE_MAX} characters are cut. To follow work in progress, poll with since set to the last entry's ts.`,input: z.object({- tail: z.int().min(1).optional().describe("return only the last n entries"),+ tail: z.int().min(1).optional().describe(`return only the last n entries (default: ${LOG_TAIL})`),level: LogLevel.optional().describe("minimum level to include: debug, info, warning, or error"),+ since: z.number().optional().describe("only entries logged after this unix time in milliseconds; pass the last entry's ts to get only new ones"),+ contains: z.string().min(1).optional().describe("only entries whose message contains this (case-insensitive)"),}),output: z.object({ entries: z.array(LogEntry) }),environment: "main",diff --git a/packages/dapi/src/tools/media-transcribe.ts b/packages/dapi/src/tools/media-transcribe.tsindex 7c1ece94..4e4a31cc 100644--- a/packages/dapi/src/tools/media-transcribe.ts+++ b/packages/dapi/src/tools/media-transcribe.ts@@ -21,8 +21,16 @@ export const mediaTranscribe = defineTool({name: "media_transcribe",title: "Transcribe speech",description:- "Transcribe the speech in a video or audio file and return the timed transcript, with word-level start/end times in seconds. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).",- input: z.object({ path: AssetPath }),- output: z.object({ segments: z.array(TranscriptSegment) }),+ "Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, `{ segments: [{ text, words: [{ text, start, end }] }] }` with word-level times in seconds; returns the file's path and its segment and word counts. Search the file for the passage you need (grep, jq) rather than reading it whole: a long recording's word timings run to tens of thousands of tokens. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).",+ input: z.object({+ path: AssetPath,+ output: z.string().optional().describe("absolute path to write the transcript JSON to (default: a fresh file under the system temp dir)"),+ }),+ output: z.object({+ path: z.string().describe("absolute path of the transcript JSON"),+ segments: z.int().describe("segments in the transcript"),+ words: z.int().describe("words across all segments"),+ }),+ result: z.object({ segments: z.array(TranscriptSegment) }),environment: "renderer",});diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.tsindex b00d26d4..e0186ea8 100644--- a/packages/dapi/src/tools/tools.test.ts+++ b/packages/dapi/src/tools/tools.test.ts@@ -10,6 +10,7 @@ import { logs } from "./logs";import { mediaFilmstrip } from "./media-filmstrip";import { mediaGrab } from "./media-grab";import { mediaListen } from "./media-listen";+import { mediaTranscribe } from "./media-transcribe";/** The messages of a failed parse, keyed by the path they point at. */function issues(result: { success: boolean; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } }) {@@ -89,6 +90,8 @@ describe("logs and export", () => {expect(logs.input.safeParse({ tail: 0 }).success).toBe(false);expect(logs.input.safeParse({ tail: 5, level: "warning" }).success).toBe(true);expect(logs.input.safeParse({ level: "verbose" }).success).toBe(false);+ expect(logs.input.safeParse({ contains: "" }).success).toBe(false);+ expect(logs.input.safeParse({ since: 1700000000000, contains: "export" }).success).toBe(true);expect(exportScene.input.safeParse({ id: "" }).success).toBe(false);});});@@ -111,6 +114,15 @@ describe("context", () => {});});+describe("media_transcribe", () => {+ it("presents the transcript as a file: the result carries segments, the output a path", () => {+ const segments = [{ text: "Hi", words: [{ text: "Hi", start: 0, end: 0.2 }] }];+ expect(mediaTranscribe.result!.safeParse({ segments }).success).toBe(true);+ expect(mediaTranscribe.output.safeParse({ path: "/tmp/t.json", segments: 1, words: 1 }).success).toBe(true);+ expect(mediaTranscribe.output.safeParse({ segments }).success).toBe(false);+ });+});+describe("image tools", () => {it("present bytes as paths: the result carries png, the output a path", () => {expect(capture.result!.safeParse([{ timecode: "0f", png: new Uint8Array(3) }]).success).toBe(true);