Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance media_transcribe tool and logging functionality

konstantin-paulus093aa671ac49
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 167c6cf9..7eb5e5f7 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -152,7 +152,10 @@ media
.command("transcribe")
.description(describe("media_transcribe"))
.argument("<path>", field("media_transcribe", "path"))
- .action((ref: string) => run("media_transcribe", { path: assetPath(ref) }));
+ .option("-o, --output <path>", field("media_transcribe", "output"))
+ .action((ref: string, opts: Omit<ToolInput<"media_transcribe">, "path">) =>
+ run("media_transcribe", { path: assetPath(ref), ...opts, output: opts.output && resolve(opts.output) }),
+ );
media
.command("grab")
@@ -229,6 +232,8 @@ program
.description(describe("logs"))
.option("-n, --tail <n>", field("logs", "tail"), numeric)
.option("-l, --level <level>", field("logs", "level"))
+ .option("--since <ms>", field("logs", "since"), numeric)
+ .option("-c, --contains <text>", field("logs", "contains"))
.action((opts: ToolInput<"logs">) => run("logs", opts));
program
diff --git a/apps/desktop/src/dapi/handlers/fonts.ts b/apps/desktop/src/dapi/handlers/fonts.ts
index 56f54b5f..25961210 100644
--- a/apps/desktop/src/dapi/handlers/fonts.ts
+++ b/apps/desktop/src/dapi/handlers/fonts.ts
@@ -4,7 +4,7 @@
import { spawnSync } from "node:child_process";
import { platform } from "node:os";
-import { DapiError } from "@diffusionstudio/dapi";
+import { DapiError, FONT_LIMIT } from "@diffusionstudio/dapi";
import type { FontFamily } from "@diffusionstudio/dapi";
import type { MainHandler } from "../handler";
@@ -70,7 +70,7 @@ function listLocalFonts(): FontFamily[] {
return JSON.parse(result.stdout.trim()) as FontFamily[];
}
-export const fonts: MainHandler<"fonts"> = async ({ family, weights, style, limit }) => {
+export const fonts: MainHandler<"fonts"> = async ({ family, weights, style, limit = FONT_LIMIT }) => {
const pattern = family?.toLowerCase();
const wanted = weights && weights.length > 0 ? new Set(weights) : null;
@@ -84,7 +84,6 @@ export const fonts: MainHandler<"fonts"> = async ({ family, weights, style, limi
});
if (variants.length === 0) continue;
families.push({ family: entry.family, variants });
- if (limit !== undefined && families.length >= limit) break;
}
- return { families };
+ return { families: families.slice(0, limit), total: families.length };
};
diff --git a/apps/desktop/src/dapi/handlers/logs.test.ts b/apps/desktop/src/dapi/handlers/logs.test.ts
new file mode 100644
index 00000000..d8496feb
--- /dev/null
+++ b/apps/desktop/src/dapi/handlers/logs.test.ts
@@ -0,0 +1,42 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { describe, expect, it } from "vitest";
+import { LOG_MESSAGE_MAX, LOG_TAIL } from "@diffusionstudio/dapi";
+import { logs } from "./logs";
+
+import type { LogEntry } from "@diffusionstudio/dapi";
+import type { MainContext } from "../handler";
+
+const entry = (ts: number, message: string, level: LogEntry["level"] = "info"): LogEntry => ({ ts, level, message, source: "" });
+
+const ctx = (entries: LogEntry[]): MainContext => ({ signal: new AbortController().signal, logs: () => entries, version: "0" });
+
+describe("logs", () => {
+ it("returns the last LOG_TAIL entries when no tail is given", async () => {
+ const buffer = Array.from({ length: LOG_TAIL + 5 }, (_, i) => entry(i, `m${i}`));
+ const { entries } = await logs({}, ctx(buffer));
+ expect(entries).toHaveLength(LOG_TAIL);
+ expect(entries[0]!.ts).toBe(5);
+ });
+
+ it("filters by level, since, and contains before taking the tail", async () => {
+ const buffer = [
+ entry(1, "Export 10%"),
+ entry(2, "export 50%"),
+ entry(3, "render done"),
+ entry(4, "EXPORT failed", "error"),
+ entry(5, "export 90%", "debug"),
+ ];
+ const { entries } = await logs({ since: 1, contains: "export", level: "info", tail: 1 }, ctx(buffer));
+ expect(entries.map((e) => e.ts)).toEqual([4]);
+ const all = await logs({ since: 1, contains: "export" }, ctx(buffer));
+ expect(all.entries.map((e) => e.ts)).toEqual([2, 4, 5]);
+ });
+
+ it("cuts long messages and says how much was dropped", async () => {
+ const { entries } = await logs({}, ctx([entry(1, "x".repeat(LOG_MESSAGE_MAX + 10))]));
+ expect(entries[0]!.message).toBe(`${"x".repeat(LOG_MESSAGE_MAX)}… (10 more chars)`);
+ });
+});
diff --git a/apps/desktop/src/dapi/handlers/logs.ts b/apps/desktop/src/dapi/handlers/logs.ts
index 7b04e6af..638b0de9 100644
--- a/apps/desktop/src/dapi/handlers/logs.ts
+++ b/apps/desktop/src/dapi/handlers/logs.ts
@@ -2,21 +2,32 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+import { LOG_MESSAGE_MAX, LOG_TAIL } from "@diffusionstudio/dapi";
+
import type { LogEntry, LogLevel } from "@diffusionstudio/dapi";
import type { MainHandler } from "../handler";
const LEVEL_RANK: Record<LogLevel, number> = { debug: 0, info: 1, warning: 2, error: 3 };
-export const logs: MainHandler<"logs"> = async ({ tail, level }, ctx) => {
- let entries = ctx.logs();
- if (level !== undefined) {
- const min = LEVEL_RANK[level];
- entries = entries.filter((e) => LEVEL_RANK[e.level] >= min);
- }
- if (tail !== undefined) entries = entries.slice(-tail);
- return { entries };
+export const logs: MainHandler<"logs"> = async ({ tail = LOG_TAIL, level, since, contains }, ctx) => {
+ const min = level === undefined ? 0 : LEVEL_RANK[level];
+ const needle = contains?.toLowerCase();
+ const entries = ctx
+ .logs()
+ .filter(
+ (e) =>
+ LEVEL_RANK[e.level] >= min &&
+ (since === undefined || e.ts > since) &&
+ (needle === undefined || e.message.toLowerCase().includes(needle)),
+ );
+ return { entries: entries.slice(-tail).map(clip) };
};
+function clip(entry: LogEntry): LogEntry {
+ const over = entry.message.length - LOG_MESSAGE_MAX;
+ return over > 0 ? { ...entry, message: `${entry.message.slice(0, LOG_MESSAGE_MAX)}… (${over} more chars)` } : entry;
+}
+
/** One log entry as a line: local time, level, message, source. */
export function formatLogEntry(entry: LogEntry): string {
const pad = (n: number, w = 2) => String(n).padStart(w, "0");
diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.ts
index 5f5115c8..a090d642 100644
--- a/apps/desktop/src/dapi/present.test.ts
+++ b/apps/desktop/src/dapi/present.test.ts
@@ -46,6 +46,14 @@ describe("present", () => {
expect(b).not.toBe(a);
});
+ it("writes a transcript to a file, unchanged, and returns its path and size", async () => {
+ const file = join(dir, "talk.json");
+ const segments = [{ text: "Hi there", words: [{ text: "Hi", start: 0, end: 0.2 }, { text: "there", start: 0.3, end: 0.6 }] }];
+ const presented = await present("media_transcribe", { path: "/c.mp4", output: file }, { segments });
+ expect(presented).toEqual({ output: { path: file, segments: 1, words: 2 }, images: [] });
+ expect(JSON.parse(readFileSync(file, "utf8"))).toEqual({ segments });
+ });
+
it("passes other results through untouched", async () => {
expect(await present("check", { id: "x" }, { stats: {}, issues: [] })).toEqual({ output: { stats: {}, issues: [] }, images: [] });
});
diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.ts
index 67d90d1f..0bdfc957 100644
--- a/apps/desktop/src/dapi/present.ts
+++ b/apps/desktop/src/dapi/present.ts
@@ -2,12 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Turns what a handler returned into what a caller receives. Tools that
-// render images hand back bytes; here they become files on disk, paths in
-// the structured result, and — when the result is small — inline images the
-// agent sees without opening anything.
-
-import { existsSync } from "node:fs";
+ import { existsSync } from "node:fs";
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
@@ -39,6 +34,8 @@ export async function present(name: ToolName, args: unknown, result: unknown): P
return presentPreview(result as ToolResult<"media_waveform">, (args as ToolArgs<"media_waveform">).output, "waveform");
case "screenshot":
return presentScreenshot(result as ToolResult<"screenshot">, (args as ToolArgs<"screenshot">).output);
+ case "media_transcribe":
+ return presentTranscript(result as ToolResult<"media_transcribe">, (args as ToolArgs<"media_transcribe">).output);
default:
return { output: result, images: [] };
}
@@ -85,6 +82,15 @@ async function presentScreenshot(result: ToolResult<"screenshot">, output: strin
return { output: presented, images: [{ path, png: result.png }] };
}
+async function presentTranscript(transcript: ToolResult<"media_transcribe">, output: string | undefined): Promise<Presented> {
+ const path = output ?? join(tmpdir(), `dapi-transcript-${randomUUID()}.json`);
+ await mkdir(dirname(path), { recursive: true });
+ await writeFile(path, JSON.stringify(transcript, null, 2));
+ const words = transcript.segments.reduce((sum, segment) => sum + segment.words.length, 0);
+ const presented: ToolOutput<"media_transcribe"> = { path, segments: transcript.segments.length, words };
+ return { output: presented, images: [] };
+}
+
function screenshotFilename(taken: Date, attempt: number): string {
const pad = (value: number) => String(value).padStart(2, "0");
const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-");
diff --git a/docs/guides/walkthroughs/podcast-clip.md b/docs/guides/walkthroughs/podcast-clip.md
index 8b9268a9..0bc66aee 100644
--- a/docs/guides/walkthroughs/podcast-clip.md
+++ b/docs/guides/walkthroughs/podcast-clip.md
@@ -36,7 +36,7 @@ Asking for a rating and a one-line summary is what makes the candidates comparab
The analysis gives you seconds; a clip needs the frame. Tighten both ends against the real audio:
-- `media_transcribe` returns word-level start/end times (use ffmpeg to shorten). Put the in-point on the first word of the hook line and the out-point after the last word of the button.
+- `media_transcribe` writes word-level start/end times to a JSON file (use ffmpeg to shorten); search it for the hook line. Put the in-point on the first word of the hook line and the out-point after the last word of the button.
- `media_waveform` with `start`/`end` around the cut shows the breaths around those words, so you can open the in-point a beat early and let the out-point land on the silence after the line instead of clipping its tail.
## 6. Download the segment and lay it out
diff --git a/docs/guides/walkthroughs/talking-head.md b/docs/guides/walkthroughs/talking-head.md
index 3d384225..436c5e7b 100644
--- a/docs/guides/walkthroughs/talking-head.md
+++ b/docs/guides/walkthroughs/talking-head.md
@@ -10,7 +10,7 @@ Write the brief first. It records the source files, the target duration (or "kee
The transcript is the spine, so transcribe everything with speech.
-- **Transcribe every video and any external audio.** `media_transcribe` returns word-level start/end times — the times you cut on. Run it on each camera take and on any separate recording (a lav or interface track you will sync to).
+- **Transcribe every video and any external audio.** `media_transcribe` writes word-level start/end times to a JSON file — the times you cut on. Run it on each camera take and on any separate recording (a lav or interface track you will sync to).
- **Render the waveform to read the gaps.** `media_waveform` marks silence in red and returns it as second ranges. A transcript marks only speech, so a gap in it can be silence *or* something to keep — a laugh, a breath. The waveform tells them apart: red is dead air to cut; a gap that still shows signal is performance to keep.
- **Resolve double takes visually.** The transcript shows repeated lines; usually the **last** take is the keeper. When takes read equally well, `media_grab` the first word of each and keep the sharpest — motion blur means a bad take.
diff --git a/docs/reference/tools/fonts.md b/docs/reference/tools/fonts.md
index d7aa766c..180d2dd8 100644
--- a/docs/reference/tools/fonts.md
+++ b/docs/reference/tools/fonts.md
@@ -14,10 +14,12 @@ List the local fonts available on this machine (macOS only). These family names
| `family` | `string` | `-f, --family <pattern>` | filter to families whose name contains this (case-insensitive) |
| `weights` | `string[]` | `-w, --weights <weights...>` | filter to variants with the given CSS weights, e.g. ["400", "700"] |
| `style` | `"normal" \| "italic"` | `-s, --style <style>` | filter to variants with the given style, normal or italic |
-| `limit` | `integer` | `-l, --limit <n>` | return at most this many families |
+| `limit` | `integer` | `-l, --limit <n>` | return at most this many families (default: 50) |
Font families listed here are valid `fontFamily` values on [`<text>`](../jsx/text.md); see [jsx/fonts.md](../jsx/fonts.md) for how a family and variant are named in a composition, and [`context`](./context.md) for the families the open project has actually registered. Runs in the app's main process, so no project needs to be open.
+A machine has hundreds of families, so filter by `family` when looking for one. `total` counts every family the filters match; when it is larger than `families.length`, the limit cut the list.
+
## Output
One JSON object:
@@ -32,5 +34,6 @@ One JSON object:
source: string; // CSS local() source
}>;
}>;
+ total: number; // families matching the filters, before the limit
}
```
diff --git a/docs/reference/tools/logs.md b/docs/reference/tools/logs.md
index 9e4e7bc2..273b1f05 100644
--- a/docs/reference/tools/logs.md
+++ b/docs/reference/tools/logs.md
@@ -11,10 +11,14 @@ Recent console output from the running app (what the devtools console shows: pag
| Field | Type | CLI | Description |
| --- | --- | --- | --- |
-| `tail` | `integer` | `-n, --tail <n>` | return only the last n entries |
+| `tail` | `integer` | `-n, --tail <n>` | return only the last n entries (default: 100) |
| `level` | `"debug" \| "info" \| "warning" \| "error"` | `-l, --level <level>` | minimum level to include: debug, info, warning, or error |
+| `since` | `number` | `--since <ms>` | only entries logged after this unix time in milliseconds; pass the last entry's `ts` to get only new ones |
+| `contains` | `string` | `-c, --contains <text>` | only entries whose message contains this (case-insensitive) |
-The buffer lives in the app's main process, so the log survives page reloads and project switches. Progress of long operations — an export's percentage, a generation landing — shows up here, so polling `logs` is how a caller follows work it started.
+The buffer lives in the app's main process, so the log survives page reloads and project switches. Progress of long operations — an export's percentage, a generation landing — shows up here, so polling `logs` is how a caller follows work it started; pass `since` with the last `ts` seen so each poll returns only what is new.
+
+`level`, `since`, and `contains` filter first, then `tail` keeps the last entries of what is left. A message longer than 4000 characters is cut, ending in `… (N more chars)`.
## Output
@@ -33,4 +37,4 @@ One JSON object, the entries oldest first:
## Errors
-Fails when `tail` is not a positive integer or `level` is not one of the four levels.
+Fails when `tail` is not a positive integer, `level` is not one of the four levels, or `contains` is empty.
diff --git a/docs/reference/tools/media/transcribe.md b/docs/reference/tools/media/transcribe.md
index 653eedca..61d244d4 100644
--- a/docs/reference/tools/media/transcribe.md
+++ b/docs/reference/tools/media/transcribe.md
@@ -1,6 +1,6 @@
# media_transcribe
-Transcribe the speech in a video or audio file and return the timed transcript, with word-level start/end times in seconds. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).
+Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, with word-level start/end times in seconds; returns the file's path and its segment and word counts. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).
| | |
| --- | --- |
@@ -12,12 +12,23 @@ Transcribe the speech in a video or audio file and return the timed transcript,
| Field | Type | CLI | Description |
| --- | --- | --- | --- |
| `path` | `string`, required | `<path>` | absolute file path or URL (works with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project) |
+| `output` | `string` | `-o, --output <path>` | absolute path to write the transcript JSON to (default: a fresh file under the system temp dir) |
-Times are in **seconds** of source/content time. The whole asset is transcribed once per app session (cached in memory, keyed by file content; an app restart or an edited file re-transcribes).
+Times are in **seconds** of source/content time. The whole asset is transcribed once per app session (cached in memory, keyed by file content; an app restart or an edited file re-transcribes). Every call writes the file again, so a cached transcript returns at once.
## Output
-One JSON object, the transcript:
+One JSON object, where the transcript went and how large it is:
+
+```ts
+{
+ path: string; // absolute path of the transcript JSON
+ segments: number; // segments in the transcript
+ words: number; // words across all segments
+}
+```
+
+The file holds the transcript as indented JSON, so it searches line by line: a grep for a phrase lands on its segment's `text`, with the word times in the lines after it. Search it (grep, jq) or read the part you need rather than loading it whole — a long recording's word timings run to tens of thousands of tokens.
```ts
{
diff --git a/docs/skills/editor.md b/docs/skills/editor.md
index e7266e6b..c556409b 100644
--- a/docs/skills/editor.md
+++ b/docs/skills/editor.md
@@ -5,7 +5,7 @@ How to understand source material before editing it. Inspect only the modalities
- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.
- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.
- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.
-- **Transcribe speech.** For speech, `media_transcribe` returns the full transcript with word-level start/end times directly — read any segment straight from it.
+- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.
- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.
# The editing loop
diff --git a/docs/skills/watch.md b/docs/skills/watch.md
index c44de813..0e1d3d1f 100644
--- a/docs/skills/watch.md
+++ b/docs/skills/watch.md
@@ -5,7 +5,7 @@ How to understand source material. Inspect only the modalities the question turn
- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.
- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.
- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.
-- **Transcribe speech.** For speech, `media_transcribe` returns the full transcript with word-level start/end times directly — read any segment straight from it.
+- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.
- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.
# Matching depth to the question
@@ -14,7 +14,7 @@ Read only as much of the footage as the answer requires — each pass costs time
- A duration or format question ends at `media_probe`.
- "Where is the quiet part" or "how is it paced" is usually answered by the waveform and filmstrip alone.
-- A question about what was said resolves fastest through `media_transcribe`; quote the transcript and its times directly.
+- A question about what was said resolves fastest through `media_transcribe`; search the transcript file for the passage and quote it with its times.
- Questions about non-speech audio (music, tone, sound events, speaker identity) are what `media_listen` is for.
- Only questions about what is *seen* need frames — and the audio pass usually tells you which moments to grab, so grab those instead of scanning blind.
- For open-ended questions ("summarize this", "what happens here"), combine passes: structure from waveform + filmstrip, content from transcript or listen, then frames at the salient moments to confirm what the picture shows.
diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.ts
index 83b45d5c..ae9bdb2c 100644
--- a/packages/dapi/src/index.ts
+++ b/packages/dapi/src/index.ts
@@ -25,6 +25,8 @@ export { DAPI_WIRE } from "./ipc";
export type { DapiCall, DapiCancel, DapiReply } from "./ipc";
export { FRAME_CAP } from "./tools/media-grab";
export { ISSUE_LOG_TAIL } from "./tools/report";
+export { LOG_TAIL, LOG_MESSAGE_MAX } from "./tools/logs";
+export { FONT_LIMIT } from "./tools/fonts";
// Named request and result types, for handlers that spell out their
// signature. Each is the parsed (output) side of the tool's schema.
diff --git a/packages/dapi/src/tool.ts b/packages/dapi/src/tool.ts
index 02466d9c..ec2fabf1 100644
--- a/packages/dapi/src/tool.ts
+++ b/packages/dapi/src/tool.ts
@@ -29,8 +29,9 @@ export interface Tool<
readonly output: Output;
/**
* What the handler returns, when that is not the output: image tools hand
- * back bytes, and the server presents them as files and inline images. Same
- * as `output` when omitted.
+ * back bytes, and the server presents them as files and inline images;
+ * transcribe hands back the transcript, and the server writes it to a file.
+ * Same as `output` when omitted.
*/
readonly result?: Result;
readonly environment: Environment;
diff --git a/packages/dapi/src/tools/fonts.ts b/packages/dapi/src/tools/fonts.ts
index 0ba24884..3d0d0cdf 100644
--- a/packages/dapi/src/tools/fonts.ts
+++ b/packages/dapi/src/tools/fonts.ts
@@ -18,17 +18,23 @@ export const FontFamily = z.object({
),
});
+/** Families returned when no limit is given. */
+export const FONT_LIMIT = 50;
+
export const fonts = defineTool({
name: "fonts",
title: "Local fonts",
description:
- "List the local fonts available on this machine (macOS only). These family names are valid `fontFamily` values on <text>; each family lists its variants.",
+ "List the local fonts available on this machine (macOS only). These family names are valid `fontFamily` values on <text>; each family lists its variants. A machine has hundreds of families, so filter by family name when looking for one; total tells whether the limit cut the list.",
input: z.object({
family: z.string().optional().describe("filter to families whose name contains this (case-insensitive)"),
weights: z.array(z.string()).optional().describe('filter to variants with the given CSS weights, e.g. ["400", "700"]'),
style: FontStyle.optional().describe("filter to variants with the given style, normal or italic"),
- limit: z.int().min(1).optional().describe("return at most this many families"),
+ limit: z.int().min(1).optional().describe(`return at most this many families (default: ${FONT_LIMIT})`),
+ }),
+ output: z.object({
+ families: z.array(FontFamily),
+ total: z.int().describe("families matching the filters before the limit; more than were returned means the list was cut"),
}),
- output: z.object({ families: z.array(FontFamily) }),
environment: "main",
});
diff --git a/packages/dapi/src/tools/logs.ts b/packages/dapi/src/tools/logs.ts
index 25c509d5..ec9f37e2 100644
--- a/packages/dapi/src/tools/logs.ts
+++ b/packages/dapi/src/tools/logs.ts
@@ -6,14 +6,21 @@ import { z } from "zod";
import { defineTool } from "../tool";
import { LogEntry, LogLevel } from "../schemas";
+/** Entries returned when no tail is given. */
+export const LOG_TAIL = 100;
+
+/** Characters a message is cut at, so one dumped object or stack cannot crowd out the rest. */
+export const LOG_MESSAGE_MAX = 4000;
+
export const logs = defineTool({
name: "logs",
title: "App logs",
- description:
- "Recent console output from the running app (what the devtools console shows: page logs, worker logs, uncaught errors), oldest first. The app buffers the last 2000 entries across reloads and project switches, so this replaces relaunching with ELECTRON_ENABLE_LOGGING=1 when debugging renderer-side behavior.",
+ description: `Recent console output from the running app (what the devtools console shows: page logs, worker logs, uncaught errors), oldest first. The app buffers the last 2000 entries across reloads and project switches, so this replaces relaunching with ELECTRON_ENABLE_LOGGING=1 when debugging renderer-side behavior. Filters apply before tail; messages longer than ${LOG_MESSAGE_MAX} characters are cut. To follow work in progress, poll with since set to the last entry's ts.`,
input: z.object({
- tail: z.int().min(1).optional().describe("return only the last n entries"),
+ tail: z.int().min(1).optional().describe(`return only the last n entries (default: ${LOG_TAIL})`),
level: LogLevel.optional().describe("minimum level to include: debug, info, warning, or error"),
+ since: z.number().optional().describe("only entries logged after this unix time in milliseconds; pass the last entry's ts to get only new ones"),
+ contains: z.string().min(1).optional().describe("only entries whose message contains this (case-insensitive)"),
}),
output: z.object({ entries: z.array(LogEntry) }),
environment: "main",
diff --git a/packages/dapi/src/tools/media-transcribe.ts b/packages/dapi/src/tools/media-transcribe.ts
index 7c1ece94..4e4a31cc 100644
--- a/packages/dapi/src/tools/media-transcribe.ts
+++ b/packages/dapi/src/tools/media-transcribe.ts
@@ -21,8 +21,16 @@ export const mediaTranscribe = defineTool({
name: "media_transcribe",
title: "Transcribe speech",
description:
- "Transcribe the speech in a video or audio file and return the timed transcript, with word-level start/end times in seconds. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).",
- input: z.object({ path: AssetPath }),
- output: z.object({ segments: z.array(TranscriptSegment) }),
+ "Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, `{ segments: [{ text, words: [{ text, start, end }] }] }` with word-level times in seconds; returns the file's path and its segment and word counts. Search the file for the passage you need (grep, jq) rather than reading it whole: a long recording's word timings run to tens of thousands of tokens. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).",
+ input: z.object({
+ path: AssetPath,
+ output: z.string().optional().describe("absolute path to write the transcript JSON to (default: a fresh file under the system temp dir)"),
+ }),
+ output: z.object({
+ path: z.string().describe("absolute path of the transcript JSON"),
+ segments: z.int().describe("segments in the transcript"),
+ words: z.int().describe("words across all segments"),
+ }),
+ result: z.object({ segments: z.array(TranscriptSegment) }),
environment: "renderer",
});
diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.ts
index b00d26d4..e0186ea8 100644
--- a/packages/dapi/src/tools/tools.test.ts
+++ b/packages/dapi/src/tools/tools.test.ts
@@ -10,6 +10,7 @@ import { logs } from "./logs";
import { mediaFilmstrip } from "./media-filmstrip";
import { mediaGrab } from "./media-grab";
import { mediaListen } from "./media-listen";
+import { mediaTranscribe } from "./media-transcribe";
/** The messages of a failed parse, keyed by the path they point at. */
function issues(result: { success: boolean; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } }) {
@@ -89,6 +90,8 @@ describe("logs and export", () => {
expect(logs.input.safeParse({ tail: 0 }).success).toBe(false);
expect(logs.input.safeParse({ tail: 5, level: "warning" }).success).toBe(true);
expect(logs.input.safeParse({ level: "verbose" }).success).toBe(false);
+ expect(logs.input.safeParse({ contains: "" }).success).toBe(false);
+ expect(logs.input.safeParse({ since: 1700000000000, contains: "export" }).success).toBe(true);
expect(exportScene.input.safeParse({ id: "" }).success).toBe(false);
});
});
@@ -111,6 +114,15 @@ describe("context", () => {
});
});
+describe("media_transcribe", () => {
+ it("presents the transcript as a file: the result carries segments, the output a path", () => {
+ const segments = [{ text: "Hi", words: [{ text: "Hi", start: 0, end: 0.2 }] }];
+ expect(mediaTranscribe.result!.safeParse({ segments }).success).toBe(true);
+ expect(mediaTranscribe.output.safeParse({ path: "/tmp/t.json", segments: 1, words: 1 }).success).toBe(true);
+ expect(mediaTranscribe.output.safeParse({ segments }).success).toBe(false);
+ });
+});
+
describe("image tools", () => {
it("present bytes as paths: the result carries png, the output a path", () => {
expect(capture.result!.safeParse([{ timecode: "0f", png: new Uint8Array(3) }]).success).toBe(true);