The VS Code of video editing, built for agents.
TypeScript
Refactor CLI tool input handling and update documentation
konstantin-paulus5c0e109cd51b
diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.tsindex 45e54a1d..48a311d3 100644--- a/apps/cli/src/cli-client.ts+++ b/apps/cli/src/cli-client.ts@@ -13,39 +13,28 @@ import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi";export const APP_NAME = "Diffusion Studio";-const DEFAULT_TIMEOUT_MS = 60000;-const GENERATE_TIMEOUT_MS = 600000;-const EXPORT_TIMEOUT_MS = 3600000;--// Long-running tools (renders, AI generation, downloads) override the-// default 60s. Keyed by name so `call` and the wrappers agree.+// Renders, AI generation, and downloads outlive the 60s default.const TIMEOUTS: Record<string, number> = {- export: EXPORT_TIMEOUT_MS,- capture: GENERATE_TIMEOUT_MS,- media_transcribe: GENERATE_TIMEOUT_MS,- media_listen: GENERATE_TIMEOUT_MS,- fetch: GENERATE_TIMEOUT_MS,+ export: 3_600_000,+ capture: 600_000,+ media_transcribe: 600_000,+ media_listen: 600_000,+ fetch: 600_000,};-export function timeoutFor(tool: string): number {- return TIMEOUTS[tool] ?? DEFAULT_TIMEOUT_MS;-}--export type CallOptions = { timeoutMs?: number };-/*** Calls one tool in the running app over an MCP session on its socket.* Typed by the catalog: the input is what the tool's schema accepts, the* output its structured content. One session per call; a command makes one* or two, and the process exits when it settles.*/-export async function call<N extends ToolName>(name: N, input: ToolInput<N>, options: CallOptions = {}): Promise<ToolOutput<N>> {- return withClient(async (client) => {- const result = await client.callTool(- { name, arguments: input as Record<string, unknown> },- undefined,- { timeout: options.timeoutMs ?? timeoutFor(name) },- );+export async function call<N extends ToolName>(name: N, input: ToolInput<N>): Promise<ToolOutput<N>> {+ const client = new Client({ name: "dapi", version });+ try {+ await client.connect(new SocketTransport(await openSocket()));+ const result = await client.callTool({ name, arguments: input as Record<string, unknown> }, undefined, {+ timeout: TIMEOUTS[name] ?? 60_000,+ });if (result.isError) {const text = (result.content as Array<{ type: string; text?: string }>).filter((block) => block.type === "text")@@ -54,22 +43,17 @@ export async function call<N extends ToolName>(name: N, input: ToolInput<N>, optthrow new Error(text || `${name} failed`);}return result.structuredContent as ToolOutput<N>;- });+ } finally {+ await client.close().catch(() => {});+ }}/** Liveness: a round-trip through the app's MCP server. */-export function ping(): Promise<void> {- return withClient(async (client) => {- await client.ping();- });-}--async function withClient<T>(fn: (client: Client) => Promise<T>): Promise<T> {- const socket = await openSocket();+export async function ping(): Promise<void> {const client = new Client({ name: "dapi", version });try {- await client.connect(new SocketTransport(socket));- return await fn(client);+ await client.connect(new SocketTransport(await openSocket()));+ await client.ping();} finally {await client.close().catch(() => {});}@@ -88,13 +72,8 @@ export function openSocket(): Promise<Socket> {});}-/** The errno of a connection failure (ENOENT/ECONNREFUSED when the app is down), if any. */-export function errnoCode(e: unknown): string | undefined {- return (e as NodeJS.ErrnoException | undefined)?.code;-}-export function isAppDown(e: unknown): boolean {- const code = errnoCode(e);+ const code = (e as NodeJS.ErrnoException | undefined)?.code;return code === "ENOENT" || code === "ECONNREFUSED";}@@ -110,43 +89,22 @@ export function launchApp(background: boolean): Promise<boolean> {}/**- * Bridges the cold-start gap after launching the app: the socket appears once- * main is ready. Retries only while the app looks down; any other error is- * the caller's.+ * Bridges the cold-start gap after launching the app: retries while the app+ * looks down, until it answers a ping (a cold app binds the socket before+ * its session is ready, so connecting alone proves nothing).*/-export async function connectWithRetry(timeoutMs = 30000): Promise<Socket> {- const start = Date.now();- let lastError: unknown = null;- while (Date.now() - start < timeoutMs) {- try {- return await openSocket();- } catch (e) {- if (!isAppDown(e)) throw e;- lastError = e;- await new Promise((r) => setTimeout(r, 200));- }- }- throw timedOut(timeoutMs, lastError);-}--function timedOut(timeoutMs: number, lastError: unknown): Error {- const detail = lastError instanceof Error ? ` (${lastError.message})` : "";- return new Error(`${APP_NAME} did not answer within ${Math.round(timeoutMs / 1000)}s of launching${detail}`);-}--/** Like `connectWithRetry`, but also proves the server answers: a cold app binds the socket before its session is ready. */export async function waitForApp(timeoutMs = 30000): Promise<void> {const start = Date.now();let lastError: unknown = null;while (Date.now() - start < timeoutMs) {try {- await ping();- return;+ return await ping();} catch (e) {if (!isAppDown(e)) throw e;lastError = e;await new Promise((r) => setTimeout(r, 200));}}- throw timedOut(timeoutMs, lastError);+ const detail = lastError instanceof Error ? ` (${lastError.message})` : "";+ throw new Error(`${APP_NAME} did not answer within ${Math.round(timeoutMs / 1000)}s of launching${detail}`);}diff --git a/apps/cli/src/help.ts b/apps/cli/src/help.tsdeleted file mode 100644index e902d2f9..00000000--- a/apps/cli/src/help.ts+++ /dev/null@@ -1,44 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--// The wrappers' help text and validation come from the catalog, so-// `dapi <cmd> --help` and the app's `tools/list` say the same thing because-// they are the same string, and a bad argument fails here with the message-// the app would have sent back.--import { z } from "zod";-import { toolByName } from "@diffusionstudio/dapi";--import type { GenericTool, ToolName } from "@diffusionstudio/dapi";--/** The tool's description, verbatim. */-export function describe(name: ToolName): string {- return toolByName(name).description;-}--/**- * An input field's description, for the option that maps onto it. The- * fallback covers fields the catalog leaves undescribed and options whose- * meaning is the CLI's own (`--separate` inverts `combine`).- */-export function field(name: ToolName, key: string, fallback?: string): string {- const tool: GenericTool = toolByName(name);- const schema = tool.input.shape[key];- if (schema === undefined && fallback === undefined) {- throw new Error(`tool ${name} has no input field "${key}"`);- }- return schema?.description ?? fallback ?? "";-}--/**- * Checks the input against the catalog before it leaves the process, so a- * bad argument fails here with the message the app would have sent back.- * Prints the issues and exits on failure.- */-export function validate(name: ToolName, input: Record<string, unknown>): void {- const result = toolByName(name).input.safeParse(input);- if (result.success) return;- console.error(z.prettifyError(result.error));- process.exit(1);-}diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 2216fcdb..5ca06ffa 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -6,126 +6,71 @@import { existsSync } from "node:fs";import { isAbsolute, resolve } from "node:path";import { Command } from "commander";+import { z } from "zod";import { version } from "../../../package.json";-import { TIME_FPS } from "@diffusionstudio/jsx";-import { ISSUE_LOG_TAIL, NonNegativeTime } from "@diffusionstudio/dapi";+import { ISSUE_LOG_TAIL, toolByName } from "@diffusionstudio/dapi";import { MCP_URL } from "@diffusionstudio/dapi/socket";import { APP_NAME, call, isAppDown, launchApp, ping, waitForApp } from "./cli-client";-import { describe, field, validate } from "./help";import { runProxy } from "./mcp-proxy";-import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi";+import type { GenericTool, ToolInput, ToolName } from "@diffusionstudio/dapi";function fail(message: string): never {console.error(message);process.exit(1);}-function handleSocketError(e: unknown): never {+function appError(e: unknown): never {if (isAppDown(e)) fail(`${APP_NAME} is not running. Launch the app first, then retry.`);fail((e as Error).message);}+/** The tool's description, verbatim. */+function describe(name: ToolName): string {+ return toolByName(name).description;+}+/**- * One wrapper's whole job: check the input against the catalog, call the- * tool, print the result the way the app returns it — its structured content,- * as one JSON object, the same thing an agent receives. Everything the app- * validates is sent as typed (strings stay strings: times like "45f" are- * parsed by the schema on both sides).+ * An input field's description, for the option that maps onto it. The+ * fallback is for the shared window fields (`start`, `end`), which the+ * catalog leaves for each wrapper to word.*/-async function run<N extends ToolName>(name: N, input: ToolInput<N>, spinner?: string): Promise<void> {- await invoke(name, input, spinner);+function field(name: ToolName, key: string, fallback?: string): string {+ const tool: GenericTool = toolByName(name);+ const schema = tool.input.shape[key];+ if (schema === undefined && fallback === undefined) throw new Error(`tool ${name} has no input field "${key}"`);+ return schema?.description ?? fallback ?? "";}-/** `run`, handing the result back for the one command that inspects it (`check`, for its exit code). */-async function invoke<N extends ToolName>(name: N, input: ToolInput<N>, spinner?: string): Promise<ToolOutput<N>> {- validate(name, input as Record<string, unknown>);- const stop = spinner ? startSpinner(spinner) : () => {};- try {- const output = await call(name, input);- stop();- console.log(JSON.stringify(output));- return output;- } catch (e) {- stop();- handleSocketError(e);- }+/**+ * Checks the input against the tool's schema, calls the tool, and prints+ * what the app returns — its structured content, as one JSON object, the+ * same thing an agent receives. Strings stay strings: times like "45f" are+ * parsed by the schema on both sides.+ */+async function run<N extends ToolName>(name: N, input: ToolInput<N>): Promise<void> {+ const parsed = toolByName(name).input.safeParse(input);+ if (!parsed.success) fail(z.prettifyError(parsed.error));+ const output = await call(name, input).catch(appError);+ console.log(JSON.stringify(output));}-// Option parsers. Numbers are converted so the schema can check them as-// numbers; an empty or non-numeric string becomes NaN, which the schema-// rejects with its own message.+// Numbers are converted so the schema can check them as numbers; an empty or+// non-numeric string becomes NaN, which the schema rejects with its own message.const numeric = (value: string): number => (value.trim() === "" ? NaN : Number(value));-/** A time as the CLI takes it, in seconds, checked the way the catalog checks it. */-function seconds(value: string, flag: string): number {- const result = NonNegativeTime.safeParse(value);- if (!result.success) fail(`${flag}: ${result.error.issues[0]?.message ?? `invalid time (got "${value}")`}`);- return result.data;-}-/*** A local file (or frames folder) that exists is sent as its absolute path;* anything else — a URL, or a library path (`b-roll/clip.mp4`) — is passed* through for the app to resolve. Library paths need an open project.*/-function resolveAssetRef(ref: string): { path: string } {- const absPath = isAbsolute(ref) ? ref : resolve(process.cwd(), ref);- if (existsSync(absPath)) return { path: absPath };- if (isAbsolute(ref)) fail(`File not found: ${absPath}`);- return { path: ref };-}--/** An output path as the app needs it: absolute, or absent for the app's default. */-function resolveOutput(path: string | undefined): string | undefined {- return path === undefined ? undefined : resolve(process.cwd(), path);-}--function startSpinner(label: string): () => void {- if (!process.stderr.isTTY) {- process.stderr.write(`${label}…\n`);- return () => {};- }- const frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];- const start = Date.now();- let i = 0;- const render = () => {- const secs = Math.floor((Date.now() - start) / 1000);- process.stderr.write(`\r${frames[i]} ${label}… ${secs}s`);- i = (i + 1) % frames.length;- };- render();- const timer = setInterval(render, 80);- return () => {- clearInterval(timer);- process.stderr.write("\r\x1b[K"); // carriage return + clear to end of line- };-}--// ----------------------------------------------------------------------------// Commands that are the CLI's own: launching, and the proxy.--type OpenOptions = { background?: boolean };--async function openProject(path: string | undefined, opts: OpenOptions): Promise<void> {- // Launching is macOS's job; elsewhere (and when the app is not installed,- // e.g. a dev checkout run from the terminal) fall through to the socket,- // which answers if the app is running and errors usefully if not.- const launched = await launchApp(opts.background ?? false);- try {- // A cold launch needs the renderer up before the app can answer; when- // nothing was launched there is nothing to wait for, so fail fast.- if (launched) await waitForApp();- else await ping();- } catch (e) {- handleSocketError(e);- }- if (path !== undefined) await run("open", { dir: resolve(path) });+function assetPath(ref: string): string {+ const abs = resolve(ref);+ if (existsSync(abs)) return abs;+ if (isAbsolute(ref)) fail(`File not found: ${abs}`);+ return ref;}-// ----------------------------------------------------------------------------// The program.-const program = new Command();program@@ -153,7 +98,11 @@ program).argument("[path]", `${field("open", "dir")} (default: none — just launch the app)`).option("-b, --background", "launch or keep the app in the background, without raising a window")- .action((path: string | undefined, opts: OpenOptions) => openProject(path, opts));+ .action(async (path: string | undefined, opts: { background?: boolean }) => {+ const launched = await launchApp(opts.background ?? false);+ await (launched ? waitForApp() : ping()).catch(appError);+ if (path !== undefined) await run("open", { dir: resolve(path) });+ });program.command("context")@@ -161,39 +110,33 @@ program.description(describe("context")).action(() => run("context", {}));-type CaptureOptions = { time?: string[]; output?: string; separate?: boolean; perSheet?: number };-program.command("capture").description(describe("capture")).argument("<id>", field("capture", "id"))- .option(- "-t, --time <time...>",- `one or more positions to capture, relative to the export's first frame, the workarea's start (0 = the export's frame 0) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`,- )- .option("-S, --separate", "write one PNG per position instead of merging them into contact sheets (combine: false)")+ .option("-t, --times <time...>", field("capture", "times"))+ .option("-S, --separate", field("capture", "separate")).option("--per-sheet <n>", field("capture", "perSheet"), numeric).option("-o, --output <dir>", field("capture", "output"))- .action((id: string, opts: CaptureOptions) => {- const frames = (opts.time ?? ["0"]).map((t) => Math.round(seconds(t, "--time") * TIME_FPS));- return run("capture", { id, frames, combine: !opts.separate, perSheet: opts.perSheet, output: resolveOutput(opts.output) });- });+ .action((id: string, opts: Omit<ToolInput<"capture">, "id">) =>+ run("capture", { id, ...opts, output: opts.output && resolve(opts.output) }),+ );program.command("export").description(describe("export")).argument("<id>", field("export", "id")).argument("[output]", field("export", "path"))- .action((id: string, output: string | undefined) => run("export", { id, path: resolveOutput(output) }, "Exporting scene"));+ .action((id: string, output: string | undefined) => run("export", { id, path: output && resolve(output) }));program.command("check").description(`${describe("check")} Exits 1 when an error-severity issue is found.`).argument("<id>", field("check", "id")).action(async (id: string) => {- const { issues } = await invoke("check", { id });- // Linter convention: issues found is a different failure than "could not run".- if (issues.some((issue) => issue.severity === "error")) process.exitCode = 1;+ const output = await call("check", { id }).catch(appError);+ console.log(JSON.stringify(output));+ if (output.issues.some((issue) => issue.severity === "error")) process.exitCode = 1;});const media = program@@ -207,71 +150,33 @@ media.command("probe").description(describe("media_probe")).argument("<path>", field("media_probe", "path"))- .action((ref: string) => run("media_probe", resolveAssetRef(ref), "Probing asset"));+ .action((ref: string) => run("media_probe", { path: assetPath(ref) }));media.command("transcribe").description(describe("media_transcribe")).argument("<path>", field("media_transcribe", "path"))- .action((ref: string) => run("media_transcribe", resolveAssetRef(ref), "Transcribing asset"));--type GrabOptions = {- time?: string[];- count?: number;- start?: string;- end?: string;- quality?: string;- uncapped?: boolean;- output?: string;- auto?: boolean;- separate?: boolean;- perSheet?: number;-};+ .action((ref: string) => run("media_transcribe", { path: assetPath(ref) }));media.command("grab").alias("sample").description(describe("media_grab")).argument("<path>", field("media_grab", "path"))- .option("-t, --time <time...>", field("media_grab", "times"))+ .option("-t, --times <time...>", field("media_grab", "times")).option("-c, --count <n>", field("media_grab", "count"), numeric).option("-a, --auto", field("media_grab", "auto")).option("-s, --start <time>", field("media_grab", "start", `with --count or --auto, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)).option("-e, --end <time>", field("media_grab", "end", `with --count or --auto, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`)).option("-q, --quality <preset>", field("media_grab", "quality"))- .option("-S, --separate", "write one PNG per frame instead of merging them into contact sheets (combine: false)")+ .option("-S, --separate", field("media_grab", "separate")).option("--per-sheet <n>", field("media_grab", "perSheet"), numeric).option("--uncapped", field("media_grab", "uncapped")).option("-o, --output <dir>", field("media_grab", "output"))- .action((ref: string, opts: GrabOptions) =>- run(- "media_grab",- {- ...resolveAssetRef(ref),- times: opts.time,- count: opts.count,- auto: opts.auto,- start: opts.start,- end: opts.end,- quality: opts.quality as ToolInput<"media_grab">["quality"],- combine: !opts.separate,- perSheet: opts.perSheet,- uncapped: opts.uncapped,- output: resolveOutput(opts.output),- },- ),+ .action((ref: string, opts: Omit<ToolInput<"media_grab">, "path">) =>+ run("media_grab", { path: assetPath(ref), ...opts, output: opts.output && resolve(opts.output) }),);-type PreviewOptions = { start?: string; end?: string; scale?: number; output?: string };--const previewInput = (ref: string, opts: PreviewOptions) => ({- ...resolveAssetRef(ref),- start: opts.start,- end: opts.end,- scale: opts.scale,- output: resolveOutput(opts.output),-});-media.command("filmstrip").alias("film")@@ -281,7 +186,9 @@ media.option("-e, --end <time>", field("media_filmstrip", "end", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)).option("-x, --scale <factor>", field("media_filmstrip", "scale"), numeric).option("-o, --output <path>", field("media_filmstrip", "output"))- .action((ref: string, opts: PreviewOptions) => run("media_filmstrip", previewInput(ref, opts), "Rendering filmstrip"));+ .action((ref: string, opts: Omit<ToolInput<"media_filmstrip">, "path">) =>+ run("media_filmstrip", { path: assetPath(ref), ...opts, output: opts.output && resolve(opts.output) }),+ );media.command("waveform")@@ -292,9 +199,9 @@ media.option("-e, --end <time>", field("media_waveform", "end", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)).option("-x, --scale <factor>", field("media_waveform", "scale"), numeric).option("-o, --output <path>", field("media_waveform", "output"))- .action((ref: string, opts: PreviewOptions) => run("media_waveform", previewInput(ref, opts), "Rendering waveform"));--type ListenOptions = { prompt?: string; start?: string; end?: string; keepVideo?: boolean };+ .action((ref: string, opts: Omit<ToolInput<"media_waveform">, "path">) =>+ run("media_waveform", { path: assetPath(ref), ...opts, output: opts.output && resolve(opts.output) }),+ );media.command("listen")@@ -303,20 +210,13 @@ media.option("-p, --prompt <str>", field("media_listen", "prompt")).option("-s, --start <time>", field("media_listen", "start")).option("-e, --end <time>", field("media_listen", "end"))- .option("--keep-video", "for a video asset, keep the video track instead of stripping to audio, so the model also reads what is on screen (expensive: uploads the full video; stripVideo: false)")- .action((ref: string, opts: ListenOptions) =>- run(- "media_listen",- { ...resolveAssetRef(ref), prompt: opts.prompt, start: opts.start, end: opts.end, stripVideo: !opts.keepVideo },- "Analyzing asset",- ),- );+ .action((ref: string, opts: Omit<ToolInput<"media_listen">, "path">) => run("media_listen", { path: assetPath(ref), ...opts }));program.command("models").description(describe("models")).argument("[type]", field("models", "type"))- .action((type: string | undefined) => run("models", { type: type as ToolInput<"models">["type"] }));+ .action((type: ToolInput<"models">["type"]) => run("models", { type }));program.command("voices")@@ -328,24 +228,18 @@ program.description(describe("whoami")).action(() => run("whoami", {}));-type LogsOptions = { tail?: number; level?: string };-program.command("logs").description(describe("logs")).option("-n, --tail <n>", field("logs", "tail"), numeric).option("-l, --level <level>", `${field("logs", "level")}: "debug", "info", "warning", or "error"`)- .action((opts: LogsOptions) => run("logs", { tail: opts.tail, level: opts.level as ToolInput<"logs">["level"] }));--type ScreenshotOptions = { output?: string };+ .action((opts: ToolInput<"logs">) => run("logs", opts));program.command("screenshot").description(describe("screenshot")).option("-o, --output <dir>", field("screenshot", "output"))- .action((opts: ScreenshotOptions) => run("screenshot", { output: resolveOutput(opts.output) }));--type ReportOptions = { body?: string; command?: string[]; logs?: number };+ .action((opts: ToolInput<"screenshot">) => run("screenshot", { output: opts.output && resolve(opts.output) }));program.command("report")@@ -353,24 +247,18 @@ program.description(describe("report")).argument("<title>", field("report", "title")).option("-b, --body <text>", field("report", "body"))- .option("-c, --command <cmd...>", `${field("report", "commands")}; repeatable`)+ .option("-c, --commands <cmd...>", `${field("report", "commands")}; repeatable`).option("--logs <n>", field("report", "logs", `trailing app log entries to attach (0 to omit; default: ${ISSUE_LOG_TAIL})`), numeric)- .action((title: string, opts: ReportOptions) => run("report", { title, body: opts.body, commands: opts.command, logs: opts.logs }));--type FontsOptions = { family?: string; weight?: string[]; style?: string; limit?: number };+ .action((title: string, opts: Omit<ToolInput<"report">, "title">) => run("report", { title, ...opts }));program.command("fonts").description(describe("fonts")).option("-f, --family <pattern>", field("fonts", "family"))- .option("-w, --weight <weights...>", `${field("fonts", "weights")}, e.g. -w 400 700`)+ .option("-w, --weights <weights...>", `${field("fonts", "weights")}, e.g. -w 400 700`).option("-s, --style <style>", `${field("fonts", "style")}: "normal" or "italic"`).option("-l, --limit <n>", field("fonts", "limit"), numeric)- .action((opts: FontsOptions) =>- run("fonts", { family: opts.family, weights: opts.weight, style: opts.style as ToolInput<"fonts">["style"], limit: opts.limit }),- );--type FetchOptions = { output?: string; format?: string; audio?: boolean };+ .action((opts: ToolInput<"fonts">) => run("fonts", opts));program.command("fetch")@@ -381,9 +269,9 @@ program.option("-a, --audio", field("fetch", "audio")).allowExcessArguments().addHelpText("after", `\nForward raw yt-dlp flags after --, e.g. dapi fetch <url> -- --sponsorblock-remove all`)- .action((url: string, opts: FetchOptions, cmd: Command) =>+ .action((url: string, opts: Omit<ToolInput<"fetch">, "url" | "raw">, cmd: Command) =>// `raw` is every operand after `url` — the yt-dlp passthrough placed after `--`.- run("fetch", { url, format: opts.format, audio: opts.audio, raw: cmd.args.slice(1), output: resolveOutput(opts.output) }, "Downloading"),+ run("fetch", { url, ...opts, raw: cmd.args.slice(1), output: opts.output && resolve(opts.output) }),);// Explicit argv convention: the packaged wrapper runs this bundle ondiff --git a/apps/cli/src/mcp-proxy.ts b/apps/cli/src/mcp-proxy.tsindex c89bf3da..92d99c68 100644--- a/apps/cli/src/mcp-proxy.ts+++ b/apps/cli/src/mcp-proxy.ts@@ -8,7 +8,7 @@// version. Stdout belongs to the protocol; anything for a human goes to// stderr.-import { APP_NAME, connectWithRetry, isAppDown, launchApp, openSocket } from "./cli-client";+import { APP_NAME, isAppDown, launchApp, openSocket, waitForApp } from "./cli-client";import type { Socket } from "node:net";@@ -22,7 +22,8 @@ export async function runProxy(): Promise<void> {if (!(await launchApp(true))) {throw new Error(`${APP_NAME} is not running. Launch the app first, then retry.`);}- socket = await connectWithRetry();+ await waitForApp();+ socket = await openSocket();}socket.on("error", (error) => {diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.tsindex 1c75de82..5f5115c8 100644--- a/apps/desktop/src/dapi/present.test.ts+++ b/apps/desktop/src/dapi/present.test.ts@@ -16,7 +16,7 @@ const png = (byte: number, size = 8) => new Uint8Array(size).fill(byte);describe("present", () => {it("writes capture frames by timecode into the requested directory and returns their paths", async () => {const out = join(dir, "frames");- const presented = await present("capture", { id: "intro", combine: true, output: out }, [+ const presented = await present("capture", { id: "intro", output: out }, [{ timecode: "0f", png: png(1) },{ timecode: "1s", png: png(2) },]);@@ -25,7 +25,7 @@ describe("present", () => {});it("picks a fresh temp directory when none is given", async () => {- const presented = await present("media_grab", { path: "/c.mp4", combine: true }, [{ timecode: "0f", png: png(3) }]);+ const presented = await present("media_grab", { path: "/c.mp4" }, [{ timecode: "0f", png: png(3) }]);const { path } = (presented.output as { images: Array<{ path: string }> }).images[0]!;expect(path).toMatch(/dapi-grab-.*[\\/]0f\.png$/);rmSync(join(path, ".."), { recursive: true, force: true });diff --git a/apps/web/src/dapi/handlers/capture.ts b/apps/web/src/dapi/handlers/capture.tsindex 13b1ee41..da4d32e8 100644--- a/apps/web/src/dapi/handlers/capture.ts+++ b/apps/web/src/dapi/handlers/capture.ts@@ -3,6 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { createImageEncoder, decodePng } from "@diffusionstudio/encoder";+import { TIME_FPS } from "@diffusionstudio/jsx";import { DapiError } from "@diffusionstudio/dapi";import { createCapture } from "@/engine/capture";import { requireScene } from "../lib/scene";@@ -13,12 +14,13 @@ import type { ToolHandler } from "../handler";// Ceiling on the height a sheet cell renders a node at.const SHEET_CAPTURE_HEIGHT = 1080;-export const capture: ToolHandler<"capture"> = async ({ id, frames, combine, perSheet }, ctx) => {+export const capture: ToolHandler<"capture"> = async ({ id, times, separate, perSheet }, ctx) => {const { world, project } = ctx.requireSession();const scene = requireScene(world, id, "capture");- // `undefined` means the export's first frame (the workarea's start).- const shots = frames && frames.length > 0 ? frames : [0];+ // Positions arrive in seconds and the encoder samples frame numbers;+ // none given means the export's first frame (the workarea's start).+ const shots = times && times.length > 0 ? times.map((t) => Math.round(t * TIME_FPS)) : [0];// The project re-rendered into a world of its own, reduced to this scene:// the same arrangement an export runs against, and the encoder's to draw.@@ -31,7 +33,7 @@ export const capture: ToolHandler<"capture"> = async ({ id, frames, combine, per// scene is drawn, not decoded, so a small one is worth rendering past// its own size; beyond SHEET_CAPTURE_HEIGHT that only costs tokens.let sheets: SheetCollector | undefined;- if (combine) {+ if (!separate) {const aspect = encoder.bounds.width / encoder.bounds.height;const height = Math.max(encoder.bounds.height, SHEET_CAPTURE_HEIGHT);sheets = new SheetCollector(shots.length, { width: height * aspect, height }, perSheet);diff --git a/apps/web/src/dapi/handlers/media-grab.ts b/apps/web/src/dapi/handlers/media-grab.tsindex ed12e9fb..7212d4f1 100644--- a/apps/web/src/dapi/handlers/media-grab.ts+++ b/apps/web/src/dapi/handlers/media-grab.ts@@ -27,7 +27,7 @@ const FRAME_QUALITY_BUDGETS: Record<FrameQuality, number> = {const AUTO_MAX_FRAMES = 30;export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {- const { times, count, start, end, quality, auto, combine, perSheet } = args;+ const { times, count, start, end, quality, auto, separate, perSheet } = args;const asset = await resolveAsset(ctx, args.path);requireAssetType(asset, ["VIDEO"], "a video");@@ -53,7 +53,7 @@ export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {requested = (times && times.length ? times : [0]).map((t) => resolveTime(t, asset.duration));}- const budget = FRAME_QUALITY_BUDGETS[quality ?? (combine ? "fullres" : "small")];+ const budget = FRAME_QUALITY_BUDGETS[quality ?? (separate ? "small" : "fullres")];const blob = await getAssetFile(asset);const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });@@ -86,7 +86,7 @@ export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {// Lay the sheets out up front: the largest cell across them sets the// decode size, so no frame is decoded bigger than it will be drawn.- const sheets = combine ? new SheetCollector(requested.length, { width: sourceWidth, height: sourceHeight }, perSheet) : undefined;+ const sheets = separate ? undefined : new SheetCollector(requested.length, { width: sourceWidth, height: sourceHeight }, perSheet);const width = sheets ? Math.min(sourceWidth, sheets.cellWidth) : sourceWidth;// Decode in ascending order (the sink's fast path), remember each@@ -96,7 +96,7 @@ export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {// No pool: each yielded canvas is fresh, so encoding it can't race the// generator's read-ahead reusing a pooled canvas.const sink = new CanvasSink(track, width < displayWidth ? { width } : undefined);- const separate: TimecodedImage[] = new Array(requested.length);+ const frames: TimecodedImage[] = new Array(requested.length);let i = 0;for await (const wrapped of sink.canvasesAtTimestamps(ordered.map(({ time }) => firstTimestamp + time))) {@@ -104,10 +104,10 @@ export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {if (!wrapped) throw new DapiError("not-found", `No frame found at ${time}s.`);const timecode = formatTimecode(time, asset.frameRate);if (sheets) await sheets.add(index, { at: time, timecode, image: wrapped.canvas });- else separate[index] = { timecode, png: await encodePng(wrapped.canvas) };+ else frames[index] = { timecode, png: await encodePng(wrapped.canvas) };}- return sheets ? sheets.result() : separate;+ return sheets ? sheets.result() : frames;} finally {input.dispose();}diff --git a/apps/web/src/dapi/handlers/media-listen.ts b/apps/web/src/dapi/handlers/media-listen.tsindex 4a5cdd9d..eb3fc85a 100644--- a/apps/web/src/dapi/handlers/media-listen.ts+++ b/apps/web/src/dapi/handlers/media-listen.ts@@ -9,14 +9,15 @@ import { requireAssetType, resolveAsset } from "../lib/assets";import type { ToolHandler } from "../handler";-export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end, stripVideo }, ctx) => {+export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end }, ctx) => {ctx.app.requireUser();const asset = await resolveAsset(ctx, path);requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");const hasWindow = start !== undefined || end !== undefined;- const audioOnly = stripVideo !== false && asset.type === "VIDEO";- const contentType = asset.type === "VIDEO" && !audioOnly ? "video/mp4" : "audio/ogg";+ // A video is stripped to its audio track: the model listens, it does not watch.+ const audioOnly = asset.type === "VIDEO";+ const contentType = "audio/ogg";// One upload per distinct analysis input, so a repeated question about the// same span reuses the transcode.diff --git a/docs/mcp-server.md b/docs/mcp-server.mdindex 7e0b37f1..37946dac 100644--- a/docs/mcp-server.md+++ b/docs/mcp-server.md@@ -111,7 +111,7 @@ Tools, mapped from today's commands:| `fetch` | `dapi fetch` | main | yt-dlp passthrough. Progress goes to MCP progress notifications instead of inherited stderr. || `report` | `dapi report` | main | Files the GitHub issue with diagnostics that main already holds. |-Argument parsing that exists today only to turn strings into numbers (`--time "45f"`, `--count`, `--per-sheet`) becomes schema: times are accepted as the same strings and parsed by `parseTime` from `@diffusionstudio/jsx` inside the schema's transform, so the CLI and the agent get identical validation and identical error messages.+Argument parsing that exists today only to turn strings into numbers (`--times "45f"`, `--count`, `--per-sheet`) becomes schema: times are accepted as the same strings and parsed by `parseTime` from `@diffusionstudio/jsx` inside the schema's transform, so the CLI and the agent get identical validation and identical error messages.### 2. Main process: the serverdiff --git a/knowledge/reference/jsx/timing.md b/knowledge/reference/jsx/timing.mdindex b795f66a..92542d0b 100644--- a/knowledge/reference/jsx/timing.md+++ b/knowledge/reference/jsx/timing.md@@ -62,4 +62,4 @@ The canonical internal unit is frames at **30 fps**; all formats are converted o| `"MM:SS"` | `"02:30"` | Minutes and seconds. || `"HH:MM:SS"` | `"01:02:30"` | Hours, minutes, seconds. |-CLI flags documented as taking a `Time` value (e.g. `--time` on [`capture`](../tools/capture.md) and [`media grab`](../tools/media/grab.md)) accept the same formats. Times in command **output** are plain seconds.+CLI flags documented as taking a `Time` value (e.g. `--times` on [`capture`](../tools/capture.md) and [`media grab`](../tools/media/grab.md)) accept the same formats. Times in command **output** are plain seconds.diff --git a/knowledge/reference/tools/capture.md b/knowledge/reference/tools/capture.mdindex 862ec840..47a30876 100644--- a/knowledge/reference/tools/capture.md+++ b/knowledge/reference/tools/capture.md@@ -1,4 +1,4 @@-# `dapi capture <id> [-t, --time <time...>]`+# `dapi capture <id> [-t, --times <time...>]`Renders single frames of a scene at one or more timeline positions and merges them into **contact sheets**: up to 12 positions per PNG, each cell labelled with the timecode of the frame actually rendered and drawn as large as the sheet allows, so a few positions arrive as one high-resolution picture instead of a directory to open one by one. `--separate` writes a PNG per position instead.@@ -9,7 +9,7 @@ Scenes only: a single element renders inside its scene, so capture the scene at## Input- `<id>`: scene id to capture (required) — the scene's `id` attribute in the project's JSX (e.g. `intro`), or its position in the file when it has none. When two files use the same id, the `file:id` form (`index.tsx:intro`) settles it. Entity numbers are not accepted; they change on every recompile.-- `-t, --time <time...>`: one or more positions to capture, relative to the export's first frame — the workarea's start, so `0` is the export's frame 0 — each a `Time` value (optional; default `0`)+- `-t, --times <time...>`: one or more positions to capture, relative to the export's first frame — the workarea's start, so `0` is the export's frame 0 — each a `Time` value (optional; default `0`)- `-S, --separate`: write one PNG per position instead of merging them into contact sheets (optional). Each is rendered at 720p height and named after its timecode (e.g. `01s12f.png`).- `--per-sheet <n>`: positions per contact sheet, 1 to 12 (optional; default as many as fit). Fewer positions per sheet means a larger cell each. Sheets are balanced, so 13 positions become 7 + 6 rather than 12 + 1.- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-capture-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites images whose name matches; with `--separate`, requested times that land on the same frame share one file.diff --git a/knowledge/reference/tools/check.md b/knowledge/reference/tools/check.mdindex e09a3f0d..79ff5599 100644--- a/knowledge/reference/tools/check.md+++ b/knowledge/reference/tools/check.md@@ -29,7 +29,7 @@ One JSON object:severity: "error" | "warning";message: string;node?: string; // source stamp of the offending node; absent for subtree-wide issues- ranges?: Array<{ start: number; end: number }>; // seconds relative to the node's start — for a scene whose workarea starts at 0, the clock `capture --time` uses+ ranges?: Array<{ start: number; end: number }>; // seconds relative to the node's start — for a scene whose workarea starts at 0, the clock `capture --times` uses}>;}```diff --git a/knowledge/reference/tools/fonts.md b/knowledge/reference/tools/fonts.mdindex 42736ebe..15470aab 100644--- a/knowledge/reference/tools/fonts.md+++ b/knowledge/reference/tools/fonts.md@@ -5,7 +5,7 @@ Lists local fonts available on this machine. macOS only. Does not require the ap## Options- `-f, --family <pattern>`: filter to families whose name contains `<pattern>` (case-insensitive)-- `-w, --weight <weights...>`: filter to variants with the given CSS weight(s), e.g. `-w 400 700`+- `-w, --weights <weights...>`: filter to variants with the given CSS weight(s), e.g. `-w 400 700`- `-s, --style <style>`: `"normal"` or `"italic"`- `-l, --limit <n>`: output at most `<n>` familiesdiff --git a/knowledge/reference/tools/media/grab.md b/knowledge/reference/tools/media/grab.mdindex f54734cc..8b204748 100644--- a/knowledge/reference/tools/media/grab.md+++ b/knowledge/reference/tools/media/grab.md@@ -5,14 +5,14 @@ Decodes one or more frames of a video asset at the given times and merges them i## Input- `<path>`: a local video file to grab frames from in place without adding it to the library, or a project library path (required; library paths need an open project).-- `-t, --time <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). A negative value is an offset back from the end of the clip, so `-1` is one second before the end and `-1f` one frame before it. Order is preserved in the output regardless of the order given. Mutually exclusive with `--count`.-- `-c, --count <n>`: instead of `--time`, grab `n` frames evenly spaced across the clip at a fixed interval of `window / n`, starting at the window start (optional; positive integer).+- `-t, --times <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). A negative value is an offset back from the end of the clip, so `-1` is one second before the end and `-1f` one frame before it. Order is preserved in the output regardless of the order given. Mutually exclusive with `--count`.+- `-c, --count <n>`: instead of `--times`, grab `n` frames evenly spaced across the clip at a fixed interval of `window / n`, starting at the window start (optional; positive integer).- `-s, --start <time>`: with `--count`, the start of the window to sample (optional; `Time` value; default `0`).- `-e, --end <time>`: with `--count`, the end of the window to sample (optional; `Time` value; default the asset duration).- `-q, --quality <preset>`: per-frame resolution preset (optional). One of `small` (384x384), `medium` (768x768), `large` (1536x1536), or `fullres` (native); each caps the total pixel count while preserving aspect ratio. Defaults to as much detail as the sheet cell can show, or `small` with `--separate`.- `-S, --separate`: write one PNG per frame instead of merging them into contact sheets (optional). The frames keep their own resolution and alpha, and each file is named after its timecode (e.g. `01s12f.png`).- `--per-sheet <n>`: frames per contact sheet, 1 to 12 (optional; default as many as fit). Fewer frames per sheet means a larger cell each. Sheets are balanced, so 13 frames become 7 + 6 rather than 12 + 1.-- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--time`) is rejected.+- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--times`) is rejected.- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-grab-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites images whose name matches; with `--separate`, requested times that land on the same frame share one file.## Timecodes@@ -42,4 +42,4 @@ A sheet's timecode is the span it covers; a frame's is its own. Sheets come in t## Errors-Exits non-zero if the path can't be resolved, the asset is not a video, any `--time` is past the asset's duration, the `--count` window is empty, `--time` and `--count` are combined, `--start`/`--end` are given without `--count`, `--per-sheet` is outside 1 to 12, more than 100 frames are requested without `--uncapped`, or a PNG can't be written.+Exits non-zero if the path can't be resolved, the asset is not a video, any `--times` value is past the asset's duration, the `--count` window is empty, `--times` and `--count` are combined, `--start`/`--end` are given without `--count`, `--per-sheet` is outside 1 to 12, more than 100 frames are requested without `--uncapped`, or a PNG can't be written.diff --git a/knowledge/reference/tools/media/listen.md b/knowledge/reference/tools/media/listen.mdindex 3fdfbde6..db5dd692 100644--- a/knowledge/reference/tools/media/listen.md+++ b/knowledge/reference/tools/media/listen.md@@ -1,6 +1,6 @@# `dapi media listen <path>`-Puts a multimodal model in front of an audio track and returns its answer. With no prompt it returns a general description of what is heard; with `--prompt` it answers that question about the audio (e.g. "who is speaking?", "what music is playing?", "summarize what is said"). Accepts an audio file or a video, but only the audio track is analyzed by default. Alias: `watch`.+Puts a multimodal model in front of an audio track and returns its answer. With no prompt it returns a general description of what is heard; with `--prompt` it answers that question about the audio (e.g. "who is speaking?", "what music is playing?", "summarize what is said"). Accepts an audio file or a video; of a video only the audio track is analyzed. Alias: `watch`.## Input@@ -8,7 +8,6 @@ Puts a multimodal model in front of an audio track and returns its answer. With- `-p, --prompt <str>`: question or instruction about the audio (optional; defaults to a general description).- `-s, --start <time>`: start of the segment to analyze, a `Time` value (optional; default `0`). Timestamps in the analysis are relative to this point.- `-e, --end <time>`: end of the segment to analyze, a `Time` value (optional; default the asset's duration).-- `--keep-video`: for a video asset, keep the video track instead of stripping it to audio, so the model can analyze what is on screen (optional; default off). Expensive: requires a full video upload.## Outputdiff --git a/knowledge/reference/tools/report.md b/knowledge/reference/tools/report.mdindex bcbb4703..b74b415b 100644--- a/knowledge/reference/tools/report.md+++ b/knowledge/reference/tools/report.md@@ -15,7 +15,7 @@ Does not require the app to be running. If the app is down or unreachable, the r## Options- `-b, --body <text>`: what happened, in markdown: what you expected, what you got, and anything the diagnostics won't show.-- `-c, --command <cmd...>`: the `dapi` command(s) that reproduce it, in order. Repeatable (`-c "dapi context" -c "dapi capture intro"`); rendered as a shell block under `## Repro`.+- `-c, --commands <cmd...>`: the `dapi` command(s) that reproduce it, in order. Repeatable (`-c "dapi context" -c "dapi capture intro"`); rendered as a shell block under `## Repro`.- `--logs <n>`: trailing app log entries to attach (default: 50). `--logs 0` omits the section, and then the app is not contacted at all.## Outputdiff --git a/packages/dapi/src/schemas.ts b/packages/dapi/src/schemas.tsindex 71f8a24a..ae9e4103 100644--- a/packages/dapi/src/schemas.ts+++ b/packages/dapi/src/schemas.ts@@ -60,10 +60,10 @@ export const outputDirField = z* rejects it otherwise, so add it to any object that spreads these fields.*/export const sheetFields = {- combine: z+ separate: z.boolean()- .default(true)- .describe("merge the images into contact sheets of up to 12 cells, each labelled with its timecode (default); false writes one image per position"),+ .optional()+ .describe("write one image per position instead of merging them into contact sheets of up to 12 cells, each labelled with its timecode"),perSheet: z.int().min(1)@@ -73,14 +73,14 @@ export const sheetFields = {};export function checkSheetOptions(- value: { combine: boolean; perSheet?: number | undefined },+ value: { separate?: boolean | undefined; perSheet?: number | undefined },ctx: z.RefinementCtx,): void {- if (value.perSheet !== undefined && !value.combine) {+ if (value.perSheet !== undefined && value.separate) {ctx.addIssue({code: "custom",path: ["perSheet"],- message: "perSheet lays out contact sheets; it cannot be combined with combine: false",+ message: "perSheet lays out contact sheets; it cannot be combined with separate: true",});}}diff --git a/packages/dapi/src/tools/capture.ts b/packages/dapi/src/tools/capture.tsindex 8ceca5c3..26b02d18 100644--- a/packages/dapi/src/tools/capture.ts+++ b/packages/dapi/src/tools/capture.ts@@ -5,19 +5,22 @@import { z } from "zod";import { defineTool } from "../tool";import { checkSheetOptions, ImageRef, outputDirField, SceneId, sheetFields, TimecodedImage } from "../schemas";+import { NonNegativeTime } from "../time";export const capture = defineTool({name: "capture",title: "Capture frames",description:- "Render single frames of a scene to PNGs — each frame is the frame an export of that scene would encode, drawn offscreen at the scene's own size. By default the positions are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and rendered as large as fits, so a few positions arrive as one high-resolution picture instead of a directory to open one by one (combine: false writes a PNG per position, at 720p height). The tool for checking composition (\"what plays at time T\": layout, overlaps, text, timing) and for verifying frames before an export. Scenes only — a single element renders inside its scene, so capture the scene at the times it plays. For a video asset's own full-resolution pixels use media_grab.",+ "Render single frames of a scene to PNGs — each frame is the frame an export of that scene would encode, drawn offscreen at the scene's own size. By default the positions are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and rendered as large as fits, so a few positions arrive as one high-resolution picture instead of a directory to open one by one (separate: true writes a PNG per position, at 720p height). The tool for checking composition (\"what plays at time T\": layout, overlaps, text, timing) and for verifying frames before an export. Scenes only — a single element renders inside its scene, so capture the scene at the times it plays. For a video asset's own full-resolution pixels use media_grab.",input: z.object({id: SceneId,- frames: z- .array(z.int().nonnegative())+ times: z+ .array(NonNegativeTime).optional()- .describe("positions to capture as frame numbers relative to the export's first frame, the workarea's start (default: [0])"),+ .describe(+ "positions to capture, relative to the export's first frame, the workarea's start (0 = the export's frame 0) — seconds (\"1.5\"), frames (\"45f\"), or \"MM:SS\" (default: [0])",+ ),...sheetFields,output: outputDirField,})diff --git a/packages/dapi/src/tools/media-grab.ts b/packages/dapi/src/tools/media-grab.tsindex 5dcbb544..9e9f8eac 100644--- a/packages/dapi/src/tools/media-grab.ts+++ b/packages/dapi/src/tools/media-grab.ts@@ -16,7 +16,7 @@ export const mediaGrab = defineTool({name: "media_grab",title: "Grab frames",description:- "Decode frames of a video file and write them as PNGs (local render, no credits). By default the frames are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and drawn as large as fits, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one (combine: false writes a PNG per frame). Grabs the asset's own pixels, unlike capture which renders the composited node. The recommended tool for understanding a video at the frame level; past ~12 frames prefer media_filmstrip.",+ "Decode frames of a video file and write them as PNGs (local render, no credits). By default the frames are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and drawn as large as fits, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one (separate: true writes a PNG per frame). Grabs the asset's own pixels, unlike capture which renders the composited node. The recommended tool for understanding a video at the frame level; past ~12 frames prefer media_filmstrip.",input: z.object({path: AssetPath,@@ -39,7 +39,7 @@ export const mediaGrab = defineTool({),...windowFields,quality: FrameQuality.optional().describe(- "frame resolution: small (384x384), medium (768x768), large (1536x1536), or fullres (native); default: as large as the sheet cell allows, or small with combine: false",+ "frame resolution: small (384x384), medium (768x768), large (1536x1536), or fullres (native); default: as large as the sheet cell allows, or small with separate: true",),...sheetFields,uncapped: zdiff --git a/packages/dapi/src/tools/media-listen.ts b/packages/dapi/src/tools/media-listen.tsindex c1d13e4d..f12c59e6 100644--- a/packages/dapi/src/tools/media-listen.ts+++ b/packages/dapi/src/tools/media-listen.ts@@ -10,7 +10,7 @@ export const mediaListen = defineTool({name: "media_listen",title: "Listen to audio",description:- "Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; by default only the audio track is analyzed. Needs a signed-in account.",+ "Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; of a video only the audio track is analyzed. Needs a signed-in account.",input: z.object({path: AssetPath,@@ -19,12 +19,6 @@ export const mediaListen = defineTool({"start of the segment to analyze (default: 0); timestamps in the analysis are relative to this point",),end: windowFields.end.describe("end of the segment to analyze (default: media duration)"),- stripVideo: z- .boolean()- .optional()- .describe(- "for a video asset, analyze the audio track only (default); false keeps the video so the model also reads what is on screen (expensive: uploads the full video)",- ),}).superRefine(checkWindow),output: z.object({diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.tsindex 0879c0a6..b00d26d4 100644--- a/packages/dapi/src/tools/tools.test.ts+++ b/packages/dapi/src/tools/tools.test.ts@@ -23,7 +23,7 @@ describe("media_grab", () => {it("parses times in every form and applies the sheet default", () => {const args = input.parse({ path: "/clip.mp4", times: ["45f", "1:10", -1, "-2f"] });expect(args.times).toEqual([1.5, 70, -1, -2 / 30]);- expect(args.combine).toBe(true);+ expect(args.separate).toBeUndefined();expect(args.perSheet).toBeUndefined();});@@ -58,21 +58,21 @@ describe("media_grab", () => {});it("rejects perSheet for separate images", () => {- expect(issues(input.safeParse({ path: "/c.mp4", combine: false, perSheet: 4 }))).toHaveProperty("perSheet");+ expect(issues(input.safeParse({ path: "/c.mp4", separate: true, perSheet: 4 }))).toHaveProperty("perSheet");expect(issues(input.safeParse({ path: "/c.mp4", perSheet: 13 }))).toHaveProperty("perSheet");expect(input.safeParse({ path: "/c.mp4", perSheet: 12 }).success).toBe(true);});});describe("capture", () => {- it("takes non-negative integer frames", () => {- expect(capture.input.parse({ id: "intro", frames: [0, 45] }).frames).toEqual([0, 45]);- expect(capture.input.safeParse({ id: "intro", frames: [1.5] }).success).toBe(false);- expect(capture.input.safeParse({ id: "intro", frames: [-1] }).success).toBe(false);+ it("takes non-negative times in every form", () => {+ expect(capture.input.parse({ id: "intro", times: [0, "45f", "1:10"] }).times).toEqual([0, 1.5, 70]);+ expect(capture.input.safeParse({ id: "intro", times: [-1] }).success).toBe(false);+ expect(capture.input.safeParse({ id: "intro", times: ["-2f"] }).success).toBe(false);});it("shares the sheet rule with media_grab", () => {- expect(issues(capture.input.safeParse({ id: "intro", combine: false, perSheet: 2 }))).toHaveProperty("perSheet");+ expect(issues(capture.input.safeParse({ id: "intro", separate: true, perSheet: 2 }))).toHaveProperty("perSheet");});});