Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor CLI tool input handling and update documentation

konstantin-paulus5c0e109cd51b
diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts
index 45e54a1d..48a311d3 100644
--- a/apps/cli/src/cli-client.ts
+++ b/apps/cli/src/cli-client.ts
@@ -13,39 +13,28 @@ import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi";
export const APP_NAME = "Diffusion Studio";
-const DEFAULT_TIMEOUT_MS = 60000;
-const GENERATE_TIMEOUT_MS = 600000;
-const EXPORT_TIMEOUT_MS = 3600000;
-
-// Long-running tools (renders, AI generation, downloads) override the
-// default 60s. Keyed by name so `call` and the wrappers agree.
+// Renders, AI generation, and downloads outlive the 60s default.
const TIMEOUTS: Record<string, number> = {
- export: EXPORT_TIMEOUT_MS,
- capture: GENERATE_TIMEOUT_MS,
- media_transcribe: GENERATE_TIMEOUT_MS,
- media_listen: GENERATE_TIMEOUT_MS,
- fetch: GENERATE_TIMEOUT_MS,
+ export: 3_600_000,
+ capture: 600_000,
+ media_transcribe: 600_000,
+ media_listen: 600_000,
+ fetch: 600_000,
};
-export function timeoutFor(tool: string): number {
- return TIMEOUTS[tool] ?? DEFAULT_TIMEOUT_MS;
-}
-
-export type CallOptions = { timeoutMs?: number };
-
/**
* Calls one tool in the running app over an MCP session on its socket.
* Typed by the catalog: the input is what the tool's schema accepts, the
* output its structured content. One session per call; a command makes one
* or two, and the process exits when it settles.
*/
-export async function call<N extends ToolName>(name: N, input: ToolInput<N>, options: CallOptions = {}): Promise<ToolOutput<N>> {
- return withClient(async (client) => {
- const result = await client.callTool(
- { name, arguments: input as Record<string, unknown> },
- undefined,
- { timeout: options.timeoutMs ?? timeoutFor(name) },
- );
+export async function call<N extends ToolName>(name: N, input: ToolInput<N>): Promise<ToolOutput<N>> {
+ const client = new Client({ name: "dapi", version });
+ try {
+ await client.connect(new SocketTransport(await openSocket()));
+ const result = await client.callTool({ name, arguments: input as Record<string, unknown> }, undefined, {
+ timeout: TIMEOUTS[name] ?? 60_000,
+ });
if (result.isError) {
const text = (result.content as Array<{ type: string; text?: string }>)
.filter((block) => block.type === "text")
@@ -54,22 +43,17 @@ export async function call<N extends ToolName>(name: N, input: ToolInput<N>, opt
throw new Error(text || `${name} failed`);
}
return result.structuredContent as ToolOutput<N>;
- });
+ } finally {
+ await client.close().catch(() => {});
+ }
}
/** Liveness: a round-trip through the app's MCP server. */
-export function ping(): Promise<void> {
- return withClient(async (client) => {
- await client.ping();
- });
-}
-
-async function withClient<T>(fn: (client: Client) => Promise<T>): Promise<T> {
- const socket = await openSocket();
+export async function ping(): Promise<void> {
const client = new Client({ name: "dapi", version });
try {
- await client.connect(new SocketTransport(socket));
- return await fn(client);
+ await client.connect(new SocketTransport(await openSocket()));
+ await client.ping();
} finally {
await client.close().catch(() => {});
}
@@ -88,13 +72,8 @@ export function openSocket(): Promise<Socket> {
});
}
-/** The errno of a connection failure (ENOENT/ECONNREFUSED when the app is down), if any. */
-export function errnoCode(e: unknown): string | undefined {
- return (e as NodeJS.ErrnoException | undefined)?.code;
-}
-
export function isAppDown(e: unknown): boolean {
- const code = errnoCode(e);
+ const code = (e as NodeJS.ErrnoException | undefined)?.code;
return code === "ENOENT" || code === "ECONNREFUSED";
}
@@ -110,43 +89,22 @@ export function launchApp(background: boolean): Promise<boolean> {
}
/**
- * Bridges the cold-start gap after launching the app: the socket appears once
- * main is ready. Retries only while the app looks down; any other error is
- * the caller's.
+ * Bridges the cold-start gap after launching the app: retries while the app
+ * looks down, until it answers a ping (a cold app binds the socket before
+ * its session is ready, so connecting alone proves nothing).
*/
-export async function connectWithRetry(timeoutMs = 30000): Promise<Socket> {
- const start = Date.now();
- let lastError: unknown = null;
- while (Date.now() - start < timeoutMs) {
- try {
- return await openSocket();
- } catch (e) {
- if (!isAppDown(e)) throw e;
- lastError = e;
- await new Promise((r) => setTimeout(r, 200));
- }
- }
- throw timedOut(timeoutMs, lastError);
-}
-
-function timedOut(timeoutMs: number, lastError: unknown): Error {
- const detail = lastError instanceof Error ? ` (${lastError.message})` : "";
- return new Error(`${APP_NAME} did not answer within ${Math.round(timeoutMs / 1000)}s of launching${detail}`);
-}
-
-/** Like `connectWithRetry`, but also proves the server answers: a cold app binds the socket before its session is ready. */
export async function waitForApp(timeoutMs = 30000): Promise<void> {
const start = Date.now();
let lastError: unknown = null;
while (Date.now() - start < timeoutMs) {
try {
- await ping();
- return;
+ return await ping();
} catch (e) {
if (!isAppDown(e)) throw e;
lastError = e;
await new Promise((r) => setTimeout(r, 200));
}
}
- throw timedOut(timeoutMs, lastError);
+ const detail = lastError instanceof Error ? ` (${lastError.message})` : "";
+ throw new Error(`${APP_NAME} did not answer within ${Math.round(timeoutMs / 1000)}s of launching${detail}`);
}
diff --git a/apps/cli/src/help.ts b/apps/cli/src/help.ts
deleted file mode 100644
index e902d2f9..00000000
--- a/apps/cli/src/help.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// The wrappers' help text and validation come from the catalog, so
-// `dapi <cmd> --help` and the app's `tools/list` say the same thing because
-// they are the same string, and a bad argument fails here with the message
-// the app would have sent back.
-
-import { z } from "zod";
-import { toolByName } from "@diffusionstudio/dapi";
-
-import type { GenericTool, ToolName } from "@diffusionstudio/dapi";
-
-/** The tool's description, verbatim. */
-export function describe(name: ToolName): string {
- return toolByName(name).description;
-}
-
-/**
- * An input field's description, for the option that maps onto it. The
- * fallback covers fields the catalog leaves undescribed and options whose
- * meaning is the CLI's own (`--separate` inverts `combine`).
- */
-export function field(name: ToolName, key: string, fallback?: string): string {
- const tool: GenericTool = toolByName(name);
- const schema = tool.input.shape[key];
- if (schema === undefined && fallback === undefined) {
- throw new Error(`tool ${name} has no input field "${key}"`);
- }
- return schema?.description ?? fallback ?? "";
-}
-
-/**
- * Checks the input against the catalog before it leaves the process, so a
- * bad argument fails here with the message the app would have sent back.
- * Prints the issues and exits on failure.
- */
-export function validate(name: ToolName, input: Record<string, unknown>): void {
- const result = toolByName(name).input.safeParse(input);
- if (result.success) return;
- console.error(z.prettifyError(result.error));
- process.exit(1);
-}
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 2216fcdb..5ca06ffa 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -6,126 +6,71 @@
import { existsSync } from "node:fs";
import { isAbsolute, resolve } from "node:path";
import { Command } from "commander";
+import { z } from "zod";
import { version } from "../../../package.json";
-import { TIME_FPS } from "@diffusionstudio/jsx";
-import { ISSUE_LOG_TAIL, NonNegativeTime } from "@diffusionstudio/dapi";
+import { ISSUE_LOG_TAIL, toolByName } from "@diffusionstudio/dapi";
import { MCP_URL } from "@diffusionstudio/dapi/socket";
import { APP_NAME, call, isAppDown, launchApp, ping, waitForApp } from "./cli-client";
-import { describe, field, validate } from "./help";
import { runProxy } from "./mcp-proxy";
-import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi";
+import type { GenericTool, ToolInput, ToolName } from "@diffusionstudio/dapi";
function fail(message: string): never {
console.error(message);
process.exit(1);
}
-function handleSocketError(e: unknown): never {
+function appError(e: unknown): never {
if (isAppDown(e)) fail(`${APP_NAME} is not running. Launch the app first, then retry.`);
fail((e as Error).message);
}
+/** The tool's description, verbatim. */
+function describe(name: ToolName): string {
+ return toolByName(name).description;
+}
+
/**
- * One wrapper's whole job: check the input against the catalog, call the
- * tool, print the result the way the app returns it — its structured content,
- * as one JSON object, the same thing an agent receives. Everything the app
- * validates is sent as typed (strings stay strings: times like "45f" are
- * parsed by the schema on both sides).
+ * An input field's description, for the option that maps onto it. The
+ * fallback is for the shared window fields (`start`, `end`), which the
+ * catalog leaves for each wrapper to word.
*/
-async function run<N extends ToolName>(name: N, input: ToolInput<N>, spinner?: string): Promise<void> {
- await invoke(name, input, spinner);
+function field(name: ToolName, key: string, fallback?: string): string {
+ const tool: GenericTool = toolByName(name);
+ const schema = tool.input.shape[key];
+ if (schema === undefined && fallback === undefined) throw new Error(`tool ${name} has no input field "${key}"`);
+ return schema?.description ?? fallback ?? "";
}
-/** `run`, handing the result back for the one command that inspects it (`check`, for its exit code). */
-async function invoke<N extends ToolName>(name: N, input: ToolInput<N>, spinner?: string): Promise<ToolOutput<N>> {
- validate(name, input as Record<string, unknown>);
- const stop = spinner ? startSpinner(spinner) : () => {};
- try {
- const output = await call(name, input);
- stop();
- console.log(JSON.stringify(output));
- return output;
- } catch (e) {
- stop();
- handleSocketError(e);
- }
+/**
+ * Checks the input against the tool's schema, calls the tool, and prints
+ * what the app returns — its structured content, as one JSON object, the
+ * same thing an agent receives. Strings stay strings: times like "45f" are
+ * parsed by the schema on both sides.
+ */
+async function run<N extends ToolName>(name: N, input: ToolInput<N>): Promise<void> {
+ const parsed = toolByName(name).input.safeParse(input);
+ if (!parsed.success) fail(z.prettifyError(parsed.error));
+ const output = await call(name, input).catch(appError);
+ console.log(JSON.stringify(output));
}
-// Option parsers. Numbers are converted so the schema can check them as
-// numbers; an empty or non-numeric string becomes NaN, which the schema
-// rejects with its own message.
+// Numbers are converted so the schema can check them as numbers; an empty or
+// non-numeric string becomes NaN, which the schema rejects with its own message.
const numeric = (value: string): number => (value.trim() === "" ? NaN : Number(value));
-/** A time as the CLI takes it, in seconds, checked the way the catalog checks it. */
-function seconds(value: string, flag: string): number {
- const result = NonNegativeTime.safeParse(value);
- if (!result.success) fail(`${flag}: ${result.error.issues[0]?.message ?? `invalid time (got "${value}")`}`);
- return result.data;
-}
-
/**
* A local file (or frames folder) that exists is sent as its absolute path;
* anything else — a URL, or a library path (`b-roll/clip.mp4`) — is passed
* through for the app to resolve. Library paths need an open project.
*/
-function resolveAssetRef(ref: string): { path: string } {
- const absPath = isAbsolute(ref) ? ref : resolve(process.cwd(), ref);
- if (existsSync(absPath)) return { path: absPath };
- if (isAbsolute(ref)) fail(`File not found: ${absPath}`);
- return { path: ref };
-}
-
-/** An output path as the app needs it: absolute, or absent for the app's default. */
-function resolveOutput(path: string | undefined): string | undefined {
- return path === undefined ? undefined : resolve(process.cwd(), path);
-}
-
-function startSpinner(label: string): () => void {
- if (!process.stderr.isTTY) {
- process.stderr.write(`${label}…\n`);
- return () => {};
- }
- const frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
- const start = Date.now();
- let i = 0;
- const render = () => {
- const secs = Math.floor((Date.now() - start) / 1000);
- process.stderr.write(`\r${frames[i]} ${label}… ${secs}s`);
- i = (i + 1) % frames.length;
- };
- render();
- const timer = setInterval(render, 80);
- return () => {
- clearInterval(timer);
- process.stderr.write("\r\x1b[K"); // carriage return + clear to end of line
- };
-}
-
-// ---------------------------------------------------------------------------
-// Commands that are the CLI's own: launching, and the proxy.
-
-type OpenOptions = { background?: boolean };
-
-async function openProject(path: string | undefined, opts: OpenOptions): Promise<void> {
- // Launching is macOS's job; elsewhere (and when the app is not installed,
- // e.g. a dev checkout run from the terminal) fall through to the socket,
- // which answers if the app is running and errors usefully if not.
- const launched = await launchApp(opts.background ?? false);
- try {
- // A cold launch needs the renderer up before the app can answer; when
- // nothing was launched there is nothing to wait for, so fail fast.
- if (launched) await waitForApp();
- else await ping();
- } catch (e) {
- handleSocketError(e);
- }
- if (path !== undefined) await run("open", { dir: resolve(path) });
+function assetPath(ref: string): string {
+ const abs = resolve(ref);
+ if (existsSync(abs)) return abs;
+ if (isAbsolute(ref)) fail(`File not found: ${abs}`);
+ return ref;
}
-// ---------------------------------------------------------------------------
-// The program.
-
const program = new Command();
program
@@ -153,7 +98,11 @@ program
)
.argument("[path]", `${field("open", "dir")} (default: none — just launch the app)`)
.option("-b, --background", "launch or keep the app in the background, without raising a window")
- .action((path: string | undefined, opts: OpenOptions) => openProject(path, opts));
+ .action(async (path: string | undefined, opts: { background?: boolean }) => {
+ const launched = await launchApp(opts.background ?? false);
+ await (launched ? waitForApp() : ping()).catch(appError);
+ if (path !== undefined) await run("open", { dir: resolve(path) });
+ });
program
.command("context")
@@ -161,39 +110,33 @@ program
.description(describe("context"))
.action(() => run("context", {}));
-type CaptureOptions = { time?: string[]; output?: string; separate?: boolean; perSheet?: number };
-
program
.command("capture")
.description(describe("capture"))
.argument("<id>", field("capture", "id"))
- .option(
- "-t, --time <time...>",
- `one or more positions to capture, relative to the export's first frame, the workarea's start (0 = the export's frame 0) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`,
- )
- .option("-S, --separate", "write one PNG per position instead of merging them into contact sheets (combine: false)")
+ .option("-t, --times <time...>", field("capture", "times"))
+ .option("-S, --separate", field("capture", "separate"))
.option("--per-sheet <n>", field("capture", "perSheet"), numeric)
.option("-o, --output <dir>", field("capture", "output"))
- .action((id: string, opts: CaptureOptions) => {
- const frames = (opts.time ?? ["0"]).map((t) => Math.round(seconds(t, "--time") * TIME_FPS));
- return run("capture", { id, frames, combine: !opts.separate, perSheet: opts.perSheet, output: resolveOutput(opts.output) });
- });
+ .action((id: string, opts: Omit<ToolInput<"capture">, "id">) =>
+ run("capture", { id, ...opts, output: opts.output && resolve(opts.output) }),
+ );
program
.command("export")
.description(describe("export"))
.argument("<id>", field("export", "id"))
.argument("[output]", field("export", "path"))
- .action((id: string, output: string | undefined) => run("export", { id, path: resolveOutput(output) }, "Exporting scene"));
+ .action((id: string, output: string | undefined) => run("export", { id, path: output && resolve(output) }));
program
.command("check")
.description(`${describe("check")} Exits 1 when an error-severity issue is found.`)
.argument("<id>", field("check", "id"))
.action(async (id: string) => {
- const { issues } = await invoke("check", { id });
- // Linter convention: issues found is a different failure than "could not run".
- if (issues.some((issue) => issue.severity === "error")) process.exitCode = 1;
+ const output = await call("check", { id }).catch(appError);
+ console.log(JSON.stringify(output));
+ if (output.issues.some((issue) => issue.severity === "error")) process.exitCode = 1;
});
const media = program
@@ -207,71 +150,33 @@ media
.command("probe")
.description(describe("media_probe"))
.argument("<path>", field("media_probe", "path"))
- .action((ref: string) => run("media_probe", resolveAssetRef(ref), "Probing asset"));
+ .action((ref: string) => run("media_probe", { path: assetPath(ref) }));
media
.command("transcribe")
.description(describe("media_transcribe"))
.argument("<path>", field("media_transcribe", "path"))
- .action((ref: string) => run("media_transcribe", resolveAssetRef(ref), "Transcribing asset"));
-
-type GrabOptions = {
- time?: string[];
- count?: number;
- start?: string;
- end?: string;
- quality?: string;
- uncapped?: boolean;
- output?: string;
- auto?: boolean;
- separate?: boolean;
- perSheet?: number;
-};
+ .action((ref: string) => run("media_transcribe", { path: assetPath(ref) }));
media
.command("grab")
.alias("sample")
.description(describe("media_grab"))
.argument("<path>", field("media_grab", "path"))
- .option("-t, --time <time...>", field("media_grab", "times"))
+ .option("-t, --times <time...>", field("media_grab", "times"))
.option("-c, --count <n>", field("media_grab", "count"), numeric)
.option("-a, --auto", field("media_grab", "auto"))
.option("-s, --start <time>", field("media_grab", "start", `with --count or --auto, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`))
.option("-e, --end <time>", field("media_grab", "end", `with --count or --auto, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`))
.option("-q, --quality <preset>", field("media_grab", "quality"))
- .option("-S, --separate", "write one PNG per frame instead of merging them into contact sheets (combine: false)")
+ .option("-S, --separate", field("media_grab", "separate"))
.option("--per-sheet <n>", field("media_grab", "perSheet"), numeric)
.option("--uncapped", field("media_grab", "uncapped"))
.option("-o, --output <dir>", field("media_grab", "output"))
- .action((ref: string, opts: GrabOptions) =>
- run(
- "media_grab",
- {
- ...resolveAssetRef(ref),
- times: opts.time,
- count: opts.count,
- auto: opts.auto,
- start: opts.start,
- end: opts.end,
- quality: opts.quality as ToolInput<"media_grab">["quality"],
- combine: !opts.separate,
- perSheet: opts.perSheet,
- uncapped: opts.uncapped,
- output: resolveOutput(opts.output),
- },
- ),
+ .action((ref: string, opts: Omit<ToolInput<"media_grab">, "path">) =>
+ run("media_grab", { path: assetPath(ref), ...opts, output: opts.output && resolve(opts.output) }),
);
-type PreviewOptions = { start?: string; end?: string; scale?: number; output?: string };
-
-const previewInput = (ref: string, opts: PreviewOptions) => ({
- ...resolveAssetRef(ref),
- start: opts.start,
- end: opts.end,
- scale: opts.scale,
- output: resolveOutput(opts.output),
-});
-
media
.command("filmstrip")
.alias("film")
@@ -281,7 +186,9 @@ media
.option("-e, --end <time>", field("media_filmstrip", "end", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`))
.option("-x, --scale <factor>", field("media_filmstrip", "scale"), numeric)
.option("-o, --output <path>", field("media_filmstrip", "output"))
- .action((ref: string, opts: PreviewOptions) => run("media_filmstrip", previewInput(ref, opts), "Rendering filmstrip"));
+ .action((ref: string, opts: Omit<ToolInput<"media_filmstrip">, "path">) =>
+ run("media_filmstrip", { path: assetPath(ref), ...opts, output: opts.output && resolve(opts.output) }),
+ );
media
.command("waveform")
@@ -292,9 +199,9 @@ media
.option("-e, --end <time>", field("media_waveform", "end", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`))
.option("-x, --scale <factor>", field("media_waveform", "scale"), numeric)
.option("-o, --output <path>", field("media_waveform", "output"))
- .action((ref: string, opts: PreviewOptions) => run("media_waveform", previewInput(ref, opts), "Rendering waveform"));
-
-type ListenOptions = { prompt?: string; start?: string; end?: string; keepVideo?: boolean };
+ .action((ref: string, opts: Omit<ToolInput<"media_waveform">, "path">) =>
+ run("media_waveform", { path: assetPath(ref), ...opts, output: opts.output && resolve(opts.output) }),
+ );
media
.command("listen")
@@ -303,20 +210,13 @@ media
.option("-p, --prompt <str>", field("media_listen", "prompt"))
.option("-s, --start <time>", field("media_listen", "start"))
.option("-e, --end <time>", field("media_listen", "end"))
- .option("--keep-video", "for a video asset, keep the video track instead of stripping to audio, so the model also reads what is on screen (expensive: uploads the full video; stripVideo: false)")
- .action((ref: string, opts: ListenOptions) =>
- run(
- "media_listen",
- { ...resolveAssetRef(ref), prompt: opts.prompt, start: opts.start, end: opts.end, stripVideo: !opts.keepVideo },
- "Analyzing asset",
- ),
- );
+ .action((ref: string, opts: Omit<ToolInput<"media_listen">, "path">) => run("media_listen", { path: assetPath(ref), ...opts }));
program
.command("models")
.description(describe("models"))
.argument("[type]", field("models", "type"))
- .action((type: string | undefined) => run("models", { type: type as ToolInput<"models">["type"] }));
+ .action((type: ToolInput<"models">["type"]) => run("models", { type }));
program
.command("voices")
@@ -328,24 +228,18 @@ program
.description(describe("whoami"))
.action(() => run("whoami", {}));
-type LogsOptions = { tail?: number; level?: string };
-
program
.command("logs")
.description(describe("logs"))
.option("-n, --tail <n>", field("logs", "tail"), numeric)
.option("-l, --level <level>", `${field("logs", "level")}: "debug", "info", "warning", or "error"`)
- .action((opts: LogsOptions) => run("logs", { tail: opts.tail, level: opts.level as ToolInput<"logs">["level"] }));
-
-type ScreenshotOptions = { output?: string };
+ .action((opts: ToolInput<"logs">) => run("logs", opts));
program
.command("screenshot")
.description(describe("screenshot"))
.option("-o, --output <dir>", field("screenshot", "output"))
- .action((opts: ScreenshotOptions) => run("screenshot", { output: resolveOutput(opts.output) }));
-
-type ReportOptions = { body?: string; command?: string[]; logs?: number };
+ .action((opts: ToolInput<"screenshot">) => run("screenshot", { output: opts.output && resolve(opts.output) }));
program
.command("report")
@@ -353,24 +247,18 @@ program
.description(describe("report"))
.argument("<title>", field("report", "title"))
.option("-b, --body <text>", field("report", "body"))
- .option("-c, --command <cmd...>", `${field("report", "commands")}; repeatable`)
+ .option("-c, --commands <cmd...>", `${field("report", "commands")}; repeatable`)
.option("--logs <n>", field("report", "logs", `trailing app log entries to attach (0 to omit; default: ${ISSUE_LOG_TAIL})`), numeric)
- .action((title: string, opts: ReportOptions) => run("report", { title, body: opts.body, commands: opts.command, logs: opts.logs }));
-
-type FontsOptions = { family?: string; weight?: string[]; style?: string; limit?: number };
+ .action((title: string, opts: Omit<ToolInput<"report">, "title">) => run("report", { title, ...opts }));
program
.command("fonts")
.description(describe("fonts"))
.option("-f, --family <pattern>", field("fonts", "family"))
- .option("-w, --weight <weights...>", `${field("fonts", "weights")}, e.g. -w 400 700`)
+ .option("-w, --weights <weights...>", `${field("fonts", "weights")}, e.g. -w 400 700`)
.option("-s, --style <style>", `${field("fonts", "style")}: "normal" or "italic"`)
.option("-l, --limit <n>", field("fonts", "limit"), numeric)
- .action((opts: FontsOptions) =>
- run("fonts", { family: opts.family, weights: opts.weight, style: opts.style as ToolInput<"fonts">["style"], limit: opts.limit }),
- );
-
-type FetchOptions = { output?: string; format?: string; audio?: boolean };
+ .action((opts: ToolInput<"fonts">) => run("fonts", opts));
program
.command("fetch")
@@ -381,9 +269,9 @@ program
.option("-a, --audio", field("fetch", "audio"))
.allowExcessArguments()
.addHelpText("after", `\nForward raw yt-dlp flags after --, e.g. dapi fetch <url> -- --sponsorblock-remove all`)
- .action((url: string, opts: FetchOptions, cmd: Command) =>
+ .action((url: string, opts: Omit<ToolInput<"fetch">, "url" | "raw">, cmd: Command) =>
// `raw` is every operand after `url` — the yt-dlp passthrough placed after `--`.
- run("fetch", { url, format: opts.format, audio: opts.audio, raw: cmd.args.slice(1), output: resolveOutput(opts.output) }, "Downloading"),
+ run("fetch", { url, ...opts, raw: cmd.args.slice(1), output: opts.output && resolve(opts.output) }),
);
// Explicit argv convention: the packaged wrapper runs this bundle on
diff --git a/apps/cli/src/mcp-proxy.ts b/apps/cli/src/mcp-proxy.ts
index c89bf3da..92d99c68 100644
--- a/apps/cli/src/mcp-proxy.ts
+++ b/apps/cli/src/mcp-proxy.ts
@@ -8,7 +8,7 @@
// version. Stdout belongs to the protocol; anything for a human goes to
// stderr.
-import { APP_NAME, connectWithRetry, isAppDown, launchApp, openSocket } from "./cli-client";
+import { APP_NAME, isAppDown, launchApp, openSocket, waitForApp } from "./cli-client";
import type { Socket } from "node:net";
@@ -22,7 +22,8 @@ export async function runProxy(): Promise<void> {
if (!(await launchApp(true))) {
throw new Error(`${APP_NAME} is not running. Launch the app first, then retry.`);
}
- socket = await connectWithRetry();
+ await waitForApp();
+ socket = await openSocket();
}
socket.on("error", (error) => {
diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.ts
index 1c75de82..5f5115c8 100644
--- a/apps/desktop/src/dapi/present.test.ts
+++ b/apps/desktop/src/dapi/present.test.ts
@@ -16,7 +16,7 @@ const png = (byte: number, size = 8) => new Uint8Array(size).fill(byte);
describe("present", () => {
it("writes capture frames by timecode into the requested directory and returns their paths", async () => {
const out = join(dir, "frames");
- const presented = await present("capture", { id: "intro", combine: true, output: out }, [
+ const presented = await present("capture", { id: "intro", output: out }, [
{ timecode: "0f", png: png(1) },
{ timecode: "1s", png: png(2) },
]);
@@ -25,7 +25,7 @@ describe("present", () => {
});
it("picks a fresh temp directory when none is given", async () => {
- const presented = await present("media_grab", { path: "/c.mp4", combine: true }, [{ timecode: "0f", png: png(3) }]);
+ const presented = await present("media_grab", { path: "/c.mp4" }, [{ timecode: "0f", png: png(3) }]);
const { path } = (presented.output as { images: Array<{ path: string }> }).images[0]!;
expect(path).toMatch(/dapi-grab-.*[\\/]0f\.png$/);
rmSync(join(path, ".."), { recursive: true, force: true });
diff --git a/apps/web/src/dapi/handlers/capture.ts b/apps/web/src/dapi/handlers/capture.ts
index 13b1ee41..da4d32e8 100644
--- a/apps/web/src/dapi/handlers/capture.ts
+++ b/apps/web/src/dapi/handlers/capture.ts
@@ -3,6 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { createImageEncoder, decodePng } from "@diffusionstudio/encoder";
+import { TIME_FPS } from "@diffusionstudio/jsx";
import { DapiError } from "@diffusionstudio/dapi";
import { createCapture } from "@/engine/capture";
import { requireScene } from "../lib/scene";
@@ -13,12 +14,13 @@ import type { ToolHandler } from "../handler";
// Ceiling on the height a sheet cell renders a node at.
const SHEET_CAPTURE_HEIGHT = 1080;
-export const capture: ToolHandler<"capture"> = async ({ id, frames, combine, perSheet }, ctx) => {
+export const capture: ToolHandler<"capture"> = async ({ id, times, separate, perSheet }, ctx) => {
const { world, project } = ctx.requireSession();
const scene = requireScene(world, id, "capture");
- // `undefined` means the export's first frame (the workarea's start).
- const shots = frames && frames.length > 0 ? frames : [0];
+ // Positions arrive in seconds and the encoder samples frame numbers;
+ // none given means the export's first frame (the workarea's start).
+ const shots = times && times.length > 0 ? times.map((t) => Math.round(t * TIME_FPS)) : [0];
// The project re-rendered into a world of its own, reduced to this scene:
// the same arrangement an export runs against, and the encoder's to draw.
@@ -31,7 +33,7 @@ export const capture: ToolHandler<"capture"> = async ({ id, frames, combine, per
// scene is drawn, not decoded, so a small one is worth rendering past
// its own size; beyond SHEET_CAPTURE_HEIGHT that only costs tokens.
let sheets: SheetCollector | undefined;
- if (combine) {
+ if (!separate) {
const aspect = encoder.bounds.width / encoder.bounds.height;
const height = Math.max(encoder.bounds.height, SHEET_CAPTURE_HEIGHT);
sheets = new SheetCollector(shots.length, { width: height * aspect, height }, perSheet);
diff --git a/apps/web/src/dapi/handlers/media-grab.ts b/apps/web/src/dapi/handlers/media-grab.ts
index ed12e9fb..7212d4f1 100644
--- a/apps/web/src/dapi/handlers/media-grab.ts
+++ b/apps/web/src/dapi/handlers/media-grab.ts
@@ -27,7 +27,7 @@ const FRAME_QUALITY_BUDGETS: Record<FrameQuality, number> = {
const AUTO_MAX_FRAMES = 30;
export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {
- const { times, count, start, end, quality, auto, combine, perSheet } = args;
+ const { times, count, start, end, quality, auto, separate, perSheet } = args;
const asset = await resolveAsset(ctx, args.path);
requireAssetType(asset, ["VIDEO"], "a video");
@@ -53,7 +53,7 @@ export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {
requested = (times && times.length ? times : [0]).map((t) => resolveTime(t, asset.duration));
}
- const budget = FRAME_QUALITY_BUDGETS[quality ?? (combine ? "fullres" : "small")];
+ const budget = FRAME_QUALITY_BUDGETS[quality ?? (separate ? "small" : "fullres")];
const blob = await getAssetFile(asset);
const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
@@ -86,7 +86,7 @@ export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {
// Lay the sheets out up front: the largest cell across them sets the
// decode size, so no frame is decoded bigger than it will be drawn.
- const sheets = combine ? new SheetCollector(requested.length, { width: sourceWidth, height: sourceHeight }, perSheet) : undefined;
+ const sheets = separate ? undefined : new SheetCollector(requested.length, { width: sourceWidth, height: sourceHeight }, perSheet);
const width = sheets ? Math.min(sourceWidth, sheets.cellWidth) : sourceWidth;
// Decode in ascending order (the sink's fast path), remember each
@@ -96,7 +96,7 @@ export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {
// No pool: each yielded canvas is fresh, so encoding it can't race the
// generator's read-ahead reusing a pooled canvas.
const sink = new CanvasSink(track, width < displayWidth ? { width } : undefined);
- const separate: TimecodedImage[] = new Array(requested.length);
+ const frames: TimecodedImage[] = new Array(requested.length);
let i = 0;
for await (const wrapped of sink.canvasesAtTimestamps(ordered.map(({ time }) => firstTimestamp + time))) {
@@ -104,10 +104,10 @@ export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => {
if (!wrapped) throw new DapiError("not-found", `No frame found at ${time}s.`);
const timecode = formatTimecode(time, asset.frameRate);
if (sheets) await sheets.add(index, { at: time, timecode, image: wrapped.canvas });
- else separate[index] = { timecode, png: await encodePng(wrapped.canvas) };
+ else frames[index] = { timecode, png: await encodePng(wrapped.canvas) };
}
- return sheets ? sheets.result() : separate;
+ return sheets ? sheets.result() : frames;
} finally {
input.dispose();
}
diff --git a/apps/web/src/dapi/handlers/media-listen.ts b/apps/web/src/dapi/handlers/media-listen.ts
index 4a5cdd9d..eb3fc85a 100644
--- a/apps/web/src/dapi/handlers/media-listen.ts
+++ b/apps/web/src/dapi/handlers/media-listen.ts
@@ -9,14 +9,15 @@ import { requireAssetType, resolveAsset } from "../lib/assets";
import type { ToolHandler } from "../handler";
-export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end, stripVideo }, ctx) => {
+export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end }, ctx) => {
ctx.app.requireUser();
const asset = await resolveAsset(ctx, path);
requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");
const hasWindow = start !== undefined || end !== undefined;
- const audioOnly = stripVideo !== false && asset.type === "VIDEO";
- const contentType = asset.type === "VIDEO" && !audioOnly ? "video/mp4" : "audio/ogg";
+ // A video is stripped to its audio track: the model listens, it does not watch.
+ const audioOnly = asset.type === "VIDEO";
+ const contentType = "audio/ogg";
// One upload per distinct analysis input, so a repeated question about the
// same span reuses the transcode.
diff --git a/docs/mcp-server.md b/docs/mcp-server.md
index 7e0b37f1..37946dac 100644
--- a/docs/mcp-server.md
+++ b/docs/mcp-server.md
@@ -111,7 +111,7 @@ Tools, mapped from today's commands:
| `fetch` | `dapi fetch` | main | yt-dlp passthrough. Progress goes to MCP progress notifications instead of inherited stderr. |
| `report` | `dapi report` | main | Files the GitHub issue with diagnostics that main already holds. |
-Argument parsing that exists today only to turn strings into numbers (`--time "45f"`, `--count`, `--per-sheet`) becomes schema: times are accepted as the same strings and parsed by `parseTime` from `@diffusionstudio/jsx` inside the schema's transform, so the CLI and the agent get identical validation and identical error messages.
+Argument parsing that exists today only to turn strings into numbers (`--times "45f"`, `--count`, `--per-sheet`) becomes schema: times are accepted as the same strings and parsed by `parseTime` from `@diffusionstudio/jsx` inside the schema's transform, so the CLI and the agent get identical validation and identical error messages.
### 2. Main process: the server
diff --git a/knowledge/reference/jsx/timing.md b/knowledge/reference/jsx/timing.md
index b795f66a..92542d0b 100644
--- a/knowledge/reference/jsx/timing.md
+++ b/knowledge/reference/jsx/timing.md
@@ -62,4 +62,4 @@ The canonical internal unit is frames at **30 fps**; all formats are converted o
| `"MM:SS"` | `"02:30"` | Minutes and seconds. |
| `"HH:MM:SS"` | `"01:02:30"` | Hours, minutes, seconds. |
-CLI flags documented as taking a `Time` value (e.g. `--time` on [`capture`](../tools/capture.md) and [`media grab`](../tools/media/grab.md)) accept the same formats. Times in command **output** are plain seconds.
+CLI flags documented as taking a `Time` value (e.g. `--times` on [`capture`](../tools/capture.md) and [`media grab`](../tools/media/grab.md)) accept the same formats. Times in command **output** are plain seconds.
diff --git a/knowledge/reference/tools/capture.md b/knowledge/reference/tools/capture.md
index 862ec840..47a30876 100644
--- a/knowledge/reference/tools/capture.md
+++ b/knowledge/reference/tools/capture.md
@@ -1,4 +1,4 @@
-# `dapi capture <id> [-t, --time <time...>]`
+# `dapi capture <id> [-t, --times <time...>]`
Renders single frames of a scene at one or more timeline positions and merges them into **contact sheets**: up to 12 positions per PNG, each cell labelled with the timecode of the frame actually rendered and drawn as large as the sheet allows, so a few positions arrive as one high-resolution picture instead of a directory to open one by one. `--separate` writes a PNG per position instead.
@@ -9,7 +9,7 @@ Scenes only: a single element renders inside its scene, so capture the scene at
## Input
- `<id>`: scene id to capture (required) — the scene's `id` attribute in the project's JSX (e.g. `intro`), or its position in the file when it has none. When two files use the same id, the `file:id` form (`index.tsx:intro`) settles it. Entity numbers are not accepted; they change on every recompile.
-- `-t, --time <time...>`: one or more positions to capture, relative to the export's first frame — the workarea's start, so `0` is the export's frame 0 — each a `Time` value (optional; default `0`)
+- `-t, --times <time...>`: one or more positions to capture, relative to the export's first frame — the workarea's start, so `0` is the export's frame 0 — each a `Time` value (optional; default `0`)
- `-S, --separate`: write one PNG per position instead of merging them into contact sheets (optional). Each is rendered at 720p height and named after its timecode (e.g. `01s12f.png`).
- `--per-sheet <n>`: positions per contact sheet, 1 to 12 (optional; default as many as fit). Fewer positions per sheet means a larger cell each. Sheets are balanced, so 13 positions become 7 + 6 rather than 12 + 1.
- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-capture-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites images whose name matches; with `--separate`, requested times that land on the same frame share one file.
diff --git a/knowledge/reference/tools/check.md b/knowledge/reference/tools/check.md
index e09a3f0d..79ff5599 100644
--- a/knowledge/reference/tools/check.md
+++ b/knowledge/reference/tools/check.md
@@ -29,7 +29,7 @@ One JSON object:
severity: "error" | "warning";
message: string;
node?: string; // source stamp of the offending node; absent for subtree-wide issues
- ranges?: Array<{ start: number; end: number }>; // seconds relative to the node's start — for a scene whose workarea starts at 0, the clock `capture --time` uses
+ ranges?: Array<{ start: number; end: number }>; // seconds relative to the node's start — for a scene whose workarea starts at 0, the clock `capture --times` uses
}>;
}
```
diff --git a/knowledge/reference/tools/fonts.md b/knowledge/reference/tools/fonts.md
index 42736ebe..15470aab 100644
--- a/knowledge/reference/tools/fonts.md
+++ b/knowledge/reference/tools/fonts.md
@@ -5,7 +5,7 @@ Lists local fonts available on this machine. macOS only. Does not require the ap
## Options
- `-f, --family <pattern>`: filter to families whose name contains `<pattern>` (case-insensitive)
-- `-w, --weight <weights...>`: filter to variants with the given CSS weight(s), e.g. `-w 400 700`
+- `-w, --weights <weights...>`: filter to variants with the given CSS weight(s), e.g. `-w 400 700`
- `-s, --style <style>`: `"normal"` or `"italic"`
- `-l, --limit <n>`: output at most `<n>` families
diff --git a/knowledge/reference/tools/media/grab.md b/knowledge/reference/tools/media/grab.md
index f54734cc..8b204748 100644
--- a/knowledge/reference/tools/media/grab.md
+++ b/knowledge/reference/tools/media/grab.md
@@ -5,14 +5,14 @@ Decodes one or more frames of a video asset at the given times and merges them i
## Input
- `<path>`: a local video file to grab frames from in place without adding it to the library, or a project library path (required; library paths need an open project).
-- `-t, --time <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). A negative value is an offset back from the end of the clip, so `-1` is one second before the end and `-1f` one frame before it. Order is preserved in the output regardless of the order given. Mutually exclusive with `--count`.
-- `-c, --count <n>`: instead of `--time`, grab `n` frames evenly spaced across the clip at a fixed interval of `window / n`, starting at the window start (optional; positive integer).
+- `-t, --times <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). A negative value is an offset back from the end of the clip, so `-1` is one second before the end and `-1f` one frame before it. Order is preserved in the output regardless of the order given. Mutually exclusive with `--count`.
+- `-c, --count <n>`: instead of `--times`, grab `n` frames evenly spaced across the clip at a fixed interval of `window / n`, starting at the window start (optional; positive integer).
- `-s, --start <time>`: with `--count`, the start of the window to sample (optional; `Time` value; default `0`).
- `-e, --end <time>`: with `--count`, the end of the window to sample (optional; `Time` value; default the asset duration).
- `-q, --quality <preset>`: per-frame resolution preset (optional). One of `small` (384x384), `medium` (768x768), `large` (1536x1536), or `fullres` (native); each caps the total pixel count while preserving aspect ratio. Defaults to as much detail as the sheet cell can show, or `small` with `--separate`.
- `-S, --separate`: write one PNG per frame instead of merging them into contact sheets (optional). The frames keep their own resolution and alpha, and each file is named after its timecode (e.g. `01s12f.png`).
- `--per-sheet <n>`: frames per contact sheet, 1 to 12 (optional; default as many as fit). Fewer frames per sheet means a larger cell each. Sheets are balanced, so 13 frames become 7 + 6 rather than 12 + 1.
-- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--time`) is rejected.
+- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--times`) is rejected.
- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-grab-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites images whose name matches; with `--separate`, requested times that land on the same frame share one file.
## Timecodes
@@ -42,4 +42,4 @@ A sheet's timecode is the span it covers; a frame's is its own. Sheets come in t
## Errors
-Exits non-zero if the path can't be resolved, the asset is not a video, any `--time` is past the asset's duration, the `--count` window is empty, `--time` and `--count` are combined, `--start`/`--end` are given without `--count`, `--per-sheet` is outside 1 to 12, more than 100 frames are requested without `--uncapped`, or a PNG can't be written.
+Exits non-zero if the path can't be resolved, the asset is not a video, any `--times` value is past the asset's duration, the `--count` window is empty, `--times` and `--count` are combined, `--start`/`--end` are given without `--count`, `--per-sheet` is outside 1 to 12, more than 100 frames are requested without `--uncapped`, or a PNG can't be written.
diff --git a/knowledge/reference/tools/media/listen.md b/knowledge/reference/tools/media/listen.md
index 3fdfbde6..db5dd692 100644
--- a/knowledge/reference/tools/media/listen.md
+++ b/knowledge/reference/tools/media/listen.md
@@ -1,6 +1,6 @@
# `dapi media listen <path>`
-Puts a multimodal model in front of an audio track and returns its answer. With no prompt it returns a general description of what is heard; with `--prompt` it answers that question about the audio (e.g. "who is speaking?", "what music is playing?", "summarize what is said"). Accepts an audio file or a video, but only the audio track is analyzed by default. Alias: `watch`.
+Puts a multimodal model in front of an audio track and returns its answer. With no prompt it returns a general description of what is heard; with `--prompt` it answers that question about the audio (e.g. "who is speaking?", "what music is playing?", "summarize what is said"). Accepts an audio file or a video; of a video only the audio track is analyzed. Alias: `watch`.
## Input
@@ -8,7 +8,6 @@ Puts a multimodal model in front of an audio track and returns its answer. With
- `-p, --prompt <str>`: question or instruction about the audio (optional; defaults to a general description).
- `-s, --start <time>`: start of the segment to analyze, a `Time` value (optional; default `0`). Timestamps in the analysis are relative to this point.
- `-e, --end <time>`: end of the segment to analyze, a `Time` value (optional; default the asset's duration).
-- `--keep-video`: for a video asset, keep the video track instead of stripping it to audio, so the model can analyze what is on screen (optional; default off). Expensive: requires a full video upload.
## Output
diff --git a/knowledge/reference/tools/report.md b/knowledge/reference/tools/report.md
index bcbb4703..b74b415b 100644
--- a/knowledge/reference/tools/report.md
+++ b/knowledge/reference/tools/report.md
@@ -15,7 +15,7 @@ Does not require the app to be running. If the app is down or unreachable, the r
## Options
- `-b, --body <text>`: what happened, in markdown: what you expected, what you got, and anything the diagnostics won't show.
-- `-c, --command <cmd...>`: the `dapi` command(s) that reproduce it, in order. Repeatable (`-c "dapi context" -c "dapi capture intro"`); rendered as a shell block under `## Repro`.
+- `-c, --commands <cmd...>`: the `dapi` command(s) that reproduce it, in order. Repeatable (`-c "dapi context" -c "dapi capture intro"`); rendered as a shell block under `## Repro`.
- `--logs <n>`: trailing app log entries to attach (default: 50). `--logs 0` omits the section, and then the app is not contacted at all.
## Output
diff --git a/packages/dapi/src/schemas.ts b/packages/dapi/src/schemas.ts
index 71f8a24a..ae9e4103 100644
--- a/packages/dapi/src/schemas.ts
+++ b/packages/dapi/src/schemas.ts
@@ -60,10 +60,10 @@ export const outputDirField = z
* rejects it otherwise, so add it to any object that spreads these fields.
*/
export const sheetFields = {
- combine: z
+ separate: z
.boolean()
- .default(true)
- .describe("merge the images into contact sheets of up to 12 cells, each labelled with its timecode (default); false writes one image per position"),
+ .optional()
+ .describe("write one image per position instead of merging them into contact sheets of up to 12 cells, each labelled with its timecode"),
perSheet: z
.int()
.min(1)
@@ -73,14 +73,14 @@ export const sheetFields = {
};
export function checkSheetOptions(
- value: { combine: boolean; perSheet?: number | undefined },
+ value: { separate?: boolean | undefined; perSheet?: number | undefined },
ctx: z.RefinementCtx,
): void {
- if (value.perSheet !== undefined && !value.combine) {
+ if (value.perSheet !== undefined && value.separate) {
ctx.addIssue({
code: "custom",
path: ["perSheet"],
- message: "perSheet lays out contact sheets; it cannot be combined with combine: false",
+ message: "perSheet lays out contact sheets; it cannot be combined with separate: true",
});
}
}
diff --git a/packages/dapi/src/tools/capture.ts b/packages/dapi/src/tools/capture.ts
index 8ceca5c3..26b02d18 100644
--- a/packages/dapi/src/tools/capture.ts
+++ b/packages/dapi/src/tools/capture.ts
@@ -5,19 +5,22 @@
import { z } from "zod";
import { defineTool } from "../tool";
import { checkSheetOptions, ImageRef, outputDirField, SceneId, sheetFields, TimecodedImage } from "../schemas";
+import { NonNegativeTime } from "../time";
export const capture = defineTool({
name: "capture",
title: "Capture frames",
description:
- "Render single frames of a scene to PNGs — each frame is the frame an export of that scene would encode, drawn offscreen at the scene's own size. By default the positions are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and rendered as large as fits, so a few positions arrive as one high-resolution picture instead of a directory to open one by one (combine: false writes a PNG per position, at 720p height). The tool for checking composition (\"what plays at time T\": layout, overlaps, text, timing) and for verifying frames before an export. Scenes only — a single element renders inside its scene, so capture the scene at the times it plays. For a video asset's own full-resolution pixels use media_grab.",
+ "Render single frames of a scene to PNGs — each frame is the frame an export of that scene would encode, drawn offscreen at the scene's own size. By default the positions are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and rendered as large as fits, so a few positions arrive as one high-resolution picture instead of a directory to open one by one (separate: true writes a PNG per position, at 720p height). The tool for checking composition (\"what plays at time T\": layout, overlaps, text, timing) and for verifying frames before an export. Scenes only — a single element renders inside its scene, so capture the scene at the times it plays. For a video asset's own full-resolution pixels use media_grab.",
input: z
.object({
id: SceneId,
- frames: z
- .array(z.int().nonnegative())
+ times: z
+ .array(NonNegativeTime)
.optional()
- .describe("positions to capture as frame numbers relative to the export's first frame, the workarea's start (default: [0])"),
+ .describe(
+ "positions to capture, relative to the export's first frame, the workarea's start (0 = the export's frame 0) — seconds (\"1.5\"), frames (\"45f\"), or \"MM:SS\" (default: [0])",
+ ),
...sheetFields,
output: outputDirField,
})
diff --git a/packages/dapi/src/tools/media-grab.ts b/packages/dapi/src/tools/media-grab.ts
index 5dcbb544..9e9f8eac 100644
--- a/packages/dapi/src/tools/media-grab.ts
+++ b/packages/dapi/src/tools/media-grab.ts
@@ -16,7 +16,7 @@ export const mediaGrab = defineTool({
name: "media_grab",
title: "Grab frames",
description:
- "Decode frames of a video file and write them as PNGs (local render, no credits). By default the frames are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and drawn as large as fits, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one (combine: false writes a PNG per frame). Grabs the asset's own pixels, unlike capture which renders the composited node. The recommended tool for understanding a video at the frame level; past ~12 frames prefer media_filmstrip.",
+ "Decode frames of a video file and write them as PNGs (local render, no credits). By default the frames are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and drawn as large as fits, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one (separate: true writes a PNG per frame). Grabs the asset's own pixels, unlike capture which renders the composited node. The recommended tool for understanding a video at the frame level; past ~12 frames prefer media_filmstrip.",
input: z
.object({
path: AssetPath,
@@ -39,7 +39,7 @@ export const mediaGrab = defineTool({
),
...windowFields,
quality: FrameQuality.optional().describe(
- "frame resolution: small (384x384), medium (768x768), large (1536x1536), or fullres (native); default: as large as the sheet cell allows, or small with combine: false",
+ "frame resolution: small (384x384), medium (768x768), large (1536x1536), or fullres (native); default: as large as the sheet cell allows, or small with separate: true",
),
...sheetFields,
uncapped: z
diff --git a/packages/dapi/src/tools/media-listen.ts b/packages/dapi/src/tools/media-listen.ts
index c1d13e4d..f12c59e6 100644
--- a/packages/dapi/src/tools/media-listen.ts
+++ b/packages/dapi/src/tools/media-listen.ts
@@ -10,7 +10,7 @@ export const mediaListen = defineTool({
name: "media_listen",
title: "Listen to audio",
description:
- "Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; by default only the audio track is analyzed. Needs a signed-in account.",
+ "Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; of a video only the audio track is analyzed. Needs a signed-in account.",
input: z
.object({
path: AssetPath,
@@ -19,12 +19,6 @@ export const mediaListen = defineTool({
"start of the segment to analyze (default: 0); timestamps in the analysis are relative to this point",
),
end: windowFields.end.describe("end of the segment to analyze (default: media duration)"),
- stripVideo: z
- .boolean()
- .optional()
- .describe(
- "for a video asset, analyze the audio track only (default); false keeps the video so the model also reads what is on screen (expensive: uploads the full video)",
- ),
})
.superRefine(checkWindow),
output: z.object({
diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.ts
index 0879c0a6..b00d26d4 100644
--- a/packages/dapi/src/tools/tools.test.ts
+++ b/packages/dapi/src/tools/tools.test.ts
@@ -23,7 +23,7 @@ describe("media_grab", () => {
it("parses times in every form and applies the sheet default", () => {
const args = input.parse({ path: "/clip.mp4", times: ["45f", "1:10", -1, "-2f"] });
expect(args.times).toEqual([1.5, 70, -1, -2 / 30]);
- expect(args.combine).toBe(true);
+ expect(args.separate).toBeUndefined();
expect(args.perSheet).toBeUndefined();
});
@@ -58,21 +58,21 @@ describe("media_grab", () => {
});
it("rejects perSheet for separate images", () => {
- expect(issues(input.safeParse({ path: "/c.mp4", combine: false, perSheet: 4 }))).toHaveProperty("perSheet");
+ expect(issues(input.safeParse({ path: "/c.mp4", separate: true, perSheet: 4 }))).toHaveProperty("perSheet");
expect(issues(input.safeParse({ path: "/c.mp4", perSheet: 13 }))).toHaveProperty("perSheet");
expect(input.safeParse({ path: "/c.mp4", perSheet: 12 }).success).toBe(true);
});
});
describe("capture", () => {
- it("takes non-negative integer frames", () => {
- expect(capture.input.parse({ id: "intro", frames: [0, 45] }).frames).toEqual([0, 45]);
- expect(capture.input.safeParse({ id: "intro", frames: [1.5] }).success).toBe(false);
- expect(capture.input.safeParse({ id: "intro", frames: [-1] }).success).toBe(false);
+ it("takes non-negative times in every form", () => {
+ expect(capture.input.parse({ id: "intro", times: [0, "45f", "1:10"] }).times).toEqual([0, 1.5, 70]);
+ expect(capture.input.safeParse({ id: "intro", times: [-1] }).success).toBe(false);
+ expect(capture.input.safeParse({ id: "intro", times: ["-2f"] }).success).toBe(false);
});
it("shares the sheet rule with media_grab", () => {
- expect(issues(capture.input.safeParse({ id: "intro", combine: false, perSheet: 2 }))).toHaveProperty("perSheet");
+ expect(issues(capture.input.safeParse({ id: "intro", separate: true, perSheet: 2 }))).toHaveProperty("perSheet");
});
});