The VS Code of video editing, built for agents.
TypeScript
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { existsSync, realpathSync } from "node:fs";
import { mkdir, mkdtemp, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, dirname, join } from "node:path";
import { randomUUID } from "node:crypto";
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
import { generatedPath } from "@diffusionstudio/dapi";
import type { GenerationResult, TimecodedImage, ToolArgs, ToolName, ToolOutput, ToolResult } from "@diffusionstudio/dapi";
/** A file the tool wrote, kept in memory only long enough to decide whether to inline it. */
export type WrittenImage = { path: string; png: Uint8Array };
export type Presented = { output: unknown; images: WrittenImage[] };
/** More than this, or any image larger than INLINE_MAX_BYTES, and the caller gets paths only. */
const INLINE_MAX_IMAGES = 4;
const INLINE_MAX_BYTES = 1 << 20;
const APP_SLUG = "diffusion-studio";
/**
* The temp dir with its real spelling. On Windows `%TEMP%` is often set with short names
*/
function tempDir(): string {
const dir = tmpdir();
try {
return realpathSync.native(dir);
} catch {
return dir;
}
}
export async function present(name: ToolName, args: unknown, result: unknown): Promise<Presented> {
switch (name) {
case "capture":
return presentImages(result as ToolResult<"capture">, (args as ToolArgs<"capture">).output, "capture");
case "grab":
return presentImages(result as ToolResult<"grab">, (args as ToolArgs<"grab">).output, "grab");
case "filmstrip":
return presentPreview(result as ToolResult<"filmstrip">, (args as ToolArgs<"filmstrip">).output, "filmstrip");
case "waveform":
return presentPreview(result as ToolResult<"waveform">, (args as ToolArgs<"waveform">).output, "waveform");
case "screenshot":
return presentScreenshot(result as ToolResult<"screenshot">, (args as ToolArgs<"screenshot">).output);
case "transcribe":
return presentTranscript(result as ToolResult<"transcribe">, (args as ToolArgs<"transcribe">).output);
case "generate":
if ((args as ToolArgs<"generate">).estimate) return { output: result, images: [] };
return presentJob(result as GenerationResult);
case "job":
return presentJob(result as GenerationResult);
default:
return { output: result, images: [] };
}
}
// Frames and contact sheets arrive in the same shape: each image is stamped
// with its timecode (`08s10f`, or `0f-08s10f` for a sheet), which is the
// filename too.
async function presentImages(images: TimecodedImage[], output: string | undefined, kind: string): Promise<Presented> {
const dir = output ?? (await mkdtemp(join(tempDir(), `dapi-${kind}-`)));
await mkdir(dir, { recursive: true });
const written: WrittenImage[] = [];
const refs: ToolOutput<"capture">["images"] = [];
for (const { timecode, png } of images) {
const path = join(dir, `${timecode}.png`);
await writeFile(path, png);
written.push({ path, png });
refs.push({ timecode, path });
}
return { output: { images: refs }, images: written };
}
/**
* Where a single-file tool writes: `output` when given, a fresh name under
* the temp dir otherwise. An `output` that names an existing directory gets
* the fresh name inside it rather than an EISDIR from writeFile.
*/
async function singleFilePath(output: string | undefined, name: string): Promise<string> {
if (output === undefined) return join(tempDir(), name);
const existing = await stat(output).catch(() => null);
if (existing?.isDirectory()) return join(output, name);
await mkdir(dirname(output), { recursive: true });
return output;
}
async function presentPreview(
result: { png: Uint8Array } & Record<string, unknown>,
output: string | undefined,
kind: string,
): Promise<Presented> {
const { png, ...rest } = result;
const path = await singleFilePath(output, `dapi-${kind}-${randomUUID()}.png`);
await writeFile(path, png);
return { output: { path, ...rest }, images: [{ path, png }] };
}
async function presentScreenshot(result: ToolResult<"screenshot">, output: string | undefined): Promise<Presented> {
const dir = output ?? tempDir();
await mkdir(dir, { recursive: true });
const taken = new Date();
let attempt = 1;
let path = join(dir, screenshotFilename(taken, attempt));
while (existsSync(path)) path = join(dir, screenshotFilename(taken, ++attempt));
await writeFile(path, result.png);
const presented: ToolOutput<"screenshot"> = { path, width: result.width, height: result.height };
return { output: presented, images: [{ path, png: result.png }] };
}
async function presentTranscript(transcript: ToolResult<"transcribe">, output: string | undefined): Promise<Presented> {
const path = await singleFilePath(output, `dapi-transcript-${randomUUID()}.json`);
await writeFile(path, JSON.stringify(transcript, null, 2));
const words = transcript.segments.reduce((sum, segment) => sum + segment.words.length, 0);
const presented: ToolOutput<"transcribe"> = { path, segments: transcript.segments.length, words };
return { output: presented, images: [] };
}
/** Files of succeeded jobs saved to disk, by job id: the first poll that sees a job succeed saves them, the rest reuse the paths. */
const jobFiles = new Map<string, Promise<string[]>>();
/** Pictures of local jobs, by job id: each is written to the temp dir, and shown inline, the first poll it comes with. */
const jobImages = new Map<string, string>();
/**
* A job's files bound for disk are saved once it has succeeded — downloaded
* from their `url`, or written from the `bytes` a local model made — to
* `saveTo` (see `generatedPath`; a file already there is written over, as the
* other tools' outputs are) or under their own names into a directory of the
* job's own under the temp dir, and their `url` or `bytes` is swapped for the
* `path`. Files the renderer saved into the library arrive with their paths.
* A local job's picture goes to the temp dir, named in `details.image`.
*/
async function presentJob({ job, saveTo, image }: GenerationResult): Promise<Presented> {
const images: WrittenImage[] = [];
let shown = job;
if (image) {
let path = jobImages.get(job.id);
if (path === undefined) {
path = join(tempDir(), `dapi-${job.id}.png`);
await writeFile(path, image);
jobImages.set(job.id, path);
images.push({ path, png: image });
}
shown = { ...job, details: { ...job.details, image: path } };
}
const pending = job.assets.filter((file) => file.url !== undefined || file.bytes !== undefined);
if (job.status !== "succeeded" || pending.length === 0) return { output: shown, images };
let paths = jobFiles.get(job.id);
if (!paths) {
const temp = join(tempDir(), `dapi-generate-${job.id}`);
const targets = job.assets.map((file, i) =>
saveTo === null ? join(temp, basename(file.filename)) : generatedPath(saveTo, file.filename, i, job.assets.length),
);
paths = saveAll(job.assets, targets);
paths.catch(() => jobFiles.delete(job.id));
jobFiles.set(job.id, paths);
}
const written = await paths;
const assets = job.assets.map(({ url: _url, bytes: _bytes, ...file }, i) => ({ ...file, path: written[i] }));
return { output: { ...shown, assets }, images };
}
async function saveAll(files: GenerationResult["job"]["assets"], targets: string[]): Promise<string[]> {
for (const [i, file] of files.entries()) {
await mkdir(dirname(targets[i]!), { recursive: true });
await writeFile(targets[i]!, file.bytes ?? (await download(file.url!, file.filename)));
}
return targets;
}
async function download(url: string, filename: string): Promise<Uint8Array> {
const response = await fetch(url);
if (!response.ok) throw new Error(`Could not download ${filename} (${response.status})`);
return new Uint8Array(await response.arrayBuffer());
}
function screenshotFilename(taken: Date, attempt: number): string {
const pad = (value: number) => String(value).padStart(2, "0");
const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-");
const time = [pad(taken.getHours()), pad(taken.getMinutes()), pad(taken.getSeconds())].join("-");
return `${APP_SLUG}_${date}_${time}${attempt > 1 ? `-${attempt}` : ""}.png`;
}
/** The MCP result: the output as text and structured content, plus the images when they are few and small. */
export function toCallToolResult({ output, images }: Presented): CallToolResult {
const content: CallToolResult["content"] = [{ type: "text", text: JSON.stringify(output) }];
const inline = images.length <= INLINE_MAX_IMAGES && images.every((image) => image.png.byteLength <= INLINE_MAX_BYTES);
if (inline) {
for (const { png } of images) {
content.push({ type: "image", data: Buffer.from(png).toString("base64"), mimeType: "image/png" });
}
}
return { content, structuredContent: output as Record<string, unknown> };
}
/** A failure the agent reads as a sentence, not a protocol error. */
export function toErrorResult(error: unknown): CallToolResult {
return { isError: true, content: [{ type: "text", text: (error as Error).message }] };
}