Skip to content

The VS Code of video editing, built for agents.

TypeScript

Implement generate and job endpoints in the web API

konstantin-paulus6a0ef3c18850
diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts
index f5ef8ef7..e2fa27cc 100644
--- a/apps/cli/src/cli-client.ts
+++ b/apps/cli/src/cli-client.ts
@@ -12,13 +12,15 @@ import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi";
export const APP_NAME = "Diffusion Studio";
-// Renders and media analysis outlive the 60s default.
+// Renders, media analysis and file transfers outlive the 60s default.
const TIMEOUTS: Record<string, number> = {
export: 3_600_000,
capture: 600_000,
media_transcribe: 600_000,
media_listen: 600_000,
media_segment: 3_600_000,
+ generate: 600_000,
+ job: 600_000,
};
/**
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 3db03d84..1d256b7b 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -5,7 +5,7 @@
import { existsSync } from "node:fs";
import { isAbsolute, resolve } from "node:path";
-import { Command } from "commander";
+import { Argument, Command } from "commander";
import { z } from "zod";
import { version } from "../../../package.json";
import { MCP_URL, toolByName } from "@diffusionstudio/dapi";
@@ -66,6 +66,24 @@ function assetPath(ref: string): string {
return ref;
}
+/** Parsed JSON, or a failure naming what was being read. */
+function json(value: string, what: string): unknown {
+ try {
+ return JSON.parse(value);
+ } catch (e) {
+ fail(`${what} is not valid JSON: ${(e as Error).message}`);
+ }
+}
+
+/** Every `{ "path": … }` file reference in a request, resolved as `assetPath` does; the rest untouched. */
+function resolveFileRefs(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(resolveFileRefs);
+ if (typeof value !== "object" || value === null) return value;
+ const entries = Object.entries(value);
+ if (entries.length === 1 && entries[0]![0] === "path" && typeof entries[0]![1] === "string") return { path: assetPath(entries[0]![1]) };
+ return Object.fromEntries(entries.map(([key, item]) => [key, resolveFileRefs(item)]));
+}
+
/** `x,y` as a point, added to the ones given before; a malformed one becomes NaN for the schema to reject. */
const point = (value: string, previous: Array<{ x: number; y: number }> = []): Array<{ x: number; y: number }> => {
const [x = NaN, y = NaN, ...rest] = value.split(",").map(numeric);
@@ -79,11 +97,12 @@ const box = (value: string): [number, number, number, number] => {
};
/**
- * A mask's destination: a path spelled as one on disk (absolute, or starting
- * with `.`) resolves against the working directory; anything else is a
- * library path, passed through for the app to resolve in the open project.
+ * Where a mask or generated files go: a path spelled as one on disk
+ * (absolute, or starting with `.`) resolves against the working directory;
+ * anything else is a library path, passed through for the app to resolve in
+ * the open project.
*/
-function maskOutput(ref: string): string {
+function libraryOrDiskPath(ref: string): string {
return isAbsolute(ref) || ref.startsWith(".") ? resolve(ref) : ref;
}
@@ -249,7 +268,46 @@ media
)
.action((ref: string, opts: Omit<ToolInput<"media_segment">, "path"> & { point?: ToolInput<"media_segment">["points"] }) => {
const { point: points, ...rest } = opts;
- return run("media_segment", { path: assetPath(ref), ...rest, points, output: opts.output && maskOutput(opts.output) });
+ return run("media_segment", { path: assetPath(ref), ...rest, points, output: opts.output && libraryOrDiskPath(opts.output) });
+ });
+
+program
+ .command("generate")
+ .alias("gen")
+ .description(describe("generate"))
+ .addArgument(new Argument("<model>", field("generate", "model")).choices(toolByName("generate").input.shape.model.options))
+ .argument(
+ "[fields]",
+ `the model's other fields as one JSON object, e.g. '{"aspectRatio":"16:9","images":[{"path":"./ref.png"}]}'; a { "path" } that exists relative to the working directory is sent as its absolute path`,
+ )
+ .option("-p, --prompt <text>", "the request's `prompt`, without JSON quoting; wins over one in the fields")
+ .option(
+ "-o, --output <path>",
+ `${field("generate", "output")}; a path starting with . resolves against the working directory`,
+ )
+ .option("--max-credits <n>", field("generate", "maxCredits"), numeric)
+ .action((model: ToolInput<"generate">["model"], fields: string | undefined, opts: { prompt?: string; output?: string; maxCredits?: number }) => {
+ const parsed = fields === undefined ? {} : json(fields, "fields");
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) fail("fields must be a JSON object.");
+ const request = resolveFileRefs(parsed) as Record<string, unknown>;
+ return run("generate", {
+ ...request,
+ model,
+ ...(opts.prompt === undefined ? {} : { prompt: opts.prompt }),
+ output: opts.output && libraryOrDiskPath(opts.output),
+ maxCredits: opts.maxCredits,
+ });
+ });
+
+program
+ .command("job")
+ .description(`${describe("job")} Exits 1 when the job failed or was canceled.`)
+ .argument("<id>", field("job", "id"))
+ .option("--cancel", field("job", "cancel"))
+ .action(async (id: string, opts: { cancel?: boolean }) => {
+ const output = await call("job", { id, cancel: opts.cancel }).catch(appError);
+ console.log(JSON.stringify(output));
+ if (output.status === "failed" || output.status === "canceled") process.exitCode = 1;
});
program
diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.ts
index 840777fe..99163a33 100644
--- a/apps/desktop/src/dapi/present.test.ts
+++ b/apps/desktop/src/dapi/present.test.ts
@@ -2,10 +2,10 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, dirname, join } from "node:path";
-import { afterAll, describe, expect, it } from "vitest";
+import { afterAll, describe, expect, it, vi } from "vitest";
import { present, toCallToolResult } from "./present";
const dir = mkdtempSync(join(tmpdir(), "dapi-present-"));
@@ -134,3 +134,65 @@ describe("toCallToolResult", () => {
expect(toCallToolResult({ output: {}, images: large }).content).toHaveLength(1);
});
});
+
+describe("present a generation job", () => {
+ const job = (id: string, status: string, assets: Array<Record<string, unknown>>) => ({
+ job: { id, status, credits: 4, etaSeconds: 10, etaRemainingSeconds: null, error: null, assets },
+ saveTo: null as string | null,
+ });
+ const file = (filename: string) => ({ url: `https://files.test/${filename}`, filename, mimeType: "image/png", size: 8 });
+
+ it("passes a job still running through as the API has it", async () => {
+ const running = job("j-running", "running", []);
+ expect((await present("job", { id: "j-running" }, running)).output).toEqual(running.job);
+ });
+
+ it("downloads a succeeded job's files once, to the output numbered and with the made extension", async () => {
+ const fetch = vi.fn(async (url: string) => new Response(new Uint8Array([url.length])));
+ vi.stubGlobal("fetch", fetch);
+ try {
+ const out = join(dir, "generated", "fox.jpg");
+ const done = { ...job("j-done", "succeeded", [file("red-fox.png"), file("red-fox-2.png")]), saveTo: out };
+ const first = await present("job", { id: "j-done" }, done);
+ const again = await present("job", { id: "j-done" }, done);
+ const assets = (first.output as { assets: Array<Record<string, unknown>> }).assets;
+ expect(assets.map((asset) => asset.path)).toEqual([join(dir, "generated", "fox-1.png"), join(dir, "generated", "fox-2.png")]);
+ expect(assets[0]).not.toHaveProperty("url");
+ expect(assets[0]).toMatchObject({ filename: "red-fox.png", mimeType: "image/png" });
+ expect(again.output).toEqual(first.output);
+ expect(fetch).toHaveBeenCalledTimes(2);
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
+
+ it("writes a single file at the output, over a file already there", async () => {
+ vi.stubGlobal("fetch", async () => new Response("new"));
+ try {
+ const out = join(dir, "single.png");
+ writeFileSync(out, "old");
+ const presented = await present("job", { id: "j-one" }, { ...job("j-one", "succeeded", [file("x.png")]), saveTo: out });
+ expect((presented.output as { assets: Array<{ path: string }> }).assets[0]!.path).toBe(out);
+ expect(readFileSync(out, "utf8")).toBe("new");
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
+
+ it("saves into a directory of the job's own under the temp dir when no directory is given", async () => {
+ vi.stubGlobal("fetch", async () => new Response(new Uint8Array([1])));
+ try {
+ const presented = await present("generate", { model: "flux-2-klein" }, job("j-temp", "succeeded", [file("a.png")]));
+ const path = (presented.output as { assets: Array<{ path: string }> }).assets[0]!.path;
+ expect(path).toMatch(/dapi-generate-j-temp[\\/]a\.png$/);
+ rmSync(dirname(path), { recursive: true, force: true });
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
+
+ it("leaves files the renderer saved into the library as they are", async () => {
+ const saved = job("j-lib", "succeeded", [{ filename: "a.png", path: "/p/assets/a.png", src: "a.png" }]);
+ expect((await present("job", { id: "j-lib" }, saved)).output).toEqual(saved.job);
+ });
+});
diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.ts
index 7fab3c19..261635ad 100644
--- a/apps/desktop/src/dapi/present.ts
+++ b/apps/desktop/src/dapi/present.ts
@@ -5,11 +5,13 @@
import { existsSync, realpathSync } from "node:fs";
import { mkdir, mkdtemp, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
+import { basename, dirname, join } from "node:path";
import { randomUUID } from "node:crypto";
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
-import type { TimecodedImage, ToolArgs, ToolName, ToolOutput, ToolResult } from "@diffusionstudio/dapi";
+import { generatedPath } from "@diffusionstudio/dapi";
+
+import type { GenerationResult, TimecodedImage, ToolArgs, ToolName, ToolOutput, ToolResult } from "@diffusionstudio/dapi";
/** A file the tool wrote, kept in memory only long enough to decide whether to inline it. */
export type WrittenImage = { path: string; png: Uint8Array };
@@ -52,6 +54,9 @@ export async function present(name: ToolName, args: unknown, result: unknown): P
return presentSegment(result as ToolResult<"media_segment">, (args as ToolArgs<"media_segment">).output);
case "context":
return presentContext(result as ToolResult<"context">);
+ case "generate":
+ case "job":
+ return presentJob(result as GenerationResult);
default:
return { output: result, images: [] };
}
@@ -162,6 +167,45 @@ async function presentContext(result: ToolResult<"context">): Promise<Presented>
return { output: { ...result, masks }, images };
}
+/** Files of succeeded jobs saved to disk, by job id: the first poll that sees a job succeed downloads them, the rest reuse the paths. */
+const jobFiles = new Map<string, Promise<string[]>>();
+
+/**
+ * A job's files bound for disk are downloaded once it has succeeded, to
+ * `saveTo` (see `generatedPath`; a file already there is written over, as
+ * the other tools' outputs are) or under the API's names into a directory of
+ * the job's own under the temp dir, and their `url` is swapped for the
+ * `path`. Files the renderer saved into the library arrive with their paths.
+ */
+async function presentJob({ job, saveTo }: GenerationResult): Promise<Presented> {
+ const pending = job.assets.filter((file) => file.url !== undefined);
+ if (job.status !== "succeeded" || pending.length === 0) return { output: job, images: [] };
+
+ let paths = jobFiles.get(job.id);
+ if (!paths) {
+ const temp = join(tempDir(), `dapi-generate-${job.id}`);
+ const targets = job.assets.map((file, i) =>
+ saveTo === null ? join(temp, basename(file.filename)) : generatedPath(saveTo, file.filename, i, job.assets.length),
+ );
+ paths = downloadAll(job.assets, targets);
+ paths.catch(() => jobFiles.delete(job.id));
+ jobFiles.set(job.id, paths);
+ }
+ const written = await paths;
+ const assets = job.assets.map(({ url: _url, ...file }, i) => ({ ...file, path: written[i] }));
+ return { output: { ...job, assets }, images: [] };
+}
+
+async function downloadAll(files: GenerationResult["job"]["assets"], targets: string[]): Promise<string[]> {
+ for (const [i, file] of files.entries()) {
+ const response = await fetch(file.url!);
+ if (!response.ok) throw new Error(`Could not download ${file.filename} (${response.status})`);
+ await mkdir(dirname(targets[i]!), { recursive: true });
+ await writeFile(targets[i]!, new Uint8Array(await response.arrayBuffer()));
+ }
+ return targets;
+}
+
function screenshotFilename(taken: Date, attempt: number): string {
const pad = (value: number) => String(value).padStart(2, "0");
const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-");
diff --git a/apps/web/src/dapi/handlers/generate.ts b/apps/web/src/dapi/handlers/generate.ts
new file mode 100644
index 00000000..8a12b37e
--- /dev/null
+++ b/apps/web/src/dapi/handlers/generate.ts
@@ -0,0 +1,215 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * `generate` and `job`: the API's generation endpoints with a local layer
+ * and nothing more. A request goes out as given, except that `{ path }` file
+ * references are uploaded and swapped for the API's asset refs; the API
+ * validates the rest. A job comes back as the API has it, except that once it
+ * has succeeded its files are saved, here into the library, or by the server
+ * to a directory on disk (see `present.ts` in the desktop app).
+ */
+
+import { getAssetFile, getLibrary } from "@diffusionstudio/runtime";
+import { isAbsoluteSource, isProjectSource, isUrlSource } from "@diffusionstudio/assets";
+import { DapiError, generatedPath } from "@diffusionstudio/dapi";
+import { TRPCClientError } from "@trpc/client";
+
+import { api } from "@/lib/api";
+import { supabase } from "@/lib/supabase";
+import { uploadFile } from "@/lib/uploads";
+import { resolveAsset } from "../lib/assets";
+
+import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
+import type { AssetRef, GenerateRequestInput, Job } from "@diffusionstudio/api-contract";
+import type { DapiErrorCode, GenerationResult } from "@diffusionstudio/dapi";
+import type { ToolContext, ToolHandler } from "../handler";
+import type { EditorSession } from "../session";
+
+export const generate: ToolHandler<"generate"> = async ({ output, maxCredits, ...request }, ctx) => {
+ await requireSignIn();
+ const target = destination(output, ctx.session());
+
+ try {
+ const job = await api.generate.mutate({
+ request: (await withUploads(request, ctx)) as GenerateRequestInput,
+ idempotencyKey: crypto.randomUUID(),
+ maxCredits,
+ });
+
+ destinations.set(job.id, target);
+ return present(job, ctx);
+ } catch (error) {
+ if (!(error instanceof TRPCClientError)) throw error;
+ const code: string | undefined = error.data?.code;
+ const mapped = code && API_ERRORS[code];
+ if (mapped) throw new DapiError(mapped, error.message, { cause: error });
+ if (code === "PAYMENT_REQUIRED") throw new Error(`Not enough credits: ${error.message}`, { cause: error });
+ throw error;
+ }
+};
+
+export const job: ToolHandler<"job"> = async ({ id, cancel }, ctx) => {
+ await requireSignIn();
+
+ try {
+ let job: Job;
+ if (cancel) {
+ job = await api.jobs.cancel.mutate({ id });
+ } else {
+ job = await api.jobs.get.query({ id });
+ }
+
+ return present(job, ctx);
+ } catch (error) {
+ if (!(error instanceof TRPCClientError)) throw error;
+ const code: string | undefined = error.data?.code;
+ const mapped = code && API_ERRORS[code];
+ if (mapped) throw new DapiError(mapped, error.message, { cause: error });
+ if (code === "PAYMENT_REQUIRED") throw new Error(`Not enough credits: ${error.message}`, { cause: error });
+ throw error;
+ }
+};
+
+// ── Requests ─────────────────────────────────────────────────
+
+type FileRef = { path: string };
+
+const isPlainObject = (value: unknown): value is Record<string, unknown> => {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** `{ path }` and nothing else: a file the caller means, as opposed to a field of the API's. */
+const isFileRef = (value: unknown): value is FileRef => {
+ return isPlainObject(value) && typeof value.path === "string" && Object.keys(value).length === 1;
+}
+
+/** The request with every `{ path }` uploaded and replaced by the ref the API names it by; the rest untouched. */
+async function withUploads(value: unknown, ctx: ToolContext): Promise<unknown> {
+ if (isFileRef(value)) return upload(value.path, ctx);
+ if (Array.isArray(value)) return Promise.all(value.map((item) => withUploads(item, ctx)));
+ if (!isPlainObject(value)) return value;
+ const entries = await Promise.all(Object.entries(value).map(async ([key, item]) => [key, await withUploads(item, ctx)] as const));
+ return Object.fromEntries(entries);
+}
+
+async function upload(path: string, ctx: ToolContext): Promise<AssetRef> {
+ const asset = await resolveAsset(ctx, path);
+ return uploadFile(await getAssetFile(asset));
+}
+
+// ── Where the files go ───────────────────────────────────────
+
+/**
+ * Into the library, at `path` (see `generatedPath`) or at its root under the
+ * API's names when that is null; or to a path on disk the server writes
+ * (null: a fresh directory under the temp dir). Settled when the job starts,
+ * so a bad `output` fails before any credits are spent.
+ */
+type Destination = { kind: "library"; path: string | null } | { kind: "disk"; path: string | null };
+
+/** Where each job started here saves its files; a job started elsewhere (the prompt box, before a reload) gets the default. */
+const destinations = new Map<string, Destination>();
+
+function destination(output: string | undefined, session: EditorSession | null): Destination {
+ if (output === undefined) return { kind: session ? "library" : "disk", path: null };
+ if (isUrlSource(output)) throw new DapiError("invalid-input", `The result cannot be saved to a URL (${output}).`);
+ if (/[\\/]$/.test(output)) throw new DapiError("invalid-input", `The output is the file to save, not a folder (got "${output}").`);
+ if (isAbsoluteSource(output)) return { kind: "disk", path: output };
+ if (!session) {
+ throw new DapiError(
+ "no-project",
+ `"${output}" is a library path, which needs an open project — open one, or pass an absolute path to save the result elsewhere.`,
+ );
+ }
+ return { kind: "library", path: output.replace(/^\.\//, "") };
+}
+
+/** Library saves in flight or done, by job id: the first poll that sees a job succeed saves it, the rest wait for that. */
+const saves = new Map<string, Promise<Asset[]>>();
+
+/**
+ * The job, its files saved into the library when that is where they go; files
+ * bound for disk keep their `url` for the server. A library destination with
+ * no project open any more falls back to the temp dir.
+ */
+async function present(job: Job, ctx: ToolContext): Promise<GenerationResult> {
+ const session = ctx.session();
+ let target = destinations.get(job.id) ?? destination(undefined, session);
+ if (target.kind === "library" && !session) target = { kind: "disk", path: null };
+ if (job.status !== "succeeded" || target.kind === "disk") {
+ return { job, saveTo: target.kind === "disk" ? target.path : null };
+ }
+
+ const library = getLibrary(session!.world);
+ let saved = saves.get(job.id);
+ if (!saved) {
+ saved = saveToLibrary(library, job, target.path);
+ saved.catch(() => saves.delete(job.id));
+ saves.set(job.id, saved);
+ }
+ const assets = await saved;
+ const dir = session!.project.dir();
+ return {
+ job: {
+ ...job,
+ assets: job.assets.map(({ url: _url, ...file }, i) => ({ ...file, path: `${dir}/${assets[i]!.source}`, src: assets[i]!.path })),
+ },
+ saveTo: null,
+ };
+}
+
+/**
+ * Downloads each file into the library, noting the job it came from, as the
+ * prompt box does. Files the library already holds from this job (saved by
+ * the prompt box, or before a reload) are not downloaded again.
+ */
+async function saveToLibrary(library: AssetLibrary, job: Job, output: string | null): Promise<Asset[]> {
+ // Newest first; the files were stored in order.
+ const existing = library.list().filter((asset) => asset.job === job.id).reverse();
+ if (existing.length >= job.assets.length) return existing;
+
+ const assets: Asset[] = [];
+ for (const [i, file] of job.assets.entries()) {
+ const response = await fetch(file.url);
+ if (!response.ok) throw new Error(`Could not download ${file.filename} (${response.status})`);
+ const path = output === null ? file.filename : generatedPath(output, file.filename, i, job.assets.length);
+ const asset = await storeAt(library, path, await response.blob());
+ assets.push(library.update(asset, { job: job.id }));
+ }
+ return assets;
+}
+
+/**
+ * Puts `blob` into the library at `path`. A file the project made already
+ * there is written over and taken in again at the same path, so everything
+ * naming it shows the new one; a linked file of the user's is never written
+ * over, and the new one gets a free name next to it instead.
+ */
+async function storeAt(library: AssetLibrary, path: string, blob: Blob): Promise<Asset> {
+ const existing = library.get(path);
+ if (existing && isProjectSource(existing.source)) {
+ await library.fs.write(existing.source, blob);
+ return library.relink(existing, existing.source);
+ }
+ const slash = path.lastIndexOf("/");
+ return library.store(blob, { folder: slash < 0 ? "" : path.slice(0, slash), name: path.slice(slash + 1) });
+}
+
+// ── The API ──────────────────────────────────────────────────
+
+async function requireSignIn(): Promise<void> {
+ const session = await supabase?.auth.getSession();
+ if (!session?.data.session) {
+ throw new DapiError("sign-in-required", "Generating needs a Diffusion Studio account — sign in to the app first.");
+ }
+}
+
+const API_ERRORS: Record<string, DapiErrorCode> = {
+ BAD_REQUEST: "invalid-input",
+ UNAUTHORIZED: "sign-in-required",
+ NOT_FOUND: "not-found",
+ CONFLICT: "busy",
+};
+
diff --git a/apps/web/src/dapi/handlers/index.ts b/apps/web/src/dapi/handlers/index.ts
index 55437dd5..bbd33a7f 100644
--- a/apps/web/src/dapi/handlers/index.ts
+++ b/apps/web/src/dapi/handlers/index.ts
@@ -16,6 +16,7 @@ import { mediaFilmstrip } from "./media-filmstrip";
import { mediaWaveform } from "./media-waveform";
import { mediaListen } from "./media-listen";
import { mediaSegment } from "./media-segment";
+import { generate, job } from "./generate";
import type { Handlers } from "../handler";
@@ -35,4 +36,6 @@ export const handlers: Handlers = {
media_waveform: mediaWaveform,
media_listen: mediaListen,
media_segment: mediaSegment,
+ generate,
+ job,
};
diff --git a/docs/reference/jsx/media.md b/docs/reference/jsx/media.md
index c537ae29..bf7dc768 100644
--- a/docs/reference/jsx/media.md
+++ b/docs/reference/jsx/media.md
@@ -27,6 +27,8 @@ A folder of pictures has a count, not a duration, so `frameRate` is what says ho
A project's assets are recorded in `assets.yml` next to its entry file: for each asset, its library `path`, where its bytes are (`source`: the absolute path of a file imported from disk — imports never move or copy files — or a project-relative path under `assets/`, whether the app wrote the bytes there or a symlink points at them), and what it was found to be. Folders are listed too, so an empty one survives a reload. Renaming or moving an asset in the panel rewrites the `src` props that named it.
+A file a model generated — in the app's prompt box or with [`generate`](../tools/generate.md) — also records the `job` it came from. [`job`](../tools/job.md) with that id returns what was asked for in `request`: the model, the prompt and every other field (an input file appears there as the API's reference to its upload, not as a library path).
+
## Adding an asset
Symlink the file into the project's `assets/` folder and it is in the library. The app is watching, so it lands while the project is open; a project that is closed takes it in on the next open. Its library path is where the link sits under `assets/` — folder and name, so `assets/b-roll/drone.mp4` is `"b-roll/drone.mp4"` and `assets/logo.png` is `"logo.png"`.
diff --git a/docs/reference/models.md b/docs/reference/models.md
new file mode 100644
index 00000000..6a7f44fb
--- /dev/null
+++ b/docs/reference/models.md
@@ -0,0 +1,95 @@
+# Models
+
+Every model [`generate`](./tools/generate.md) runs, and the fields each takes. A request is `model` plus these fields; the API validates it and answers a field it cannot take with what to fix. Models of a kind share one set of fields (every image model takes `prompt`, `images`, `aspectRatio`, …), and each model narrows them: the values it offers are in its row. Leave a field out for the model's default. Files are `{ "path": … }` (see [generate](./tools/generate.md#files)).
+
+| Kind | Models | Makes |
+| --- | --- | --- |
+| [Image](#image) | `nano-banana-pro`, `gpt-image-2.5-sunburst`, `nano-banana-2`, `seedream-5.0-pro`, `flux-2-pro`, `grok-imagine-image-2.0`, `flux-2-klein`, `krea-2-large` | images from a prompt, or edits of reference images |
+| [Video](#video) | `veo-3.1`, `veo-3.1-fast`, `kling-3-pro`, `kling-o3-pro`, `seedance-2.5`, `wan-3.0`, `minimax-h3`, `flux-3-video`, `grok-imagine-video-1.5` | a clip from a prompt, optionally from a start (and end) frame |
+| [Audio](#audio) | `elevenlabs-music`, `elevenlabs-sfx` | music, sound effects |
+| [Voice](#voice) | `elevenlabs-v3`, `gemini-3.8-flash-tts` | speech |
+| [Tools](#tools) | `remove-background`, `upscale-image`, `upscale-video`, `transcribe`, `analyze` | one file in, one file out |
+
+## Image
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `prompt` | `string`, required | what to make, or how to change the reference images; up to 32,000 characters |
+| `images` | `{ path }[]` | reference images, or images to edit; up to the model's limit below |
+| `aspectRatio` | `string` | output width:height, from the model's row |
+| `resolution` | `"1K" \| "2K" \| "4K"` | long edge of about 1024, 2048 or 4096 px, from the model's row; higher costs more |
+| `count` | `integer` | images to make, 1–10 (default 1); each is its own file and is charged |
+
+| Model | | References | Aspect ratios | Resolutions |
+| --- | --- | --- | --- | --- |
+| `nano-banana-pro` | Nano Banana Pro — pro control, readable text, ultra consistent | 14 | 16:9, 9:16, 1:1, 4:3, 3:4 | 1K, 2K, 4K |
+| `gpt-image-2.5-sunburst` | GPT Image 2.5 Sunburst — precise detail, strong text, true edits | 16 | 16:9, 9:16, 1:1, 4:3, 3:4 | 1K |
+| `nano-banana-2` | Nano Banana 2 — fast, high quality, flexible edits | 14 | 16:9, 9:16, 1:1, 4:3, 3:4 | 1K, 2K, 4K |
+| `seedream-5.0-pro` | Seedream 5.0 Pro — lifelike scenes, precise multi-image edits | 14 | 16:9, 9:16, 1:1, 4:3, 3:4 | 1K, 2K |
+| `flux-2-pro` | FLUX.2 Pro — frontier quality, consistent references | 8 | 16:9, 9:16, 1:1, 4:3, 3:4 | 1K |
+| `grok-imagine-image-2.0` | Grok Imagine 2.0 — fast, vivid, photoreal | 3 | 16:9, 9:16, 1:1, 4:3, 3:4 | 1K, 2K |
+| `flux-2-klein` | FLUX.2 Klein — low budget, high quality, fast | 4 | 16:9, 9:16, 1:1, 4:3, 3:4 | 1K |
+| `krea-2-large` | Krea 2 Large — raw, textured, artistic looks | 1 | 16:9, 9:16, 1:1, 4:3 | 1K |
+
+The aspect ratios listed work on every model in its row. Some models take more (`2:3`, `3:2`, `4:5`, `5:4`, `21:9`, `9:21`, `1:2`, `2:1`, `1:4`, `4:1`, `1:8`, `8:1`, `9:19.5`, `19.5:9`, `9:20`, `20:9`), and the API says when one does not.
+
+## Video
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `prompt` | `string`, required | the shot: subject, action, camera, sound; up to 4,000 characters |
+| `startFrame` | `{ path }` | an image the clip starts on |
+| `endFrame` | `{ path }` | an image the clip ends on; only models with *end* in their row |
+| `duration` | `integer` | seconds, from the model's row |
+| `aspectRatio` | `string` | output width:height, from the model's row; a model without any takes its proportions from the start frame |
+| `resolution` | `"480p" \| "720p" \| "1080p" \| "2K"` | from the model's row; higher costs more |
+
+| Model | | Frames | Durations (s) | Aspect ratios | Resolutions |
+| --- | --- | --- | --- | --- | --- |
+| `veo-3.1` | Veo 3.1 — realistic physics, complex scenes | start, end | 4, 6, 8 | 16:9, 9:16 | 720p, 1080p (8 s only) |
+| `veo-3.1-fast` | Veo 3.1 Fast — the same engine, lower cost | start, end | 4, 6, 8 | 16:9, 9:16 | 720p, 1080p (8 s only) |
+| `kling-3-pro` | Kling 3.0 — cinematic motion with built-in audio | start, end | 3–15 | 16:9, 9:16, 1:1 | 720p |
+| `kling-o3-pro` | Kling 3.0 Omni — multi-modal reasoning, strong scenes | start | 3–15 | from the start frame | 1080p |
+| `seedance-2.5` | Seedance 2.5 — rich motion, lip-synced audio, up to 30 s | start, end | 4–30 | 16:9, 9:16, 1:1, 4:3, 3:4 | 480p, 720p |
+| `wan-3.0` | Wan 3.0 — expressive motion, clips up to 30 s | start | 2–30 | 16:9, 9:16, 1:1, 4:3, 3:4 | 480p, 720p, 1080p |
+| `minimax-h3` | MiniMax H3 — native 2K with built-in audio | start, end | 5–15 | 16:9, 9:16, 1:1, 4:3, 3:4 | 2K |
+| `flux-3-video` | FLUX.3 Video — sharp detail with native audio | start, end | 5–20 | 16:9, 9:16, 1:1, 4:3, 3:4 | 720p, 1080p |
+| `grok-imagine-video-1.5` | Grok Imagine Video 1.5 — quick clips from text or a frame | start | 1–15 | 16:9, 9:16, 1:1, 4:3, 3:4 | 480p, 720p, 1080p |
+
+Durations given as a range take any whole second in it.
+
+## Audio
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `prompt` | `string`, required | the music (genre, mood, instruments, structure, lyrics) or the sound; up to 4,100 characters |
+| `duration` | `number` | seconds, within the model's range |
+
+| Model | | Duration (s) |
+| --- | --- | --- |
+| `elevenlabs-music` | ElevenLabs Music — full compositions, stems, and lyrics | 3–600 (default 30) |
+| `elevenlabs-sfx` | ElevenLabs SFX — sound effects, seamless looping | up to 30; left out, the model picks the length |
+
+## Voice
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `prompt` | `string`, required | the text to speak; up to 5,000 characters. ElevenLabs v3 takes audio tags in brackets (`[whispers]`, `[laughs]`); Gemini takes a style instruction before the text (`Say cheerfully: …`) |
+| `voice` | `string`, required | one of the model's voices, from its voice list (linked below) |
+
+| Model | | Voices |
+| --- | --- | --- |
+| `elevenlabs-v3` | ElevenLabs v3 — expressive speech in many voices | [an ElevenLabs voice id](./voices/elevenlabs-v3.md) |
+| `gemini-3.8-flash-tts` | Gemini 3.8 Flash TTS — natural speech, styled in the prompt | [a Gemini voice name](./voices/gemini-3.8-flash-tts.md) |
+
+## Tools
+
+Each takes one file and returns one.
+
+| Model | Fields | Returns |
+| --- | --- | --- |
+| `remove-background` | `image`: `{ path }`, required | a PNG of the image with a transparent background |
+| `upscale-image` | `image`: `{ path }`, required | a PNG of the image upscaled 2× |
+| `upscale-video` | `video`: `{ path }`, required | an MP4 of the video upscaled to 2K at 30 fps |
+| `transcribe` | `audio`: `{ path }`, required (audio or video); `languageCode`: e.g. `en`, `en_us`, `de` (detected when left out) | `transcript.json`: `[{ text, speaker?, words: [{ text, start, end }] }]`, times in seconds, speakers labelled `A`, `B`, … when told apart |
+| `analyze` | `media`: `{ path }`, required (image, video or audio); `prompt`: what to look for, up to 10,000 characters (a general description when left out) | the analysis, as a file |
diff --git a/docs/reference/tools/README.md b/docs/reference/tools/README.md
index c623dcb5..c56ce76a 100644
--- a/docs/reference/tools/README.md
+++ b/docs/reference/tools/README.md
@@ -44,6 +44,8 @@ Every tool runs inside the app, so the app has to be running. Over MCP that is a
| [`media_waveform`](./media/waveform.md) | `diffusion media waveform` | Waveform preview |
| [`media_listen`](./media/listen.md) | `diffusion media listen` | Listen to audio |
| [`media_segment`](./media/segment.md) | `diffusion media segment` | Segment object |
+| [`generate`](./generate.md) | `diffusion generate` | Generate media |
+| [`job`](./job.md) | `diffusion job` | Generation job |
| [`logs`](./logs.md) | `diffusion logs` | App logs |
| [`screenshot`](./screenshot.md) | `diffusion screenshot` | Window screenshot |
| [`window`](./window.md) | `diffusion window` | App window |
@@ -55,6 +57,7 @@ How the surface is divided:
- **The project loop.** [`open`](./open.md) a folder, edit its JSX, [`context`](./context.md) for what the source cannot say, [`capture`](./capture.md) and [`check`](./check.md) to verify, [`export`](./export.md) when asked.
- **Media inspection** (`media_*`): a file by path, without adding it to the project. Absolute paths and URLs work with or without an open project; library paths (`b-roll/clip.mp4`) need one.
- **Masks.** [`media_segment`](./media/segment.md) segments and tracks an object in footage and writes the mask file a `<mask src>` names.
+- **Generation.** [`generate`](./generate.md) runs a model of the Diffusion Studio API (images, video, music, speech, and tools such as upscaling) on the account's credits; poll [`job`](./job.md) until it has ended, and its files are saved into the library or a directory. The models and their fields are in [models.md](../models.md).
- **What a declaration may name.** [`fonts`](./fonts.md).
- **The app and the machine.** [`logs`](./logs.md), [`screenshot`](./screenshot.md), [`window`](./window.md), [`report`](./report.md).
diff --git a/docs/reference/tools/generate.md b/docs/reference/tools/generate.md
new file mode 100644
index 00000000..f7740670
--- /dev/null
+++ b/docs/reference/tools/generate.md
@@ -0,0 +1,84 @@
+# generate
+
+Start a job on a generative model of the Diffusion Studio API — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis — and return the job at once, with its estimated run time and the credits it costs. Poll [`job`](./job.md) with its id until it has ended; the files it made are then saved and their paths returned. Generating needs a signed-in account and spends its credits.
+
+| | |
+| --- | --- |
+| MCP tool | `generate` |
+| CLI | `diffusion generate <model> [fields] [options]` |
+| CLI aliases | `diffusion gen` |
+
+## Input
+
+The input is the API's request — `model` plus that model's fields — with two fields of the tool's own. Every model and the fields it takes are listed in [models.md](../models.md).
+
+| Field | Type | CLI | Description |
+| --- | --- | --- | --- |
+| `model` | `string`, required | `<model>` | the model to run, one of the ids in [models.md](../models.md) |
+| *the model's fields* | | `[fields]`, `-p, --prompt <text>` | `prompt`, `images`, `aspectRatio`, `duration`, `voice`, … as the model takes them. On the CLI one JSON object; `--prompt` sets `prompt` without JSON quoting |
+| `output` | `string` | `-o, --output <path>` | file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: the library's root under the API's name with a project open, its files under `assets/`; else a fresh directory under the system temp dir). On the CLI a path starting with `.` resolves against the working directory. |
+| `maxCredits` | `number` | `--max-credits <n>` | refuse to start the job if it costs more credits than this |
+
+The model's fields are not checked here: they go to the API as given, and the API answers a request it cannot run with what to fix (`invalid-input`, e.g. an aspect ratio the model does not offer). Each model narrows its kind's bounds, so read its entry in [models.md](../models.md) rather than guessing.
+
+## Files
+
+Where a field takes a file (`images`, `startFrame`, `endFrame`, `image`, `video`, `audio`, `media`), put a `{ "path": … }` object: an absolute path or URL (with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project). The file is uploaded before the job starts — once per file, however many jobs name it — and the reference becomes the API's own. The API's references (`{ "kind": "asset", "id" }`, `{ "kind": "url", "url" }`) pass through unchanged. On the CLI a `path` that exists relative to the working directory is sent as its absolute path.
+
+```json
+{
+ "model": "nano-banana-pro",
+ "prompt": "The fox from the first image in the snowy forest of the second, golden hour",
+ "images": [{ "path": "/Users/me/refs/fox.png" }, { "path": "b-roll/forest.jpg" }],
+ "aspectRatio": "16:9",
+ "resolution": "2K"
+}
+```
+
+```bash
+diffusion generate flux-2-klein --prompt "A red fox at dawn, misty meadow, telephoto"
+diffusion generate nano-banana-pro --prompt "The fox in the forest" '{"images":[{"path":"./fox.png"},{"path":"./forest.jpg"}],"aspectRatio":"16:9"}'
+diffusion generate veo-3.1-fast --prompt "The fox turns and runs into the trees" '{"startFrame":{"path":"./fox.png"},"duration":6}' -o ./fox-run.mp4
+diffusion generate elevenlabs-v3 --prompt "Welcome back to the channel." '{"voice":"JBFqnCBsd6RMkjVDRZzb"}'
+diffusion generate remove-background '{"image":{"path":"./fox.png"}}'
+```
+
+## Where the files go
+
+`output` names the file the result is saved as, chosen when the job starts:
+
+- **The extension is the model's.** What a model makes decides the type, so an extension that does not match is replaced and a missing one is added: `fox.jpg` becomes `fox.png` when the model made a PNG, `clips/run` becomes `clips/run.mp4`. Read the returned `path`, not the one you asked for.
+- **Several files are numbered.** A job that makes more than one (an image `count` above 1) saves `fox-1.png`, `fox-2.png`, …; a single file is saved at the path itself.
+- **A library path** (`b-roll/fox.png`, needs an open project) puts the file into the project's library, under `assets/`, named by `src` like any other asset, and notes the job it came from, as the app's own generations do. A file the project made already at that path is replaced, and everything naming it shows the new one; a file linked from elsewhere on disk is never written over, and the result gets a free name next to it instead.
+- **An absolute path** saves the file there, replacing a file already there.
+- **Left out**, the file goes into the open project's library at its root under the API's name (`red-fox-at-dawn.png`, numbered if taken); with no project open, into a fresh directory of the job's own under the system temp dir, as [`media_transcribe`](./media/transcribe.md) and the other inspection tools do.
+
+The files are saved by the first [`job`](./job.md) call that sees the job succeed; later calls return the same paths. A file saved into the library records the job in `assets.yml` (its `job` field), so what made it can be looked up later.
+
+## Output
+
+The API's job, as it is when the call returns — almost always `queued` — and the same shape [`job`](./job.md) returns. Every field is the API's, passed through unchanged, except that each file in `assets` has its download `url` replaced by where it was saved. The fields to act on:
+
+```ts
+{
+ id: string; // for `job`
+ status: string; // queued, running → succeeded, failed or canceled; stop polling once it has ended
+ etaSeconds: number; // estimated seconds the model runs for
+ etaRemainingSeconds: number | null; // estimated seconds left while running
+ credits: number; // charged; refunded if the job fails or is canceled
+ error: { code: string; message: string } | null; // why it failed or was canceled
+ assets: Array<{ // the files it made, empty until it succeeds
+ path: string; // absolute path of the saved file
+ src?: string; // its library path, for `src`, when it went into the library
+ // …and the API's metadata: filename, mimeType, size, width/height, duration
+ }>;
+}
+```
+
+## Errors
+
+- `sign-in-required` — the app is not signed in to a Diffusion Studio account.
+- `invalid-input` — the API rejected the request; the message names the field and what it takes.
+- `no-project` — `output` or a file is a library path and no project is open.
+- `not-found` — a `{ path }` names no file.
+- Not enough credits for the job (or more than `maxCredits`): the message says so.
diff --git a/docs/reference/tools/job.md b/docs/reference/tools/job.md
new file mode 100644
index 00000000..7c092b42
--- /dev/null
+++ b/docs/reference/tools/job.md
@@ -0,0 +1,56 @@
+# job
+
+The state of a job started with [`generate`](./generate.md): its status, progress estimate and cost, and, once it has succeeded, the files it made, saved where `generate` was told. `cancel` stops a job and refunds its credits.
+
+| | |
+| --- | --- |
+| MCP tool | `job` |
+| CLI | `diffusion job <id>` |
+
+## Input
+
+| Field | Type | CLI | Description |
+| --- | --- | --- | --- |
+| `id` | `string`, required | `<id>` | job id, as `generate` returned it |
+| `cancel` | `boolean` | `--cancel` | cancel the job, refunding its credits |
+
+## Polling
+
+A job is `queued`, then `running`, then ends `succeeded`, `failed` or `canceled`. Once it has ended, nothing about it changes again: stop polling. While it works, the job says how long to wait — `etaRemainingSeconds` while running, `etaSeconds` (the whole run) while queued — so look once around then rather than in a tight loop; a call made early simply returns the job still working. Images take seconds, video minutes.
+
+The first call that sees the job succeed downloads its files to where [`generate`](./generate.md#where-the-files-go) was told and returns their paths in `assets`; later calls return the same paths without downloading again. A job whose files are in the open project's library already is never downloaded again: `assets` lists the files of it still there, so a lookup changes nothing. A job started with `generate` before the app restarted, with none of its files in the library, is saved to the default place: the open project's library, or the temp dir with none open.
+
+From a shell, `diffusion job` exits `1` once the job has failed or was canceled, and prints the job either way:
+
+```bash
+id=$(diffusion generate flux-2-klein --prompt "A red fox at dawn" | jq -r .id)
+sleep 15
+diffusion job "$id" | jq '{status, etaRemainingSeconds, assets: [.assets[].path]}'
+```
+
+## Looking up a generated file
+
+Every file the library holds from a model — made in the app's prompt box or with `generate` — records the job it came from in the project's `assets.yml`, as `job`. Pass that id here to see how it was made: `request` holds the model, the prompt and every other field, as the API took them. An input file appears there as the API's reference to its upload (`{ "kind": "asset", "id": <sha256> }`), not as a library path.
+
+```yaml
+# assets.yml
+assets:
+ - id: eb9b19faf784b692
+ path: red-fox-at-dawn.png
+ source: assets/red-fox-at-dawn.png
+ # …
+ job: b22bbf6f-4c94-4a5b-a4c7-98c696706906
+```
+
+```bash
+diffusion job b22bbf6f-4c94-4a5b-a4c7-98c696706906 | jq .request
+```
+
+## Output
+
+The API's job, with the files it made saved locally: each file's `url` is replaced by its absolute `path`, and by its library `src` when it went into the project's library. See [`generate`](./generate.md#output) for the fields to act on. A failed or canceled job says why in `error`, and its credits are refunded.
+
+## Errors
+
+- `not-found` — no job of this account has the id.
+- `sign-in-required` — the app is not signed in to a Diffusion Studio account.
diff --git a/docs/reference/voices/elevenlabs-v3.md b/docs/reference/voices/elevenlabs-v3.md
new file mode 100644
index 00000000..27dd476c
--- /dev/null
+++ b/docs/reference/voices/elevenlabs-v3.md
@@ -0,0 +1,33 @@
+# ElevenLabs v3 voices
+
+The voices [`elevenlabs-v3`](../models.md#voice) speaks in: pass the id as `voice`. The prompt is the text to speak, up to 5,000 characters; audio tags in brackets steer the delivery (`[whispers]`, `[laughs]`, `[excited]`).
+
+```json
+{ "model": "elevenlabs-v3", "voice": "JBFqnCBsd6RMkjVDRZzb", "prompt": "[softly] Welcome back. Let's pick up where we left off." }
+```
+
+| `voice` | Name | Sounds |
+| --- | --- | --- |
+| `CwhRBWXzGAHq8TQ4Fs17` | Roger | Laid-back, casual, resonant. |
+| `EXAVITQu4vr4xnSDxMaL` | Sarah | Mature, reassuring, confident. |
+| `FGY2WhTYpPnrIDTdsKH5` | Laura | Enthusiast, quirky attitude. |
+| `IKne3meq5aSn9XLyUdCD` | Charlie | Deep, confident, energetic. |
+| `JBFqnCBsd6RMkjVDRZzb` | George | Warm, captivating storyteller. |
+| `N2lVS1w4EtoT3dr4eOWO` | Callum | Husky trickster. |
+| `SAz9YHcvj6GT2YYXdXww` | River | Relaxed, neutral, informative. |
+| `SOYHLrjzK2X1ezoPC6cr` | Harry | Fierce warrior. |
+| `TX3LPaxmHKxFdv7VOQHJ` | Liam | Energetic, social media creator. |
+| `Xb7hH8MSUJpSbSDYk0k2` | Alice | Clear, engaging educator. |
+| `XrExE9yKIg1WjnnlVkGX` | Matilda | Knowledgeable, professional. |
+| `bIHbv24MWmeRgasZH58o` | Will | Relaxed optimist. |
+| `cgSgspJ2msm6clMCkdW9` | Jessica | Playful, bright, warm. |
+| `cjVigY5qzO86Huf0OWal` | Eric | Smooth, trustworthy. |
+| `hpp4J3VqNfWAUOO0d1Us` | Bella | Professional, bright, warm. |
+| `iP95p4xoKVk53GoZ742B` | Chris | Charming, down-to-earth. |
+| `nPczCjzI2devNBz1zQrb` | Brian | Deep, resonant, comforting. |
+| `onwK4e9ZLuTAKqWW03F9` | Daniel | Steady broadcaster. |
+| `pFZP5JQG7iQjIQuC4Bku` | Lily | Velvety actress. |
+| `pNInz6obpgDQGcFmaJgB` | Adam | Dominant, firm. |
+| `pqHfZKP75CvOlQylNhV4` | Bill | Wise, mature, balanced. |
+| `wBXNqKUATyqu0RtYt25i` | Adam | Natural, versatile narrator. |
+| `YtJ0uNlI0V9EhriZURew` | Lyle | Smooth, laid-back storyteller. |
diff --git a/docs/reference/voices/gemini-3.8-flash-tts.md b/docs/reference/voices/gemini-3.8-flash-tts.md
new file mode 100644
index 00000000..c6441bc9
--- /dev/null
+++ b/docs/reference/voices/gemini-3.8-flash-tts.md
@@ -0,0 +1,40 @@
+# Gemini 3.8 Flash TTS voices
+
+The voices [`gemini-3.8-flash-tts`](../models.md#voice) speaks in: pass the name as `voice`. The prompt is the text to speak, up to 5,000 characters; a style instruction before it steers the delivery (`Say cheerfully: …`).
+
+```json
+{ "model": "gemini-3.8-flash-tts", "voice": "Kore", "prompt": "Say calmly: Welcome back. Let's pick up where we left off." }
+```
+
+| `voice` | Sounds |
+| --- | --- |
+| `Zephyr` | Bright, light, sunny. |
+| `Puck` | Upbeat, cheeky, energetic. |
+| `Charon` | Informative, measured newsreader. |
+| `Kore` | Firm, confident, composed. |
+| `Fenrir` | Excitable, animated, eager. |
+| `Leda` | Youthful, fresh, expressive. |
+| `Orus` | Firm, grounded, decisive. |
+| `Aoede` | Breezy, airy, carefree. |
+| `Callirrhoe` | Easy-going, relaxed, warm. |
+| `Autonoe` | Bright, clear, upbeat narrator. |
+| `Enceladus` | Breathy, hushed, intimate. |
+| `Iapetus` | Clear, crisp, articulate. |
+| `Umbriel` | Easy-going, friendly, laid-back. |
+| `Algieba` | Smooth, deep, velvety. |
+| `Despina` | Smooth, soft, polished. |
+| `Erinome` | Clear, precise, poised. |
+| `Algenib` | Gravelly, deep, rugged. |
+| `Rasalgethi` | Informative, professional presenter. |
+| `Laomedeia` | Upbeat, cheerful, animated. |
+| `Achernar` | Soft, gentle, soothing. |
+| `Alnilam` | Firm, strong, authoritative. |
+| `Schedar` | Even, steady, balanced. |
+| `Gacrux` | Mature, seasoned, assured. |
+| `Pulcherrima` | Forward, bold, direct. |
+| `Achird` | Friendly, approachable, kind. |
+| `Zubenelgenubi` | Casual, conversational, chill. |
+| `Vindemiatrix` | Gentle, calm, reassuring. |
+| `Sadachbia` | Lively, spirited, dynamic. |
+| `Sadaltager` | Knowledgeable, thoughtful educator. |
+| `Sulafat` | Warm, caring, inviting. |
diff --git a/packages/dapi/package.json b/packages/dapi/package.json
index d7de5715..a90d0f11 100644
--- a/packages/dapi/package.json
+++ b/packages/dapi/package.json
@@ -1,7 +1,7 @@
{
"name": "@diffusionstudio/dapi",
"version": "0.1.0",
- "description": "The dapi tool catalog: every tool the Diffusion Studio app exposes to agents and the CLI, as a name, a description, and zod input/output schemas. The one definition the app serves, the CLI derives its commands from, and the renderer validates against. Depends on zod and the Time parser only.",
+ "description": "The dapi tool catalog: every tool the Diffusion Studio app exposes to agents and the CLI, as a name, a description, and zod input/output schemas. The one definition the app serves, the CLI derives its commands from, and the renderer validates against. Depends on zod, the Time parser, and the API contract (for its model ids).",
"license": "MPL-2.0",
"type": "module",
"sideEffects": false,
@@ -18,6 +18,7 @@
"zod"
],
"dependencies": {
+ "@diffusionstudio/api-contract": "^0.3.0",
"@diffusionstudio/jsx": "*",
"zod": "^4.4.3"
},
diff --git a/packages/dapi/src/catalog.ts b/packages/dapi/src/catalog.ts
index 081cc01e..c41ea563 100644
--- a/packages/dapi/src/catalog.ts
+++ b/packages/dapi/src/catalog.ts
@@ -19,14 +19,15 @@ import { mediaFilmstrip } from "./tools/media-filmstrip";
import { mediaWaveform } from "./tools/media-waveform";
import { mediaListen } from "./tools/media-listen";
import { mediaSegment } from "./tools/media-segment";
+import { generate, job } from "./tools/generate";
import { fonts } from "./tools/fonts";
import { report } from "./tools/report";
import { appWindow } from "./tools/window";
/**
* Every tool, in the order a listing shows them: the project loop first
- * (open, look, capture, check, export), then media inspection, then the
- * app and machine utilities.
+ * (open, look, capture, check, export), then media inspection, then
+ * generation, then the app and machine utilities.
*/
export const catalog = [
open,
@@ -41,6 +42,8 @@ export const catalog = [
mediaWaveform,
mediaListen,
mediaSegment,
+ generate,
+ job,
logs,
screenshot,
appWindow,
diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.ts
index aa447f78..1c87b585 100644
--- a/packages/dapi/src/index.ts
+++ b/packages/dapi/src/index.ts
@@ -29,6 +29,7 @@ export { SEGMENT_MODELS, WEAK_IOU } from "./tools/media-segment";
export { ISSUE_LOG_TAIL } from "./tools/report";
export { LOG_TAIL, LOG_MESSAGE_MAX } from "./tools/logs";
export { FONT_LIMIT } from "./tools/fonts";
+export { generatedPath } from "./tools/generate";
// Named request and result types, for handlers that spell out their
// signature. Each is the parsed (output) side of the tool's schema.
@@ -80,5 +81,9 @@ export type MediaListenRequest = ToolArgs<"media_listen">;
export type MediaListenResult = ToolResult<"media_listen">;
export type MediaSegmentRequest = ToolArgs<"media_segment">;
export type MediaSegmentResult = ToolResult<"media_segment">;
+export type GenerateRequest = ToolArgs<"generate">;
+export type GenerationResult = ToolResult<"generate">;
+export type GenerationJob = ToolOutput<"generate">;
+export type JobRequest = ToolArgs<"job">;
export type FontsRequest = ToolArgs<"fonts">;
export type ReportRequest = ToolArgs<"report">;
diff --git a/packages/dapi/src/tools/generate.ts b/packages/dapi/src/tools/generate.ts
new file mode 100644
index 00000000..3a61e700
--- /dev/null
+++ b/packages/dapi/src/tools/generate.ts
@@ -0,0 +1,97 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Generation on the Diffusion Studio API, nearly as the API has it: a request
+// is the API's own (`model` plus that model's fields), passed through for the
+// API to validate, and a job is the API's `Job`. What the tools add is what
+// only a local client can do: send local files, and save what a job made.
+
+import { z } from "zod";
+import { MODEL_IDS } from "@diffusionstudio/api-contract";
+import { defineTool } from "../tool";
+
+const JPEG = /^\.jpe?g$/i;
+
+/**
+ * Where file `index` of the `count` a job made is saved when the caller named
+ * the output `output`: that path, with the extension of what the model made
+ * (`filename`'s) in place of a different one or added when there is none, and
+ * numbered (`-1`, `-2`, …) when the job made several. Either separator works,
+ * so the renderer and the server agree on library and OS paths alike.
+ */
+export function generatedPath(output: string, filename: string, index: number, count: number): string {
+ const slash = Math.max(output.lastIndexOf("/"), output.lastIndexOf("\\"));
+ const base = output.slice(slash + 1);
+ const dot = base.lastIndexOf(".");
+ const stem = dot > 0 ? base.slice(0, dot) : base;
+ const given = dot > 0 ? base.slice(dot) : "";
+ const made = /\.[^./\\]+$/.exec(filename)?.[0] ?? given;
+ const same = given.toLowerCase() === made.toLowerCase() || (JPEG.test(given) && JPEG.test(made));
+ return `${output.slice(0, slash + 1)}${stem}${count > 1 ? `-${index + 1}` : ""}${same ? given : made}`;
+}
+
+/** A file a job made, once it is on this machine; the API's file with its `url` swapped for where it was saved. */
+export const GeneratedFile = z.looseObject({
+ path: z.string().describe("absolute path of the file"),
+ src: z.string().optional().describe("the file's library path, for `src`; present when it went into the project's library"),
+});
+
+/**
+ * The API's `Job`, its files saved locally. Only what the tools rely on is
+ * spelled out; every other field is the API's, passed through as it is.
+ */
+export const GenerationJob = z.looseObject({
+ id: z.string().describe("job id, for `job`"),
+ status: z.string().describe("queued or running while the job works; succeeded, failed or canceled once it has ended — stop polling then"),
+ assets: z.array(GeneratedFile).describe("the files the job made, saved: empty until it succeeds"),
+});
+
+/**
+ * What the renderer hands back: the job as the API has it, each file either
+ * saved into the library already (`path`, `src`) or still at its `url`, for
+ * the server to download to `saveTo` (see `generatedPath`; null: a fresh
+ * directory under the system temp dir, under the API's names).
+ */
+export const GenerationResult = z.object({
+ job: z.looseObject({
+ id: z.string(),
+ status: z.string(),
+ assets: z.array(z.looseObject({ url: z.string().optional(), path: z.string().optional(), filename: z.string() })),
+ }),
+ saveTo: z.string().nullable(),
+});
+
+export const generate = defineTool({
+ name: "generate",
+ title: "Generate media",
+ description:
+ "Start a job on a generative model of the Diffusion Studio API — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis — and return the job at once, with its estimated run time and the credits it costs. The input is the API's request: `model` plus that model's fields (`prompt`, `images`, `aspectRatio`, `duration`, `voice`, …), listed per model in the docs at `reference/models.md`. The fields are passed through as given and the API validates them, answering with what to fix. Where a field takes a file, put `{ \"path\": \"…\" }` — an absolute path, a URL, or a library path — and it is uploaded first. Poll `job` with the returned id until its status is succeeded, failed or canceled; the estimate says when to look. Generating needs a signed-in account and spends its credits.",
+ input: z.looseObject({
+ model: z.enum(MODEL_IDS).describe("the model to run; its fields are listed in `reference/models.md`"),
+ output: z
+ .string()
+ .optional()
+ .describe(
+ "file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: the library's root under the API's name with a project open, its files under `assets/`; else a fresh directory under the system temp dir)",
+ ),
+ maxCredits: z.number().positive().optional().describe("refuse to start the job if it costs more credits than this"),
+ }),
+ output: GenerationJob,
+ result: GenerationResult,
+ environment: "renderer",
+});
+
+export const job = defineTool({
+ name: "job",
+ title: "Generation job",
+ description:
+ "The state of a job started with `generate`: its status, progress estimate and cost, and, once it has succeeded, the files it made, saved where `generate` was told (the first call to see it succeed saves them; later calls return the same paths). Poll until the status is succeeded, failed or canceled, waiting about `etaRemainingSeconds` (or `etaSeconds` while queued) between calls. `cancel` stops a job and refunds its credits.",
+ input: z.object({
+ id: z.string().min(1).describe("job id, as `generate` returned it"),
+ cancel: z.boolean().optional().describe("cancel the job, refunding its credits"),
+ }),
+ output: GenerationJob,
+ result: GenerationResult,
+ environment: "renderer",
+});
diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.ts
index 4540b95c..cab9840f 100644
--- a/packages/dapi/src/tools/tools.test.ts
+++ b/packages/dapi/src/tools/tools.test.ts
@@ -12,6 +12,7 @@ import { mediaGrab } from "./media-grab";
import { mediaListen } from "./media-listen";
import { mediaSegment } from "./media-segment";
import { mediaTranscribe } from "./media-transcribe";
+import { generate, generatedPath } from "./generate";
/** The messages of a failed parse, keyed by the path they point at. */
function issues(result: { success: boolean; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } }) {
@@ -191,3 +192,38 @@ describe("media_segment", () => {
expect(input.safeParse({ ...at, box: [0, 0, 1, 1], model: "huge" }).success).toBe(false);
});
});
+
+describe("generate", () => {
+ const input = generate.input;
+
+ it("passes the model's fields through unchecked, for the API to validate", () => {
+ const fields = { prompt: "a fox", images: [{ path: "./ref.png" }, { kind: "url", url: "https://x.test/a.png" }], duration: 99, anything: { at: "all" } };
+ expect(input.parse({ model: "nano-banana-pro", ...fields })).toEqual({ model: "nano-banana-pro", ...fields });
+ });
+
+ it("knows the models", () => {
+ expect(issues(input.safeParse({ model: "dall-e-1", prompt: "a fox" }))).toHaveProperty("model");
+ });
+
+ it("lets a job's fields beyond the documented ones through to the caller", () => {
+ const job = { id: "j", model: "flux-2-klein", status: "succeeded", etaSeconds: 9, etaRemainingSeconds: null, credits: 2, error: null, createdAt: "2026-10-06T00:00:00Z", assets: [{ path: "/a.png", filename: "a.png", width: 1024 }] };
+ expect(generate.output.parse(job)).toEqual(job);
+ });
+});
+
+describe("generatedPath", () => {
+ it("saves a single file at the output, keeping an extension that matches", () => {
+ expect(generatedPath("b-roll/fox.png", "red-fox.png", 0, 1)).toBe("b-roll/fox.png");
+ expect(generatedPath("/out/fox.JPEG", "red-fox.jpg", 0, 1)).toBe("/out/fox.JPEG");
+ });
+
+ it("corrects or adds the extension to what the model made", () => {
+ expect(generatedPath("/out/fox.jpg", "red-fox.png", 0, 1)).toBe("/out/fox.png");
+ expect(generatedPath("clips/run", "fox-run.mp4", 0, 1)).toBe("clips/run.mp4");
+ expect(generatedPath("C:\\out\\v1.2\\fox", "a.webp", 0, 1)).toBe("C:\\out\\v1.2\\fox.webp");
+ });
+
+ it("numbers the files of a job that made several", () => {
+ expect([0, 1, 2].map((i) => generatedPath("fox.png", "a.png", i, 3))).toEqual(["fox-1.png", "fox-2.png", "fox-3.png"]);
+ });
+});