The VS Code of video editing, built for agents.
TypeScript
Implement estimate option in generate tool for cost and time prediction
konstantin-paulusfe3f1d68e76d
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 10504359..a62111f4 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -252,7 +252,8 @@ program`${field("generate", "output")}; a path starting with . resolves against the working directory`,).option("--max-credits <n>", field("generate", "maxCredits"), numeric)- .action((model: ToolInput<"generate">["model"], fields: string | undefined, opts: { prompt?: string; output?: string; maxCredits?: number }) => {+ .option("--estimate", field("generate", "estimate"))+ .action((model: ToolInput<"generate">["model"], fields: string | undefined, opts: { prompt?: string; output?: string; maxCredits?: number; estimate?: boolean }) => {const parsed = fields === undefined ? {} : json(fields, "fields");if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) fail("fields must be a JSON object.");const request = resolveFileRefs(parsed) as Record<string, unknown>;@@ -262,6 +263,7 @@ program...(opts.prompt === undefined ? {} : { prompt: opts.prompt }),output: opts.output && libraryOrDiskPath(opts.output),maxCredits: opts.maxCredits,+ estimate: opts.estimate,});});diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.tsindex d8b3d0c4..daad6cbf 100644--- a/apps/desktop/src/dapi/present.ts+++ b/apps/desktop/src/dapi/present.ts@@ -51,6 +51,8 @@ export async function present(name: ToolName, args: unknown, result: unknown): Pcase "media_transcribe":return presentTranscript(result as ToolResult<"media_transcribe">, (args as ToolArgs<"media_transcribe">).output);case "generate":+ if ((args as ToolArgs<"generate">).estimate) return { output: result, images: [] };+ return presentJob(result as GenerationResult);case "job":return presentJob(result as GenerationResult);default:diff --git a/apps/web/src/dapi/handlers/generate.ts b/apps/web/src/dapi/handlers/generate.tsindex f3110904..282d4f64 100644--- a/apps/web/src/dapi/handlers/generate.ts+++ b/apps/web/src/dapi/handlers/generate.ts@@ -31,8 +31,18 @@ export const generate: ToolHandler<"generate"> = async (args, ctx) => {return startLocalJob(args, destination(args.output, ctx.session()), ctx);}- const { output, maxCredits, ...request } = args;+ const { output, maxCredits, estimate, ...request } = args;await requireSignIn();++ if (estimate) {+ try {+ const { credits, etaSeconds } = await api.estimate.query((await withUploads(request, ctx)) as GenerateRequestInput);+ return { credits, etaSeconds };+ } catch (error) {+ throw apiError(error);+ }+ }+const target = destination(output, ctx.session());try {@@ -45,12 +55,7 @@ export const generate: ToolHandler<"generate"> = async (args, ctx) => {destinations.set(job.id, target);return present(job, ctx);} catch (error) {- if (!(error instanceof TRPCClientError)) throw error;- const code: string | undefined = error.data?.code;- const mapped = code && API_ERRORS[code];- if (mapped) throw new DapiError(mapped, error.message, { cause: error });- if (code === "PAYMENT_REQUIRED") throw new Error(`Not enough credits: ${error.message}`, { cause: error });- throw error;+ throw apiError(error);}};@@ -71,12 +76,7 @@ export const job: ToolHandler<"job"> = async ({ id, cancel }, ctx) => {return present(job, ctx);} catch (error) {- if (!(error instanceof TRPCClientError)) throw error;- const code: string | undefined = error.data?.code;- const mapped = code && API_ERRORS[code];- if (mapped) throw new DapiError(mapped, error.message, { cause: error });- if (code === "PAYMENT_REQUIRED") throw new Error(`Not enough credits: ${error.message}`, { cause: error });- throw error;+ throw apiError(error);}};@@ -218,3 +218,13 @@ const API_ERRORS: Record<string, DapiErrorCode> = {CONFLICT: "busy",};+/** An API call's failure as the tools report it: a tool error where one fits, a sentence for credits, else as it came. */+function apiError(error: unknown): unknown {+ if (!(error instanceof TRPCClientError)) return error;+ const code: string | undefined = error.data?.code;+ const mapped = code && API_ERRORS[code];+ if (mapped) return new DapiError(mapped, error.message, { cause: error });+ if (code === "PAYMENT_REQUIRED") return new Error(`Not enough credits: ${error.message}`, { cause: error });+ return error;+}+diff --git a/apps/web/src/dapi/local/jobs.ts b/apps/web/src/dapi/local/jobs.tsindex 2c0ad959..c70210e9 100644--- a/apps/web/src/dapi/local/jobs.ts+++ b/apps/web/src/dapi/local/jobs.ts@@ -12,7 +12,7 @@ import { localModels } from "./models";import type { World } from "koota";import type { Accessor } from "solid-js";import type { JobErrorCode } from "@diffusionstudio/api-contract";-import type { GenerateRequest, GenerationResult, LocalModelId } from "@diffusionstudio/dapi";+import type { GenerateRequest, GenerationEstimate, GenerationResult, LocalModelId } from "@diffusionstudio/dapi";import type { ToolContext } from "../handler";import type { Destination } from "../lib/outputs";import type { EditorSession } from "../session";@@ -61,7 +61,17 @@ const jobs = new Map<string, Entry>();* Answers at once with the job queued, or, for work the model says is quick* (`inline`), once it has ended.*/-export async function startLocalJob(args: GenerateRequest, target: Destination, ctx: ToolContext): Promise<GenerationResult> {+export async function startLocalJob(args: GenerateRequest, target: Destination, ctx: ToolContext): Promise<GenerationResult | GenerationEstimate> {+ if (args.estimate) {+ const model = args.model as LocalModelId;+ const parsed = LOCAL_MODELS[model].safeParse(args);+ if (!parsed.success) {+ throw new DapiError("invalid-input", z.prettifyError(parsed.error));+ }+ const plan = await localModels[model].prepare(parsed.data, ctx);+ return { credits: 0, etaSeconds: round(plan.etaSeconds) };+ }+const model = args.model as LocalModelId;const parsed = LOCAL_MODELS[model].safeParse(args);if (!parsed.success) {@@ -204,7 +214,7 @@ async function save(entry: Entry, files: LocalFile[], current: Accessor<EditorSe}/** The request as the job reports it, as the API's jobs do: the model's fields, not the tool's. */-function modelFields({ output: _output, maxCredits: _maxCredits, ...fields }: Record<string, unknown>): Record<string, unknown> {+function modelFields({ output: _output, maxCredits: _maxCredits, estimate: _estimate, ...fields }: Record<string, unknown>): Record<string, unknown> {return fields;}diff --git a/docs/reference/tools/README.md b/docs/reference/tools/README.mdindex d69c7278..75a77b37 100644--- a/docs/reference/tools/README.md+++ b/docs/reference/tools/README.md@@ -55,7 +55,7 @@ How the surface is divided:- **The project loop.** [`open`](./open.md) a folder, edit its JSX, [`context`](./context.md) for what the source cannot say, [`capture`](./capture.md) and [`check`](./check.md) to verify, [`export`](./export.md) when asked.- **Media inspection** (`media_*`): a file by path, without adding it to the project. Absolute paths and URLs work with or without an open project; library paths (`b-roll/clip.mp4`) need one.-- **Generation.** [`generate`](./generate.md) runs a model of the Diffusion Studio API (images, video, music, speech, and tools such as upscaling) on the account's credits, or a model on this machine for free — [`sam-2.1`](./media/segment.md) segments and tracks an object in footage, making the mask file a `<mask src>` names; poll [`job`](./job.md) until it has ended, and its files are saved into the library or a directory. The models and their fields are in [models.md](../models.md).+- **Generation.** [`generate`](./generate.md) runs a model of the Diffusion Studio API (images, video, music, speech, and tools such as upscaling) on the account's credits, or a model on this machine for free (with `estimate`, it only prices the request) — [`sam-2.1`](./media/segment.md) segments and tracks an object in footage, making the mask file a `<mask src>` names; poll [`job`](./job.md) until it has ended, and its files are saved into the library or a directory. The models and their fields are in [models.md](../models.md).- **What a declaration may name.** [`fonts`](./fonts.md).- **The app and the machine.** [`logs`](./logs.md), [`screenshot`](./screenshot.md), [`window`](./window.md), [`report`](./report.md).diff --git a/docs/reference/tools/generate.md b/docs/reference/tools/generate.mdindex 66b94d6c..070a0c58 100644--- a/docs/reference/tools/generate.md+++ b/docs/reference/tools/generate.md@@ -1,6 +1,6 @@# generate-Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking ([`sam-2.1`](./media/segment.md)) on this machine — and return the job at once, with its estimated run time and the credits it costs. Poll [`job`](./job.md) with its id until it has ended; the files it made are then saved and their paths returned. API models need a signed-in account and spend its credits; [local models](../models.md#local) run on the GPU, free, and need no account.+Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking ([`sam-2.1`](./media/segment.md)) on this machine — and return the job at once, with its estimated run time and the credits it costs. Poll [`job`](./job.md) with its id until it has ended; the files it made are then saved and their paths returned. With [`estimate`](#estimating), nothing starts: the answer is only what the job would cost and how long it would run. API models need a signed-in account and spend its credits; [local models](../models.md#local) run on the GPU, free, and need no account.| | || --- | --- |@@ -18,6 +18,7 @@ The input is the model's request — `model` plus that model's fields — with t| *the model's fields* | | `[fields]`, `-p, --prompt <text>` | `prompt`, `images`, `aspectRatio`, `duration`, `voice`, … as the model takes them. On the CLI one JSON object; `--prompt` sets `prompt` without JSON quoting || `output` | `string` | `-o, --output <path>` | file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: the library's root under the API's name with a project open, its files under `assets/`; else a fresh directory under the system temp dir). On the CLI a path starting with `.` resolves against the working directory. || `maxCredits` | `number` | `--max-credits <n>` | refuse to start the job if it costs more credits than this |+| `estimate` | `boolean` | `--estimate` | don't start the job: answer with only its `credits` and `etaSeconds`, spending nothing (see [Estimating](#estimating)) |An API model's fields are not checked here: they go to the API as given, and the API answers a request it cannot run with what to fix (`invalid-input`, e.g. an aspect ratio the model does not offer). Each model narrows its kind's bounds, so read its entry in [models.md](../models.md) rather than guessing. A [local model](../models.md#local)'s fields are checked by the app, the same way, before its job starts.@@ -43,6 +44,17 @@ diffusion generate elevenlabs-v3 --prompt "Welcome back to the channel." '{"voicdiffusion generate remove-background '{"image":{"path":"./fox.png"}}'```+## Estimating++With `estimate`, the request is priced instead of run: no job starts, no credits are spent, and the answer is the job's price and run time alone. Ask before a job whose price matters — a long video, a large `count`, a tool priced by its input's length — or to compare models for the same request.++```bash+diffusion generate veo-3.1-fast --prompt "The fox turns and runs into the trees" '{"duration":8}' --estimate+# prints { "credits": …, "etaSeconds": … }+```++Files (`{ "path": … }`) are uploaded first, as for a job, since a tool such as `transcribe` or `upscale-video` is priced by the length of its input; starting the job afterwards reuses the upload. `output` and `maxCredits` are ignored. A [local model](../models.md#local) costs `0`, its `etaSeconds` planned from the file as its job would be. An API model still needs a signed-in account.+## Where the files go`output` names the file the result is saved as, chosen when the job starts:@@ -57,7 +69,7 @@ The files are saved by the first [`job`](./job.md) call that sees the job succee## Output-The API's job, as it is when the call returns — almost always `queued`; a local model's quick work, such as a `sam-2.1` preview, already ended — and the same shape [`job`](./job.md) returns. Every field is the API's, passed through unchanged, except that each file in `assets` has its download `url` replaced by where it was saved. The fields to act on:+With `estimate`, `{ credits, etaSeconds }` and nothing else. Otherwise, the API's job, as it is when the call returns — almost always `queued`; a local model's quick work, such as a `sam-2.1` preview, already ended — and the same shape [`job`](./job.md) returns. Every field is the API's, passed through unchanged, except that each file in `assets` has its download `url` replaced by where it was saved. The fields to act on:```ts{diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.tsindex d8722de7..d82d4148 100644--- a/packages/dapi/src/index.ts+++ b/packages/dapi/src/index.ts@@ -89,8 +89,9 @@ export type MediaWaveformResult = ToolResult<"media_waveform">;export type MediaListenRequest = ToolArgs<"media_listen">;export type MediaListenResult = ToolResult<"media_listen">;export type GenerateRequest = ToolArgs<"generate">;-export type GenerationResult = ToolResult<"generate">;-export type GenerationJob = ToolOutput<"generate">;+export type GenerationResult = ToolResult<"job">;+export type GenerationJob = ToolOutput<"job">;+export type GenerationEstimate = Extract<ToolResult<"generate">, { credits: number }>;export type JobRequest = ToolArgs<"job">;export type FontsRequest = ToolArgs<"fonts">;export type ReportRequest = ToolArgs<"report">;diff --git a/packages/dapi/src/local/fields.ts b/packages/dapi/src/local/fields.tsindex 2a958e35..a2be1423 100644--- a/packages/dapi/src/local/fields.ts+++ b/packages/dapi/src/local/fields.ts@@ -16,4 +16,5 @@ export const FileRef = z.strictObject({ path: AssetPath });export const requestFields = {output: z.string().optional(),maxCredits: z.number().optional(),+ estimate: z.boolean().optional(),};diff --git a/packages/dapi/src/tools/generate.ts b/packages/dapi/src/tools/generate.tsindex bf8c0519..8a780854 100644--- a/packages/dapi/src/tools/generate.ts+++ b/packages/dapi/src/tools/generate.ts@@ -83,6 +83,9 @@ export const GenerationResult = z.object({image: Bytes.optional(),});+/** What `generate` answers with `estimate`: the job's price and run time, and no job. */+export const GenerationEstimate = z.object({ credits: z.number(), etaSeconds: z.number() });+/*** Every field a model takes, by name only, from the API's request schemas and* the local models'. Declared for MCP clients that send only the properties a@@ -100,7 +103,7 @@ export const generate = defineTool({name: "generate",title: "Generate media",description:- "Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking (`sam-2.1`) on this machine — and return the job at once, with its estimated run time and the credits it costs. The input is the model's request: `model` plus that model's fields (`prompt`, `images`, `aspectRatio`, `duration`, `voice`, …), listed per model in the docs at `reference/models.md`. An API model's fields are passed through as given and the API validates them, answering with what to fix; a local model's are checked here. Where a field takes a file, put `{ \"path\": \"…\" }` — an absolute path, a URL, or a library path — and it is uploaded first (a local model reads it in place). Poll `job` with the returned id until its status is succeeded, failed or canceled; the estimate says when to look. API models need a signed-in account and spend its credits; local models run on the GPU, free, and need no account.",+ "Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking (`sam-2.1`) on this machine — and return the job at once, with its estimated run time and the credits it costs. The input is the model's request: `model` plus that model's fields (`prompt`, `images`, `aspectRatio`, `duration`, `voice`, …), listed per model in the docs at `reference/models.md`. An API model's fields are passed through as given and the API validates them, answering with what to fix; a local model's are checked here. Where a field takes a file, put `{ \"path\": \"…\" }` — an absolute path, a URL, or a library path — and it is uploaded first (a local model reads it in place). Poll `job` with the returned id until its status is succeeded, failed or canceled; the estimate says when to look. With `estimate`, nothing starts: the answer is only what the job would cost and how long it would run. API models need a signed-in account and spend its credits; local models run on the GPU, free, and need no account.",input: z.looseObject({model: z.enum([...MODEL_IDS, ...LOCAL_MODEL_IDS]).describe("the model to run; its fields are listed in `reference/models.md`"),...modelFields,@@ -111,9 +114,16 @@ export const generate = defineTool({"file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: with a project open, the library's root under the model's name for the file, its files under `assets/`, or the model's own folder where `reference/models.md` gives one; else a fresh directory under the system temp dir)",),maxCredits: z.number().positive().optional().describe("refuse to start the job if it costs more credits than this"),+ estimate: z+ .boolean()+ .optional()+ .describe("don't start the job: answer with only its `credits` and `etaSeconds`, spending nothing"),}),- output: GenerationJob,- result: GenerationResult,+ output: GenerationJob.partial({ id: true, status: true, assets: true }).extend({+ credits: z.number().describe("credits the job costs: charged, and refunded if it fails or is canceled; with `estimate`, what it would cost"),+ etaSeconds: z.number().describe("estimated seconds the job runs for"),+ }),+ result: z.union([GenerationResult, GenerationEstimate]),environment: "renderer",});diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.tsindex e75c5b3d..0bd7b662 100644--- a/packages/dapi/src/tools/tools.test.ts+++ b/packages/dapi/src/tools/tools.test.ts@@ -218,6 +218,13 @@ describe("generate", () => {expect(issues(input.safeParse({ model: "dall-e-1", prompt: "a fox" }))).toHaveProperty("model");});+ it("answers an estimate with the job's price alone, as an object schema", () => {+ expect(input.parse({ model: "veo-3.1-fast", prompt: "a fox", estimate: true })).toEqual({ model: "veo-3.1-fast", prompt: "a fox", estimate: true });+ expect(generate.output.parse({ credits: 38, etaSeconds: 90 })).toEqual({ credits: 38, etaSeconds: 90 });+ expect(toolJsonSchemas(generate).outputSchema).toHaveProperty("type", "object");+ expect(job.output.safeParse({ credits: 38, etaSeconds: 90 }).success).toBe(false);+ });+it("lets a job's fields beyond the documented ones through to the caller", () => {const job = { id: "j", model: "flux-2-klein", status: "succeeded", etaSeconds: 9, etaRemainingSeconds: null, credits: 2, error: null, createdAt: "2026-10-06T00:00:00Z", assets: [{ path: "/a.png", filename: "a.png", width: 1024 }] };expect(generate.output.parse(job)).toEqual(job);