Skip to content

The VS Code of video editing, built for agents.

TypeScript

Implement estimate option in generate tool for cost and time prediction

konstantin-paulusfe3f1d68e76d
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 10504359..a62111f4 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -252,7 +252,8 @@ program
`${field("generate", "output")}; a path starting with . resolves against the working directory`,
)
.option("--max-credits <n>", field("generate", "maxCredits"), numeric)
- .action((model: ToolInput<"generate">["model"], fields: string | undefined, opts: { prompt?: string; output?: string; maxCredits?: number }) => {
+ .option("--estimate", field("generate", "estimate"))
+ .action((model: ToolInput<"generate">["model"], fields: string | undefined, opts: { prompt?: string; output?: string; maxCredits?: number; estimate?: boolean }) => {
const parsed = fields === undefined ? {} : json(fields, "fields");
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) fail("fields must be a JSON object.");
const request = resolveFileRefs(parsed) as Record<string, unknown>;
@@ -262,6 +263,7 @@ program
...(opts.prompt === undefined ? {} : { prompt: opts.prompt }),
output: opts.output && libraryOrDiskPath(opts.output),
maxCredits: opts.maxCredits,
+ estimate: opts.estimate,
});
});
diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.ts
index d8b3d0c4..daad6cbf 100644
--- a/apps/desktop/src/dapi/present.ts
+++ b/apps/desktop/src/dapi/present.ts
@@ -51,6 +51,8 @@ export async function present(name: ToolName, args: unknown, result: unknown): P
case "media_transcribe":
return presentTranscript(result as ToolResult<"media_transcribe">, (args as ToolArgs<"media_transcribe">).output);
case "generate":
+ if ((args as ToolArgs<"generate">).estimate) return { output: result, images: [] };
+ return presentJob(result as GenerationResult);
case "job":
return presentJob(result as GenerationResult);
default:
diff --git a/apps/web/src/dapi/handlers/generate.ts b/apps/web/src/dapi/handlers/generate.ts
index f3110904..282d4f64 100644
--- a/apps/web/src/dapi/handlers/generate.ts
+++ b/apps/web/src/dapi/handlers/generate.ts
@@ -31,8 +31,18 @@ export const generate: ToolHandler<"generate"> = async (args, ctx) => {
return startLocalJob(args, destination(args.output, ctx.session()), ctx);
}
- const { output, maxCredits, ...request } = args;
+ const { output, maxCredits, estimate, ...request } = args;
await requireSignIn();
+
+ if (estimate) {
+ try {
+ const { credits, etaSeconds } = await api.estimate.query((await withUploads(request, ctx)) as GenerateRequestInput);
+ return { credits, etaSeconds };
+ } catch (error) {
+ throw apiError(error);
+ }
+ }
+
const target = destination(output, ctx.session());
try {
@@ -45,12 +55,7 @@ export const generate: ToolHandler<"generate"> = async (args, ctx) => {
destinations.set(job.id, target);
return present(job, ctx);
} catch (error) {
- if (!(error instanceof TRPCClientError)) throw error;
- const code: string | undefined = error.data?.code;
- const mapped = code && API_ERRORS[code];
- if (mapped) throw new DapiError(mapped, error.message, { cause: error });
- if (code === "PAYMENT_REQUIRED") throw new Error(`Not enough credits: ${error.message}`, { cause: error });
- throw error;
+ throw apiError(error);
}
};
@@ -71,12 +76,7 @@ export const job: ToolHandler<"job"> = async ({ id, cancel }, ctx) => {
return present(job, ctx);
} catch (error) {
- if (!(error instanceof TRPCClientError)) throw error;
- const code: string | undefined = error.data?.code;
- const mapped = code && API_ERRORS[code];
- if (mapped) throw new DapiError(mapped, error.message, { cause: error });
- if (code === "PAYMENT_REQUIRED") throw new Error(`Not enough credits: ${error.message}`, { cause: error });
- throw error;
+ throw apiError(error);
}
};
@@ -218,3 +218,13 @@ const API_ERRORS: Record<string, DapiErrorCode> = {
CONFLICT: "busy",
};
+/** An API call's failure as the tools report it: a tool error where one fits, a sentence for credits, else as it came. */
+function apiError(error: unknown): unknown {
+ if (!(error instanceof TRPCClientError)) return error;
+ const code: string | undefined = error.data?.code;
+ const mapped = code && API_ERRORS[code];
+ if (mapped) return new DapiError(mapped, error.message, { cause: error });
+ if (code === "PAYMENT_REQUIRED") return new Error(`Not enough credits: ${error.message}`, { cause: error });
+ return error;
+}
+
diff --git a/apps/web/src/dapi/local/jobs.ts b/apps/web/src/dapi/local/jobs.ts
index 2c0ad959..c70210e9 100644
--- a/apps/web/src/dapi/local/jobs.ts
+++ b/apps/web/src/dapi/local/jobs.ts
@@ -12,7 +12,7 @@ import { localModels } from "./models";
import type { World } from "koota";
import type { Accessor } from "solid-js";
import type { JobErrorCode } from "@diffusionstudio/api-contract";
-import type { GenerateRequest, GenerationResult, LocalModelId } from "@diffusionstudio/dapi";
+import type { GenerateRequest, GenerationEstimate, GenerationResult, LocalModelId } from "@diffusionstudio/dapi";
import type { ToolContext } from "../handler";
import type { Destination } from "../lib/outputs";
import type { EditorSession } from "../session";
@@ -61,7 +61,17 @@ const jobs = new Map<string, Entry>();
* Answers at once with the job queued, or, for work the model says is quick
* (`inline`), once it has ended.
*/
-export async function startLocalJob(args: GenerateRequest, target: Destination, ctx: ToolContext): Promise<GenerationResult> {
+export async function startLocalJob(args: GenerateRequest, target: Destination, ctx: ToolContext): Promise<GenerationResult | GenerationEstimate> {
+ if (args.estimate) {
+ const model = args.model as LocalModelId;
+ const parsed = LOCAL_MODELS[model].safeParse(args);
+ if (!parsed.success) {
+ throw new DapiError("invalid-input", z.prettifyError(parsed.error));
+ }
+ const plan = await localModels[model].prepare(parsed.data, ctx);
+ return { credits: 0, etaSeconds: round(plan.etaSeconds) };
+ }
+
const model = args.model as LocalModelId;
const parsed = LOCAL_MODELS[model].safeParse(args);
if (!parsed.success) {
@@ -204,7 +214,7 @@ async function save(entry: Entry, files: LocalFile[], current: Accessor<EditorSe
}
/** The request as the job reports it, as the API's jobs do: the model's fields, not the tool's. */
-function modelFields({ output: _output, maxCredits: _maxCredits, ...fields }: Record<string, unknown>): Record<string, unknown> {
+function modelFields({ output: _output, maxCredits: _maxCredits, estimate: _estimate, ...fields }: Record<string, unknown>): Record<string, unknown> {
return fields;
}
diff --git a/docs/reference/tools/README.md b/docs/reference/tools/README.md
index d69c7278..75a77b37 100644
--- a/docs/reference/tools/README.md
+++ b/docs/reference/tools/README.md
@@ -55,7 +55,7 @@ How the surface is divided:
- **The project loop.** [`open`](./open.md) a folder, edit its JSX, [`context`](./context.md) for what the source cannot say, [`capture`](./capture.md) and [`check`](./check.md) to verify, [`export`](./export.md) when asked.
- **Media inspection** (`media_*`): a file by path, without adding it to the project. Absolute paths and URLs work with or without an open project; library paths (`b-roll/clip.mp4`) need one.
-- **Generation.** [`generate`](./generate.md) runs a model of the Diffusion Studio API (images, video, music, speech, and tools such as upscaling) on the account's credits, or a model on this machine for free — [`sam-2.1`](./media/segment.md) segments and tracks an object in footage, making the mask file a `<mask src>` names; poll [`job`](./job.md) until it has ended, and its files are saved into the library or a directory. The models and their fields are in [models.md](../models.md).
+- **Generation.** [`generate`](./generate.md) runs a model of the Diffusion Studio API (images, video, music, speech, and tools such as upscaling) on the account's credits, or a model on this machine for free (with `estimate`, it only prices the request) — [`sam-2.1`](./media/segment.md) segments and tracks an object in footage, making the mask file a `<mask src>` names; poll [`job`](./job.md) until it has ended, and its files are saved into the library or a directory. The models and their fields are in [models.md](../models.md).
- **What a declaration may name.** [`fonts`](./fonts.md).
- **The app and the machine.** [`logs`](./logs.md), [`screenshot`](./screenshot.md), [`window`](./window.md), [`report`](./report.md).
diff --git a/docs/reference/tools/generate.md b/docs/reference/tools/generate.md
index 66b94d6c..070a0c58 100644
--- a/docs/reference/tools/generate.md
+++ b/docs/reference/tools/generate.md
@@ -1,6 +1,6 @@
# generate
-Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking ([`sam-2.1`](./media/segment.md)) on this machine — and return the job at once, with its estimated run time and the credits it costs. Poll [`job`](./job.md) with its id until it has ended; the files it made are then saved and their paths returned. API models need a signed-in account and spend its credits; [local models](../models.md#local) run on the GPU, free, and need no account.
+Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking ([`sam-2.1`](./media/segment.md)) on this machine — and return the job at once, with its estimated run time and the credits it costs. Poll [`job`](./job.md) with its id until it has ended; the files it made are then saved and their paths returned. With [`estimate`](#estimating), nothing starts: the answer is only what the job would cost and how long it would run. API models need a signed-in account and spend its credits; [local models](../models.md#local) run on the GPU, free, and need no account.
| | |
| --- | --- |
@@ -18,6 +18,7 @@ The input is the model's request — `model` plus that model's fields — with t
| *the model's fields* | | `[fields]`, `-p, --prompt <text>` | `prompt`, `images`, `aspectRatio`, `duration`, `voice`, … as the model takes them. On the CLI one JSON object; `--prompt` sets `prompt` without JSON quoting |
| `output` | `string` | `-o, --output <path>` | file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: the library's root under the API's name with a project open, its files under `assets/`; else a fresh directory under the system temp dir). On the CLI a path starting with `.` resolves against the working directory. |
| `maxCredits` | `number` | `--max-credits <n>` | refuse to start the job if it costs more credits than this |
+| `estimate` | `boolean` | `--estimate` | don't start the job: answer with only its `credits` and `etaSeconds`, spending nothing (see [Estimating](#estimating)) |
An API model's fields are not checked here: they go to the API as given, and the API answers a request it cannot run with what to fix (`invalid-input`, e.g. an aspect ratio the model does not offer). Each model narrows its kind's bounds, so read its entry in [models.md](../models.md) rather than guessing. A [local model](../models.md#local)'s fields are checked by the app, the same way, before its job starts.
@@ -43,6 +44,17 @@ diffusion generate elevenlabs-v3 --prompt "Welcome back to the channel." '{"voic
diffusion generate remove-background '{"image":{"path":"./fox.png"}}'
```
+## Estimating
+
+With `estimate`, the request is priced instead of run: no job starts, no credits are spent, and the answer is the job's price and run time alone. Ask before a job whose price matters — a long video, a large `count`, a tool priced by its input's length — or to compare models for the same request.
+
+```bash
+diffusion generate veo-3.1-fast --prompt "The fox turns and runs into the trees" '{"duration":8}' --estimate
+# prints { "credits": …, "etaSeconds": … }
+```
+
+Files (`{ "path": … }`) are uploaded first, as for a job, since a tool such as `transcribe` or `upscale-video` is priced by the length of its input; starting the job afterwards reuses the upload. `output` and `maxCredits` are ignored. A [local model](../models.md#local) costs `0`, its `etaSeconds` planned from the file as its job would be. An API model still needs a signed-in account.
+
## Where the files go
`output` names the file the result is saved as, chosen when the job starts:
@@ -57,7 +69,7 @@ The files are saved by the first [`job`](./job.md) call that sees the job succee
## Output
-The API's job, as it is when the call returns — almost always `queued`; a local model's quick work, such as a `sam-2.1` preview, already ended — and the same shape [`job`](./job.md) returns. Every field is the API's, passed through unchanged, except that each file in `assets` has its download `url` replaced by where it was saved. The fields to act on:
+With `estimate`, `{ credits, etaSeconds }` and nothing else. Otherwise, the API's job, as it is when the call returns — almost always `queued`; a local model's quick work, such as a `sam-2.1` preview, already ended — and the same shape [`job`](./job.md) returns. Every field is the API's, passed through unchanged, except that each file in `assets` has its download `url` replaced by where it was saved. The fields to act on:
```ts
{
diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.ts
index d8722de7..d82d4148 100644
--- a/packages/dapi/src/index.ts
+++ b/packages/dapi/src/index.ts
@@ -89,8 +89,9 @@ export type MediaWaveformResult = ToolResult<"media_waveform">;
export type MediaListenRequest = ToolArgs<"media_listen">;
export type MediaListenResult = ToolResult<"media_listen">;
export type GenerateRequest = ToolArgs<"generate">;
-export type GenerationResult = ToolResult<"generate">;
-export type GenerationJob = ToolOutput<"generate">;
+export type GenerationResult = ToolResult<"job">;
+export type GenerationJob = ToolOutput<"job">;
+export type GenerationEstimate = Extract<ToolResult<"generate">, { credits: number }>;
export type JobRequest = ToolArgs<"job">;
export type FontsRequest = ToolArgs<"fonts">;
export type ReportRequest = ToolArgs<"report">;
diff --git a/packages/dapi/src/local/fields.ts b/packages/dapi/src/local/fields.ts
index 2a958e35..a2be1423 100644
--- a/packages/dapi/src/local/fields.ts
+++ b/packages/dapi/src/local/fields.ts
@@ -16,4 +16,5 @@ export const FileRef = z.strictObject({ path: AssetPath });
export const requestFields = {
output: z.string().optional(),
maxCredits: z.number().optional(),
+ estimate: z.boolean().optional(),
};
diff --git a/packages/dapi/src/tools/generate.ts b/packages/dapi/src/tools/generate.ts
index bf8c0519..8a780854 100644
--- a/packages/dapi/src/tools/generate.ts
+++ b/packages/dapi/src/tools/generate.ts
@@ -83,6 +83,9 @@ export const GenerationResult = z.object({
image: Bytes.optional(),
});
+/** What `generate` answers with `estimate`: the job's price and run time, and no job. */
+export const GenerationEstimate = z.object({ credits: z.number(), etaSeconds: z.number() });
+
/**
* Every field a model takes, by name only, from the API's request schemas and
* the local models'. Declared for MCP clients that send only the properties a
@@ -100,7 +103,7 @@ export const generate = defineTool({
name: "generate",
title: "Generate media",
description:
- "Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking (`sam-2.1`) on this machine — and return the job at once, with its estimated run time and the credits it costs. The input is the model's request: `model` plus that model's fields (`prompt`, `images`, `aspectRatio`, `duration`, `voice`, …), listed per model in the docs at `reference/models.md`. An API model's fields are passed through as given and the API validates them, answering with what to fix; a local model's are checked here. Where a field takes a file, put `{ \"path\": \"…\" }` — an absolute path, a URL, or a library path — and it is uploaded first (a local model reads it in place). Poll `job` with the returned id until its status is succeeded, failed or canceled; the estimate says when to look. API models need a signed-in account and spend its credits; local models run on the GPU, free, and need no account.",
+ "Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking (`sam-2.1`) on this machine — and return the job at once, with its estimated run time and the credits it costs. The input is the model's request: `model` plus that model's fields (`prompt`, `images`, `aspectRatio`, `duration`, `voice`, …), listed per model in the docs at `reference/models.md`. An API model's fields are passed through as given and the API validates them, answering with what to fix; a local model's are checked here. Where a field takes a file, put `{ \"path\": \"…\" }` — an absolute path, a URL, or a library path — and it is uploaded first (a local model reads it in place). Poll `job` with the returned id until its status is succeeded, failed or canceled; the estimate says when to look. With `estimate`, nothing starts: the answer is only what the job would cost and how long it would run. API models need a signed-in account and spend its credits; local models run on the GPU, free, and need no account.",
input: z.looseObject({
model: z.enum([...MODEL_IDS, ...LOCAL_MODEL_IDS]).describe("the model to run; its fields are listed in `reference/models.md`"),
...modelFields,
@@ -111,9 +114,16 @@ export const generate = defineTool({
"file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: with a project open, the library's root under the model's name for the file, its files under `assets/`, or the model's own folder where `reference/models.md` gives one; else a fresh directory under the system temp dir)",
),
maxCredits: z.number().positive().optional().describe("refuse to start the job if it costs more credits than this"),
+ estimate: z
+ .boolean()
+ .optional()
+ .describe("don't start the job: answer with only its `credits` and `etaSeconds`, spending nothing"),
}),
- output: GenerationJob,
- result: GenerationResult,
+ output: GenerationJob.partial({ id: true, status: true, assets: true }).extend({
+ credits: z.number().describe("credits the job costs: charged, and refunded if it fails or is canceled; with `estimate`, what it would cost"),
+ etaSeconds: z.number().describe("estimated seconds the job runs for"),
+ }),
+ result: z.union([GenerationResult, GenerationEstimate]),
environment: "renderer",
});
diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.ts
index e75c5b3d..0bd7b662 100644
--- a/packages/dapi/src/tools/tools.test.ts
+++ b/packages/dapi/src/tools/tools.test.ts
@@ -218,6 +218,13 @@ describe("generate", () => {
expect(issues(input.safeParse({ model: "dall-e-1", prompt: "a fox" }))).toHaveProperty("model");
});
+ it("answers an estimate with the job's price alone, as an object schema", () => {
+ expect(input.parse({ model: "veo-3.1-fast", prompt: "a fox", estimate: true })).toEqual({ model: "veo-3.1-fast", prompt: "a fox", estimate: true });
+ expect(generate.output.parse({ credits: 38, etaSeconds: 90 })).toEqual({ credits: 38, etaSeconds: 90 });
+ expect(toolJsonSchemas(generate).outputSchema).toHaveProperty("type", "object");
+ expect(job.output.safeParse({ credits: 38, etaSeconds: 90 }).success).toBe(false);
+ });
+
it("lets a job's fields beyond the documented ones through to the caller", () => {
const job = { id: "j", model: "flux-2-klein", status: "succeeded", etaSeconds: 9, etaRemainingSeconds: null, credits: 2, error: null, createdAt: "2026-10-06T00:00:00Z", assets: [{ path: "/a.png", filename: "a.png", width: 1024 }] };
expect(generate.output.parse(job)).toEqual(job);