Skip to content

The VS Code of video editing, built for agents.

TypeScript
8.3 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// Generation on the Diffusion Studio API, nearly as the API has it: a request
// is the API's own (`model` plus that model's fields), passed through for the
// API to validate, and a job is the API's `Job`. What the tools add is what
// only a local client can do: send local files, save what a job made, and
// run the models that work on this machine (see ../local) as jobs of the same
// shape.

import { z } from "zod";
import { MODEL_IDS, generateRequest } from "@diffusionstudio/api-contract";
import { defineTool } from "../tool";
import { Bytes } from "../schemas";
import { LOCAL_MODELS, LOCAL_MODEL_IDS } from "../local";
import { requestFields } from "../local/fields";

const JPEG = /^\.jpe?g$/i;

/**
 * Where file `index` of the `count` a job made is saved when the caller named
 * the output `output`: that path, with the extension of what the model made
 * (`filename`'s) in place of a different one or added when there is none, and
 * numbered (`-1`, `-2`, …) when the job made several. Either separator works,
 * so the renderer and the server agree on library and OS paths alike.
 */
export function generatedPath(output: string, filename: string, index: number, count: number): string {
  const slash = Math.max(output.lastIndexOf("/"), output.lastIndexOf("\\"));
  const base = output.slice(slash + 1);
  const dot = base.lastIndexOf(".");
  const stem = dot > 0 ? base.slice(0, dot) : base;
  const given = dot > 0 ? base.slice(dot) : "";
  const made = /\.[^./\\]+$/.exec(filename)?.[0] ?? given;
  const same = given.toLowerCase() === made.toLowerCase() || (JPEG.test(given) && JPEG.test(made));
  return `${output.slice(0, slash + 1)}${stem}${count > 1 ? `-${index + 1}` : ""}${same ? given : made}`;
}

/** A file a job made, once it is on this machine; the API's file with its `url` swapped for where it was saved. */
export const GeneratedFile = z.looseObject({
  path: z.string().describe("absolute path of the file"),
  src: z.string().optional().describe("the file's library path, for `src`; present when it went into the project's library"),
});

/**
 * The API's `Job`, its files saved locally. Only what the tools rely on is
 * spelled out; every other field is the API's, passed through as it is.
 */
export const GenerationJob = z.looseObject({
  id: z.string().describe("job id, for `job`"),
  status: z.string().describe("queued or running while the job works; succeeded, failed or canceled once it has ended — stop polling then"),
  assets: z.array(GeneratedFile).describe("the files the job made, saved: empty until it succeeds"),
  progress: z
    .number()
    .nullable()
    .optional()
    .describe("on a local model's job, 0..1 through its current `phase` (a model's download, then the work); null when there is none to report"),
  details: z
    .record(z.string(), z.unknown())
    .optional()
    .describe("on a local model's job, what the model reports beyond its files, model by model (see `reference/models.md`)"),
});

/**
 * What the renderer hands back: the job as the API has it, each file either
 * saved into the library already (`path`, `src`), still at its `url`, or, from
 * a local model, held as `bytes`: the last two for the server to download or
 * write to `saveTo` (see `generatedPath`; null: a fresh directory under the
 * system temp dir, under the files' own names). `image` is a picture of what
 * a local job made, for the caller to look at: the server writes it to the
 * temp dir once, shows it inline that once, and names it in `details.image`.
 */
export const GenerationResult = z.object({
  job: z.looseObject({
    id: z.string(),
    status: z.string(),
    assets: z.array(
      z.looseObject({ url: z.string().optional(), bytes: Bytes.optional(), path: z.string().optional(), filename: z.string() }),
    ),
    details: z.record(z.string(), z.unknown()).optional(),
  }),
  saveTo: z.string().nullable(),
  image: Bytes.optional(),
});

/** What `generate` answers with `estimate`: the job's price and run time, and no job. */
export const GenerationEstimate = z.object({ credits: z.number(), etaSeconds: z.number() });

/**
 * Every field a model takes, by name only, from the API's request schemas and
 * the local models'. Declared for MCP clients that send only the properties a
 * schema lists; each is `unknown`, so the API (or a local model's own schema)
 * still does all the checking.
 */
const modelFields = Object.fromEntries(
  [...generateRequest.options, ...Object.values(LOCAL_MODELS)]
    .flatMap((schema) => Object.keys(schema.shape))
    .filter((key) => key !== "model" && !(key in requestFields))
    .map((key) => [key, z.unknown().optional()]),
);

export const generate = defineTool({
  name: "generate",
  title: "Generate media",
  description:
    "Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking (`sam-2.1`) on this machine — and return the job at once, with its estimated run time and the credits it costs. The input is the model's request: `model` plus that model's fields (`prompt`, `images`, `aspectRatio`, `duration`, `voice`, …), listed per model in the docs at `reference/models.md`. An API model's fields are passed through as given and the API validates them, answering with what to fix; a local model's are checked here. Where a field takes a file, put `{ \"path\": \"…\" }` — an absolute path, a URL, or a library path — and it is uploaded first (a local model reads it in place). Poll `job` with the returned id until its status is succeeded, failed or canceled; the estimate says when to look. With `estimate`, nothing starts: the answer is only what the job would cost and how long it would run. API models need a signed-in account and spend its credits; local models run on the GPU, free, and need no account.",
  input: z.looseObject({
    model: z.enum([...MODEL_IDS, ...LOCAL_MODEL_IDS]).describe("the model to run; its fields are listed in `reference/models.md`"),
    ...modelFields,
    output: z
      .string()
      .optional()
      .describe(
        "file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: with a project open, the library's root under the model's name for the file, its files under `assets/`, or the model's own folder where `reference/models.md` gives one; else a fresh directory under the system temp dir)",
      ),
    maxCredits: z.number().positive().optional().describe("refuse to start the job if it costs more credits than this"),
    estimate: z
      .boolean()
      .optional()
      .describe("don't start the job: answer with only its `credits` and `etaSeconds`, spending nothing"),
  }),
  output: GenerationJob.partial({ id: true, status: true, assets: true }).extend({
    credits: z.number().describe("credits the job costs: charged, and refunded if it fails or is canceled; with `estimate`, what it would cost"),
    etaSeconds: z.number().describe("estimated seconds the job runs for"),
  }),
  result: z.union([GenerationResult, GenerationEstimate]),
  environment: "renderer",
});

export const job = defineTool({
  name: "job",
  title: "Generation job",
  description:
    "The state of a job started with `generate`: its status, progress estimate and cost, and, once it has succeeded, the files it made, saved where `generate` was told (the first call to see it succeed saves them; later calls return the same paths), and, from a local model, its `details`. Poll until the status is succeeded, failed or canceled, waiting about `etaRemainingSeconds` (or `etaSeconds` while queued) between calls. `cancel` stops a job and refunds its credits. A local model's job lives in the app: it is gone once the app restarts.",
  input: z.object({
    id: z.string().min(1).describe("job id, as `generate` returned it"),
    cancel: z.boolean().optional().describe("cancel the job, refunding its credits"),
  }),
  output: GenerationJob,
  result: GenerationResult,
  environment: "renderer",
});