Skip to content

The VS Code of video editing, built for agents.

TypeScript

Remove media segment functionality and related tests

konstantin-pauluse18ab8456272
diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts
index e2fa27cc..53e07ad4 100644
--- a/apps/cli/src/cli-client.ts
+++ b/apps/cli/src/cli-client.ts
@@ -18,7 +18,6 @@ const TIMEOUTS: Record<string, number> = {
capture: 600_000,
media_transcribe: 600_000,
media_listen: 600_000,
- media_segment: 3_600_000,
generate: 600_000,
job: 600_000,
};
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index fb33dabd..10504359 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -84,20 +84,8 @@ function resolveFileRefs(value: unknown): unknown {
return Object.fromEntries(entries.map(([key, item]) => [key, resolveFileRefs(item)]));
}
-/** `x,y` as a point, added to the ones given before; a malformed one becomes NaN for the schema to reject. */
-const point = (value: string, previous: Array<{ x: number; y: number }> = []): Array<{ x: number; y: number }> => {
- const [x = NaN, y = NaN, ...rest] = value.split(",").map(numeric);
- return [...previous, rest.length > 0 ? { x: NaN, y: NaN } : { x, y }];
-};
-
-/** `x0,y0,x1,y1` as a box; the wrong count of numbers becomes NaN for the schema to reject. */
-const box = (value: string): [number, number, number, number] => {
- const bounds = value.split(",").map(numeric);
- return bounds.length === 4 ? (bounds as [number, number, number, number]) : [NaN, NaN, NaN, NaN];
-};
-
/**
- * Where a mask or generated files go: a path spelled as one on disk
+ * Where generated files go: a path spelled as one on disk
* (absolute, or starting with `.`) resolves against the working directory;
* anything else is a library path, passed through for the app to resolve in
* the open project.
@@ -249,28 +237,6 @@ media
.option("-e, --end <time>", field("media_listen", "end"))
.action((ref: string, opts: Omit<ToolInput<"media_listen">, "path">) => run("media_listen", { path: assetPath(ref), ...opts }));
-media
- .command("segment")
- .alias("mask")
- .description(describe("media_segment"))
- .argument("<path>", field("media_segment", "path"))
- .requiredOption("-t, --time <time>", field("media_segment", "time"))
- .option("-p, --point <x,y>", `${field("media_segment", "points")}; repeat for more`, point)
- .option("-x, --exclude <x,y>", `${field("media_segment", "exclude")}; repeat for more`, point)
- .option("-b, --box <x0,y0,x1,y1>", field("media_segment", "box"), box)
- .option("--preview", field("media_segment", "preview"))
- .option("-s, --start <time>", field("media_segment", "start"))
- .option("-e, --end <time>", field("media_segment", "end"))
- .option("-m, --model <size>", field("media_segment", "model"))
- .option(
- "-o, --output <path>",
- `${field("media_segment", "output")}; a path starting with . resolves against the working directory`,
- )
- .action((ref: string, opts: Omit<ToolInput<"media_segment">, "path"> & { point?: ToolInput<"media_segment">["points"] }) => {
- const { point: points, ...rest } = opts;
- return run("media_segment", { path: assetPath(ref), ...rest, points, output: opts.output && libraryOrDiskPath(opts.output) });
- });
-
program
.command("generate")
.alias("gen")
diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.ts
index 99163a33..1ad4da8f 100644
--- a/apps/desktop/src/dapi/present.test.ts
+++ b/apps/desktop/src/dapi/present.test.ts
@@ -69,51 +69,6 @@ describe("present", () => {
expect(JSON.parse(readFileSync(file, "utf8"))).toEqual({ segments });
});
- it("writes a segment's mask where output says and its picture to the temp dir", async () => {
- const found = {
- model: "tiny" as const, frameRate: 30, start: 0, end: 1, frames: 30, bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [],
- };
- const file = join(dir, "masks", "skater.mask");
- const presented = await present("media_segment", { path: "/c.mp4", time: 0, output: file }, { png: png(8), mask: png(9, 4), ...found });
- const output = presented.output as { image: string; path: string };
- expect(output).toEqual({ image: output.image, path: file, ...found });
- expect(output.image).toMatch(/dapi-segment-.*\.png$/);
- expect(readFileSync(file)).toEqual(Buffer.from(png(9, 4)));
- expect(presented.images).toEqual([{ path: output.image, png: png(8) }]);
- });
-
- it("keeps the path of a mask the app put into the library, and writes none for a preview", async () => {
- const found = {
- model: "tiny" as const, frameRate: 30, start: 1, end: 1.033, frames: 1, bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [],
- };
- const stored = await present("media_segment", { path: "/c.mp4", time: 1 }, { png: png(8), path: "/p/assets/masks/a.mask", src: "masks/a.mask", ...found });
- expect(stored.output).toMatchObject({ path: "/p/assets/masks/a.mask", src: "masks/a.mask" });
- const preview = await present("media_segment", { path: "/c.mp4", time: 1, preview: true }, { png: png(8), ...found });
- expect(preview.output).not.toHaveProperty("path");
- });
-
- it("has no picture for a track still running in the background", async () => {
- const span = { model: "tiny" as const, frameRate: 30, start: 0, end: 2, frames: 60 };
- const running = await present("media_segment", { path: "/c.mp4", time: 1 }, { path: "/p/assets/masks/a.mask", src: "masks/a.mask", state: "tracking" as const, ...span });
- expect(running).toEqual({ output: { path: "/p/assets/masks/a.mask", src: "masks/a.mask", state: "tracking", ...span }, images: [] });
- });
-
- it("writes a done track's contact sheet once, inline only the first time context reports it", async () => {
- const row = {
- id: "track-1", src: "masks/a.mask", video: "a.mp4", state: "done" as const, progress: 1,
- model: "tiny" as const, frameRate: 30, start: 0, end: 2, frames: 60, bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [],
- };
- const base = { rootDir: "/p", projectDir: "/p/a", currentTime: null, fontFamilies: [] };
- const first = await present("context", {}, { ...base, masks: [{ ...row, png: png(8) }] });
- const image = (first.output as { masks: { image: string }[] }).masks[0]!.image;
- expect(image).toMatch(/dapi-segment-.*\.png$/);
- expect(first.images).toEqual([{ path: image, png: png(8) }]);
- expect(first.output).toEqual({ ...base, masks: [{ ...row, image }] });
-
- const again = await present("context", {}, { ...base, masks: [{ ...row, png: png(8) }] });
- expect(again).toEqual({ output: { ...base, masks: [{ ...row, image }] }, images: [] });
- });
-
it("passes other results through untouched", async () => {
expect(await present("check", { id: "x" }, { stats: {}, issues: [] })).toEqual({ output: { stats: {}, issues: [] }, images: [] });
});
@@ -195,4 +150,31 @@ describe("present a generation job", () => {
const saved = job("j-lib", "succeeded", [{ filename: "a.png", path: "/p/assets/a.png", src: "a.png" }]);
expect((await present("job", { id: "j-lib" }, saved)).output).toEqual(saved.job);
});
+ it("writes a local job's files from their bytes, to the output, once", async () => {
+ const out = join(dir, "masks", "skater.mask");
+ const done = { ...job("local-mask", "succeeded", [{ filename: "Tracking 1.mask", mimeType: "", size: 4, bytes: png(9, 4) }]), saveTo: out };
+ const first = await present("job", { id: "local-mask" }, done);
+ writeFileSync(out, "edited since");
+ const again = await present("job", { id: "local-mask" }, done);
+ const assets = (first.output as { assets: Array<Record<string, unknown>> }).assets;
+ expect(assets).toEqual([{ filename: "Tracking 1.mask", mimeType: "", size: 4, path: out }]);
+ expect(again.output).toEqual(first.output);
+ expect(readFileSync(out, "utf8")).toBe("edited since");
+ });
+
+ it("writes a local job's picture to the temp dir, inline only the first time, and names it in its details", async () => {
+ const details = { frameRate: 30, start: 1, end: 1.033, frames: 1, bbox: null, lost: [] };
+ const preview = { ...job("local-preview", "succeeded", []), image: png(8) };
+ preview.job = { ...preview.job, details } as typeof preview.job;
+ const first = await present("generate", { model: "sam-2.1" }, preview);
+ const image = (first.output as { details: { image: string } }).details.image;
+ expect(image).toMatch(/dapi-local-preview\.png$/);
+ expect(first.output).toEqual({ ...preview.job, details: { ...details, image } });
+ expect(first.images).toEqual([{ path: image, png: png(8) }]);
+ expect(readFileSync(image)).toEqual(Buffer.from(png(8)));
+
+ const again = await present("job", { id: "local-preview" }, preview);
+ expect(again).toEqual({ output: first.output, images: [] });
+ rmSync(image, { force: true });
+ });
});
diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.ts
index 261635ad..d8b3d0c4 100644
--- a/apps/desktop/src/dapi/present.ts
+++ b/apps/desktop/src/dapi/present.ts
@@ -50,10 +50,6 @@ export async function present(name: ToolName, args: unknown, result: unknown): P
return presentScreenshot(result as ToolResult<"screenshot">, (args as ToolArgs<"screenshot">).output);
case "media_transcribe":
return presentTranscript(result as ToolResult<"media_transcribe">, (args as ToolArgs<"media_transcribe">).output);
- case "media_segment":
- return presentSegment(result as ToolResult<"media_segment">, (args as ToolArgs<"media_segment">).output);
- case "context":
- return presentContext(result as ToolResult<"context">);
case "generate":
case "job":
return presentJob(result as GenerationResult);
@@ -123,63 +119,37 @@ async function presentTranscript(transcript: ToolResult<"media_transcribe">, out
return { output: presented, images: [] };
}
-/**
- * The picture always goes to the temp dir; `output` is where the mask goes.
- * A mask the renderer put into the project's library comes with its path; one
- * that comes as bytes is written here, to `output` or the temp dir. A track
- * still running in the background has no picture yet (see `presentContext`).
- */
-async function presentSegment(result: ToolResult<"media_segment">, output: string | undefined): Promise<Presented> {
- const { png, mask, ...found } = result;
- let path = found.path;
- if (mask) {
- path = await singleFilePath(output, `dapi-mask-${randomUUID()}.mask`);
- await writeFile(path, mask);
- }
- const images: WrittenImage[] = [];
- let image: string | undefined;
- if (png) {
- image = join(tempDir(), `dapi-segment-${randomUUID()}.png`);
- await writeFile(image, png);
- images.push({ path: image, png });
- }
- const presented: ToolOutput<"media_segment"> = { ...found, ...(image === undefined ? {} : { image }), ...(path === undefined ? {} : { path }) };
- return { output: presented, images };
-}
+/** Files of succeeded jobs saved to disk, by job id: the first poll that sees a job succeed saves them, the rest reuse the paths. */
+const jobFiles = new Map<string, Promise<string[]>>();
-/** Contact sheets of background tracks already written, by track id: each is written, and shown inline, once. */
-const trackSheets = new Map<string, string>();
+/** Pictures of local jobs, by job id: each is written to the temp dir, and shown inline, the first poll it comes with. */
+const jobImages = new Map<string, string>();
-/** A done track's contact sheet goes to the temp dir the first poll it shows up in, and arrives inline with that poll. */
-async function presentContext(result: ToolResult<"context">): Promise<Presented> {
+/**
+ * A job's files bound for disk are saved once it has succeeded — downloaded
+ * from their `url`, or written from the `bytes` a local model made — to
+ * `saveTo` (see `generatedPath`; a file already there is written over, as the
+ * other tools' outputs are) or under their own names into a directory of the
+ * job's own under the temp dir, and their `url` or `bytes` is swapped for the
+ * `path`. Files the renderer saved into the library arrive with their paths.
+ * A local job's picture goes to the temp dir, named in `details.image`.
+ */
+async function presentJob({ job, saveTo, image }: GenerationResult): Promise<Presented> {
const images: WrittenImage[] = [];
- const masks: ToolOutput<"context">["masks"] = [];
- for (const { png, ...row } of result.masks) {
- let image = trackSheets.get(row.id);
- if (png && image === undefined) {
- image = join(tempDir(), `dapi-segment-${randomUUID()}.png`);
- await writeFile(image, png);
- trackSheets.set(row.id, image);
- images.push({ path: image, png });
+ let shown = job;
+ if (image) {
+ let path = jobImages.get(job.id);
+ if (path === undefined) {
+ path = join(tempDir(), `dapi-${job.id}.png`);
+ await writeFile(path, image);
+ jobImages.set(job.id, path);
+ images.push({ path, png: image });
}
- masks.push(image === undefined ? row : { ...row, image });
+ shown = { ...job, details: { ...job.details, image: path } };
}
- return { output: { ...result, masks }, images };
-}
-/** Files of succeeded jobs saved to disk, by job id: the first poll that sees a job succeed downloads them, the rest reuse the paths. */
-const jobFiles = new Map<string, Promise<string[]>>();
-
-/**
- * A job's files bound for disk are downloaded once it has succeeded, to
- * `saveTo` (see `generatedPath`; a file already there is written over, as
- * the other tools' outputs are) or under the API's names into a directory of
- * the job's own under the temp dir, and their `url` is swapped for the
- * `path`. Files the renderer saved into the library arrive with their paths.
- */
-async function presentJob({ job, saveTo }: GenerationResult): Promise<Presented> {
- const pending = job.assets.filter((file) => file.url !== undefined);
- if (job.status !== "succeeded" || pending.length === 0) return { output: job, images: [] };
+ const pending = job.assets.filter((file) => file.url !== undefined || file.bytes !== undefined);
+ if (job.status !== "succeeded" || pending.length === 0) return { output: shown, images };
let paths = jobFiles.get(job.id);
if (!paths) {
@@ -187,25 +157,29 @@ async function presentJob({ job, saveTo }: GenerationResult): Promise<Presented>
const targets = job.assets.map((file, i) =>
saveTo === null ? join(temp, basename(file.filename)) : generatedPath(saveTo, file.filename, i, job.assets.length),
);
- paths = downloadAll(job.assets, targets);
+ paths = saveAll(job.assets, targets);
paths.catch(() => jobFiles.delete(job.id));
jobFiles.set(job.id, paths);
}
const written = await paths;
- const assets = job.assets.map(({ url: _url, ...file }, i) => ({ ...file, path: written[i] }));
- return { output: { ...job, assets }, images: [] };
+ const assets = job.assets.map(({ url: _url, bytes: _bytes, ...file }, i) => ({ ...file, path: written[i] }));
+ return { output: { ...shown, assets }, images };
}
-async function downloadAll(files: GenerationResult["job"]["assets"], targets: string[]): Promise<string[]> {
+async function saveAll(files: GenerationResult["job"]["assets"], targets: string[]): Promise<string[]> {
for (const [i, file] of files.entries()) {
- const response = await fetch(file.url!);
- if (!response.ok) throw new Error(`Could not download ${file.filename} (${response.status})`);
await mkdir(dirname(targets[i]!), { recursive: true });
- await writeFile(targets[i]!, new Uint8Array(await response.arrayBuffer()));
+ await writeFile(targets[i]!, file.bytes ?? (await download(file.url!, file.filename)));
}
return targets;
}
+async function download(url: string, filename: string): Promise<Uint8Array> {
+ const response = await fetch(url);
+ if (!response.ok) throw new Error(`Could not download ${filename} (${response.status})`);
+ return new Uint8Array(await response.arrayBuffer());
+}
+
function screenshotFilename(taken: Date, attempt: number): string {
const pad = (value: number) => String(value).padStart(2, "0");
const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-");
diff --git a/apps/desktop/src/dapi/tools-session.test.ts b/apps/desktop/src/dapi/tools-session.test.ts
index d52840ed..8c0d4016 100644
--- a/apps/desktop/src/dapi/tools-session.test.ts
+++ b/apps/desktop/src/dapi/tools-session.test.ts
@@ -9,7 +9,7 @@ import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
import { JSON_SCHEMA_DIALECT, tools } from "@diffusionstudio/dapi";
import { serveCatalog } from "./tools-session";
-const CONTEXT = { rootDir: null, projectDir: null, currentTime: null, fontFamilies: [], masks: [] };
+const CONTEXT = { rootDir: null, projectDir: null, currentTime: null, fontFamilies: [] };
let client: Client;
const calls: Array<{ name: string; args: unknown }> = [];
diff --git a/apps/web/src/dapi/handlers/context.ts b/apps/web/src/dapi/handlers/context.ts
index 81cbd149..e1f2a4e2 100644
--- a/apps/web/src/dapi/handlers/context.ts
+++ b/apps/web/src/dapi/handlers/context.ts
@@ -4,7 +4,6 @@
import { Computed, Fonts, FrameRate, getActiveEntity } from "@diffusionstudio/runtime";
import { getProjectsRoot } from "@/projects";
-import { maskTrackRows } from "./media-segment";
import type { ToolHandler } from "../handler";
@@ -13,15 +12,15 @@ import type { ToolHandler } from "../handler";
* scenes, what is selected, which scene is active, the work area are all in
* the file, and a caller that wants them reads it. What is left over is which
* folder new projects go in, which project folder the app has open, where its
- * playhead sits, which font families are actually registered in the world
- * drawing it, and how background mask tracks are getting on.
+ * playhead sits, and which font families are actually registered in the
+ * world drawing it.
* With no project open only the root is left to report.
*/
export const context: ToolHandler<"context"> = async (_, ctx) => {
const rootDir = await getProjectsRoot();
const open = ctx.session();
- if (!open) return { rootDir, projectDir: null, currentTime: null, fontFamilies: [], masks: [] };
+ if (!open) return { rootDir, projectDir: null, currentTime: null, fontFamilies: [] };
const { world, project } = open;
const frameRate = world.get(FrameRate)?.value || 30;
@@ -36,6 +35,5 @@ export const context: ToolHandler<"context"> = async (_, ctx) => {
// What text can be drawn with right now: registered in the world, not
// merely named in the source. The editor default is always among them.
fontFamilies: [...new Set(["Inter", ...(world.get(Fonts)?.list ?? []).map((f) => f.family)])],
- masks: maskTrackRows(world),
};
};
diff --git a/apps/web/src/dapi/handlers/generate.ts b/apps/web/src/dapi/handlers/generate.ts
index 365122ba..f3110904 100644
--- a/apps/web/src/dapi/handlers/generate.ts
+++ b/apps/web/src/dapi/handlers/generate.ts
@@ -4,8 +4,7 @@
import { getAssetFile, getLibrary } from "@diffusionstudio/runtime";
-import { isAbsoluteSource, isProjectSource, isUrlSource } from "@diffusionstudio/assets";
-import { DapiError, generatedPath } from "@diffusionstudio/dapi";
+import { DapiError, generatedPath, isLocalJobId, isLocalModel } from "@diffusionstudio/dapi";
import { TRPCClientError } from "@trpc/client";
import { api } from "@/lib/api";
@@ -13,15 +12,26 @@ import { supabase } from "@/lib/supabase";
import { uploadFile } from "@/lib/uploads";
import { encodeSceneAudio, hasAudio } from "@/engine/scene-audio";
import { resolveAsset } from "../lib/assets";
+import { destination, storeAt } from "../lib/outputs";
import { requireScene } from "../lib/scene";
+import { localJob, startLocalJob } from "../local/jobs";
import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
import type { AssetRef, GenerateRequestInput, Job } from "@diffusionstudio/api-contract";
import type { DapiErrorCode, GenerationResult } from "@diffusionstudio/dapi";
import type { ToolContext, ToolHandler } from "../handler";
-import type { EditorSession } from "../session";
+import type { Destination } from "../lib/outputs";
-export const generate: ToolHandler<"generate"> = async ({ output, maxCredits, ...request }, ctx) => {
+/**
+ * Starts a job: on the API, or, for a model that runs on this machine, in the
+ * app (see ../local). Both answer with the API's `Job`, and `job` polls both.
+ */
+export const generate: ToolHandler<"generate"> = async (args, ctx) => {
+ if (isLocalModel(args.model)) {
+ return startLocalJob(args, destination(args.output, ctx.session()), ctx);
+ }
+
+ const { output, maxCredits, ...request } = args;
await requireSignIn();
const target = destination(output, ctx.session());
@@ -45,6 +55,10 @@ export const generate: ToolHandler<"generate"> = async ({ output, maxCredits, ..
};
export const job: ToolHandler<"job"> = async ({ id, cancel }, ctx) => {
+ if (isLocalJobId(id)) {
+ return localJob(id, cancel ?? false);
+ }
+
await requireSignIn();
try {
@@ -130,31 +144,9 @@ async function uploadScene(id: string, field: string | undefined, ctx: ToolConte
// ── Where the files go ───────────────────────────────────────
-/**
- * Into the library, at `path` (see `generatedPath`) or at its root under the
- * API's names when that is null; or to a path on disk the server writes
- * (null: a fresh directory under the temp dir). Settled when the job starts,
- * so a bad `output` fails before any credits are spent.
- */
-type Destination = { kind: "library"; path: string | null } | { kind: "disk"; path: string | null };
-
/** Where each job started here saves its files; a job started elsewhere (the prompt box, before a reload) gets the default. */
const destinations = new Map<string, Destination>();
-function destination(output: string | undefined, session: EditorSession | null): Destination {
- if (output === undefined) return { kind: session ? "library" : "disk", path: null };
- if (isUrlSource(output)) throw new DapiError("invalid-input", `The result cannot be saved to a URL (${output}).`);
- if (/[\\/]$/.test(output)) throw new DapiError("invalid-input", `The output is the file to save, not a folder (got "${output}").`);
- if (isAbsoluteSource(output)) return { kind: "disk", path: output };
- if (!session) {
- throw new DapiError(
- "no-project",
- `"${output}" is a library path, which needs an open project — open one, or pass an absolute path to save the result elsewhere.`,
- );
- }
- return { kind: "library", path: output.replace(/^\.\//, "") };
-}
-
/** Library saves in flight or done, by job id: the first poll that sees a job succeed saves it, the rest wait for that. */
const saves = new Map<string, Promise<Asset[]>>();
@@ -210,22 +202,6 @@ async function saveToLibrary(library: AssetLibrary, job: Job, output: string | n
return assets;
}
-/**
- * Puts `blob` into the library at `path`. A file the project made already
- * there is written over and taken in again at the same path, so everything
- * naming it shows the new one; a linked file of the user's is never written
- * over, and the new one gets a free name next to it instead.
- */
-async function storeAt(library: AssetLibrary, path: string, blob: Blob): Promise<Asset> {
- const existing = library.get(path);
- if (existing && isProjectSource(existing.source)) {
- await library.fs.write(existing.source, blob);
- return library.relink(existing, existing.source);
- }
- const slash = path.lastIndexOf("/");
- return library.store(blob, { folder: slash < 0 ? "" : path.slice(0, slash), name: path.slice(slash + 1) });
-}
-
// ── The API ──────────────────────────────────────────────────
async function requireSignIn(): Promise<void> {
diff --git a/apps/web/src/dapi/handlers/index.ts b/apps/web/src/dapi/handlers/index.ts
index bbd33a7f..284d1ac4 100644
--- a/apps/web/src/dapi/handlers/index.ts
+++ b/apps/web/src/dapi/handlers/index.ts
@@ -15,7 +15,6 @@ import { mediaTranscribe } from "./media-transcribe";
import { mediaFilmstrip } from "./media-filmstrip";
import { mediaWaveform } from "./media-waveform";
import { mediaListen } from "./media-listen";
-import { mediaSegment } from "./media-segment";
import { generate, job } from "./generate";
import type { Handlers } from "../handler";
@@ -35,7 +34,6 @@ export const handlers: Handlers = {
media_filmstrip: mediaFilmstrip,
media_waveform: mediaWaveform,
media_listen: mediaListen,
- media_segment: mediaSegment,
generate,
job,
};
diff --git a/apps/web/src/dapi/lib/outputs.ts b/apps/web/src/dapi/lib/outputs.ts
new file mode 100644
index 00000000..b7e76743
--- /dev/null
+++ b/apps/web/src/dapi/lib/outputs.ts
@@ -0,0 +1,48 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { isAbsoluteSource, isProjectSource, isUrlSource } from "@diffusionstudio/assets";
+import { DapiError } from "@diffusionstudio/dapi";
+
+import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
+import type { EditorSession } from "../session";
+
+/**
+ * Where the files a job makes go: into the library, at `path` (see
+ * `generatedPath`) or under the model's own names when that is null; or to a
+ * path on disk the server writes (null: a fresh directory under the temp
+ * dir). Settled when the job starts, so a bad `output` fails before any
+ * credits are spent or any frame is worked on.
+ */
+export type Destination = { kind: "library"; path: string | null } | { kind: "disk"; path: string | null };
+
+export function destination(output: string | undefined, session: EditorSession | null): Destination {
+ if (output === undefined) return { kind: session ? "library" : "disk", path: null };
+ if (isUrlSource(output)) throw new DapiError("invalid-input", `The result cannot be saved to a URL (${output}).`);
+ if (/[\\/]$/.test(output)) throw new DapiError("invalid-input", `The output is the file to save, not a folder (got "${output}").`);
+ if (isAbsoluteSource(output)) return { kind: "disk", path: output };
+ if (!session) {
+ throw new DapiError(
+ "no-project",
+ `"${output}" is a library path, which needs an open project — open one, or pass an absolute path to save the result elsewhere.`,
+ );
+ }
+ return { kind: "library", path: output.replace(/^\.\//, "") };
+}
+
+/**
+ * Puts `blob` into the library at `path`. A file the project made already
+ * there is written over and taken in again at the same path, so everything
+ * naming it shows the new one; a linked file of the user's is never written
+ * over, and the new one gets a free name next to it instead.
+ */
+export async function storeAt(library: AssetLibrary, path: string, blob: Blob): Promise<Asset> {
+ const existing = library.get(path);
+ if (existing && isProjectSource(existing.source)) {
+ await library.fs.write(existing.source, blob);
+ return library.relink(existing, existing.source);
+ }
+ const slash = path.lastIndexOf("/");
+ return library.store(blob, { folder: slash < 0 ? "" : path.slice(0, slash), name: path.slice(slash + 1) });
+}
diff --git a/apps/web/src/dapi/local/jobs.ts b/apps/web/src/dapi/local/jobs.ts
new file mode 100644
index 00000000..2c0ad959
--- /dev/null
+++ b/apps/web/src/dapi/local/jobs.ts
@@ -0,0 +1,222 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { z } from "zod";
+import { getLibrary } from "@diffusionstudio/runtime";
+import { DapiError, LOCAL_JOB_PREFIX, LOCAL_MODELS, generatedPath, isDapiError } from "@diffusionstudio/dapi";
+
+import { storeAt } from "../lib/outputs";
+import { localModels } from "./models";
+
+import type { World } from "koota";
+import type { Accessor } from "solid-js";
+import type { JobErrorCode } from "@diffusionstudio/api-contract";
+import type { GenerateRequest, GenerationResult, LocalModelId } from "@diffusionstudio/dapi";
+import type { ToolContext } from "../handler";
+import type { Destination } from "../lib/outputs";
+import type { EditorSession } from "../session";
+import type { LocalFile, LocalPlan } from "./model";
+
+type Status = "queued" | "running" | "succeeded" | "failed" | "canceled";
+
+type JobFile = GenerationResult["job"]["assets"][number];
+
+/** A job of a local model, in the API's `Job` shape, with `progress` and `details` of its own. */
+type LocalJob = {
+ id: string;
+ model: LocalModelId;
+ request: Record<string, unknown>;
+ status: Status;
+ phase: string | null;
+ progress: number | null;
+ credits: 0;
+ etaSeconds: number;
+ etaRemainingSeconds: number | null;
+ name: string | null;
+ assets: JobFile[];
+ details: Record<string, unknown>;
+ error: { code: JobErrorCode; message: string } | null;
+ createdAt: string;
+ startedAt: string | null;
+ completedAt: string | null;
+ expiresAt: null;
+};
+
+type Entry = {
+ job: LocalJob;
+ target: Destination;
+ /** The world whose library the files go into: closing its project cancels the job. */
+ world: World | null;
+ controller: AbortController;
+ image?: Uint8Array;
+};
+
+/** Every local job started since the app loaded, by id. They live as long as the page: a reload forgets them. */
+const jobs = new Map<string, Entry>();
+
+/**
+ * Checks the request against the model's schema, settles the job with the
+ * model (a bad file or time fails here, before any work), and starts it.
+ * Answers at once with the job queued, or, for work the model says is quick
+ * (`inline`), once it has ended.
+ */
+export async function startLocalJob(args: GenerateRequest, target: Destination, ctx: ToolContext): Promise<GenerationResult> {
+ const model = args.model as LocalModelId;
+ const parsed = LOCAL_MODELS[model].safeParse(args);
+ if (!parsed.success) {
+ throw new DapiError("invalid-input", z.prettifyError(parsed.error));
+ }
+ const plan = await localModels[model].prepare(parsed.data, ctx);
+ const session = ctx.session();
+ const entry: Entry = {
+ job: {
+ id: `${LOCAL_JOB_PREFIX}${crypto.randomUUID()}`,
+ model,
+ request: modelFields(parsed.data),
+ status: "queued",
+ phase: "queued",
+ progress: null,
+ credits: 0,
+ etaSeconds: round(plan.etaSeconds),
+ etaRemainingSeconds: null,
+ name: null,
+ assets: [],
+ details: plan.details ?? {},
+ error: null,
+ createdAt: new Date().toISOString(),
+ startedAt: null,
+ completedAt: null,
+ expiresAt: null,
+ },
+ target,
+ world: target.kind === "library" ? session!.world : null,
+ controller: new AbortController(),
+ };
+ jobs.set(entry.job.id, entry);
+
+ const done = execute(entry, plan, ctx.session);
+ if (plan.inline) {
+ // The caller waits for it, so the caller going away stops it.
+ const abort = () => entry.controller.abort("The call was canceled.");
+ ctx.signal.addEventListener("abort", abort, { once: true });
+ await done.finally(() => ctx.signal.removeEventListener("abort", abort));
+ }
+ return present(entry);
+}
+
+/** A local job as it stands, canceled first when `cancel`. */
+export function localJob(id: string, cancel: boolean): GenerationResult {
+ const entry = jobs.get(id);
+ if (!entry) {
+ throw new DapiError("not-found", `No local job ${id}: a local model's jobs live in the app, and are gone once it restarts or reloads.`);
+ }
+ if (cancel && !hasEnded(entry.job)) {
+ entry.controller.abort("Canceled.");
+ end(entry.job, "canceled", { code: "CANCELED", message: "Canceled." });
+ }
+ return present(entry);
+}
+
+function present({ job, target, image }: Entry): GenerationResult {
+ return {
+ job: { ...job, assets: job.assets.map((file) => ({ ...file })), details: { ...job.details } },
+ saveTo: target.kind === "disk" ? target.path : null,
+ ...(image ? { image } : {}),
+ };
+}
+
+/** Runs the plan and files each step on the job; never rejects, a failure is the job's. */
+async function execute(entry: Entry, plan: LocalPlan, current: Accessor<EditorSession | null>): Promise<void> {
+ const { job, controller } = entry;
+ const { signal } = controller;
+ // A project closed in the meantime stops the job at its next report, and
+ // nothing is written into a library no longer open.
+ const stillOpen = () => {
+ if (entry.world && current()?.world !== entry.world) controller.abort("The project was closed before the job finished.");
+ return !signal.aborted;
+ };
+ const start = () => {
+ if (job.status !== "queued") return;
+ job.status = "running";
+ job.phase = "running";
+ job.startedAt = new Date().toISOString();
+ };
+
+ try {
+ const output = await plan.run({
+ signal,
+ start: () => {
+ if (stillOpen()) start();
+ },
+ report: (phase, progress, etaRemainingSeconds) => {
+ if (!stillOpen()) return;
+ start();
+ Object.assign(job, { phase, progress, etaRemainingSeconds: etaRemainingSeconds === null ? null : round(etaRemainingSeconds) });
+ },
+ });
+ if (!stillOpen()) {
+ signal.throwIfAborted();
+ }
+ start();
+ Object.assign(job, { phase: "finalizing", progress: null, etaRemainingSeconds: null });
+ const assets = await save(entry, output.files, current);
+ if (hasEnded(job)) return;
+ job.assets = assets;
+ job.details = { ...job.details, ...output.details };
+ entry.image = output.image;
+ end(job, "succeeded", null);
+ } catch (error) {
+ if (hasEnded(job)) return;
+ if (signal.aborted) {
+ end(job, "canceled", { code: "CANCELED", message: String(signal.reason) });
+ return;
+ }
+ const message = error instanceof Error ? error.message : String(error);
+ end(job, "failed", { code: isDapiError(error) && error.code === "invalid-input" ? "INVALID_INPUT" : "INTERNAL", message });
+ }
+}
+
+/**
+ * The files where the job was told: into the library — at `output`, its
+ * extension the file's, written over a file the project made there, or under
+ * the model's name in its folder, next to anything already there — or held as
+ * bytes for the server to write to disk.
+ */
+async function save(entry: Entry, files: LocalFile[], current: Accessor<EditorSession | null>): Promise<JobFile[]> {
+ const { target } = entry;
+ const described = (file: LocalFile) => ({ filename: file.filename, mimeType: file.blob.type, size: file.blob.size });
+ if (target.kind === "disk") {
+ return Promise.all(files.map(async (file) => ({ ...described(file), bytes: new Uint8Array(await file.blob.arrayBuffer()) })));
+ }
+
+ const session = current()!;
+ const library = getLibrary(session.world);
+ const saved: JobFile[] = [];
+ for (const [i, file] of files.entries()) {
+ const asset =
+ target.path === null
+ ? await library.store(file.blob, { folder: file.folder ?? "", name: file.filename })
+ : await storeAt(library, generatedPath(target.path, file.filename, i, files.length), file.blob);
+ saved.push({ ...described(file), path: `${session.project.dir()}/${asset.source}`, src: asset.path });
+ }
+ return saved;
+}
+
+/** The request as the job reports it, as the API's jobs do: the model's fields, not the tool's. */
+function modelFields({ output: _output, maxCredits: _maxCredits, ...fields }: Record<string, unknown>): Record<string, unknown> {
+ return fields;
+}
+
+function end(job: LocalJob, status: Status, error: LocalJob["error"]): void {
+ Object.assign(job, { status, error, phase: null, etaRemainingSeconds: null, completedAt: new Date().toISOString() });
+ if (status === "succeeded") job.progress = 1;
+}
+
+function hasEnded(job: LocalJob): boolean {
+ return job.status === "succeeded" || job.status === "failed" || job.status === "canceled";
+}
+
+function round(seconds: number): number {
+ return Math.max(0, Math.round(seconds * 10) / 10);
+}
diff --git a/apps/web/src/dapi/local/model.ts b/apps/web/src/dapi/local/model.ts
new file mode 100644
index 00000000..802f550e
--- /dev/null
+++ b/apps/web/src/dapi/local/model.ts
@@ -0,0 +1,56 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// What a model that runs on this machine implements to be a `generate` model.
+// The runner (./jobs) owns everything a job has in common: its id and status,
+// timestamps, cancelling, errors, and where its files are saved. A model
+// says what the job will be and does the work.
+
+import type { LocalModelId, LocalRequest } from "@diffusionstudio/dapi";
+import type { ToolContext } from "../handler";
+
+export interface LocalModel<M extends LocalModelId> {
+ /**
+ * Checks what the request's schema cannot (the file it names, times against
+ * the file's length) and plans the job. Throws a `DapiError` for a request
+ * it cannot run, before any job starts. Where the files go (`output`) is
+ * the runner's business, not the model's.
+ */
+ prepare(request: LocalRequest<M>, ctx: ToolContext): Promise<LocalPlan>;
+}
+
+export type LocalPlan = {
+ /** Seconds the job is expected to run for, a download of the model included. */
+ etaSeconds: number;
+ /** The job's `details` from the start: what is known before any work. */
+ details?: Record<string, unknown>;
+ /** Work quick enough that `generate` runs it before answering, with the job ended. */
+ inline?: boolean;
+ run(job: LocalRun): Promise<LocalOutput>;
+};
+
+export type LocalRun = {
+ /** Fires when the job is canceled, the caller of an inline job goes away, or the project its files go into closes. */
+ signal: AbortSignal;
+ /** The model is the job's now: a queued job is running from here on. */
+ start(): void;
+ /** Where the job stands: its phase, how far through it (0..1, null when unknown), and the seconds left in all (null when unknown). */
+ report(phase: string, progress: number | null, etaRemainingSeconds: number | null): void;
+};
+
+export type LocalOutput = {
+ files: LocalFile[];
+ /** Added to the job's `details`. */
+ details?: Record<string, unknown>;
+ /** A picture of what the job made, for the caller to look at once: shown inline, its path in `details.image`. */
+ image?: Uint8Array;
+};
+
+export type LocalFile = {
+ blob: Blob;
+ /** The file's name, its extension the type `output` is corrected to. */
+ filename: string;
+ /** The library folder the file goes into when `output` names none (default: the library's root). */
+ folder?: string;
+};
diff --git a/apps/web/src/dapi/local/models.ts b/apps/web/src/dapi/local/models.ts
new file mode 100644
index 00000000..f4d42112
--- /dev/null
+++ b/apps/web/src/dapi/local/models.ts
@@ -0,0 +1,13 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { sam } from "./sam";
+
+import type { LocalModelId } from "@diffusionstudio/dapi";
+import type { LocalModel } from "./model";
+
+/** Every local model the catalog names, by id: one missing fails the type check. */
+export const localModels: { readonly [M in LocalModelId]: LocalModel<M> } = {
+ "sam-2.1": sam,
+};
diff --git a/apps/web/src/dapi/handlers/media-segment.ts b/apps/web/src/dapi/local/sam.ts
similarity index 54%
rename from apps/web/src/dapi/handlers/media-segment.ts
rename to apps/web/src/dapi/local/sam.ts
index eda673ec..ea19a2d7 100644
--- a/apps/web/src/dapi/handlers/media-segment.ts
+++ b/apps/web/src/dapi/local/sam.ts
@@ -3,25 +3,25 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { CanvasSink } from "mediabunny";
-import { ASSETS_DIR, MASK_EXTENSION, isAbsoluteSource, isProjectSource, isUrlSource, traceMask } from "@diffusionstudio/assets";
+import { MASK_EXTENSION, traceMask } from "@diffusionstudio/assets";
import { DapiError, WEAK_IOU } from "@diffusionstudio/dapi";
import { encodePng } from "@diffusionstudio/encoder";
import { FrameRate, formatTimecode, getLibrary, getVideoTrack } from "@diffusionstudio/runtime";
+import { downloadSize, sam2Model } from "@diffusionstudio/sam2/models";
-import { encodeObjectMask, nextTrackingName, objectMaskFolder, pendingMaskPaths } from "@/engine/object-mask/commit";
+import { encodeObjectMask, nextTrackingName, objectMaskFolder } from "@/engine/object-mask/commit";
import { segmentFootage } from "@/engine/object-mask/tracking";
import { requireAssetType, resolveAsset } from "../lib/assets";
import { SheetCollector } from "../lib/sheets";
-import type { World } from "koota";
-import type { Accessor } from "solid-js";
import type { InputVideoTrack } from "mediabunny";
-import type { Asset, AssetLibrary, MaskFrame, VideoAsset } from "@diffusionstudio/assets";
-import type { MediaSegmentRequest, MediaSegmentResult, ToolResult } from "@diffusionstudio/dapi";
+import type { MaskFrame, VideoAsset } from "@diffusionstudio/assets";
+import type { SamDetails, SamRequest } from "@diffusionstudio/dapi";
import type { Sam2Point } from "@diffusionstudio/sam2";
+import type { Sam2ModelId } from "@diffusionstudio/sam2/models";
import type { FootageSegmentRequest, FootageSegments } from "@/engine/object-mask/tracking";
-import type { EditorSession } from "../session";
-import type { ToolHandler } from "../handler";
+import type { ToolContext } from "../handler";
+import type { LocalModel, LocalOutput, LocalRun } from "./model";
type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
@@ -36,79 +36,115 @@ const SHEET_FRAME_WIDTH = 512;
const OUTSIDE = "rgba(255, 0, 200, 0.5)";
const BOX = "#FFC400";
+/** Seconds a tracked frame takes, by size, as measured on an M1: what a job's estimate starts from. */
+const SECONDS_PER_FRAME: Record<Sam2ModelId, number> = { tiny: 0.25, small: 2, "base-plus": 2.6, large: 4.5 };
+/** A preview: encoding the prompted frame, the slow part of it, about. */
+const PREVIEW_SECONDS = 2;
+/** The rate a first use's model download is reckoned at, in bytes a second. */
+const DOWNLOAD_RATE = 10_000_000;
+
+type Prompt = Pick<SamRequest, "points" | "exclude" | "box">;
+type Found = Required<Pick<SamDetails, "bbox" | "area" | "score" | "iou" | "lost" | "weak">>;
+type SegmentRequest = Omit<FootageSegmentRequest, "signal" | "onStart" | "onDownload" | "onProgress">;
+
/**
- * Segments an object in a video and tracks it through the span asked for, on
- * the object mask tool's model and queue, without a word in the app. A mask
- * bound for the project's library is tracked in the background: the call
- * answers with where it will be at once, and `context` reports the track (see
- * `maskTrackRows`). One the server writes, and a preview of the prompted
- * frame alone, are segmented before the call returns, with a picture of what
- * was found.
+ * SAM 2.1: segments an object in a video and tracks it through the span
+ * asked for, on the object mask tool's model and queue, without a word in the
+ * app, making a mask file. A preview segments the prompted frame alone and
+ * makes only a picture of it, quick enough that `generate` waits for it.
*/
-export const mediaSegment: ToolHandler<"media_segment"> = async (args, ctx) => {
- const asset = await resolveAsset(ctx, args.path);
- requireAssetType(asset, ["VIDEO"], "a video");
- const session = ctx.session();
- if (args.time > asset.duration) {
- throw new DapiError("invalid-input", `time ${args.time}s is past the video's end (${asset.duration.toFixed(2)}s).`);
- }
+export const sam: LocalModel<"sam-2.1"> = {
+ async prepare(request, ctx) {
+ const asset = await resolveAsset(ctx, request.video.path);
+ requireAssetType(asset, ["VIDEO"], "a video");
+ if (request.time > asset.duration) {
+ throw new DapiError("invalid-input", `time ${request.time}s is past the video's end (${asset.duration.toFixed(2)}s).`);
+ }
- // Frames are counted on the grid the mask file will hold: the project's
- // rate, which `<mask>` plays it back at, or the file's own with none open.
- const fps = session?.world.get(FrameRate)?.value ?? (asset.frameRate || 30);
- const total = Math.max(1, Math.ceil(asset.duration * fps - 1e-6));
- let first = Math.min(Math.round(args.time * fps), total - 1);
- let count = 1;
- if (!args.preview) {
- const from = args.start === undefined ? 0 : Math.round(args.start * fps);
- const to = args.end === undefined ? total : Math.min(total, Math.round(args.end * fps));
- if (from >= total) {
- throw new DapiError("invalid-input", `start ${args.start}s is past the video's end (${asset.duration.toFixed(2)}s).`);
+ // Frames are counted on the grid the mask file will hold: the project's
+ // rate, which `<mask>` plays it back at, or the file's own with none open.
+ const fps = ctx.session()?.world.get(FrameRate)?.value ?? (asset.frameRate || 30);
+ const total = Math.max(1, Math.ceil(asset.duration * fps - 1e-6));
+ let first = Math.min(Math.round(request.time * fps), total - 1);
+ let count = 1;
+ if (!request.preview) {
+ const from = request.start === undefined ? 0 : Math.round(request.start * fps);
+ const to = request.end === undefined ? total : Math.min(total, Math.round(request.end * fps));
+ if (from >= total) {
+ throw new DapiError("invalid-input", `start ${request.start}s is past the video's end (${asset.duration.toFixed(2)}s).`);
+ }
+ count = Math.max(1, to - from);
+ first = from;
}
- count = Math.max(1, to - from);
- first = from;
- }
- const seedFrame = Math.min(Math.max(Math.round(args.time * fps), first), first + count - 1);
- const target = args.preview ? null : destination(args.output, session);
-
- const model = args.model ?? "tiny";
- const span: Span = { model, frameRate: fps, start: round(first / fps), end: round((first + count) / fps), frames: count };
- const request: SegmentRequest = {
- asset,
- fps,
- model,
- points: prompts(args),
- seedFrame,
- first,
- count,
- preview: args.preview ?? false,
- };
- if (target?.kind === "library") return startTrack(session!, ctx.session, args, request, span, target.path);
+ const segment: SegmentRequest = {
+ asset,
+ fps,
+ model: request.size,
+ points: prompts(request),
+ seedFrame: Math.min(Math.max(Math.round(request.time * fps), first), first + count - 1),
+ first,
+ count,
+ preview: request.preview ?? false,
+ };
+ const estimate = await estimateSeconds(segment);
+ const span = { frameRate: fps, start: round(first / fps), end: round((first + count) / fps), frames: count };
+ return {
+ etaSeconds: estimate.download + estimate.work,
+ details: span,
+ inline: segment.preview,
+ run: (job) => run(segment, request, estimate, job, ctx),
+ };
+ },
+};
- let segments: FootageSegments;
- try {
- segments = await segmentFootage({ ...request, signal: ctx.signal });
- } catch (error) {
- if (isAbort(error)) throw new DapiError("canceled", "The call was canceled.");
- throw error;
- }
+type Estimate = { download: number; work: number };
+
+/** Seconds the job is reckoned to take: the model's download, when it is not cached whole, then the frames. */
+async function estimateSeconds(request: SegmentRequest): Promise<Estimate> {
+ const work = request.preview ? PREVIEW_SECONDS : request.count * SECONDS_PER_FRAME[request.model];
+ const { cachedSam2Models } = await import("@diffusionstudio/sam2");
+ const cached = await cachedSam2Models().catch(() => new Set<Sam2ModelId>());
+ return { download: cached.has(request.model) ? 0 : downloadSize(sam2Model(request.model)) / DOWNLOAD_RATE, work };
+}
+
+async function run(request: SegmentRequest, prompt: Prompt, estimate: Estimate, job: LocalRun, ctx: ToolContext): Promise<LocalOutput> {
+ // The rate frames are tracked at is measured from when the model is ready.
+ let since = performance.now();
+ const segments = await segmentFootage({
+ ...request,
+ signal: job.signal,
+ onStart: () => {
+ since = performance.now();
+ job.start();
+ },
+ onDownload: (fraction) => {
+ since = performance.now();
+ job.report("downloading", fraction, fraction === null ? null : (1 - fraction) * estimate.download + estimate.work);
+ },
+ onProgress: (completed, total) => {
+ const perFrame = (performance.now() - since) / 1000 / completed;
+ job.report("tracking", completed / total, perFrame * (total - completed));
+ },
+ });
+ const { asset, first, fps, seedFrame } = request;
const found = describeFound(segments, seedFrame - first, first, fps);
const video = await decodableTrack(asset);
- if (args.preview) {
- const png = await previewImage(video, segments, seedFrame, segments.masks[0] ?? null, args);
- return { png, ...span, ...found };
+ if (request.preview) {
+ const image = await previewImage(video, segments, seedFrame, segments.masks[0] ?? null, prompt);
+ return { files: [], details: found, image };
}
- const png = await contactSheet(video, segments, asset, first, fps);
- const file = encodeTrack(segments, request);
- return { png, mask: new Uint8Array(await file.arrayBuffer()), state: "done", ...span, ...found };
-};
-
-type Span = Pick<MediaSegmentResult, "model" | "frameRate" | "start" | "end" | "frames">;
-type Found = Required<Pick<MediaSegmentResult, "bbox" | "area" | "score" | "iou" | "lost" | "weak">>;
-type SegmentRequest = Omit<FootageSegmentRequest, "signal" | "onDownload" | "onProgress">;
+ const image = await contactSheet(video, segments, asset, first, fps);
+ // Filed as the object mask tool files its masks: `masks/<video>/Tracking <n>.mask`.
+ const folder = objectMaskFolder(asset);
+ const session = ctx.session();
+ const filename = session
+ ? nextTrackingName(getLibrary(session.world), folder)
+ : `${folder.slice(folder.lastIndexOf("/") + 1)}.${MASK_EXTENSION}`;
+ return { files: [{ blob: encodeTrack(segments, request), filename, folder }], details: found, image };
+}
/** What the masks say about the object: on the seed frame, and the spans where it was lost or weak. */
function describeFound(segments: FootageSegments, seedIndex: number, first: number, fps: number): Found {
@@ -140,104 +176,8 @@ async function decodableTrack(asset: VideoAsset): Promise<InputVideoTrack> {
return video;
}
-function isAbort(error: unknown): boolean {
- return error instanceof DOMException && error.name === "AbortError";
-}
-
-// ── Tracks in the background ─────────────────────────────────
-
-/** A background track as `context` reports it. */
-type TrackRow = ToolResult<"context">["masks"][number];
-
-/** The tracks started in each project's world, oldest first; they go with the world. */
-const tracks = new WeakMap<World, TrackRow[]>();
-
-/** The background tracks started in `world`, for `context`. */
-export function maskTrackRows(world: World): TrackRow[] {
- return (tracks.get(world) ?? []).map((row) => ({ ...row }));
-}
-
-/**
- * Starts tracking into the library and answers at once with where the mask
- * will be. The path is promised to the track while it runs, so the tool and
- * other tracks name their masks around it.
- */
-function startTrack(
- session: EditorSession,
- current: Accessor<EditorSession | null>,
- args: MediaSegmentRequest,
- request: SegmentRequest,
- span: Span,
- path: string | null,
-): MediaSegmentResult {
- const library = getLibrary(session.world);
- const folder = objectMaskFolder(request.asset);
- const src = path ?? `${folder}/${nextTrackingName(library, folder)}`;
- const existing = library.get(src);
- const source = existing && isReplaceableMask(existing) ? existing.source : `${ASSETS_DIR}/${src}`;
-
- const row: TrackRow = { id: crypto.randomUUID(), src, video: args.path, state: "loading", progress: null, ...span };
- const rows = tracks.get(session.world) ?? [];
- rows.push(row);
- tracks.set(session.world, rows);
-
- pendingMaskPaths.add(src);
- void runTrack(row, session, current, request, path !== null).finally(() => pendingMaskPaths.delete(src));
- return { path: `${session.project.dir()}/${source}`, src, state: "tracking", ...span };
-}
-
-/**
- * Tracks, then writes the mask into the library, filing each step on `row`;
- * `replace` when the caller named the path, so a mask already there is
- * written over. A project closed in the meantime stops the track at its next
- * frame, and nothing is written into a library no longer open.
- */
-async function runTrack(
- row: TrackRow,
- session: EditorSession,
- current: Accessor<EditorSession | null>,
- request: SegmentRequest,
- replace: boolean,
-): Promise<void> {
- const controller = new AbortController();
- const stillOpen = () => {
- if (current()?.world !== session.world) controller.abort();
- return !controller.signal.aborted;
- };
-
- try {
- const segments = await segmentFootage({
- ...request,
- signal: controller.signal,
- onDownload: (fraction) => {
- if (stillOpen()) Object.assign(row, { state: "loading", progress: fraction });
- },
- onProgress: (completed, total) => {
- if (stillOpen()) Object.assign(row, { state: "tracking", progress: completed / total });
- },
- });
-
- const video = await decodableTrack(request.asset);
- const png = await contactSheet(video, segments, request.asset, request.first, request.fps);
- const file = encodeTrack(segments, request);
- if (!stillOpen()) return;
-
- const stored = await storeMask(getLibrary(session.world), row.src, file, replace);
- Object.assign(row, {
- state: "done",
- progress: 1,
- src: stored.path,
- png,
- ...describeFound(segments, request.seedFrame - request.first, request.first, request.fps),
- });
- } catch (error) {
- if (isAbort(error)) return;
- Object.assign(row, { state: "failed", progress: null, error: error instanceof Error ? error.message : String(error) });
- }
-}
-
/** The prompt as SAM 2 takes it: points on the object, points off it, then a box's corners. */
-function prompts(args: MediaSegmentRequest): Sam2Point[] {
+function prompts(args: Prompt): Sam2Point[] {
const points: Sam2Point[] = [
...(args.points ?? []).map(({ x, y }) => ({ x, y, label: 1 as const })),
...(args.exclude ?? []).map(({ x, y }) => ({ x, y, label: 0 as const })),
@@ -249,59 +189,6 @@ function prompts(args: MediaSegmentRequest): Sam2Point[] {
return points;
}
-// ── Where the mask goes ──────────────────────────────────────
-
-/**
- * Where a tracked mask is written: into the library — at `path`, or at the
- * tool's own `masks/<video>/Tracking <n>.mask` when that is null — or back to
- * the server, which writes absolute paths and the temp dir. Settled before
- * tracking, so a bad `output` fails before the long part.
- */
-type Destination = { kind: "library"; path: string | null } | { kind: "server" };
-
-function destination(output: string | undefined, session: EditorSession | null): Destination {
- if (output === undefined) return session ? { kind: "library", path: null } : { kind: "server" };
- if (isAbsoluteSource(output)) return { kind: "server" };
- if (isUrlSource(output)) throw new DapiError("invalid-input", `The mask cannot be written to a URL (${output}).`);
- if (!session) {
- throw new DapiError(
- "no-project",
- `"${output}" is a library path, which needs an open project — open one, or pass an absolute path to write the mask elsewhere.`,
- );
- }
-
- const path = output.replace(/^\.\//, "").replace(/\/+$/, "");
- const extension = /\.([^./]+)$/.exec(path)?.[1];
- if (extension !== undefined && extension.toLowerCase() !== MASK_EXTENSION) {
- throw new DapiError("invalid-input", `A mask file ends in .${MASK_EXTENSION} (got "${output}").`);
- }
- const named = extension === undefined ? `${path}.${MASK_EXTENSION}` : path;
- const existing = getLibrary(session.world).get(named);
- if (existing && !isReplaceableMask(existing)) {
- throw new DapiError("invalid-input", `"${named}" in the library is not a mask file of this project's, so it cannot be replaced.`);
- }
- return { kind: "library", path: named };
-}
-
-function isReplaceableMask(asset: Asset): boolean {
- return asset.type === "MASK" && isProjectSource(asset.source);
-}
-
-/**
- * Puts the mask file into the library at `path`. When `replace`, a mask
- * already there has its file written over and is taken in again at the same
- * path, as a restore does, so every `<mask>` naming it shows the new frames.
- */
-async function storeMask(library: AssetLibrary, path: string, file: Blob, replace: boolean): Promise<Asset> {
- const existing = library.get(path);
- if (replace && existing && isReplaceableMask(existing)) {
- await library.fs.write(existing.source, file);
- return library.relink(existing, existing.source);
- }
- const slash = path.lastIndexOf("/");
- return library.store(file, { folder: slash < 0 ? "" : path.slice(0, slash), name: path.slice(slash + 1) });
-}
-
// ── What was found ───────────────────────────────────────────
/** A frame without a mask, or one where the model sees no object. */
@@ -358,7 +245,7 @@ async function previewImage(
segments: FootageSegments,
seedFrame: number,
mask: MaskFrame | null,
- args: MediaSegmentRequest,
+ args: Prompt,
): Promise<Uint8Array> {
const width = Math.min(PREVIEW_WIDTH, await video.getDisplayWidth());
const sink = new CanvasSink(video, { width });
@@ -436,7 +323,7 @@ function drawMask(ctx: Ctx2D, mask: MaskFrame, grid: number, width: number, heig
* points off it black, each labelled `+n` or `−n` in the order it was given,
* and the box as a dashed amber frame.
*/
-function drawPrompts(ctx: Ctx2D, args: MediaSegmentRequest, width: number, height: number, scale: number): void {
+function drawPrompts(ctx: Ctx2D, args: Prompt, width: number, height: number, scale: number): void {
ctx.save();
if (args.box) {
const [x0, y0, x1, y1] = args.box;
diff --git a/apps/web/src/engine/object-mask/commit.tsx b/apps/web/src/engine/object-mask/commit.tsx
index 493a8686..f005b5bb 100644
--- a/apps/web/src/engine/object-mask/commit.tsx
+++ b/apps/web/src/engine/object-mask/commit.tsx
@@ -88,18 +88,11 @@ export function objectMaskFolder(video: VideoAsset): string {
return `${MASKS_FOLDER}/${assetName(video).replace(/\.[^.]+$/, '')}`;
}
-/**
- * Library paths promised to masks still being tracked in the background (see
- * the agent's `media_segment`): taken, though nothing is there yet.
- */
-export const pendingMaskPaths = new Set<string>();
-
-/** `Tracking <n>.mask`, `n` the first count not yet taken in `folder`, nor promised to a track still running. */
+/** `Tracking <n>.mask`, `n` the first count not yet taken in `folder`. */
export function nextTrackingName(library: AssetLibrary, folder: string): string {
for (let n = 1; ; n++) {
const name = `Tracking ${n}.${MASK_EXTENSION}`;
- const path = `${folder}/${name}`;
- if (!library.get(path) && !pendingMaskPaths.has(path)) return name;
+ if (!library.get(`${folder}/${name}`)) return name;
}
}
diff --git a/apps/web/src/engine/object-mask/tracking.ts b/apps/web/src/engine/object-mask/tracking.ts
index 3a91bd4f..46fc3ba3 100644
--- a/apps/web/src/engine/object-mask/tracking.ts
+++ b/apps/web/src/engine/object-mask/tracking.ts
@@ -311,7 +311,7 @@ export function getMaskRestoreOf(asset: MaskAsset | null | undefined): MaskResto
/**
* An object to segment in footage on its own, rather than on a clip: what
- * the agent's `media_segment` asks. Frames are counted on a grid of `fps`,
+ * the agent's `sam-2.1` jobs ask. Frames are counted on a grid of `fps`,
* the one the mask file will hold.
*/
export type FootageSegmentRequest = {
@@ -327,6 +327,8 @@ export type FootageSegmentRequest = {
/** Segments the seed frame alone, without tracking: the frame stays encoded, so the next prompt on it is quick. */
preview: boolean;
signal: AbortSignal;
+ /** Its turn has come: the tool's own sessions and earlier requests are done with the model. */
+ onStart?: () => void;
/** The model's download, 0 to 1, while it is on its way. */
onDownload?: (progress: number | null) => void;
/** Frames masked so far, of `count`. */
@@ -356,6 +358,7 @@ export function segmentFootage(request: FootageSegmentRequest): Promise<FootageS
busy = true;
try {
signal.throwIfAborted();
+ request.onStart?.();
const session = await openSession(asset, fps, request.model, request.onDownload);
signal.throwIfAborted();
const model = session.model.model;
diff --git a/docs/reference/jsx/styles.md b/docs/reference/jsx/styles.md
index 7bd4a446..ea911e60 100644
--- a/docs/reference/jsx/styles.md
+++ b/docs/reference/jsx/styles.md
@@ -68,7 +68,7 @@ A filter over the parent's **rendered pixels** — its fills, strokes and childr
## `<mask>`
-A matte limiting the `<effect>` holding it: a **picture** whose alpha says where the effect applies, fitted into the node's box the way the node fits its footage. It is what the editor's object mask tool makes — pick an object in a video clip, track it, and the frames it found are written to the library and named by `src` — and what [`media_segment`](../tools/media/segment.md) makes from a tool call. Several under one effect intersect. Without a `src` a `<mask>` does nothing.
+A matte limiting the `<effect>` holding it: a **picture** whose alpha says where the effect applies, fitted into the node's box the way the node fits its footage. It is what the editor's object mask tool makes — pick an object in a video clip, track it, and the frames it found are written to the library and named by `src` — and what [`generate`](../tools/generate.md)'s [`sam-2.1`](../tools/media/segment.md) makes from a tool call. Several under one effect intersect. Without a `src` a `<mask>` does nothing.
Under an `"opacity"` effect the mask is the cut-out: the clip shows inside the mask and goes transparent outside, as a mask on Premiere's Opacity does. Under any other effect the effect shows through inside the mask and the picture is untouched outside: an inverted mask on a `"blur"` blurs the background behind a person.
diff --git a/docs/reference/models.md b/docs/reference/models.md
index c3e431df..514a1575 100644
--- a/docs/reference/models.md
+++ b/docs/reference/models.md
@@ -1,6 +1,6 @@
# Models
-Every model [`generate`](./tools/generate.md) runs, and the fields each takes. A request is `model` plus these fields; the API validates it and answers a field it cannot take with what to fix. Models of a kind share one set of fields (every image model takes `prompt`, `images`, `aspectRatio`, …), and each model narrows them: the values it offers are in its row. Leave a field out for the model's default. Files are `{ "path": … }` (see [generate](./tools/generate.md#files)).
+Every model [`generate`](./tools/generate.md) runs, and the fields each takes. A request is `model` plus these fields; the API validates it and answers a field it cannot take with what to fix ([local models](#local) are checked by the app the same way). Models of a kind share one set of fields (every image model takes `prompt`, `images`, `aspectRatio`, …), and each model narrows them: the values it offers are in its row. Leave a field out for the model's default. Files are `{ "path": … }` (see [generate](./tools/generate.md#files)).
| Kind | Models | Makes |
| --- | --- | --- |
@@ -9,6 +9,7 @@ Every model [`generate`](./tools/generate.md) runs, and the fields each takes. A
| [Audio](#audio) | `elevenlabs-music`, `elevenlabs-sfx` | music, sound effects |
| [Voice](#voice) | `elevenlabs-v3`, `gemini-3.8-flash-tts` | speech |
| [Tools](#tools) | `remove-background`, `upscale-image`, `upscale-video`, `transcribe`, `analyze` | one file in, one file out |
+| [Local](#local) | `sam-2.1` | runs on this machine, free: an object's mask, tracked through footage |
## Image
@@ -93,3 +94,11 @@ Each takes one file and returns one.
| `upscale-video` | `video`: `{ path }`, required | an MP4 of the video upscaled to 2K at 30 fps |
| `transcribe` | `audio`: `{ path }` (audio or video) or `{ scene }` (the scene's mix, see [Scenes](./tools/generate.md#scenes)), required; `languageCode`: e.g. `en`, `en_us`, `de` (detected when left out) | `transcript.json`: `[{ text, speaker?, words: [{ text, start, end }] }]`, times in seconds, speakers labelled `A`, `B`, … when told apart |
| `analyze` | `media`: `{ path }`, required (image, video or audio); `prompt`: what to look for, up to 10,000 characters (a general description when left out) | the analysis, as a file |
+
+## Local
+
+These run in the app, on this machine's GPU: no account, no credits. A job is shaped like the API's and polled the same way with [`job`](./tools/job.md), with two fields of its own — `progress` (0..1 through its current `phase`) and `details` (what the model found) — and lasts only as long as the app: a job id is gone once the app restarts. The first use of a model downloads it.
+
+| Model | | Makes | Fields |
+| --- | --- | --- | --- |
+| `sam-2.1` | SAM 2.1 — segments an object in a video and tracks it through the footage; `size` tiny (default), small, base-plus or large | a mask file (`.mask`) for a [`<mask src>`](./jsx/styles.md#mask) | [segment.md](./tools/media/segment.md) |
diff --git a/docs/reference/tools/README.md b/docs/reference/tools/README.md
index c56ce76a..d69c7278 100644
--- a/docs/reference/tools/README.md
+++ b/docs/reference/tools/README.md
@@ -43,7 +43,6 @@ Every tool runs inside the app, so the app has to be running. Over MCP that is a
| [`media_filmstrip`](./media/filmstrip.md) | `diffusion media filmstrip` | Filmstrip preview |
| [`media_waveform`](./media/waveform.md) | `diffusion media waveform` | Waveform preview |
| [`media_listen`](./media/listen.md) | `diffusion media listen` | Listen to audio |
-| [`media_segment`](./media/segment.md) | `diffusion media segment` | Segment object |
| [`generate`](./generate.md) | `diffusion generate` | Generate media |
| [`job`](./job.md) | `diffusion job` | Generation job |
| [`logs`](./logs.md) | `diffusion logs` | App logs |
@@ -56,8 +55,7 @@ How the surface is divided:
- **The project loop.** [`open`](./open.md) a folder, edit its JSX, [`context`](./context.md) for what the source cannot say, [`capture`](./capture.md) and [`check`](./check.md) to verify, [`export`](./export.md) when asked.
- **Media inspection** (`media_*`): a file by path, without adding it to the project. Absolute paths and URLs work with or without an open project; library paths (`b-roll/clip.mp4`) need one.
-- **Masks.** [`media_segment`](./media/segment.md) segments and tracks an object in footage and writes the mask file a `<mask src>` names.
-- **Generation.** [`generate`](./generate.md) runs a model of the Diffusion Studio API (images, video, music, speech, and tools such as upscaling) on the account's credits; poll [`job`](./job.md) until it has ended, and its files are saved into the library or a directory. The models and their fields are in [models.md](../models.md).
+- **Generation.** [`generate`](./generate.md) runs a model of the Diffusion Studio API (images, video, music, speech, and tools such as upscaling) on the account's credits, or a model on this machine for free — [`sam-2.1`](./media/segment.md) segments and tracks an object in footage, making the mask file a `<mask src>` names; poll [`job`](./job.md) until it has ended, and its files are saved into the library or a directory. The models and their fields are in [models.md](../models.md).
- **What a declaration may name.** [`fonts`](./fonts.md).
- **The app and the machine.** [`logs`](./logs.md), [`screenshot`](./screenshot.md), [`window`](./window.md), [`report`](./report.md).
diff --git a/docs/reference/tools/context.md b/docs/reference/tools/context.md
index 589e5c1a..8b0998f9 100644
--- a/docs/reference/tools/context.md
+++ b/docs/reference/tools/context.md
@@ -1,6 +1,6 @@
# context
-Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, and the progress of `media_segment` tracks running in the background. Poll it to wait for tracks without blocking.
+Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, and the registered font families.
| | |
| --- | --- |
@@ -26,21 +26,10 @@ One JSON object:
projectDir: string | null; // absolute open project folder — where the JSX being edited lives; null when none is open
currentTime: number | null; // playhead in the active scene, in seconds; null if no scene is active
fontFamilies: string[]; // families registered in the running world, valid as `fontFamily`
- masks: { // media_segment tracks started while this project is open, oldest first
- id: string; // the track's id, the same across polls
- src: string; // the mask's library path, for <mask src>; the file is there once done
- video: string; // the footage tracked, as media_segment was given it
- state: "loading" | "tracking" | "done" | "failed";
- progress: number | null; // 0..1: the model's download while loading, the frames masked while tracking; 1 once done
- error?: string; // what it failed with, on `failed` rows
- image?: string; // absolute path of a contact sheet of tracked frames, on `done` rows
- model, frameRate, start, end, frames; // the span, as media_segment returned it
- bbox?, area?, score?, iou?, lost?, weak?; // what was found, on `done` rows — see media_segment
- }[];
}
```
-With no project open (the app sits at the dashboard) the report is just `{ rootDir, projectDir: null }` with an empty `masks` list: there is no playhead, no world, and no fonts to speak of. Open one with [`open`](./open.md).
+With no project open (the app sits at the dashboard) the report is just `{ rootDir, projectDir: null }`: there is no playhead, no world, and no fonts to speak of. Open one with [`open`](./open.md).
`rootDir` is reported whether or not a project is open — it is where a caller with nothing open goes to create or find one.
@@ -49,5 +38,3 @@ With no project open (the app sits at the dashboard) the report is just `{ rootD
`projectDir` is the folder the app is editing, which is not necessarily the one a command was run from: check it before writing to source files.
`fontFamilies` is what text can be drawn with right now — loaded into the world, not merely named in the source — and always includes the editor default. For every family a text can name, see [`fonts`](./fonts.md).
-
-`masks` is how a caller waits for a [`media_segment`](./media/segment.md) track that went into the library: the call returns as soon as it starts, so poll this until the row whose `src` it returned is `done` or `failed`. `loading` covers the model loading (its download, the first time, is `progress`) and waiting behind the editor's object mask tool or another track. A `done` row carries the same findings `media_segment` returns for a finished track, and its contact sheet as `image` — over MCP it also arrives inline, once, with the first poll that reports the row done. Rows last as long as the project stays open; closing it stops its tracks.
diff --git a/docs/reference/tools/generate.md b/docs/reference/tools/generate.md
index f7740670..66b94d6c 100644
--- a/docs/reference/tools/generate.md
+++ b/docs/reference/tools/generate.md
@@ -1,6 +1,6 @@
# generate
-Start a job on a generative model of the Diffusion Studio API — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis — and return the job at once, with its estimated run time and the credits it costs. Poll [`job`](./job.md) with its id until it has ended; the files it made are then saved and their paths returned. Generating needs a signed-in account and spends its credits.
+Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking ([`sam-2.1`](./media/segment.md)) on this machine — and return the job at once, with its estimated run time and the credits it costs. Poll [`job`](./job.md) with its id until it has ended; the files it made are then saved and their paths returned. API models need a signed-in account and spend its credits; [local models](../models.md#local) run on the GPU, free, and need no account.
| | |
| --- | --- |
@@ -10,7 +10,7 @@ Start a job on a generative model of the Diffusion Studio API — images, video,
## Input
-The input is the API's request — `model` plus that model's fields — with two fields of the tool's own. Every model and the fields it takes are listed in [models.md](../models.md).
+The input is the model's request — `model` plus that model's fields — with two fields of the tool's own. Every model and the fields it takes are listed in [models.md](../models.md).
| Field | Type | CLI | Description |
| --- | --- | --- | --- |
@@ -19,11 +19,11 @@ The input is the API's request — `model` plus that model's fields — with two
| `output` | `string` | `-o, --output <path>` | file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: the library's root under the API's name with a project open, its files under `assets/`; else a fresh directory under the system temp dir). On the CLI a path starting with `.` resolves against the working directory. |
| `maxCredits` | `number` | `--max-credits <n>` | refuse to start the job if it costs more credits than this |
-The model's fields are not checked here: they go to the API as given, and the API answers a request it cannot run with what to fix (`invalid-input`, e.g. an aspect ratio the model does not offer). Each model narrows its kind's bounds, so read its entry in [models.md](../models.md) rather than guessing.
+An API model's fields are not checked here: they go to the API as given, and the API answers a request it cannot run with what to fix (`invalid-input`, e.g. an aspect ratio the model does not offer). Each model narrows its kind's bounds, so read its entry in [models.md](../models.md) rather than guessing. A [local model](../models.md#local)'s fields are checked by the app, the same way, before its job starts.
## Files
-Where a field takes a file (`images`, `startFrame`, `endFrame`, `image`, `video`, `audio`, `media`), put a `{ "path": … }` object: an absolute path or URL (with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project). The file is uploaded before the job starts — once per file, however many jobs name it — and the reference becomes the API's own. The API's references (`{ "kind": "asset", "id" }`, `{ "kind": "url", "url" }`) pass through unchanged. On the CLI a `path` that exists relative to the working directory is sent as its absolute path.
+Where a field takes a file (`images`, `startFrame`, `endFrame`, `image`, `video`, `audio`, `media`), put a `{ "path": … }` object: an absolute path or URL (with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project). The file is uploaded before the job starts — once per file, however many jobs name it — and the reference becomes the API's own; a local model reads it in place. The API's references (`{ "kind": "asset", "id" }`, `{ "kind": "url", "url" }`) pass through unchanged. On the CLI a `path` that exists relative to the working directory is sent as its absolute path.
```json
{
@@ -51,13 +51,13 @@ diffusion generate remove-background '{"image":{"path":"./fox.png"}}'
- **Several files are numbered.** A job that makes more than one (an image `count` above 1) saves `fox-1.png`, `fox-2.png`, …; a single file is saved at the path itself.
- **A library path** (`b-roll/fox.png`, needs an open project) puts the file into the project's library, under `assets/`, named by `src` like any other asset, and notes the job it came from, as the app's own generations do. A file the project made already at that path is replaced, and everything naming it shows the new one; a file linked from elsewhere on disk is never written over, and the result gets a free name next to it instead.
- **An absolute path** saves the file there, replacing a file already there.
-- **Left out**, the file goes into the open project's library at its root under the API's name (`red-fox-at-dawn.png`, numbered if taken); with no project open, into a fresh directory of the job's own under the system temp dir, as [`media_transcribe`](./media/transcribe.md) and the other inspection tools do.
+- **Left out**, the file goes into the open project's library at its root under the API's name (`red-fox-at-dawn.png`, numbered if taken), or where a local model files its own (`sam-2.1`: `masks/<video>/Tracking <n>.mask`); with no project open, into a fresh directory of the job's own under the system temp dir, as [`media_transcribe`](./media/transcribe.md) and the other inspection tools do.
-The files are saved by the first [`job`](./job.md) call that sees the job succeed; later calls return the same paths. A file saved into the library records the job in `assets.yml` (its `job` field), so what made it can be looked up later.
+The files are saved by the first [`job`](./job.md) call that sees the job succeed (a local model's library files as soon as it does); later calls return the same paths. A file an API model made that went into the library records the job in `assets.yml` (its `job` field), so what made it can be looked up later.
## Output
-The API's job, as it is when the call returns — almost always `queued` — and the same shape [`job`](./job.md) returns. Every field is the API's, passed through unchanged, except that each file in `assets` has its download `url` replaced by where it was saved. The fields to act on:
+The API's job, as it is when the call returns — almost always `queued`; a local model's quick work, such as a `sam-2.1` preview, already ended — and the same shape [`job`](./job.md) returns. Every field is the API's, passed through unchanged, except that each file in `assets` has its download `url` replaced by where it was saved. The fields to act on:
```ts
{
@@ -67,6 +67,8 @@ The API's job, as it is when the call returns — almost always `queued` — and
etaRemainingSeconds: number | null; // estimated seconds left while running
credits: number; // charged; refunded if the job fails or is canceled
error: { code: string; message: string } | null; // why it failed or was canceled
+ progress?: number | null; // a local model's job: 0..1 through its current `phase`
+ details?: object; // a local model's job: what it found, model by model (see models.md)
assets: Array<{ // the files it made, empty until it succeeds
path: string; // absolute path of the saved file
src?: string; // its library path, for `src`, when it went into the library
@@ -77,8 +79,8 @@ The API's job, as it is when the call returns — almost always `queued` — and
## Errors
-- `sign-in-required` — the app is not signed in to a Diffusion Studio account.
-- `invalid-input` — the API rejected the request; the message names the field and what it takes.
+- `sign-in-required` — an API model, and the app is not signed in to a Diffusion Studio account.
+- `invalid-input` — the API (or, for a local model, the app) rejected the request; the message names the field and what it takes.
- `no-project` — `output` or a file is a library path and no project is open.
- `not-found` — a `{ path }` names no file.
- Not enough credits for the job (or more than `maxCredits`): the message says so.
diff --git a/docs/reference/tools/job.md b/docs/reference/tools/job.md
index 7c092b42..2df1106c 100644
--- a/docs/reference/tools/job.md
+++ b/docs/reference/tools/job.md
@@ -1,6 +1,6 @@
# job
-The state of a job started with [`generate`](./generate.md): its status, progress estimate and cost, and, once it has succeeded, the files it made, saved where `generate` was told. `cancel` stops a job and refunds its credits.
+The state of a job started with [`generate`](./generate.md): its status, progress estimate and cost, and, once it has succeeded, the files it made, saved where `generate` was told, and, from a [local model](../models.md#local), its `details`. `cancel` stops a job and refunds its credits.
| | |
| --- | --- |
@@ -16,7 +16,7 @@ The state of a job started with [`generate`](./generate.md): its status, progres
## Polling
-A job is `queued`, then `running`, then ends `succeeded`, `failed` or `canceled`. Once it has ended, nothing about it changes again: stop polling. While it works, the job says how long to wait — `etaRemainingSeconds` while running, `etaSeconds` (the whole run) while queued — so look once around then rather than in a tight loop; a call made early simply returns the job still working. Images take seconds, video minutes.
+A job is `queued`, then `running`, then ends `succeeded`, `failed` or `canceled`. Once it has ended, nothing about it changes again: stop polling. While it works, the job says how long to wait — `etaRemainingSeconds` while running, `etaSeconds` (the whole run) while queued — so look once around then rather than in a tight loop; a call made early simply returns the job still working. Images take seconds, video minutes. A local model's job also reports its `phase` and `progress` (0..1 through it).
The first call that sees the job succeed downloads its files to where [`generate`](./generate.md#where-the-files-go) was told and returns their paths in `assets`; later calls return the same paths without downloading again. A job whose files are in the open project's library already is never downloaded again: `assets` lists the files of it still there, so a lookup changes nothing. A job started with `generate` before the app restarted, with none of its files in the library, is saved to the default place: the open project's library, or the temp dir with none open.
@@ -28,6 +28,10 @@ sleep 15
diffusion job "$id" | jq '{status, etaRemainingSeconds, assets: [.assets[].path]}'
```
+## Local jobs
+
+A [local model](../models.md#local)'s job runs in the app, and its id starts with `local-`. It is polled and canceled like any other, costs nothing, and needs no account, but it lives only as long as the app: once the app restarts or reloads, `job` answers `not-found` for it. Its files are saved when it succeeds, and its `details` — what the model found, with a picture as `details.image` that over MCP also arrives inline, once — come with every poll from then on. Closing the project its files go into cancels it.
+
## Looking up a generated file
Every file the library holds from a model — made in the app's prompt box or with `generate` — records the job it came from in the project's `assets.yml`, as `job`. Pass that id here to see how it was made: `request` holds the model, the prompt and every other field, as the API took them. An input file appears there as the API's reference to its upload (`{ "kind": "asset", "id": <sha256> }`), not as a library path.
@@ -52,5 +56,5 @@ The API's job, with the files it made saved locally: each file's `url` is replac
## Errors
-- `not-found` — no job of this account has the id.
-- `sign-in-required` — the app is not signed in to a Diffusion Studio account.
+- `not-found` — no job of this account has the id, or a local job's id from before the app restarted.
+- `sign-in-required` — an API job, and the app is not signed in to a Diffusion Studio account.
diff --git a/docs/reference/tools/media/segment.md b/docs/reference/tools/media/segment.md
index 1f48cc4c..92965ba8 100644
--- a/docs/reference/tools/media/segment.md
+++ b/docs/reference/tools/media/segment.md
@@ -1,56 +1,52 @@
-# media_segment
+# sam-2.1
-Segment an object in a video with SAM 2.1 and track it through the footage, writing a mask file (`.mask`) that a [`<mask src>`](../../jsx/styles.md#mask) names: the file the editor's object mask tool makes (local model on the GPU, no credits). Prompt the object on the frame at `time` with `points` on it, `exclude` points off it, and/or a `box` around it, all in 0..1 of the frame (x right, y down). `preview: true` segments that frame alone and writes only an image: the frame with the mask tinted and outlined, the prompts drawn, and a 0.1 grid to read coordinates off. Refine the prompts with previews (fast once the frame is encoded), then make the same call without `preview` to track. Tracking covers `start` to `end`, the whole file by default, at roughly 0.25 s a frame with the tiny model and several times that with the larger ones, so pass the span the clip actually plays. With a project open and the mask going into its library, a track runs in the background: the call returns the mask's paths at once with `state: "tracking"`, and [`context`](../context.md) reports its progress, then a contact sheet of tracked frames and the spans where the object was lost or the mask is weak. A mask written elsewhere is tracked before the call returns, with the same findings. The first use of a model downloads it (83 MB for tiny, up to 475 MB for large).
+SAM 2.1 segments an object in a video and tracks it through the footage, making the mask file (`.mask`) a [`<mask src>`](../../jsx/styles.md#mask) names — the file the editor's object mask tool makes. It runs in the app, on this machine's GPU: no account, no credits. Like every [local model](../../models.md#local), its job is polled with [`job`](../job.md) and lasts only as long as the app.
| | |
| --- | --- |
-| MCP tool | `media_segment` |
-| CLI | `diffusion media segment <path> --time <time> [options]` |
-| CLI aliases | `diffusion media mask` |
+| Model | `sam-2.1`, run with [`generate`](../generate.md) |
+| CLI | `diffusion generate sam-2.1 '<fields>' [-o <path>]` |
## Input
-| Field | Type | CLI | Description |
+`model: "sam-2.1"` plus these fields; `output` is [`generate`](../generate.md#where-the-files-go)'s.
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `video` | `{ path }`, required | the footage |
+| `time` | `Time`, required | the frame the prompts are placed on, in source time — seconds (`"1.5"`), frames (`"45f"`), or `"MM:SS"` |
+| `points` | `{ x, y }[]` | points on the object, each in 0..1 of the frame |
+| `exclude` | `{ x, y }[]` | points on what is not the object, to take it out of the mask |
+| `box` | `[x0, y0, x1, y1]` | a box around the object, in 0..1 of the frame |
+| `preview` | `boolean` | segment only the frame at `time` and make no mask: for checking a prompt before tracking |
+| `start` | `Time` | start of the span to track (default: 0) |
+| `end` | `Time` | end of the span to track (default: the file's end) |
+| `size` | `string` | the model's size, from the table below (default: `tiny`) |
+
+| Size | | Download | Per tracked frame |
| --- | --- | --- | --- |
-| `path` | `string`, required | `<path>` | absolute file path or URL (works with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project) |
-| `time` | `Time`, required | `-t, --time <time>` | the frame the prompts are placed on, in source time — seconds ("1.5"), frames ("45f"), or "MM:SS" |
-| `points` | `{ x, y }[]` | `-p, --point <x,y>` | points on the object, each { x, y } in 0..1 of the frame; repeat the option for more |
-| `exclude` | `{ x, y }[]` | `-x, --exclude <x,y>` | points on what is not the object, to take it out of the mask; each { x, y } in 0..1 of the frame; repeat the option for more |
-| `box` | `[x0, y0, x1, y1]` | `-b, --box <x0,y0,x1,y1>` | a box around the object, in 0..1 of the frame |
-| `preview` | `boolean` | `--preview` | segment only the frame at `time` and write no mask: for checking a prompt before tracking |
-| `start` | `Time` | `-s, --start <time>` | start of the span to track (default: 0) |
-| `end` | `Time` | `-e, --end <time>` | end of the span to track (default: the file's end) |
-| `model` | `"tiny" \| "small" \| "base-plus" \| "large"` | `-m, --model <size>` | SAM 2.1 size: tiny (fastest), small, base-plus, or large (slowest; finest edges, best on small objects) (default: tiny) |
-| `output` | `string` | `-o, --output <path>` | where the mask file goes: a library path like `masks/skater.mask` (needs an open project; a mask already there is replaced, so every `<mask>` naming it follows) or an absolute file path (default: `masks/<video>/Tracking <n>.mask` in the library with a project open, else a fresh file under the system temp dir). On the CLI a path starting with `.` resolves against the working directory. |
+| `tiny` | fastest; the default | 83 MB | about 0.25 s |
+| `small` | | 112 MB | about 2 s |
+| `base-plus` | | 183 MB | about 2.6 s |
+| `large` | finest edges, best on small objects | 475 MB | about 4.5 s |
## Prompting
-A prompt needs at least one point on the object or a box around it; `exclude` points only take things away. Coordinates are fractions of the frame as it displays (rotation applied), the same space [`media_grab`](./grab.md) frames are drawn in. A box is usually the most reliable first prompt: draw it around the object as seen in a grabbed frame. When the mask spills onto something next to the object, add an `exclude` point on that thing; when it misses part of the object, add a point on the missing part.
+A prompt needs at least one point on the object or a box around it; `exclude` points only take things away. Coordinates are fractions of the frame as it displays (rotation applied, x right, y down), the same space [`media_grab`](./grab.md) frames are drawn in. A box is usually the most reliable first prompt: draw it around the object as seen in a grabbed frame. When the mask spills onto something next to the object, add an `exclude` point on that thing; when it misses part of the object, add a point on the missing part.
-The loop is preview, look, adjust:
+With `preview: true`, `generate` segments that frame alone and answers with the job already succeeded: no file, and as `details.image` the frame with everything outside the mask washed magenta, its edge outlined, the prompts drawn (`+n` on the object, `−n` off it, the box dashed) and a 0.1 grid to read coordinates off. The first preview of a frame encodes it, which takes a moment; further previews on the same frame, file and size run the mask decoder alone and return at once. A preview makes no mask, so it takes neither a span nor an `output`.
```bash
-diffusion media segment footage/skater.mp4 --time 2.4 --box 0.4,0.2,0.65,0.95 --preview
-diffusion media segment footage/skater.mp4 --time 2.4 --box 0.4,0.2,0.65,0.95 --exclude 0.5,0.9 --preview
-diffusion media segment footage/skater.mp4 --time 2.4 --box 0.4,0.2,0.65,0.95 --exclude 0.5,0.9 --start 2 --end 9 -o masks/skater.mask
+diffusion generate sam-2.1 '{"video":{"path":"footage/skater.mp4"},"time":2.4,"box":[0.4,0.2,0.65,0.95],"preview":true}'
+diffusion generate sam-2.1 '{"video":{"path":"footage/skater.mp4"},"time":2.4,"box":[0.4,0.2,0.65,0.95],"exclude":[{"x":0.5,"y":0.9}],"preview":true}'
+diffusion generate sam-2.1 '{"video":{"path":"footage/skater.mp4"},"time":2.4,"box":[0.4,0.2,0.65,0.95],"exclude":[{"x":0.5,"y":0.9}],"start":2,"end":9}' -o masks/skater.mask
```
-The first preview of a frame encodes it, which takes a moment; further previews on the same frame, file and model run the mask decoder alone and return at once. A preview writes no mask, so it takes neither a span nor an `output`.
-
## Tracking
-Without `preview`, the object is segmented on the prompted frame and followed forward and backward through every frame from `start` to `end`, one mask frame per frame at the project's frame rate (the file's own with no project open). `time` must lie in the span. Tracking runs silently — the app shows nothing — and shares the model with the editor's object mask tool, so a track waits for the tool's own work to finish first, and tracks run one after another.
-
-**Into the project's library** (a project is open and `output` is a library path or left out), the track runs in the background. The call returns at once with the mask's `path` and `src`, the span it covers, and `state: "tracking"` — no picture and no findings yet. Poll [`context`](../context.md) until the track's row in `masks` (the one whose `src` matches) is `done`: that row carries the findings below and the contact sheet as `image`, and the mask file is in the library from then on. Name it in a `<mask>` once it is done. A `failed` row says why in `error`; closing the project stops its tracks.
-
-```bash
-diffusion media segment footage/skater.mp4 --time 2.4 --box 0.4,0.2,0.65,0.95 --start 2 --end 9 -o masks/skater.mask
-diffusion context
-```
-
-**Anywhere else** (an absolute `output`, or no project open), the call waits for the whole track and returns everything at once, `state: "done"`: the CLI allows it an hour, and an MCP client should give the call a comparable timeout.
+Without `preview`, the object is segmented on the prompted frame and followed forward and backward through every frame from `start` to `end`, one mask frame per frame at the project's frame rate (the file's own with no project open), so pass the span the clip actually plays. `time` must lie in the span. The job runs silently — the app shows nothing — and shares the model with the editor's object mask tool: it is `queued` while the tool or an earlier job has the model, then `running`, its `phase` `downloading` on a first use and `tracking` after, with `etaRemainingSeconds` measured from the frames done so far. Closing the project a mask is bound for cancels its job.
-The mask carries its recipe (footage, prompt, span, model), so the editor can make it again if its file goes missing. Name it on the clip it was made from, with `start` as its `sourceIn`, so it stays on the footage whatever the trim:
+Left without an `output`, the mask goes into the open project's library as `masks/<video>/Tracking <n>.mask`, as the object mask tool files its own, or, with no project open, into a directory of the job's own under the system temp dir. The mask carries its recipe (footage, prompt, span, model), so the editor can make it again if its file goes missing. Name it on the clip it was made from, with `details.start` as its `sourceIn`, so it stays on the footage whatever the trim:
```tsx
<video src="footage/skater.mp4" sourceIn={2} width={1920} height={1080}>
@@ -62,20 +58,15 @@ The mask carries its recipe (footage, prompt, span, model), so the editor can ma
## Output
-One JSON object:
+The span is in `details` from the start; what was found arrives once the job has succeeded, with a picture — the preview, or a contact sheet of up to 6 tracked frames with their masks — that over MCP also arrives inline, once, with the first poll that sees it.
```ts
-{
- image?: string; // PNG: the prompted frame with its mask on a preview, a contact sheet of up to 6 tracked frames otherwise; absent while a track runs
- path?: string; // absolute path of the mask file; absent on a preview
- src?: string; // the mask's library path, for <mask src>; when it goes into the project's library
- state?: "tracking" | "done"; // on a track: "tracking" while it runs in the background, "done" when the mask is written
- model: "tiny" | "small" | "base-plus" | "large";
+details: {
frameRate: number; // frames per second the mask holds
start: number; // source time of the first masked frame, seconds: the <mask>'s sourceIn
end: number; // source time just past the last masked frame, seconds
frames: number; // frames masked (1 on a preview)
- // What was found; absent while a track runs, when context's row reports it instead.
+ image?: string; // PNG: the prompted frame with its mask on a preview, the contact sheet otherwise
bbox?: [x0, y0, x1, y1] | null; // on the prompted frame, the object's bounds in 0..1 of the frame
area?: number; // on the prompted frame, the share of the frame the object covers, 0..1
score?: number; // on the prompted frame, the object score: at or below 0 the model sees no object
@@ -85,8 +76,8 @@ One JSON object:
}
```
-`lost` covers occlusion and the object leaving the frame as well as a track that slipped; look at those moments with [`media_grab`](./grab.md) before trusting or re-prompting them. Over MCP the image also arrives inline when it is at most a megabyte.
+`lost` covers occlusion and the object leaving the frame as well as a track that slipped; look at those moments with [`media_grab`](./grab.md) before trusting or re-prompting them.
## Errors
-Fails when the path can't be resolved or the asset is not a video, `time` or `start` is past the video's end, no point or box prompts the object, a box's bounds are out of order or outside 0..1, `time` lies outside `start`–`end`, a preview is given a span or an `output`, `output` is a library path with no project open or names something in the library that is not one of the project's masks, `output` has an extension other than `.mask`, the model cannot load (no WebGPU, or the download failed), or the call is canceled. A background track fails on its row in `context` instead.
+`generate` refuses the request (`invalid-input`) when no point or box prompts the object, a box's bounds are out of order or outside 0..1, `time` lies outside `start`–`end` or `time` or `start` is past the video's end, a preview is given a span or an `output`, or a field is one the model does not take; and when the path can't be resolved or is not a video. A job fails when the model cannot load (no WebGPU, or the download failed).
diff --git a/docs/skills/editor.md b/docs/skills/editor.md
index 816fe949..57a0ab61 100644
--- a/docs/skills/editor.md
+++ b/docs/skills/editor.md
@@ -46,7 +46,7 @@ How to confirm a change actually produced what you intended. A clean save does n
- Use the built-in tags for the media a composition is made of (audio, video, images, captions).
- Hoist the properties that define the composition's look — title copy, font family and size, accent colors, key padding — into top-level consts annotated with `@inspect`, so they become live controls in the app's inspector.
- For anything 3D, use Three.js drawn into a `<surface>` tag.
-- To cut an object out of footage, or confine an effect to it, track it with `media_segment` (preview the prompt first) and name the mask file in a `<mask>` under the clip's `<effect>`. A track into the library runs in the background: keep working, and poll `context` until its row in `masks` is done before naming it.
+- To cut an object out of footage, or confine an effect to it, track it with `generate`'s local `sam-2.1` model (preview the prompt first, see [sam-2.1](../reference/tools/media/segment.md)) and name the mask file in a `<mask>` under the clip's `<effect>`. The track runs as a job: keep working, and poll `job` until it has succeeded before naming the mask.
- For motion graphics, overlays and UI-heavy graphics, the `<html>` tag driven by a paused [anime.js](https://animejs.com) timeline
- Before animating anything, read the [easings reference](../guides/motion/easings.md) and choose easings deliberately — default or linear easing is what makes motion read as a slideshow.
- For tasks that don't need an editing UI, keep the app in the background: from a shell, `diffusion open -b <dir>` launches it that way. Over MCP the app is already running, and `open` only opens the folder.
diff --git a/packages/dapi/src/catalog.ts b/packages/dapi/src/catalog.ts
index c41ea563..4b2eb96b 100644
--- a/packages/dapi/src/catalog.ts
+++ b/packages/dapi/src/catalog.ts
@@ -18,7 +18,6 @@ import { mediaTranscribe } from "./tools/media-transcribe";
import { mediaFilmstrip } from "./tools/media-filmstrip";
import { mediaWaveform } from "./tools/media-waveform";
import { mediaListen } from "./tools/media-listen";
-import { mediaSegment } from "./tools/media-segment";
import { generate, job } from "./tools/generate";
import { fonts } from "./tools/fonts";
import { report } from "./tools/report";
@@ -41,7 +40,6 @@ export const catalog = [
mediaFilmstrip,
mediaWaveform,
mediaListen,
- mediaSegment,
generate,
job,
logs,
diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.ts
index 1c87b585..d8722de7 100644
--- a/packages/dapi/src/index.ts
+++ b/packages/dapi/src/index.ts
@@ -25,16 +25,26 @@ export type { ToolJsonSchemas } from "./json-schema";
export { DAPI_WIRE } from "./ipc";
export type { DapiCall, DapiCancel, DapiReply } from "./ipc";
export { FRAME_CAP } from "./tools/media-grab";
-export { SEGMENT_MODELS, WEAK_IOU } from "./tools/media-segment";
export { ISSUE_LOG_TAIL } from "./tools/report";
export { LOG_TAIL, LOG_MESSAGE_MAX } from "./tools/logs";
export { FONT_LIMIT } from "./tools/fonts";
export { generatedPath } from "./tools/generate";
+export {
+ LOCAL_JOB_PREFIX,
+ LOCAL_MODELS,
+ LOCAL_MODEL_IDS,
+ SAM_SIZES,
+ WEAK_IOU,
+ isLocalJobId,
+ isLocalModel,
+ samDetails,
+ samRequest,
+} from "./local";
+export type { LocalModelId, LocalRequest, SamDetails, SamRequest } from "./local";
// Named request and result types, for handlers that spell out their
// signature. Each is the parsed (output) side of the tool's schema.
import type { ImageRef as ImageRefSchema, LogEntry as LogEntrySchema, LogLevel as LogLevelSchema, TimecodedImage as TimecodedImageSchema } from "./schemas";
-import type { MaskRow as MaskRowType } from "./tools/context";
import type { CheckIssue as CheckIssueSchema, CheckIssueCode as CheckIssueCodeSchema } from "./tools/check";
import type { ExportFormat as ExportFormatSchema, ExportSettings as ExportSettingsSchema } from "./tools/export";
import type { FrameQuality as FrameQualitySchema } from "./tools/media-grab";
@@ -46,7 +56,6 @@ export type LogLevel = z.output<typeof LogLevelSchema>;
export type LogEntry = z.output<typeof LogEntrySchema>;
export type TimecodedImage = z.output<typeof TimecodedImageSchema>;
export type ImageRef = z.output<typeof ImageRefSchema>;
-export type MaskRow = MaskRowType;
export type CheckIssueCode = z.output<typeof CheckIssueCodeSchema>;
export type CheckIssue = z.output<typeof CheckIssueSchema>;
export type ExportFormat = z.output<typeof ExportFormatSchema>;
@@ -79,8 +88,6 @@ export type MediaWaveformRequest = ToolArgs<"media_waveform">;
export type MediaWaveformResult = ToolResult<"media_waveform">;
export type MediaListenRequest = ToolArgs<"media_listen">;
export type MediaListenResult = ToolResult<"media_listen">;
-export type MediaSegmentRequest = ToolArgs<"media_segment">;
-export type MediaSegmentResult = ToolResult<"media_segment">;
export type GenerateRequest = ToolArgs<"generate">;
export type GenerationResult = ToolResult<"generate">;
export type GenerationJob = ToolOutput<"generate">;
diff --git a/packages/dapi/src/local/fields.ts b/packages/dapi/src/local/fields.ts
new file mode 100644
index 00000000..2a958e35
--- /dev/null
+++ b/packages/dapi/src/local/fields.ts
@@ -0,0 +1,19 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { z } from "zod";
+import { AssetPath } from "../schemas";
+
+/** A file a request names, as the API's requests do: `{ path }`. */
+export const FileRef = z.strictObject({ path: AssetPath });
+
+/**
+ * The fields `generate` adds to every request, the tool's rather than the
+ * model's. `generate`'s own schema checks them; a local model's request
+ * schema lets them through so it can be strict about the rest.
+ */
+export const requestFields = {
+ output: z.string().optional(),
+ maxCredits: z.number().optional(),
+};
diff --git a/packages/dapi/src/local/index.ts b/packages/dapi/src/local/index.ts
new file mode 100644
index 00000000..e6b127d5
--- /dev/null
+++ b/packages/dapi/src/local/index.ts
@@ -0,0 +1,42 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Models the app runs on this machine, behind the same `generate` and `job`
+// as the API's: a request is `model` plus the model's fields, and a job is
+// shaped like the API's `Job`. Unlike the API's, a local model's request is
+// checked here, by its schema, since no server is there to say what to fix.
+//
+// A model is named for itself (`sam-2.1`), never for its task: task names
+// (`transcribe`, `remove-background`) are the API's, and the two lists must
+// not share an id.
+
+import { samRequest } from "./sam";
+
+import type { z } from "zod";
+
+/** Each local model's request schema, by model id. */
+export const LOCAL_MODELS = {
+ "sam-2.1": samRequest,
+} as const;
+
+export type LocalModelId = keyof typeof LOCAL_MODELS;
+
+/** A local model's request, parsed. */
+export type LocalRequest<M extends LocalModelId = LocalModelId> = z.output<(typeof LOCAL_MODELS)[M]>;
+
+export const LOCAL_MODEL_IDS = Object.keys(LOCAL_MODELS) as LocalModelId[];
+
+export function isLocalModel(model: string): model is LocalModelId {
+ return Object.hasOwn(LOCAL_MODELS, model);
+}
+
+/** Ids of jobs run on this machine start with this; the API's are UUIDs. */
+export const LOCAL_JOB_PREFIX = "local-";
+
+export function isLocalJobId(id: string): boolean {
+ return id.startsWith(LOCAL_JOB_PREFIX);
+}
+
+export { SAM_SIZES, WEAK_IOU, samDetails, samRequest } from "./sam";
+export type { SamDetails, SamRequest } from "./sam";
diff --git a/packages/dapi/src/local/sam.ts b/packages/dapi/src/local/sam.ts
new file mode 100644
index 00000000..cc0114cb
--- /dev/null
+++ b/packages/dapi/src/local/sam.ts
@@ -0,0 +1,106 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// SAM 2.1, the editor's object mask model: segments an object in a video and
+// tracks it through the footage, making the mask file (`.mask`) a `<mask src>`
+// names.
+
+import { z } from "zod";
+import { checkWindow } from "../schemas";
+import { NonNegativeTime, TIME_FORMS } from "../time";
+import { FileRef, requestFields } from "./fields";
+
+/** The SAM 2.1 sizes the app offers, fastest first: the ids of its model catalogue (`SAM2_MODELS`). */
+export const SAM_SIZES = ["tiny", "small", "base-plus", "large"] as const;
+
+/** A tracked frame the model rates below this is reported as weak. */
+export const WEAK_IOU = 0.5;
+
+const Unit = z.number().min(0).max(1);
+
+const Point = z.strictObject({ x: Unit, y: Unit });
+
+const Bounds = z.tuple([z.number(), z.number(), z.number(), z.number()]);
+
+/** A span of source time, `[start, end)` in seconds. */
+const Range = z.tuple([z.number(), z.number()]);
+
+export const samRequest = z
+ .strictObject({
+ model: z.literal("sam-2.1"),
+ video: FileRef.describe("the footage, `{ path }`"),
+ size: z
+ .enum(SAM_SIZES)
+ .default("tiny")
+ .describe("tiny (fastest), small, base-plus, or large (slowest; finest edges, best on small objects) (default: tiny)"),
+ time: NonNegativeTime.describe(`the frame the prompts are placed on, in source time — ${TIME_FORMS}`),
+ points: z.array(Point).optional().describe("points on the object, each { x, y } in 0..1 of the frame"),
+ exclude: z
+ .array(Point)
+ .optional()
+ .describe("points on what is not the object, to take it out of the mask; each { x, y } in 0..1 of the frame"),
+ box: Bounds.optional().describe("a box around the object, [x0, y0, x1, y1] in 0..1 of the frame"),
+ preview: z
+ .boolean()
+ .optional()
+ .describe("segment only the frame at `time` and write no mask: for checking a prompt before tracking"),
+ start: NonNegativeTime.optional().describe(`start of the span to track — ${TIME_FORMS} (default: 0)`),
+ end: NonNegativeTime.optional().describe(`end of the span to track — ${TIME_FORMS} (default: the file's end)`),
+ ...requestFields,
+ })
+ .superRefine((value, ctx) => {
+ if (!value.points?.length && !value.box) {
+ ctx.addIssue({ code: "custom", path: ["points"], message: "prompt the object with points on it or a box around it" });
+ }
+ if (value.box && (value.box[0] >= value.box[2] || value.box[1] >= value.box[3])) {
+ ctx.addIssue({ code: "custom", path: ["box"], message: "a box is [x0, y0, x1, y1] in 0..1 of the frame, with x0 < x1 and y0 < y1" });
+ }
+ if (value.box?.some((bound) => bound < 0 || bound > 1)) {
+ ctx.addIssue({ code: "custom", path: ["box"], message: "a box's bounds are in 0..1 of the frame" });
+ }
+ if (value.preview) {
+ for (const key of ["start", "end", "output"] as const) {
+ if (value[key] !== undefined) {
+ ctx.addIssue({ code: "custom", path: [key], message: `a preview segments one frame and writes no mask; ${key} applies to tracking only` });
+ }
+ }
+ return;
+ }
+ checkWindow(value, ctx);
+ if (value.start !== undefined && value.time < value.start) {
+ ctx.addIssue({ code: "custom", path: ["time"], message: `time (${value.time}s) is before start (${value.start}s); the prompted frame must be tracked` });
+ }
+ if (value.end !== undefined && value.time >= value.end) {
+ ctx.addIssue({ code: "custom", path: ["time"], message: `time (${value.time}s) is not before end (${value.end}s); the prompted frame must be tracked` });
+ }
+ });
+
+/**
+ * A job's `details`: the span, known when the job starts, and what was found,
+ * once it has succeeded.
+ */
+export const samDetails = z.object({
+ frameRate: z.number().describe("frames per second the mask holds: the project's, or the file's own with no project open"),
+ start: z.number().describe("source time of the first masked frame, in seconds: the `sourceIn` of a `<mask>` naming the file"),
+ end: z.number().describe("source time just past the last masked frame, in seconds"),
+ frames: z.int().describe("frames masked"),
+ image: z
+ .string()
+ .optional()
+ .describe("absolute path of a PNG: the prompted frame with its mask on a preview, a contact sheet of tracked frames otherwise"),
+ bbox: Bounds.nullable()
+ .optional()
+ .describe("on the prompted frame, the object's bounds as [x0, y0, x1, y1] in 0..1 of the frame; null when nothing was segmented"),
+ area: z.number().optional().describe("on the prompted frame, the share of the frame the object covers, 0..1"),
+ score: z.number().optional().describe("on the prompted frame, the model's object score: at or below 0 it sees no object there"),
+ iou: z.number().optional().describe("on the prompted frame, the model's estimate of the mask's quality, 0..1"),
+ lost: z
+ .array(Range)
+ .optional()
+ .describe("source-time spans, [start, end) in seconds, where no object was found: occluded, out of frame, or lost by the track"),
+ weak: z.array(Range).optional().describe(`source-time spans, [start, end) in seconds, where the model rated its mask below ${WEAK_IOU}`),
+});
+
+export type SamRequest = z.output<typeof samRequest>;
+export type SamDetails = z.output<typeof samDetails>;
diff --git a/packages/dapi/src/tools/context.ts b/packages/dapi/src/tools/context.ts
index 87399f38..f680f624 100644
--- a/packages/dapi/src/tools/context.ts
+++ b/packages/dapi/src/tools/context.ts
@@ -4,41 +4,17 @@
import { z } from "zod";
import { defineTool } from "../tool";
-import { Bytes } from "../schemas";
-import { segmentFound, segmentSpan } from "./media-segment";
-
-const maskRow = {
- id: z.string().describe("the track's id, the same across polls"),
- src: z.string().describe("the mask's library path, for `<mask src>`: the file is there once the row is done"),
- video: z.string().describe("the footage tracked, as media_segment was given it"),
- state: z.enum(["loading", "tracking", "done", "failed"]).describe(
- "`loading` while the model loads or another track has it, `tracking` while frames are masked, then `done` or `failed`",
- ),
- progress: z
- .number()
- .nullable()
- .describe("0..1: the model's download while loading (null when there is none to report), the frames masked while tracking; 1 once done"),
- error: z.string().optional().describe("what the track failed with, on failed rows"),
- ...segmentSpan,
- ...z.object(segmentFound).partial().shape,
-};
-
-const MaskRow = z.object({
- ...maskRow,
- image: z.string().optional().describe("absolute path of a contact sheet of tracked frames with their masks, on done rows"),
-});
/**
* What the project's source cannot say: the JSX already holds the scenes,
* the selection, and the work area, so the report is only the folders, the
- * playhead, the fonts actually registered, and where background mask tracks
- * stand.
+ * playhead, and the fonts actually registered.
*/
export const context = defineTool({
name: "context",
title: "App context",
description:
- "Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, and the progress of `media_segment` tracks running in the background. Poll it to wait for tracks without blocking.",
+ "Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, and the registered font families.",
input: z.object({}),
output: z.object({
rootDir: z.string().nullable().describe("folder new projects are created in; null until one has been chosen"),
@@ -50,17 +26,6 @@ export const context = defineTool({
fontFamilies: z
.array(z.string())
.describe("families registered in the world drawing the project; the editor default is always among them"),
- masks: z.array(MaskRow).describe("media_segment tracks started while this project is open, oldest first"),
- }),
- result: z.object({
- rootDir: z.string().nullable(),
- projectDir: z.string().nullable(),
- currentTime: z.number().nullable(),
- fontFamilies: z.array(z.string()),
- /** A done row's contact sheet comes as bytes, which the server writes to a file once per track. */
- masks: z.array(z.object({ ...maskRow, png: Bytes.optional() })),
}),
environment: "renderer",
});
-
-export type MaskRow = z.output<typeof MaskRow>;
diff --git a/packages/dapi/src/tools/generate.ts b/packages/dapi/src/tools/generate.ts
index 3a61e700..c9bc601b 100644
--- a/packages/dapi/src/tools/generate.ts
+++ b/packages/dapi/src/tools/generate.ts
@@ -5,11 +5,15 @@
// Generation on the Diffusion Studio API, nearly as the API has it: a request
// is the API's own (`model` plus that model's fields), passed through for the
// API to validate, and a job is the API's `Job`. What the tools add is what
-// only a local client can do: send local files, and save what a job made.
+// only a local client can do: send local files, save what a job made, and
+// run the models that work on this machine (see ../local) as jobs of the same
+// shape.
import { z } from "zod";
import { MODEL_IDS } from "@diffusionstudio/api-contract";
import { defineTool } from "../tool";
+import { Bytes } from "../schemas";
+import { LOCAL_MODEL_IDS } from "../local";
const JPEG = /^\.jpe?g$/i;
@@ -45,35 +49,51 @@ export const GenerationJob = z.looseObject({
id: z.string().describe("job id, for `job`"),
status: z.string().describe("queued or running while the job works; succeeded, failed or canceled once it has ended — stop polling then"),
assets: z.array(GeneratedFile).describe("the files the job made, saved: empty until it succeeds"),
+ progress: z
+ .number()
+ .nullable()
+ .optional()
+ .describe("on a local model's job, 0..1 through its current `phase` (a model's download, then the work); null when there is none to report"),
+ details: z
+ .record(z.string(), z.unknown())
+ .optional()
+ .describe("on a local model's job, what the model reports beyond its files, model by model (see `reference/models.md`)"),
});
/**
* What the renderer hands back: the job as the API has it, each file either
- * saved into the library already (`path`, `src`) or still at its `url`, for
- * the server to download to `saveTo` (see `generatedPath`; null: a fresh
- * directory under the system temp dir, under the API's names).
+ * saved into the library already (`path`, `src`), still at its `url`, or, from
+ * a local model, held as `bytes`: the last two for the server to download or
+ * write to `saveTo` (see `generatedPath`; null: a fresh directory under the
+ * system temp dir, under the files' own names). `image` is a picture of what
+ * a local job made, for the caller to look at: the server writes it to the
+ * temp dir once, shows it inline that once, and names it in `details.image`.
*/
export const GenerationResult = z.object({
job: z.looseObject({
id: z.string(),
status: z.string(),
- assets: z.array(z.looseObject({ url: z.string().optional(), path: z.string().optional(), filename: z.string() })),
+ assets: z.array(
+ z.looseObject({ url: z.string().optional(), bytes: Bytes.optional(), path: z.string().optional(), filename: z.string() }),
+ ),
+ details: z.record(z.string(), z.unknown()).optional(),
}),
saveTo: z.string().nullable(),
+ image: Bytes.optional(),
});
export const generate = defineTool({
name: "generate",
title: "Generate media",
description:
- "Start a job on a generative model of the Diffusion Studio API — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis — and return the job at once, with its estimated run time and the credits it costs. The input is the API's request: `model` plus that model's fields (`prompt`, `images`, `aspectRatio`, `duration`, `voice`, …), listed per model in the docs at `reference/models.md`. The fields are passed through as given and the API validates them, answering with what to fix. Where a field takes a file, put `{ \"path\": \"…\" }` — an absolute path, a URL, or a library path — and it is uploaded first. Poll `job` with the returned id until its status is succeeded, failed or canceled; the estimate says when to look. Generating needs a signed-in account and spends its credits.",
+ "Start a job on a generative model — images, video, music, sound effects, speech, and tools such as background removal, upscaling, transcription and analysis on the Diffusion Studio API, and object segmentation and tracking (`sam-2.1`) on this machine — and return the job at once, with its estimated run time and the credits it costs. The input is the model's request: `model` plus that model's fields (`prompt`, `images`, `aspectRatio`, `duration`, `voice`, …), listed per model in the docs at `reference/models.md`. An API model's fields are passed through as given and the API validates them, answering with what to fix; a local model's are checked here. Where a field takes a file, put `{ \"path\": \"…\" }` — an absolute path, a URL, or a library path — and it is uploaded first (a local model reads it in place). Poll `job` with the returned id until its status is succeeded, failed or canceled; the estimate says when to look. API models need a signed-in account and spend its credits; local models run on the GPU, free, and need no account.",
input: z.looseObject({
- model: z.enum(MODEL_IDS).describe("the model to run; its fields are listed in `reference/models.md`"),
+ model: z.enum([...MODEL_IDS, ...LOCAL_MODEL_IDS]).describe("the model to run; its fields are listed in `reference/models.md`"),
output: z
.string()
.optional()
.describe(
- "file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: the library's root under the API's name with a project open, its files under `assets/`; else a fresh directory under the system temp dir)",
+ "file the result is saved as: a library path like `b-roll/fox.png` (needs an open project) or an absolute path; its extension is corrected to what the model made, several files are numbered (`fox-1.png`, `fox-2.png`), and a file already there is replaced (default: with a project open, the library's root under the model's name for the file, its files under `assets/`, or the model's own folder where `reference/models.md` gives one; else a fresh directory under the system temp dir)",
),
maxCredits: z.number().positive().optional().describe("refuse to start the job if it costs more credits than this"),
}),
@@ -86,7 +106,7 @@ export const job = defineTool({
name: "job",
title: "Generation job",
description:
- "The state of a job started with `generate`: its status, progress estimate and cost, and, once it has succeeded, the files it made, saved where `generate` was told (the first call to see it succeed saves them; later calls return the same paths). Poll until the status is succeeded, failed or canceled, waiting about `etaRemainingSeconds` (or `etaSeconds` while queued) between calls. `cancel` stops a job and refunds its credits.",
+ "The state of a job started with `generate`: its status, progress estimate and cost, and, once it has succeeded, the files it made, saved where `generate` was told (the first call to see it succeed saves them; later calls return the same paths), and, from a local model, its `details`. Poll until the status is succeeded, failed or canceled, waiting about `etaRemainingSeconds` (or `etaSeconds` while queued) between calls. `cancel` stops a job and refunds its credits. A local model's job lives in the app: it is gone once the app restarts.",
input: z.object({
id: z.string().min(1).describe("job id, as `generate` returned it"),
cancel: z.boolean().optional().describe("cancel the job, refunding its credits"),
diff --git a/packages/dapi/src/tools/media-segment.ts b/packages/dapi/src/tools/media-segment.ts
deleted file mode 100644
index 27e0c183..00000000
--- a/packages/dapi/src/tools/media-segment.ts
+++ /dev/null
@@ -1,137 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { z } from "zod";
-import { defineTool } from "../tool";
-import { AssetPath, Bytes, checkWindow } from "../schemas";
-import { NonNegativeTime, TIME_FORMS } from "../time";
-
-/** The SAM 2.1 sizes the app offers, fastest first: the ids of its model catalogue (`SAM2_MODELS`). */
-export const SEGMENT_MODELS = ["tiny", "small", "base-plus", "large"] as const;
-
-/** A tracked frame the model rates below this is reported as weak. */
-export const WEAK_IOU = 0.5;
-
-const Unit = z.number().min(0).max(1);
-
-const Point = z.object({ x: Unit, y: Unit });
-
-/** A span of source time, `[start, end)` in seconds. */
-const Range = z.tuple([z.number(), z.number()]);
-
-const Bounds = z.tuple([z.number(), z.number(), z.number(), z.number()]);
-
-/** The track asked for, known before it runs. */
-export const segmentSpan = {
- model: z.enum(SEGMENT_MODELS),
- frameRate: z.number().describe("frames per second the mask holds: the project's, or the file's own with no project open"),
- start: z.number().describe("source time of the first masked frame, in seconds: the `sourceIn` of a `<mask>` naming the file"),
- end: z.number().describe("source time just past the last masked frame, in seconds"),
- frames: z.int().describe("frames masked"),
-};
-
-/** What segmenting found, the same whether the server or the renderer writes the files; absent while a track runs. */
-export const segmentFound = {
- bbox: Bounds.nullable().describe(
- "on the prompted frame, the object's bounds as [x0, y0, x1, y1] in 0..1 of the frame; null when nothing was segmented",
- ),
- area: z.number().describe("on the prompted frame, the share of the frame the object covers, 0..1"),
- score: z.number().describe("on the prompted frame, the model's object score: at or below 0 it sees no object there"),
- iou: z.number().describe("on the prompted frame, the model's estimate of the mask's quality, 0..1"),
- lost: z
- .array(Range)
- .describe("source-time spans, [start, end) in seconds, where no object was found: occluded, out of frame, or lost by the track"),
- weak: z.array(Range).describe(`source-time spans, [start, end) in seconds, where the model rated its mask below ${WEAK_IOU}`),
-};
-
-const optionalFound = z.object(segmentFound).partial().shape;
-
-export const mediaSegment = defineTool({
- name: "media_segment",
- title: "Segment object",
- description:
- "Segment an object in a video with SAM 2.1 and track it through the footage, writing a mask file (`.mask`) that a `<mask src>` names: the file the editor's object mask tool makes (local model on the GPU, no credits). " +
- "Prompt the object on the frame at `time` with `points` on it, `exclude` points off it, and/or a `box` around it, all in 0..1 of the frame (x right, y down). " +
- "`preview: true` segments that frame alone and writes only an image: the frame with the mask tinted and outlined, the prompts drawn, and a 0.1 grid to read coordinates off. Refine the prompts with previews (fast once the frame is encoded), then make the same call without `preview` to track. " +
- "Tracking covers `start` to `end`, the whole file by default, at roughly 0.25 s a frame with the tiny model and several times that with the larger ones, so pass the span the clip actually plays. " +
- "With a project open and the mask going into its library, a track runs in the background: the call returns the mask's paths at once with `state: \"tracking\"`, and `context` reports its progress, then a contact sheet of tracked frames and the spans where the object was lost or the mask is weak. A mask written elsewhere is tracked before the call returns, with the same findings. The first use of a model downloads it (83 MB for tiny, up to 475 MB for large).",
- input: z
- .object({
- path: AssetPath,
- time: NonNegativeTime.describe(`the frame the prompts are placed on, in source time — ${TIME_FORMS}`),
- points: z.array(Point).optional().describe("points on the object, each { x, y } in 0..1 of the frame"),
- exclude: z
- .array(Point)
- .optional()
- .describe("points on what is not the object, to take it out of the mask; each { x, y } in 0..1 of the frame"),
- box: Bounds.optional().describe("a box around the object, [x0, y0, x1, y1] in 0..1 of the frame"),
- preview: z
- .boolean()
- .optional()
- .describe("segment only the frame at `time` and write no mask: for checking a prompt before tracking"),
- start: NonNegativeTime.optional().describe(`start of the span to track — ${TIME_FORMS} (default: 0)`),
- end: NonNegativeTime.optional().describe(`end of the span to track — ${TIME_FORMS} (default: the file's end)`),
- model: z
- .enum(SEGMENT_MODELS)
- .optional()
- .describe("SAM 2.1 size: tiny (fastest), small, base-plus, or large (slowest; finest edges, best on small objects) (default: tiny)"),
- output: z
- .string()
- .optional()
- .describe(
- "where the mask file goes: a library path like `masks/skater.mask` (needs an open project; a mask already there is replaced, so every `<mask>` naming it follows) or an absolute file path (default: `masks/<video>/Tracking <n>.mask` in the library with a project open, else a fresh file under the system temp dir)",
- ),
- })
- .superRefine((value, ctx) => {
- if (!value.points?.length && !value.box) {
- ctx.addIssue({ code: "custom", path: ["points"], message: "prompt the object with points on it or a box around it" });
- }
- if (value.box && (value.box[0] >= value.box[2] || value.box[1] >= value.box[3])) {
- ctx.addIssue({ code: "custom", path: ["box"], message: "a box is [x0, y0, x1, y1] in 0..1 of the frame, with x0 < x1 and y0 < y1" });
- }
- if (value.box?.some((bound) => bound < 0 || bound > 1)) {
- ctx.addIssue({ code: "custom", path: ["box"], message: "a box's bounds are in 0..1 of the frame" });
- }
- if (value.preview) {
- for (const key of ["start", "end", "output"] as const) {
- if (value[key] !== undefined) {
- ctx.addIssue({ code: "custom", path: [key], message: `a preview segments one frame and writes no mask; ${key} applies to tracking only` });
- }
- }
- return;
- }
- checkWindow(value, ctx);
- if (value.start !== undefined && value.time < value.start) {
- ctx.addIssue({ code: "custom", path: ["time"], message: `time (${value.time}s) is before start (${value.start}s); the prompted frame must be tracked` });
- }
- if (value.end !== undefined && value.time >= value.end) {
- ctx.addIssue({ code: "custom", path: ["time"], message: `time (${value.time}s) is not before end (${value.end}s); the prompted frame must be tracked` });
- }
- }),
- output: z.object({
- image: z
- .string()
- .optional()
- .describe("absolute path of the PNG: the prompted frame with its mask on a preview, a contact sheet of tracked frames otherwise; absent while a track runs"),
- path: z.string().optional().describe("absolute path of the mask file; absent on a preview"),
- src: z.string().optional().describe("the mask's library path, for `<mask src>`; present when it goes into the project's library"),
- state: z
- .enum(["tracking", "done"])
- .optional()
- .describe("on a track: `tracking` while it runs in the background (poll `context` until its row in `masks` is done), `done` when the mask is written"),
- ...segmentSpan,
- ...optionalFound,
- }),
- result: z.object({
- png: Bytes.optional(),
- /** The mask file, for the server to write where the renderer cannot: an absolute path, or the temp dir. */
- mask: Bytes.optional(),
- path: z.string().optional(),
- src: z.string().optional(),
- state: z.enum(["tracking", "done"]).optional(),
- ...segmentSpan,
- ...optionalFound,
- }),
- environment: "renderer",
-});
diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.ts
index cab9840f..f7274fe0 100644
--- a/packages/dapi/src/tools/tools.test.ts
+++ b/packages/dapi/src/tools/tools.test.ts
@@ -10,9 +10,10 @@ import { logs } from "./logs";
import { mediaFilmstrip } from "./media-filmstrip";
import { mediaGrab } from "./media-grab";
import { mediaListen } from "./media-listen";
-import { mediaSegment } from "./media-segment";
import { mediaTranscribe } from "./media-transcribe";
-import { generate, generatedPath } from "./generate";
+import { generate, generatedPath, job } from "./generate";
+import { MODEL_IDS } from "@diffusionstudio/api-contract";
+import { LOCAL_MODEL_IDS, isLocalJobId, isLocalModel, samRequest } from "../local";
/** The messages of a failed parse, keyed by the path they point at. */
function issues(result: { success: boolean; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } }) {
@@ -101,7 +102,7 @@ describe("logs and export", () => {
describe("context", () => {
it("reports the same shape with and without an open project", () => {
expect(
- context.output.safeParse({ rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [], masks: [] }).success,
+ context.output.safeParse({ rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [] }).success,
).toBe(true);
expect(
context.output.safeParse({
@@ -109,22 +110,10 @@ describe("context", () => {
projectDir: "/p/a",
currentTime: null,
fontFamilies: ["Inter"],
- masks: [],
}).success,
).toBe(true);
expect(context.output.safeParse({ rootDir: "/p", projectDir: "/p/a" }).success).toBe(false);
});
-
- it("reports background mask tracks, with a contact sheet as bytes in the result and a path in the output", () => {
- const span = { model: "tiny", frameRate: 30, start: 0, end: 2, frames: 60 };
- const running = { id: "t1", src: "masks/a/Tracking 1.mask", video: "a.mp4", state: "tracking", progress: 0.4, ...span };
- const found = { bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [] };
- const base = { rootDir: "/p", projectDir: "/p/a", currentTime: null, fontFamilies: [] };
- expect(context.output.safeParse({ ...base, masks: [running] }).success).toBe(true);
- expect(context.result!.safeParse({ ...base, masks: [{ ...running, state: "done", progress: 1, png: new Uint8Array(3), ...found }] }).success).toBe(true);
- expect(context.output.safeParse({ ...base, masks: [{ ...running, state: "done", progress: 1, image: "/tmp/s.png", ...found }] }).success).toBe(true);
- expect(context.output.safeParse({ ...base, masks: [{ ...running, state: "paused" }] }).success).toBe(false);
- });
});
describe("media_transcribe", () => {
@@ -144,52 +133,67 @@ describe("image tools", () => {
});
});
-describe("media_segment", () => {
- const input = mediaSegment.input;
- const at = { path: "/c.mp4", time: "1.5" };
+describe("sam-2.1", () => {
+ const at = { model: "sam-2.1", video: { path: "/c.mp4" }, time: "1.5" };
- it("takes points, exclusions and a box, with the time in every form", () => {
- const args = input.parse({ ...at, points: [{ x: 0.5, y: 0.4 }], exclude: [{ x: 0.5, y: 0.9 }], box: [0.2, 0.1, 0.8, 0.95] });
+ it("takes points, exclusions and a box, with the time in every form, and is tiny unless sized", () => {
+ const args = samRequest.parse({ ...at, points: [{ x: 0.5, y: 0.4 }], exclude: [{ x: 0.5, y: 0.9 }], box: [0.2, 0.1, 0.8, 0.95] });
expect(args.time).toBe(1.5);
- expect(input.parse({ path: "/c.mp4", time: "45f", box: [0, 0, 1, 1] }).time).toBe(1.5);
+ expect(args.size).toBe("tiny");
+ expect(samRequest.parse({ ...at, time: "45f", box: [0, 0, 1, 1] }).time).toBe(1.5);
});
it("needs the object prompted by points or a box", () => {
- expect(issues(input.safeParse(at))).toHaveProperty("points");
- expect(issues(input.safeParse({ ...at, exclude: [{ x: 0.1, y: 0.1 }] }))).toHaveProperty("points");
- expect(input.safeParse({ ...at, points: [{ x: 0, y: 1 }] }).success).toBe(true);
+ expect(issues(samRequest.safeParse(at))).toHaveProperty("points");
+ expect(issues(samRequest.safeParse({ ...at, exclude: [{ x: 0.1, y: 0.1 }] }))).toHaveProperty("points");
+ expect(samRequest.safeParse({ ...at, points: [{ x: 0, y: 1 }] }).success).toBe(true);
});
it("keeps prompts inside the frame and boxes the right way round", () => {
- expect(input.safeParse({ ...at, points: [{ x: 1.2, y: 0.5 }] }).success).toBe(false);
- expect(issues(input.safeParse({ ...at, box: [0.8, 0.1, 0.2, 0.9] }))).toHaveProperty("box");
- expect(issues(input.safeParse({ ...at, box: [-0.1, 0.1, 0.2, 0.9] }))).toHaveProperty("box");
+ expect(samRequest.safeParse({ ...at, points: [{ x: 1.2, y: 0.5 }] }).success).toBe(false);
+ expect(issues(samRequest.safeParse({ ...at, box: [0.8, 0.1, 0.2, 0.9] }))).toHaveProperty("box");
+ expect(issues(samRequest.safeParse({ ...at, box: [-0.1, 0.1, 0.2, 0.9] }))).toHaveProperty("box");
});
it("rejects a span or an output on a preview", () => {
const preview = { ...at, points: [{ x: 0.5, y: 0.5 }], preview: true };
- expect(input.safeParse(preview).success).toBe(true);
- expect(issues(input.safeParse({ ...preview, start: 1 }))).toHaveProperty("start");
- expect(issues(input.safeParse({ ...preview, output: "masks/a.mask" }))).toHaveProperty("output");
+ expect(samRequest.safeParse(preview).success).toBe(true);
+ expect(issues(samRequest.safeParse({ ...preview, start: 1 }))).toHaveProperty("start");
+ expect(issues(samRequest.safeParse({ ...preview, output: "masks/a.mask" }))).toHaveProperty("output");
});
it("tracks a span that holds the prompted frame", () => {
const track = { ...at, points: [{ x: 0.5, y: 0.5 }] };
- expect(input.safeParse({ ...track, start: 1, end: "0:03" }).success).toBe(true);
- expect(issues(input.safeParse({ ...track, start: 2 }))).toHaveProperty("time");
- expect(issues(input.safeParse({ ...track, end: 1.5 }))).toHaveProperty("time");
- expect(issues(input.safeParse({ ...track, start: 3, end: 2 }))).toHaveProperty("end");
+ expect(samRequest.safeParse({ ...track, start: 1, end: "0:03", output: "masks/a.mask", maxCredits: 5 }).success).toBe(true);
+ expect(issues(samRequest.safeParse({ ...track, start: 2 }))).toHaveProperty("time");
+ expect(issues(samRequest.safeParse({ ...track, end: 1.5 }))).toHaveProperty("time");
+ expect(issues(samRequest.safeParse({ ...track, start: 3, end: 2 }))).toHaveProperty("end");
});
- it("answers a background track with its paths and span alone", () => {
- const span = { model: "tiny", frameRate: 30, start: 0, end: 2, frames: 60 };
- expect(mediaSegment.output.safeParse({ path: "/p/assets/masks/a.mask", src: "masks/a.mask", state: "tracking", ...span }).success).toBe(true);
- expect(mediaSegment.output.safeParse({ path: "/p/assets/masks/a.mask", state: "queued", ...span }).success).toBe(false);
+ it("names only the sizes the app offers, and no field it does not take", () => {
+ const boxed = { ...at, box: [0, 0, 1, 1] };
+ expect(samRequest.safeParse({ ...boxed, size: "base-plus" }).success).toBe(true);
+ expect(samRequest.safeParse({ ...boxed, size: "huge" }).success).toBe(false);
+ expect(samRequest.safeParse({ ...boxed, boxes: [] }).success).toBe(false);
+ expect(samRequest.safeParse({ ...boxed, video: "/c.mp4" }).success).toBe(false);
});
+});
- it("names only the models the app offers", () => {
- expect(input.safeParse({ ...at, box: [0, 0, 1, 1], model: "base-plus" }).success).toBe(true);
- expect(input.safeParse({ ...at, box: [0, 0, 1, 1], model: "huge" }).success).toBe(false);
+describe("local models", () => {
+ it("share no id with the API's", () => {
+ expect(LOCAL_MODEL_IDS.filter((id) => (MODEL_IDS as string[]).includes(id))).toEqual([]);
+ expect(isLocalModel("sam-2.1")).toBe(true);
+ expect(isLocalModel("flux-2-klein")).toBe(false);
+ expect(isLocalModel("toString")).toBe(false);
+ });
+
+ it("run as jobs whose ids say so", () => {
+ expect(generate.input.safeParse({ model: "sam-2.1", video: { path: "/c.mp4" } }).success).toBe(true);
+ expect(isLocalJobId("local-3f2a")).toBe(true);
+ expect(isLocalJobId("b22bbf6f-4c94-4a5b-a4c7-98c696706906")).toBe(false);
+ const running = { id: "local-1", status: "running", phase: "tracking", progress: 0.4, assets: [], details: { frames: 60 } };
+ expect(job.output.parse(running)).toEqual(running);
+ expect(generate.result!.safeParse({ job: { id: "local-1", status: "succeeded", assets: [{ bytes: new Uint8Array(3), filename: "a.mask" }] }, saveTo: null, image: new Uint8Array(3) }).success).toBe(true);
});
});