Skip to content

The VS Code of video editing, built for agents.

TypeScript

Implement Background Track Handling in Media Segment Tool

konstantin-paulus8c5529eacfe2
diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.ts
index 729e9cb8..52bb7d1b 100644
--- a/apps/desktop/src/dapi/present.test.ts
+++ b/apps/desktop/src/dapi/present.test.ts
@@ -92,6 +92,28 @@ describe("present", () => {
expect(preview.output).not.toHaveProperty("path");
});
+ it("has no picture for a track still running in the background", async () => {
+ const span = { model: "tiny" as const, frameRate: 30, start: 0, end: 2, frames: 60 };
+ const running = await present("media_segment", { path: "/c.mp4", time: 1 }, { path: "/p/assets/masks/a.mask", src: "masks/a.mask", state: "tracking" as const, ...span });
+ expect(running).toEqual({ output: { path: "/p/assets/masks/a.mask", src: "masks/a.mask", state: "tracking", ...span }, images: [] });
+ });
+
+ it("writes a done track's contact sheet once, inline only the first time context reports it", async () => {
+ const row = {
+ id: "track-1", src: "masks/a.mask", video: "a.mp4", state: "done" as const, progress: 1,
+ model: "tiny" as const, frameRate: 30, start: 0, end: 2, frames: 60, bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [],
+ };
+ const base = { rootDir: "/p", projectDir: "/p/a", currentTime: null, fontFamilies: [], generations: [] };
+ const first = await present("context", {}, { ...base, masks: [{ ...row, png: png(8) }] });
+ const image = (first.output as { masks: { image: string }[] }).masks[0]!.image;
+ expect(image).toMatch(/dapi-segment-.*\.png$/);
+ expect(first.images).toEqual([{ path: image, png: png(8) }]);
+ expect(first.output).toEqual({ ...base, masks: [{ ...row, image }] });
+
+ const again = await present("context", {}, { ...base, masks: [{ ...row, png: png(8) }] });
+ expect(again).toEqual({ output: { ...base, masks: [{ ...row, image }] }, images: [] });
+ });
+
it("passes other results through untouched", async () => {
expect(await present("check", { id: "x" }, { stats: {}, issues: [] })).toEqual({ output: { stats: {}, issues: [] }, images: [] });
});
diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.ts
index 1772786b..7fab3c19 100644
--- a/apps/desktop/src/dapi/present.ts
+++ b/apps/desktop/src/dapi/present.ts
@@ -50,6 +50,8 @@ export async function present(name: ToolName, args: unknown, result: unknown): P
return presentTranscript(result as ToolResult<"media_transcribe">, (args as ToolArgs<"media_transcribe">).output);
case "media_segment":
return presentSegment(result as ToolResult<"media_segment">, (args as ToolArgs<"media_segment">).output);
+ case "context":
+ return presentContext(result as ToolResult<"context">);
default:
return { output: result, images: [] };
}
@@ -119,19 +121,45 @@ async function presentTranscript(transcript: ToolResult<"media_transcribe">, out
/**
* The picture always goes to the temp dir; `output` is where the mask goes.
* A mask the renderer put into the project's library comes with its path; one
- * that comes as bytes is written here, to `output` or the temp dir.
+ * that comes as bytes is written here, to `output` or the temp dir. A track
+ * still running in the background has no picture yet (see `presentContext`).
*/
async function presentSegment(result: ToolResult<"media_segment">, output: string | undefined): Promise<Presented> {
const { png, mask, ...found } = result;
- const image = join(tempDir(), `dapi-segment-${randomUUID()}.png`);
- await writeFile(image, png);
let path = found.path;
if (mask) {
path = await singleFilePath(output, `dapi-mask-${randomUUID()}.mask`);
await writeFile(path, mask);
}
- const presented: ToolOutput<"media_segment"> = { image, ...found, ...(path === undefined ? {} : { path }) };
- return { output: presented, images: [{ path: image, png }] };
+ const images: WrittenImage[] = [];
+ let image: string | undefined;
+ if (png) {
+ image = join(tempDir(), `dapi-segment-${randomUUID()}.png`);
+ await writeFile(image, png);
+ images.push({ path: image, png });
+ }
+ const presented: ToolOutput<"media_segment"> = { ...found, ...(image === undefined ? {} : { image }), ...(path === undefined ? {} : { path }) };
+ return { output: presented, images };
+}
+
+/** Contact sheets of background tracks already written, by track id: each is written, and shown inline, once. */
+const trackSheets = new Map<string, string>();
+
+/** A done track's contact sheet goes to the temp dir the first poll it shows up in, and arrives inline with that poll. */
+async function presentContext(result: ToolResult<"context">): Promise<Presented> {
+ const images: WrittenImage[] = [];
+ const masks: ToolOutput<"context">["masks"] = [];
+ for (const { png, ...row } of result.masks) {
+ let image = trackSheets.get(row.id);
+ if (png && image === undefined) {
+ image = join(tempDir(), `dapi-segment-${randomUUID()}.png`);
+ await writeFile(image, png);
+ trackSheets.set(row.id, image);
+ images.push({ path: image, png });
+ }
+ masks.push(image === undefined ? row : { ...row, image });
+ }
+ return { output: { ...result, masks }, images };
}
function screenshotFilename(taken: Date, attempt: number): string {
diff --git a/apps/web/src/components/ui/sonner.tsx b/apps/web/src/components/ui/sonner.tsx
index a520432b..6f51afdf 100644
--- a/apps/web/src/components/ui/sonner.tsx
+++ b/apps/web/src/components/ui/sonner.tsx
@@ -31,10 +31,8 @@ export const Toaster = (props: Parameters<typeof Sonner>[0]) => {
description: 'text-xs',
content: 'flex gap-0.5',
icon: 'size-6 mb-1',
-
- // actionButton: 'bg-zinc-400',
- // cancelButton: 'bg-orange-400',
- // closeButton: 'bg-lime-400',
+ actionButton: 'mt-2 text-xs text-muted-foreground hover:text-foreground transition-colors cursor-pointer',
+ cancelButton: 'mt-2 text-xs text-muted-foreground hover:text-foreground transition-colors cursor-pointer',
},
}}
{...props}
diff --git a/apps/web/src/dapi/handlers/context.ts b/apps/web/src/dapi/handlers/context.ts
index 822b3a54..cef1cb06 100644
--- a/apps/web/src/dapi/handlers/context.ts
+++ b/apps/web/src/dapi/handlers/context.ts
@@ -4,6 +4,7 @@
import { AssetId, Computed, Fonts, FrameRate, Generating, getActiveEntity, Library, Name, PendingSource, Source, SourceError } from "@diffusionstudio/runtime";
import { getProjectsRoot } from "@/projects";
+import { maskTrackRows } from "./media-segment";
import type { Entity, World } from "koota";
import type { GenerationRow } from "@diffusionstudio/dapi";
@@ -15,13 +16,14 @@ import type { ToolHandler } from "../handler";
* the file, and a caller that wants them reads it. What is left over is which
* folder new projects go in, which project folder the app has open, where its
* playhead sits, which font families are actually registered in the world
- * drawing it. With no project open only the root is left to report.
+ * drawing it, and how generations and background mask tracks are getting on.
+ * With no project open only the root is left to report.
*/
export const context: ToolHandler<"context"> = async (_, ctx) => {
const rootDir = await getProjectsRoot();
const open = ctx.session();
- if (!open) return { rootDir, projectDir: null, currentTime: null, fontFamilies: [], generations: [] };
+ if (!open) return { rootDir, projectDir: null, currentTime: null, fontFamilies: [], generations: [], masks: [] };
const { world, project } = open;
const frameRate = world.get(FrameRate)?.value || 30;
@@ -37,6 +39,7 @@ export const context: ToolHandler<"context"> = async (_, ctx) => {
// merely named in the source. The editor default is always among them.
fontFamilies: [...new Set(["Inter", ...(world.get(Fonts)?.list ?? []).map((f) => f.family)])],
generations: collectGenerations(world),
+ masks: maskTrackRows(world),
};
};
diff --git a/apps/web/src/dapi/handlers/media-segment.ts b/apps/web/src/dapi/handlers/media-segment.ts
index c67c0efb..eda673ec 100644
--- a/apps/web/src/dapi/handlers/media-segment.ts
+++ b/apps/web/src/dapi/handlers/media-segment.ts
@@ -3,23 +3,23 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { CanvasSink } from "mediabunny";
-import { toast } from "somoto";
-import { MASK_EXTENSION, assetName, isAbsoluteSource, isProjectSource, isUrlSource, traceMask } from "@diffusionstudio/assets";
+import { ASSETS_DIR, MASK_EXTENSION, isAbsoluteSource, isProjectSource, isUrlSource, traceMask } from "@diffusionstudio/assets";
import { DapiError, WEAK_IOU } from "@diffusionstudio/dapi";
import { encodePng } from "@diffusionstudio/encoder";
import { FrameRate, formatTimecode, getLibrary, getVideoTrack } from "@diffusionstudio/runtime";
-import { sam2Model } from "@diffusionstudio/sam2/models";
-import { encodeObjectMask, nextTrackingName, objectMaskFolder } from "@/engine/object-mask/commit";
+import { encodeObjectMask, nextTrackingName, objectMaskFolder, pendingMaskPaths } from "@/engine/object-mask/commit";
import { segmentFootage } from "@/engine/object-mask/tracking";
import { requireAssetType, resolveAsset } from "../lib/assets";
import { SheetCollector } from "../lib/sheets";
+import type { World } from "koota";
+import type { Accessor } from "solid-js";
import type { InputVideoTrack } from "mediabunny";
import type { Asset, AssetLibrary, MaskFrame, VideoAsset } from "@diffusionstudio/assets";
-import type { MediaSegmentRequest } from "@diffusionstudio/dapi";
+import type { MediaSegmentRequest, MediaSegmentResult, ToolResult } from "@diffusionstudio/dapi";
import type { Sam2Point } from "@diffusionstudio/sam2";
-import type { FootageSegments } from "@/engine/object-mask/tracking";
+import type { FootageSegmentRequest, FootageSegments } from "@/engine/object-mask/tracking";
import type { EditorSession } from "../session";
import type { ToolHandler } from "../handler";
@@ -38,9 +38,12 @@ const BOX = "#FFC400";
/**
* Segments an object in a video and tracks it through the span asked for, on
- * the object mask tool's model and queue; the mask goes into the project's
- * library (or back to the server to write when it goes elsewhere), with a
- * picture of what was found. A preview segments the prompted frame alone.
+ * the object mask tool's model and queue, without a word in the app. A mask
+ * bound for the project's library is tracked in the background: the call
+ * answers with where it will be at once, and `context` reports the track (see
+ * `maskTrackRows`). One the server writes, and a preview of the prompted
+ * frame alone, are segmented before the call returns, with a picture of what
+ * was found.
*/
export const mediaSegment: ToolHandler<"media_segment"> = async (args, ctx) => {
const asset = await resolveAsset(ctx, args.path);
@@ -68,70 +71,170 @@ export const mediaSegment: ToolHandler<"media_segment"> = async (args, ctx) => {
const seedFrame = Math.min(Math.max(Math.round(args.time * fps), first), first + count - 1);
const target = args.preview ? null : destination(args.output, session);
- const progress = new Progress(asset, args.preview ?? false);
- progress.tracked(0, count);
+ const model = args.model ?? "tiny";
+ const span: Span = { model, frameRate: fps, start: round(first / fps), end: round((first + count) / fps), frames: count };
+ const request: SegmentRequest = {
+ asset,
+ fps,
+ model,
+ points: prompts(args),
+ seedFrame,
+ first,
+ count,
+ preview: args.preview ?? false,
+ };
+
+ if (target?.kind === "library") return startTrack(session!, ctx.session, args, request, span, target.path);
+
let segments: FootageSegments;
try {
- segments = await segmentFootage({
- asset,
- fps,
- model: args.model ?? "tiny",
- points: prompts(args),
- seedFrame,
- first: args.preview ? seedFrame : first,
- count,
- preview: args.preview ?? false,
- signal: AbortSignal.any([ctx.signal, progress.signal]),
- onDownload: (fraction) => progress.download(sam2Model(args.model ?? "tiny").label, fraction),
- onProgress: (completed, of) => progress.tracked(completed, of),
- });
+ segments = await segmentFootage({ ...request, signal: ctx.signal });
} catch (error) {
- if (error instanceof DOMException && error.name === "AbortError") {
- throw new DapiError("canceled", progress.signal.aborted ? "Segmenting was canceled in the app." : "The call was canceled.");
- }
+ if (isAbort(error)) throw new DapiError("canceled", "The call was canceled.");
throw error;
- } finally {
- progress.done();
}
+ const found = describeFound(segments, seedFrame - first, first, fps);
+ const video = await decodableTrack(asset);
+ if (args.preview) {
+ const png = await previewImage(video, segments, seedFrame, segments.masks[0] ?? null, args);
+ return { png, ...span, ...found };
+ }
+
+ const png = await contactSheet(video, segments, asset, first, fps);
+ const file = encodeTrack(segments, request);
+ return { png, mask: new Uint8Array(await file.arrayBuffer()), state: "done", ...span, ...found };
+};
+
+type Span = Pick<MediaSegmentResult, "model" | "frameRate" | "start" | "end" | "frames">;
+type Found = Required<Pick<MediaSegmentResult, "bbox" | "area" | "score" | "iou" | "lost" | "weak">>;
+type SegmentRequest = Omit<FootageSegmentRequest, "signal" | "onDownload" | "onProgress">;
+
+/** What the masks say about the object: on the seed frame, and the spans where it was lost or weak. */
+function describeFound(segments: FootageSegments, seedIndex: number, first: number, fps: number): Found {
const { masks, grid } = segments;
- const seed = masks[seedFrame - first] ?? null;
- const found = {
- model: args.model ?? "tiny",
- frameRate: fps,
- start: round(first / fps),
- end: round((first + count) / fps),
- frames: count,
+ const seed = masks[seedIndex] ?? null;
+ return {
...measure(seed, grid),
score: seed?.score ?? 0,
iou: seed?.iou ?? 0,
lost: spans(masks, first, fps, (mask) => isLost(mask)),
weak: spans(masks, first, fps, (mask) => !isLost(mask) && mask!.iou < WEAK_IOU),
};
+}
+
+/** The tracked frames as a mask file, with the recipe that made them. */
+function encodeTrack(segments: FootageSegments, request: SegmentRequest): Blob {
+ return encodeObjectMask(request.asset, request.fps, segments.grid, segments.masks, {
+ model: segments.model.repo,
+ source: request.asset.id,
+ first: request.first,
+ seedFrame: request.seedFrame,
+ points: request.points,
+ });
+}
+async function decodableTrack(asset: VideoAsset): Promise<InputVideoTrack> {
const video = await getVideoTrack(asset);
if (!video) throw new DapiError("wrong-kind", `Asset ${asset.id} has no decodable video track.`);
+ return video;
+}
- if (args.preview) {
- const png = await previewImage(video, segments, seedFrame, seed, args);
- return { png, ...found };
- }
+function isAbort(error: unknown): boolean {
+ return error instanceof DOMException && error.name === "AbortError";
+}
- const png = await contactSheet(video, segments, asset, first, fps);
- const file = encodeObjectMask(asset, fps, grid, masks, {
- model: segments.model.repo,
- source: asset.id,
- first,
- seedFrame,
- points: prompts(args),
- });
+// ── Tracks in the background ─────────────────────────────────
- if (target?.kind !== "library") return { png, mask: new Uint8Array(await file.arrayBuffer()), ...found };
+/** A background track as `context` reports it. */
+type TrackRow = ToolResult<"context">["masks"][number];
- const library = getLibrary(session!.world);
- const stored = await storeMask(library, target.path, asset, file);
- return { png, path: `${session!.project.dir()}/${stored.source}`, src: stored.path, ...found };
-};
+/** The tracks started in each project's world, oldest first; they go with the world. */
+const tracks = new WeakMap<World, TrackRow[]>();
+
+/** The background tracks started in `world`, for `context`. */
+export function maskTrackRows(world: World): TrackRow[] {
+ return (tracks.get(world) ?? []).map((row) => ({ ...row }));
+}
+
+/**
+ * Starts tracking into the library and answers at once with where the mask
+ * will be. The path is promised to the track while it runs, so the tool and
+ * other tracks name their masks around it.
+ */
+function startTrack(
+ session: EditorSession,
+ current: Accessor<EditorSession | null>,
+ args: MediaSegmentRequest,
+ request: SegmentRequest,
+ span: Span,
+ path: string | null,
+): MediaSegmentResult {
+ const library = getLibrary(session.world);
+ const folder = objectMaskFolder(request.asset);
+ const src = path ?? `${folder}/${nextTrackingName(library, folder)}`;
+ const existing = library.get(src);
+ const source = existing && isReplaceableMask(existing) ? existing.source : `${ASSETS_DIR}/${src}`;
+
+ const row: TrackRow = { id: crypto.randomUUID(), src, video: args.path, state: "loading", progress: null, ...span };
+ const rows = tracks.get(session.world) ?? [];
+ rows.push(row);
+ tracks.set(session.world, rows);
+
+ pendingMaskPaths.add(src);
+ void runTrack(row, session, current, request, path !== null).finally(() => pendingMaskPaths.delete(src));
+ return { path: `${session.project.dir()}/${source}`, src, state: "tracking", ...span };
+}
+
+/**
+ * Tracks, then writes the mask into the library, filing each step on `row`;
+ * `replace` when the caller named the path, so a mask already there is
+ * written over. A project closed in the meantime stops the track at its next
+ * frame, and nothing is written into a library no longer open.
+ */
+async function runTrack(
+ row: TrackRow,
+ session: EditorSession,
+ current: Accessor<EditorSession | null>,
+ request: SegmentRequest,
+ replace: boolean,
+): Promise<void> {
+ const controller = new AbortController();
+ const stillOpen = () => {
+ if (current()?.world !== session.world) controller.abort();
+ return !controller.signal.aborted;
+ };
+
+ try {
+ const segments = await segmentFootage({
+ ...request,
+ signal: controller.signal,
+ onDownload: (fraction) => {
+ if (stillOpen()) Object.assign(row, { state: "loading", progress: fraction });
+ },
+ onProgress: (completed, total) => {
+ if (stillOpen()) Object.assign(row, { state: "tracking", progress: completed / total });
+ },
+ });
+
+ const video = await decodableTrack(request.asset);
+ const png = await contactSheet(video, segments, request.asset, request.first, request.fps);
+ const file = encodeTrack(segments, request);
+ if (!stillOpen()) return;
+
+ const stored = await storeMask(getLibrary(session.world), row.src, file, replace);
+ Object.assign(row, {
+ state: "done",
+ progress: 1,
+ src: stored.path,
+ png,
+ ...describeFound(segments, request.seedFrame - request.first, request.first, request.fps),
+ });
+ } catch (error) {
+ if (isAbort(error)) return;
+ Object.assign(row, { state: "failed", progress: null, error: error instanceof Error ? error.message : String(error) });
+ }
+}
/** The prompt as SAM 2 takes it: points on the object, points off it, then a box's corners. */
function prompts(args: MediaSegmentRequest): Sam2Point[] {
@@ -185,18 +288,13 @@ function isReplaceableMask(asset: Asset): boolean {
}
/**
- * Puts the mask file into the library. A mask already at `path` has its file
- * written over and is taken in again at the same path, as a restore does, so
- * every `<mask>` naming it shows the new frames.
+ * Puts the mask file into the library at `path`. When `replace`, a mask
+ * already there has its file written over and is taken in again at the same
+ * path, as a restore does, so every `<mask>` naming it shows the new frames.
*/
-async function storeMask(library: AssetLibrary, path: string | null, video: VideoAsset, file: Blob): Promise<Asset> {
- if (path === null) {
- const folder = objectMaskFolder(video);
- return library.store(file, { folder, name: nextTrackingName(library, folder) });
- }
-
+async function storeMask(library: AssetLibrary, path: string, file: Blob, replace: boolean): Promise<Asset> {
const existing = library.get(path);
- if (existing && isReplaceableMask(existing)) {
+ if (replace && existing && isReplaceableMask(existing)) {
await library.fs.write(existing.source, file);
return library.relink(existing, existing.source);
}
@@ -417,50 +515,3 @@ function drawLabel(ctx: Ctx2D, text: string, x: number, y: number, align: "left"
ctx.fillText(text, left + padX, y + 0.5);
ctx.restore();
}
-
-// ── Progress in the app ──────────────────────────────────────
-
-/**
- * What the app shows while a call runs long: the model's download, and a
- * track's frames, in a toast with a Cancel button. A preview shows only a
- * download; it is over too quickly otherwise.
- */
-class Progress {
- private readonly controller = new AbortController();
- private readonly id = `media-segment-${crypto.randomUUID()}`;
- private shown: string | null = null;
-
- constructor(
- private readonly asset: VideoAsset,
- private readonly preview: boolean,
- ) {}
-
- /** Fires when the Cancel button is pressed. */
- get signal(): AbortSignal {
- return this.controller.signal;
- }
-
- download(label: string, fraction: number | null): void {
- this.show(fraction === null ? `Loading ${label}…` : `Downloading ${label}… ${Math.floor(fraction * 100)}%`);
- }
-
- tracked(completed: number, total: number): void {
- if (this.preview) return;
- this.show(`Tracking an object in ${assetName(this.asset)}… ${Math.floor((completed / total) * 100)}%`);
- }
-
- done(): void {
- if (this.shown !== null) toast.dismiss(this.id);
- }
-
- // A percent at a time: downloads and tracks report far more often.
- private show(message: string): void {
- if (message === this.shown) return;
- this.shown = message;
- toast.loading(message, {
- id: this.id,
- duration: Infinity,
- cancel: { label: "Cancel", onClick: () => this.controller.abort() },
- });
- }
-}
diff --git a/apps/web/src/engine/object-mask/commit.tsx b/apps/web/src/engine/object-mask/commit.tsx
index f005b5bb..493a8686 100644
--- a/apps/web/src/engine/object-mask/commit.tsx
+++ b/apps/web/src/engine/object-mask/commit.tsx
@@ -88,11 +88,18 @@ export function objectMaskFolder(video: VideoAsset): string {
return `${MASKS_FOLDER}/${assetName(video).replace(/\.[^.]+$/, '')}`;
}
-/** `Tracking <n>.mask`, `n` the first count not yet taken in `folder`. */
+/**
+ * Library paths promised to masks still being tracked in the background (see
+ * the agent's `media_segment`): taken, though nothing is there yet.
+ */
+export const pendingMaskPaths = new Set<string>();
+
+/** `Tracking <n>.mask`, `n` the first count not yet taken in `folder`, nor promised to a track still running. */
export function nextTrackingName(library: AssetLibrary, folder: string): string {
for (let n = 1; ; n++) {
const name = `Tracking ${n}.${MASK_EXTENSION}`;
- if (!library.get(`${folder}/${name}`)) return name;
+ const path = `${folder}/${name}`;
+ if (!library.get(path) && !pendingMaskPaths.has(path)) return name;
}
}
diff --git a/docs/reference/tools/context.md b/docs/reference/tools/context.md
index 48c585f1..6483c3e7 100644
--- a/docs/reference/tools/context.md
+++ b/docs/reference/tools/context.md
@@ -1,6 +1,6 @@
# context
-Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, and where its `generate.*` declarations stand. Poll it to wait for generations without blocking.
+Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, where its `generate.*` declarations stand, and the progress of `media_segment` tracks running in the background. Poll it to wait for generations and tracks without blocking.
| | |
| --- | --- |
@@ -33,10 +33,21 @@ One JSON object:
error?: string; // what it failed with, on `failed` rows
asset?: string; // the library path it landed as, on `done` rows
}[];
+ masks: { // media_segment tracks started while this project is open, oldest first
+ id: string; // the track's id, the same across polls
+ src: string; // the mask's library path, for <mask src>; the file is there once done
+ video: string; // the footage tracked, as media_segment was given it
+ state: "loading" | "tracking" | "done" | "failed";
+ progress: number | null; // 0..1: the model's download while loading, the frames masked while tracking; 1 once done
+ error?: string; // what it failed with, on `failed` rows
+ image?: string; // absolute path of a contact sheet of tracked frames, on `done` rows
+ model, frameRate, start, end, frames; // the span, as media_segment returned it
+ bbox?, area?, score?, iou?, lost?, weak?; // what was found, on `done` rows — see media_segment
+ }[];
}
```
-With no project open (the app sits at the dashboard) the report is just `{ rootDir, projectDir: null }`: there is no playhead, no world, and no fonts to speak of. Open one with [`open`](./open.md).
+With no project open (the app sits at the dashboard) the report is just `{ rootDir, projectDir: null }` with empty lists: there is no playhead, no world, and no fonts to speak of. Open one with [`open`](./open.md).
`rootDir` is reported whether or not a project is open — it is where a caller with nothing open goes to create or find one.
@@ -47,3 +58,5 @@ With no project open (the app sits at the dashboard) the report is just `{ rootD
`fontFamilies` is what text can be drawn with right now — loaded into the world, not merely named in the source — and always includes the editor default. For every family installed on the machine, see [`fonts`](./fonts.md).
`generations` is how a caller waits for `generate.*` declarations without blocking: generation is asynchronous, so poll this until nothing is `generating`. A `done` row's `asset` is a library path, ready for [`media_probe`](./media/probe.md) and its siblings; a `failed` row's `error` is the message the library recorded for the generation, which is what keeps it from being generated again (see [jsx/errors.md](../jsx/errors.md#failed-sources)).
+
+`masks` is how a caller waits for a [`media_segment`](./media/segment.md) track that went into the library: the call returns as soon as it starts, so poll this until the row whose `src` it returned is `done` or `failed`. `loading` covers the model loading (its download, the first time, is `progress`) and waiting behind the editor's object mask tool or another track. A `done` row carries the same findings `media_segment` returns for a finished track, and its contact sheet as `image` — over MCP it also arrives inline, once, with the first poll that reports the row done. Rows last as long as the project stays open; closing it stops its tracks.
diff --git a/docs/reference/tools/media/segment.md b/docs/reference/tools/media/segment.md
index f93e1c00..1f48cc4c 100644
--- a/docs/reference/tools/media/segment.md
+++ b/docs/reference/tools/media/segment.md
@@ -1,6 +1,6 @@
# media_segment
-Segment an object in a video with SAM 2.1 and track it through the footage, writing a mask file (`.mask`) that a [`<mask src>`](../../jsx/styles.md#mask) names: the file the editor's object mask tool makes (local model on the GPU, no credits). Prompt the object on the frame at `time` with `points` on it, `exclude` points off it, and/or a `box` around it, all in 0..1 of the frame (x right, y down). `preview: true` segments that frame alone and writes only an image: the frame with the mask tinted and outlined, the prompts drawn, and a 0.1 grid to read coordinates off. Refine the prompts with previews (fast once the frame is encoded), then make the same call without `preview` to track. Tracking covers `start` to `end`, the whole file by default, at roughly 0.25 s a frame with the tiny model and several times that with the larger ones, so pass the span the clip actually plays. Returns the mask's path (and its library path when it went into the project), a contact sheet of tracked frames, and the spans where the object was lost or the mask is weak. The first use of a model downloads it (83 MB for tiny, up to 475 MB for large).
+Segment an object in a video with SAM 2.1 and track it through the footage, writing a mask file (`.mask`) that a [`<mask src>`](../../jsx/styles.md#mask) names: the file the editor's object mask tool makes (local model on the GPU, no credits). Prompt the object on the frame at `time` with `points` on it, `exclude` points off it, and/or a `box` around it, all in 0..1 of the frame (x right, y down). `preview: true` segments that frame alone and writes only an image: the frame with the mask tinted and outlined, the prompts drawn, and a 0.1 grid to read coordinates off. Refine the prompts with previews (fast once the frame is encoded), then make the same call without `preview` to track. Tracking covers `start` to `end`, the whole file by default, at roughly 0.25 s a frame with the tiny model and several times that with the larger ones, so pass the span the clip actually plays. With a project open and the mask going into its library, a track runs in the background: the call returns the mask's paths at once with `state: "tracking"`, and [`context`](../context.md) reports its progress, then a contact sheet of tracked frames and the spans where the object was lost or the mask is weak. A mask written elsewhere is tracked before the call returns, with the same findings. The first use of a model downloads it (83 MB for tiny, up to 475 MB for large).
| | |
| --- | --- |
@@ -39,7 +39,16 @@ The first preview of a frame encodes it, which takes a moment; further previews
## Tracking
-Without `preview`, the object is segmented on the prompted frame and followed forward and backward through every frame from `start` to `end`, one mask frame per frame at the project's frame rate (the file's own with no project open). `time` must lie in the span. The app shows the progress with a Cancel button, and the tool waits for the whole track: the CLI allows it an hour, and an MCP client should give the call a comparable timeout. Tracking shares the model with the editor's object mask tool, so a call waits for the tool's own work to finish first.
+Without `preview`, the object is segmented on the prompted frame and followed forward and backward through every frame from `start` to `end`, one mask frame per frame at the project's frame rate (the file's own with no project open). `time` must lie in the span. Tracking runs silently — the app shows nothing — and shares the model with the editor's object mask tool, so a track waits for the tool's own work to finish first, and tracks run one after another.
+
+**Into the project's library** (a project is open and `output` is a library path or left out), the track runs in the background. The call returns at once with the mask's `path` and `src`, the span it covers, and `state: "tracking"` — no picture and no findings yet. Poll [`context`](../context.md) until the track's row in `masks` (the one whose `src` matches) is `done`: that row carries the findings below and the contact sheet as `image`, and the mask file is in the library from then on. Name it in a `<mask>` once it is done. A `failed` row says why in `error`; closing the project stops its tracks.
+
+```bash
+diffusion media segment footage/skater.mp4 --time 2.4 --box 0.4,0.2,0.65,0.95 --start 2 --end 9 -o masks/skater.mask
+diffusion context
+```
+
+**Anywhere else** (an absolute `output`, or no project open), the call waits for the whole track and returns everything at once, `state: "done"`: the CLI allows it an hour, and an MCP client should give the call a comparable timeout.
The mask carries its recipe (footage, prompt, span, model), so the editor can make it again if its file goes missing. Name it on the clip it was made from, with `start` as its `sourceIn`, so it stays on the footage whatever the trim:
@@ -57,20 +66,22 @@ One JSON object:
```ts
{
- image: string; // PNG: the prompted frame with its mask on a preview, a contact sheet of up to 6 tracked frames otherwise
+ image?: string; // PNG: the prompted frame with its mask on a preview, a contact sheet of up to 6 tracked frames otherwise; absent while a track runs
path?: string; // absolute path of the mask file; absent on a preview
- src?: string; // the mask's library path, for <mask src>; when it went into the project's library
+ src?: string; // the mask's library path, for <mask src>; when it goes into the project's library
+ state?: "tracking" | "done"; // on a track: "tracking" while it runs in the background, "done" when the mask is written
model: "tiny" | "small" | "base-plus" | "large";
frameRate: number; // frames per second the mask holds
start: number; // source time of the first masked frame, seconds: the <mask>'s sourceIn
end: number; // source time just past the last masked frame, seconds
frames: number; // frames masked (1 on a preview)
- bbox: [x0, y0, x1, y1] | null; // on the prompted frame, the object's bounds in 0..1 of the frame
- area: number; // on the prompted frame, the share of the frame the object covers, 0..1
- score: number; // on the prompted frame, the object score: at or below 0 the model sees no object
- iou: number; // on the prompted frame, the model's estimate of the mask's quality, 0..1
- lost: [number, number][]; // [start, end) spans, seconds, where no object was found
- weak: [number, number][]; // [start, end) spans, seconds, where the model rated its mask below 0.5
+ // What was found; absent while a track runs, when context's row reports it instead.
+ bbox?: [x0, y0, x1, y1] | null; // on the prompted frame, the object's bounds in 0..1 of the frame
+ area?: number; // on the prompted frame, the share of the frame the object covers, 0..1
+ score?: number; // on the prompted frame, the object score: at or below 0 the model sees no object
+ iou?: number; // on the prompted frame, the model's estimate of the mask's quality, 0..1
+ lost?: [number, number][]; // [start, end) spans, seconds, where no object was found
+ weak?: [number, number][]; // [start, end) spans, seconds, where the model rated its mask below 0.5
}
```
@@ -78,4 +89,4 @@ One JSON object:
## Errors
-Fails when the path can't be resolved or the asset is not a video, `time` or `start` is past the video's end, no point or box prompts the object, a box's bounds are out of order or outside 0..1, `time` lies outside `start`–`end`, a preview is given a span or an `output`, `output` is a library path with no project open or names something in the library that is not one of the project's masks, `output` has an extension other than `.mask`, the model cannot load (no WebGPU, or the download failed), or the call is canceled in the app.
+Fails when the path can't be resolved or the asset is not a video, `time` or `start` is past the video's end, no point or box prompts the object, a box's bounds are out of order or outside 0..1, `time` lies outside `start`–`end`, a preview is given a span or an `output`, `output` is a library path with no project open or names something in the library that is not one of the project's masks, `output` has an extension other than `.mask`, the model cannot load (no WebGPU, or the download failed), or the call is canceled. A background track fails on its row in `context` instead.
diff --git a/docs/skills/editor.md b/docs/skills/editor.md
index 57f9f46e..9d47b6f5 100644
--- a/docs/skills/editor.md
+++ b/docs/skills/editor.md
@@ -46,7 +46,7 @@ How to confirm a change actually produced what you intended. A clean save does n
- Use the built-in tags for the media a composition is made of (audio, video, images, captions).
- Hoist the properties that define the composition's look — title copy, font family and size, accent colors, key padding — into top-level consts annotated with `@inspect`, so they become live controls in the app's inspector.
- For anything 3D, use Three.js drawn into a `<surface>` tag.
-- To cut an object out of footage, or confine an effect to it, track it with `media_segment` (preview the prompt first) and name the mask file in a `<mask>` under the clip's `<effect>`.
+- To cut an object out of footage, or confine an effect to it, track it with `media_segment` (preview the prompt first) and name the mask file in a `<mask>` under the clip's `<effect>`. A track into the library runs in the background: keep working, and poll `context` until its row in `masks` is done before naming it.
- For motion graphics, overlays and UI-heavy graphics, the `<html>` tag driven by a paused [anime.js](https://animejs.com) timeline
- Before animating anything, read the [easings reference](../guides/motion/easings.md) and choose easings deliberately — default or linear easing is what makes motion read as a slideshow.
- Add auto captions last, after everything else is assembled, so they transcribe the finished audio at its final placement.
diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.ts
index 41ee3178..98f4fc45 100644
--- a/packages/dapi/src/index.ts
+++ b/packages/dapi/src/index.ts
@@ -33,7 +33,7 @@ export { FONT_LIMIT } from "./tools/fonts";
// Named request and result types, for handlers that spell out their
// signature. Each is the parsed (output) side of the tool's schema.
import type { ImageRef as ImageRefSchema, LogEntry as LogEntrySchema, LogLevel as LogLevelSchema, TimecodedImage as TimecodedImageSchema } from "./schemas";
-import type { GenerationRow as GenerationRowType } from "./tools/context";
+import type { GenerationRow as GenerationRowType, MaskRow as MaskRowType } from "./tools/context";
import type { CheckIssue as CheckIssueSchema, CheckIssueCode as CheckIssueCodeSchema } from "./tools/check";
import type { ExportFormat as ExportFormatSchema, ExportSettings as ExportSettingsSchema } from "./tools/export";
import type { ModelInfo as ModelInfoSchema } from "./tools/models";
@@ -48,6 +48,7 @@ export type LogEntry = z.output<typeof LogEntrySchema>;
export type TimecodedImage = z.output<typeof TimecodedImageSchema>;
export type ImageRef = z.output<typeof ImageRefSchema>;
export type GenerationRow = GenerationRowType;
+export type MaskRow = MaskRowType;
export type CheckIssueCode = z.output<typeof CheckIssueCodeSchema>;
export type CheckIssue = z.output<typeof CheckIssueSchema>;
export type ExportFormat = z.output<typeof ExportFormatSchema>;
@@ -61,7 +62,7 @@ export type FontFamily = z.output<typeof FontFamilySchema>;
export type OpenRequest = ToolArgs<"open">;
export type OpenResult = ToolResult<"open">;
-export type ContextResult = ToolOutput<"context">;
+export type ContextResult = ToolResult<"context">;
export type CaptureRequest = ToolArgs<"capture">;
export type CaptureResult = ToolResult<"capture">;
export type CheckRequest = ToolArgs<"check">;
diff --git a/packages/dapi/src/tools/context.ts b/packages/dapi/src/tools/context.ts
index 82bb450e..dd923c89 100644
--- a/packages/dapi/src/tools/context.ts
+++ b/packages/dapi/src/tools/context.ts
@@ -4,6 +4,8 @@
import { z } from "zod";
import { defineTool } from "../tool";
+import { Bytes } from "../schemas";
+import { segmentFound, segmentSpan } from "./media-segment";
const GenerationRow = z.object({
element: z
@@ -19,16 +21,38 @@ const GenerationRow = z.object({
.describe("the library path the generation landed as, on done rows — ready for media_probe and its siblings"),
});
+const maskRow = {
+ id: z.string().describe("the track's id, the same across polls"),
+ src: z.string().describe("the mask's library path, for `<mask src>`: the file is there once the row is done"),
+ video: z.string().describe("the footage tracked, as media_segment was given it"),
+ state: z.enum(["loading", "tracking", "done", "failed"]).describe(
+ "`loading` while the model loads or another track has it, `tracking` while frames are masked, then `done` or `failed`",
+ ),
+ progress: z
+ .number()
+ .nullable()
+ .describe("0..1: the model's download while loading (null when there is none to report), the frames masked while tracking; 1 once done"),
+ error: z.string().optional().describe("what the track failed with, on failed rows"),
+ ...segmentSpan,
+ ...z.object(segmentFound).partial().shape,
+};
+
+const MaskRow = z.object({
+ ...maskRow,
+ image: z.string().optional().describe("absolute path of a contact sheet of tracked frames with their masks, on done rows"),
+});
+
/**
* What the project's source cannot say: the JSX already holds the scenes,
* the selection, and the work area, so the report is only the folders, the
- * playhead, the fonts actually registered, and where generations stand.
+ * playhead, the fonts actually registered, and where generations and
+ * background mask tracks stand.
*/
export const context = defineTool({
name: "context",
title: "App context",
description:
- "Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, and where its `generate.*` declarations stand. Poll it to wait for generations without blocking.",
+ "Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, where its `generate.*` declarations stand, and the progress of `media_segment` tracks running in the background. Poll it to wait for generations and tracks without blocking.",
input: z.object({}),
output: z.object({
rootDir: z.string().nullable().describe("folder new projects are created in; null until one has been chosen"),
@@ -41,8 +65,19 @@ export const context = defineTool({
.array(z.string())
.describe("families registered in the world drawing the project; the editor default is always among them"),
generations: z.array(GenerationRow),
+ masks: z.array(MaskRow).describe("media_segment tracks started while this project is open, oldest first"),
+ }),
+ result: z.object({
+ rootDir: z.string().nullable(),
+ projectDir: z.string().nullable(),
+ currentTime: z.number().nullable(),
+ fontFamilies: z.array(z.string()),
+ generations: z.array(GenerationRow),
+ /** A done row's contact sheet comes as bytes, which the server writes to a file once per track. */
+ masks: z.array(z.object({ ...maskRow, png: Bytes.optional() })),
}),
environment: "renderer",
});
export type GenerationRow = z.output<typeof GenerationRow>;
+export type MaskRow = z.output<typeof MaskRow>;
diff --git a/packages/dapi/src/tools/media-segment.ts b/packages/dapi/src/tools/media-segment.ts
index c23bbbef..27e0c183 100644
--- a/packages/dapi/src/tools/media-segment.ts
+++ b/packages/dapi/src/tools/media-segment.ts
@@ -22,13 +22,17 @@ const Range = z.tuple([z.number(), z.number()]);
const Bounds = z.tuple([z.number(), z.number(), z.number(), z.number()]);
-/** What the call found, the same whether the server or the renderer writes the files. */
-const found = {
+/** The track asked for, known before it runs. */
+export const segmentSpan = {
model: z.enum(SEGMENT_MODELS),
frameRate: z.number().describe("frames per second the mask holds: the project's, or the file's own with no project open"),
start: z.number().describe("source time of the first masked frame, in seconds: the `sourceIn` of a `<mask>` naming the file"),
end: z.number().describe("source time just past the last masked frame, in seconds"),
frames: z.int().describe("frames masked"),
+};
+
+/** What segmenting found, the same whether the server or the renderer writes the files; absent while a track runs. */
+export const segmentFound = {
bbox: Bounds.nullable().describe(
"on the prompted frame, the object's bounds as [x0, y0, x1, y1] in 0..1 of the frame; null when nothing was segmented",
),
@@ -41,6 +45,8 @@ const found = {
weak: z.array(Range).describe(`source-time spans, [start, end) in seconds, where the model rated its mask below ${WEAK_IOU}`),
};
+const optionalFound = z.object(segmentFound).partial().shape;
+
export const mediaSegment = defineTool({
name: "media_segment",
title: "Segment object",
@@ -49,7 +55,7 @@ export const mediaSegment = defineTool({
"Prompt the object on the frame at `time` with `points` on it, `exclude` points off it, and/or a `box` around it, all in 0..1 of the frame (x right, y down). " +
"`preview: true` segments that frame alone and writes only an image: the frame with the mask tinted and outlined, the prompts drawn, and a 0.1 grid to read coordinates off. Refine the prompts with previews (fast once the frame is encoded), then make the same call without `preview` to track. " +
"Tracking covers `start` to `end`, the whole file by default, at roughly 0.25 s a frame with the tiny model and several times that with the larger ones, so pass the span the clip actually plays. " +
- "Returns the mask's path (and its library path when it went into the project), a contact sheet of tracked frames, and the spans where the object was lost or the mask is weak. The first use of a model downloads it (83 MB for tiny, up to 475 MB for large).",
+ "With a project open and the mask going into its library, a track runs in the background: the call returns the mask's paths at once with `state: \"tracking\"`, and `context` reports its progress, then a contact sheet of tracked frames and the spans where the object was lost or the mask is weak. A mask written elsewhere is tracked before the call returns, with the same findings. The first use of a model downloads it (83 MB for tiny, up to 475 MB for large).",
input: z
.object({
path: AssetPath,
@@ -106,18 +112,26 @@ export const mediaSegment = defineTool({
output: z.object({
image: z
.string()
- .describe("absolute path of the PNG: the prompted frame with its mask on a preview, a contact sheet of tracked frames otherwise"),
+ .optional()
+ .describe("absolute path of the PNG: the prompted frame with its mask on a preview, a contact sheet of tracked frames otherwise; absent while a track runs"),
path: z.string().optional().describe("absolute path of the mask file; absent on a preview"),
- src: z.string().optional().describe("the mask's library path, for `<mask src>`; present when it was written into the project's library"),
- ...found,
+ src: z.string().optional().describe("the mask's library path, for `<mask src>`; present when it goes into the project's library"),
+ state: z
+ .enum(["tracking", "done"])
+ .optional()
+ .describe("on a track: `tracking` while it runs in the background (poll `context` until its row in `masks` is done), `done` when the mask is written"),
+ ...segmentSpan,
+ ...optionalFound,
}),
result: z.object({
- png: Bytes,
+ png: Bytes.optional(),
/** The mask file, for the server to write where the renderer cannot: an absolute path, or the temp dir. */
mask: Bytes.optional(),
path: z.string().optional(),
src: z.string().optional(),
- ...found,
+ state: z.enum(["tracking", "done"]).optional(),
+ ...segmentSpan,
+ ...optionalFound,
}),
environment: "renderer",
});
diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.ts
index abe62e95..5a9e4aa5 100644
--- a/packages/dapi/src/tools/tools.test.ts
+++ b/packages/dapi/src/tools/tools.test.ts
@@ -100,7 +100,7 @@ describe("logs and export", () => {
describe("context", () => {
it("reports the same shape with and without an open project", () => {
expect(
- context.output.safeParse({ rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [], generations: [] }).success,
+ context.output.safeParse({ rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [], generations: [], masks: [] }).success,
).toBe(true);
expect(
context.output.safeParse({
@@ -109,10 +109,22 @@ describe("context", () => {
currentTime: null,
fontFamilies: ["Inter"],
generations: [{ element: "index.tsx:3", name: null, state: "done", asset: "gen/a.mp4" }],
+ masks: [],
}).success,
).toBe(true);
expect(context.output.safeParse({ rootDir: "/p", projectDir: "/p/a" }).success).toBe(false);
});
+
+ it("reports background mask tracks, with a contact sheet as bytes in the result and a path in the output", () => {
+ const span = { model: "tiny", frameRate: 30, start: 0, end: 2, frames: 60 };
+ const running = { id: "t1", src: "masks/a/Tracking 1.mask", video: "a.mp4", state: "tracking", progress: 0.4, ...span };
+ const found = { bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [] };
+ const base = { rootDir: "/p", projectDir: "/p/a", currentTime: null, fontFamilies: [], generations: [] };
+ expect(context.output.safeParse({ ...base, masks: [running] }).success).toBe(true);
+ expect(context.result!.safeParse({ ...base, masks: [{ ...running, state: "done", progress: 1, png: new Uint8Array(3), ...found }] }).success).toBe(true);
+ expect(context.output.safeParse({ ...base, masks: [{ ...running, state: "done", progress: 1, image: "/tmp/s.png", ...found }] }).success).toBe(true);
+ expect(context.output.safeParse({ ...base, masks: [{ ...running, state: "paused" }] }).success).toBe(false);
+ });
});
describe("media_transcribe", () => {
@@ -169,6 +181,12 @@ describe("media_segment", () => {
expect(issues(input.safeParse({ ...track, start: 3, end: 2 }))).toHaveProperty("end");
});
+ it("answers a background track with its paths and span alone", () => {
+ const span = { model: "tiny", frameRate: 30, start: 0, end: 2, frames: 60 };
+ expect(mediaSegment.output.safeParse({ path: "/p/assets/masks/a.mask", src: "masks/a.mask", state: "tracking", ...span }).success).toBe(true);
+ expect(mediaSegment.output.safeParse({ path: "/p/assets/masks/a.mask", state: "queued", ...span }).success).toBe(false);
+ });
+
it("names only the models the app offers", () => {
expect(input.safeParse({ ...at, box: [0, 0, 1, 1], model: "base-plus" }).success).toBe(true);
expect(input.safeParse({ ...at, box: [0, 0, 1, 1], model: "huge" }).success).toBe(false);