The VS Code of video editing, built for agents.
TypeScript
Add Media Segment Tool for Object Tracking in Video
konstantin-paulus67e783e5c87d
diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.tsindex 015a8844..9aab75fb 100644--- a/apps/cli/src/cli-client.ts+++ b/apps/cli/src/cli-client.ts@@ -18,6 +18,7 @@ const TIMEOUTS: Record<string, number> = {capture: 600_000,media_transcribe: 600_000,media_listen: 600_000,+ media_segment: 3_600_000,};/**diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 266e19f6..d3e4a1b0 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -66,6 +66,27 @@ function assetPath(ref: string): string {return ref;}+/** `x,y` as a point, added to the ones given before; a malformed one becomes NaN for the schema to reject. */+const point = (value: string, previous: Array<{ x: number; y: number }> = []): Array<{ x: number; y: number }> => {+ const [x = NaN, y = NaN, ...rest] = value.split(",").map(numeric);+ return [...previous, rest.length > 0 ? { x: NaN, y: NaN } : { x, y }];+};++/** `x0,y0,x1,y1` as a box; the wrong count of numbers becomes NaN for the schema to reject. */+const box = (value: string): [number, number, number, number] => {+ const bounds = value.split(",").map(numeric);+ return bounds.length === 4 ? (bounds as [number, number, number, number]) : [NaN, NaN, NaN, NaN];+};++/**+ * A mask's destination: a path spelled as one on disk (absolute, or starting+ * with `.`) resolves against the working directory; anything else is a+ * library path, passed through for the app to resolve in the open project.+ */+function maskOutput(ref: string): string {+ return isAbsolute(ref) || ref.startsWith(".") ? resolve(ref) : ref;+}+const program = new Command();program@@ -207,6 +228,28 @@ media.option("-e, --end <time>", field("media_listen", "end")).action((ref: string, opts: Omit<ToolInput<"media_listen">, "path">) => run("media_listen", { path: assetPath(ref), ...opts }));+media+ .command("segment")+ .alias("mask")+ .description(describe("media_segment"))+ .argument("<path>", field("media_segment", "path"))+ .requiredOption("-t, --time <time>", field("media_segment", "time"))+ .option("-p, --point <x,y>", `${field("media_segment", "points")}; repeat for more`, point)+ .option("-x, --exclude <x,y>", `${field("media_segment", "exclude")}; repeat for more`, point)+ .option("-b, --box <x0,y0,x1,y1>", field("media_segment", "box"), box)+ .option("--preview", field("media_segment", "preview"))+ .option("-s, --start <time>", field("media_segment", "start"))+ .option("-e, --end <time>", field("media_segment", "end"))+ .option("-m, --model <size>", field("media_segment", "model"))+ .option(+ "-o, --output <path>",+ `${field("media_segment", "output")}; a path starting with . resolves against the working directory`,+ )+ .action((ref: string, opts: Omit<ToolInput<"media_segment">, "path"> & { point?: ToolInput<"media_segment">["points"] }) => {+ const { point: points, ...rest } = opts;+ return run("media_segment", { path: assetPath(ref), ...rest, points, output: opts.output && maskOutput(opts.output) });+ });+program.command("models").description(describe("models"))diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.tsindex d11113e3..729e9cb8 100644--- a/apps/desktop/src/dapi/present.test.ts+++ b/apps/desktop/src/dapi/present.test.ts@@ -69,6 +69,29 @@ describe("present", () => {expect(JSON.parse(readFileSync(file, "utf8"))).toEqual({ segments });});+ it("writes a segment's mask where output says and its picture to the temp dir", async () => {+ const found = {+ model: "tiny" as const, frameRate: 30, start: 0, end: 1, frames: 30, bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [],+ };+ const file = join(dir, "masks", "skater.mask");+ const presented = await present("media_segment", { path: "/c.mp4", time: 0, output: file }, { png: png(8), mask: png(9, 4), ...found });+ const output = presented.output as { image: string; path: string };+ expect(output).toEqual({ image: output.image, path: file, ...found });+ expect(output.image).toMatch(/dapi-segment-.*\.png$/);+ expect(readFileSync(file)).toEqual(Buffer.from(png(9, 4)));+ expect(presented.images).toEqual([{ path: output.image, png: png(8) }]);+ });++ it("keeps the path of a mask the app put into the library, and writes none for a preview", async () => {+ const found = {+ model: "tiny" as const, frameRate: 30, start: 1, end: 1.033, frames: 1, bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [],+ };+ const stored = await present("media_segment", { path: "/c.mp4", time: 1 }, { png: png(8), path: "/p/assets/masks/a.mask", src: "masks/a.mask", ...found });+ expect(stored.output).toMatchObject({ path: "/p/assets/masks/a.mask", src: "masks/a.mask" });+ const preview = await present("media_segment", { path: "/c.mp4", time: 1, preview: true }, { png: png(8), ...found });+ expect(preview.output).not.toHaveProperty("path");+ });+it("passes other results through untouched", async () => {expect(await present("check", { id: "x" }, { stats: {}, issues: [] })).toEqual({ output: { stats: {}, issues: [] }, images: [] });});diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.tsindex c699f093..1772786b 100644--- a/apps/desktop/src/dapi/present.ts+++ b/apps/desktop/src/dapi/present.ts@@ -48,6 +48,8 @@ export async function present(name: ToolName, args: unknown, result: unknown): Preturn presentScreenshot(result as ToolResult<"screenshot">, (args as ToolArgs<"screenshot">).output);case "media_transcribe":return presentTranscript(result as ToolResult<"media_transcribe">, (args as ToolArgs<"media_transcribe">).output);+ case "media_segment":+ return presentSegment(result as ToolResult<"media_segment">, (args as ToolArgs<"media_segment">).output);default:return { output: result, images: [] };}@@ -114,6 +116,24 @@ async function presentTranscript(transcript: ToolResult<"media_transcribe">, outreturn { output: presented, images: [] };}+/**+ * The picture always goes to the temp dir; `output` is where the mask goes.+ * A mask the renderer put into the project's library comes with its path; one+ * that comes as bytes is written here, to `output` or the temp dir.+ */+async function presentSegment(result: ToolResult<"media_segment">, output: string | undefined): Promise<Presented> {+ const { png, mask, ...found } = result;+ const image = join(tempDir(), `dapi-segment-${randomUUID()}.png`);+ await writeFile(image, png);+ let path = found.path;+ if (mask) {+ path = await singleFilePath(output, `dapi-mask-${randomUUID()}.mask`);+ await writeFile(path, mask);+ }+ const presented: ToolOutput<"media_segment"> = { image, ...found, ...(path === undefined ? {} : { path }) };+ return { output: presented, images: [{ path: image, png }] };+}+function screenshotFilename(taken: Date, attempt: number): string {const pad = (value: number) => String(value).padStart(2, "0");const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-");diff --git a/apps/web/src/dapi/handlers/index.ts b/apps/web/src/dapi/handlers/index.tsindex 47f6f4ea..ce1b8489 100644--- a/apps/web/src/dapi/handlers/index.ts+++ b/apps/web/src/dapi/handlers/index.ts@@ -17,6 +17,7 @@ import { mediaTranscribe } from "./media-transcribe";import { mediaFilmstrip } from "./media-filmstrip";import { mediaWaveform } from "./media-waveform";import { mediaListen } from "./media-listen";+import { mediaSegment } from "./media-segment";import type { Handlers } from "../handler";@@ -37,4 +38,5 @@ export const handlers: Handlers = {media_filmstrip: mediaFilmstrip,media_waveform: mediaWaveform,media_listen: mediaListen,+ media_segment: mediaSegment,};diff --git a/apps/web/src/dapi/handlers/media-segment.ts b/apps/web/src/dapi/handlers/media-segment.tsnew file mode 100644index 00000000..c67c0efb--- /dev/null+++ b/apps/web/src/dapi/handlers/media-segment.ts@@ -0,0 +1,466 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { CanvasSink } from "mediabunny";+import { toast } from "somoto";+import { MASK_EXTENSION, assetName, isAbsoluteSource, isProjectSource, isUrlSource, traceMask } from "@diffusionstudio/assets";+import { DapiError, WEAK_IOU } from "@diffusionstudio/dapi";+import { encodePng } from "@diffusionstudio/encoder";+import { FrameRate, formatTimecode, getLibrary, getVideoTrack } from "@diffusionstudio/runtime";+import { sam2Model } from "@diffusionstudio/sam2/models";++import { encodeObjectMask, nextTrackingName, objectMaskFolder } from "@/engine/object-mask/commit";+import { segmentFootage } from "@/engine/object-mask/tracking";+import { requireAssetType, resolveAsset } from "../lib/assets";+import { SheetCollector } from "../lib/sheets";++import type { InputVideoTrack } from "mediabunny";+import type { Asset, AssetLibrary, MaskFrame, VideoAsset } from "@diffusionstudio/assets";+import type { MediaSegmentRequest } from "@diffusionstudio/dapi";+import type { Sam2Point } from "@diffusionstudio/sam2";+import type { FootageSegments } from "@/engine/object-mask/tracking";+import type { EditorSession } from "../session";+import type { ToolHandler } from "../handler";++type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;++/** A preview's frame is drawn this wide at most: detail enough to judge an edge, small enough to arrive inline. */+const PREVIEW_WIDTH = 1024;+/** Tracked frames on the contact sheet, spread evenly over the span. */+const SHEET_FRAMES = 6;+/** A sheet cell's frame at most this wide, so the sheet stays small enough to arrive inline. */+const SHEET_FRAME_WIDTH = 512;++/** What is outside the mask is washed with: a magenta footage rarely has, strong enough to show a hole. */+const OUTSIDE = "rgba(255, 0, 200, 0.5)";+const BOX = "#FFC400";++/**+ * Segments an object in a video and tracks it through the span asked for, on+ * the object mask tool's model and queue; the mask goes into the project's+ * library (or back to the server to write when it goes elsewhere), with a+ * picture of what was found. A preview segments the prompted frame alone.+ */+export const mediaSegment: ToolHandler<"media_segment"> = async (args, ctx) => {+ const asset = await resolveAsset(ctx, args.path);+ requireAssetType(asset, ["VIDEO"], "a video");+ const session = ctx.session();+ if (args.time > asset.duration) {+ throw new DapiError("invalid-input", `time ${args.time}s is past the video's end (${asset.duration.toFixed(2)}s).`);+ }++ // Frames are counted on the grid the mask file will hold: the project's+ // rate, which `<mask>` plays it back at, or the file's own with none open.+ const fps = session?.world.get(FrameRate)?.value ?? (asset.frameRate || 30);+ const total = Math.max(1, Math.ceil(asset.duration * fps - 1e-6));+ let first = Math.min(Math.round(args.time * fps), total - 1);+ let count = 1;+ if (!args.preview) {+ const from = args.start === undefined ? 0 : Math.round(args.start * fps);+ const to = args.end === undefined ? total : Math.min(total, Math.round(args.end * fps));+ if (from >= total) {+ throw new DapiError("invalid-input", `start ${args.start}s is past the video's end (${asset.duration.toFixed(2)}s).`);+ }+ count = Math.max(1, to - from);+ first = from;+ }+ const seedFrame = Math.min(Math.max(Math.round(args.time * fps), first), first + count - 1);+ const target = args.preview ? null : destination(args.output, session);++ const progress = new Progress(asset, args.preview ?? false);+ progress.tracked(0, count);+ let segments: FootageSegments;+ try {+ segments = await segmentFootage({+ asset,+ fps,+ model: args.model ?? "tiny",+ points: prompts(args),+ seedFrame,+ first: args.preview ? seedFrame : first,+ count,+ preview: args.preview ?? false,+ signal: AbortSignal.any([ctx.signal, progress.signal]),+ onDownload: (fraction) => progress.download(sam2Model(args.model ?? "tiny").label, fraction),+ onProgress: (completed, of) => progress.tracked(completed, of),+ });+ } catch (error) {+ if (error instanceof DOMException && error.name === "AbortError") {+ throw new DapiError("canceled", progress.signal.aborted ? "Segmenting was canceled in the app." : "The call was canceled.");+ }+ throw error;+ } finally {+ progress.done();+ }++ const { masks, grid } = segments;+ const seed = masks[seedFrame - first] ?? null;+ const found = {+ model: args.model ?? "tiny",+ frameRate: fps,+ start: round(first / fps),+ end: round((first + count) / fps),+ frames: count,+ ...measure(seed, grid),+ score: seed?.score ?? 0,+ iou: seed?.iou ?? 0,+ lost: spans(masks, first, fps, (mask) => isLost(mask)),+ weak: spans(masks, first, fps, (mask) => !isLost(mask) && mask!.iou < WEAK_IOU),+ };++ const video = await getVideoTrack(asset);+ if (!video) throw new DapiError("wrong-kind", `Asset ${asset.id} has no decodable video track.`);++ if (args.preview) {+ const png = await previewImage(video, segments, seedFrame, seed, args);+ return { png, ...found };+ }++ const png = await contactSheet(video, segments, asset, first, fps);+ const file = encodeObjectMask(asset, fps, grid, masks, {+ model: segments.model.repo,+ source: asset.id,+ first,+ seedFrame,+ points: prompts(args),+ });++ if (target?.kind !== "library") return { png, mask: new Uint8Array(await file.arrayBuffer()), ...found };++ const library = getLibrary(session!.world);+ const stored = await storeMask(library, target.path, asset, file);+ return { png, path: `${session!.project.dir()}/${stored.source}`, src: stored.path, ...found };+};++/** The prompt as SAM 2 takes it: points on the object, points off it, then a box's corners. */+function prompts(args: MediaSegmentRequest): Sam2Point[] {+ const points: Sam2Point[] = [+ ...(args.points ?? []).map(({ x, y }) => ({ x, y, label: 1 as const })),+ ...(args.exclude ?? []).map(({ x, y }) => ({ x, y, label: 0 as const })),+ ];+ if (args.box) {+ const [x0, y0, x1, y1] = args.box;+ points.push({ x: x0, y: y0, label: 2 }, { x: x1, y: y1, label: 3 });+ }+ return points;+}++// ── Where the mask goes ──────────────────────────────────────++/**+ * Where a tracked mask is written: into the library — at `path`, or at the+ * tool's own `masks/<video>/Tracking <n>.mask` when that is null — or back to+ * the server, which writes absolute paths and the temp dir. Settled before+ * tracking, so a bad `output` fails before the long part.+ */+type Destination = { kind: "library"; path: string | null } | { kind: "server" };++function destination(output: string | undefined, session: EditorSession | null): Destination {+ if (output === undefined) return session ? { kind: "library", path: null } : { kind: "server" };+ if (isAbsoluteSource(output)) return { kind: "server" };+ if (isUrlSource(output)) throw new DapiError("invalid-input", `The mask cannot be written to a URL (${output}).`);+ if (!session) {+ throw new DapiError(+ "no-project",+ `"${output}" is a library path, which needs an open project — open one, or pass an absolute path to write the mask elsewhere.`,+ );+ }++ const path = output.replace(/^\.\//, "").replace(/\/+$/, "");+ const extension = /\.([^./]+)$/.exec(path)?.[1];+ if (extension !== undefined && extension.toLowerCase() !== MASK_EXTENSION) {+ throw new DapiError("invalid-input", `A mask file ends in .${MASK_EXTENSION} (got "${output}").`);+ }+ const named = extension === undefined ? `${path}.${MASK_EXTENSION}` : path;+ const existing = getLibrary(session.world).get(named);+ if (existing && !isReplaceableMask(existing)) {+ throw new DapiError("invalid-input", `"${named}" in the library is not a mask file of this project's, so it cannot be replaced.`);+ }+ return { kind: "library", path: named };+}++function isReplaceableMask(asset: Asset): boolean {+ return asset.type === "MASK" && isProjectSource(asset.source);+}++/**+ * Puts the mask file into the library. A mask already at `path` has its file+ * written over and is taken in again at the same path, as a restore does, so+ * every `<mask>` naming it shows the new frames.+ */+async function storeMask(library: AssetLibrary, path: string | null, video: VideoAsset, file: Blob): Promise<Asset> {+ if (path === null) {+ const folder = objectMaskFolder(video);+ return library.store(file, { folder, name: nextTrackingName(library, folder) });+ }++ const existing = library.get(path);+ if (existing && isReplaceableMask(existing)) {+ await library.fs.write(existing.source, file);+ return library.relink(existing, existing.source);+ }+ const slash = path.lastIndexOf("/");+ return library.store(file, { folder: slash < 0 ? "" : path.slice(0, slash), name: path.slice(slash + 1) });+}++// ── What was found ───────────────────────────────────────────++/** A frame without a mask, or one where the model sees no object. */+function isLost(mask: MaskFrame | null): boolean {+ return !mask || mask.score <= 0 || !mask.field.some((cell) => cell > 0);+}++/** The runs of frames `test` holds for, as `[start, end)` source seconds. */+function spans(masks: readonly (MaskFrame | null)[], first: number, fps: number, test: (mask: MaskFrame | null) => boolean): [number, number][] {+ const found: [number, number][] = [];+ let from = -1;+ for (let i = 0; i <= masks.length; i++) {+ const holds = i < masks.length && test(masks[i] ?? null);+ if (holds && from < 0) from = i;+ if (!holds && from >= 0) {+ found.push([round((first + from) / fps), round((first + i) / fps)]);+ from = -1;+ }+ }+ return found;+}++/** The object's bounds and share of the frame, from the cells that are in. */+function measure(mask: MaskFrame | null, grid: number): { bbox: [number, number, number, number] | null; area: number } {+ if (!mask) return { bbox: null, area: 0 };+ let cells = 0;+ let x0 = grid;+ let y0 = grid;+ let x1 = -1;+ let y1 = -1;+ for (let y = 0; y < grid; y++) {+ for (let x = 0; x < grid; x++) {+ if (mask.field[y * grid + x]! <= 0) continue;+ cells++;+ x0 = Math.min(x0, x);+ y0 = Math.min(y0, y);+ x1 = Math.max(x1, x);+ y1 = Math.max(y1, y);+ }+ }+ if (cells === 0) return { bbox: null, area: 0 };+ return { bbox: [round(x0 / grid), round(y0 / grid), round((x1 + 1) / grid), round((y1 + 1) / grid)], area: round(cells / (grid * grid)) };+}++function round(value: number): number {+ return Math.round(value * 1000) / 1000;+}++// ── Pictures ─────────────────────────────────────────────────++/** The prompted frame with its mask, the prompts, and a tenths grid to read coordinates off. */+async function previewImage(+ video: InputVideoTrack,+ segments: FootageSegments,+ seedFrame: number,+ mask: MaskFrame | null,+ args: MediaSegmentRequest,+): Promise<Uint8Array> {+ const width = Math.min(PREVIEW_WIDTH, await video.getDisplayWidth());+ const sink = new CanvasSink(video, { width });+ const wrapped = await sink.getCanvas(segments.seconds(seedFrame));+ if (!wrapped) throw new DapiError("not-found", "The prompted frame could not be decoded.");++ const { canvas } = wrapped;+ const ctx = canvas.getContext("2d") as Ctx2D;+ const scale = canvas.width / PREVIEW_WIDTH;+ if (mask) drawMask(ctx, mask, segments.grid, canvas.width, canvas.height, scale);+ drawGrid(ctx, canvas.width, canvas.height);+ drawPrompts(ctx, args, canvas.width, canvas.height, scale);+ drawGridLabels(ctx, canvas.width, canvas.height, scale);+ return encodePng(canvas);+}++/** Tracked frames spread over the span, each with its mask, on one sheet labelled with their source timecodes. */+async function contactSheet(video: InputVideoTrack, segments: FootageSegments, asset: VideoAsset, first: number, fps: number): Promise<Uint8Array> {+ const { masks, grid } = segments;+ const picks = masks.length <= SHEET_FRAMES+ ? masks.map((_, i) => i)+ : Array.from({ length: SHEET_FRAMES }, (_, k) => Math.round((k * (masks.length - 1)) / (SHEET_FRAMES - 1)));++ const width = Math.min(SHEET_FRAME_WIDTH, await video.getDisplayWidth());+ const height = Math.round((width * (await video.getDisplayHeight())) / (await video.getDisplayWidth()));+ const sheets = new SheetCollector(picks.length, { width, height }, SHEET_FRAMES);+ const sink = new CanvasSink(video, { width });++ let k = 0;+ for await (const wrapped of sink.canvasesAtTimestamps(picks.map((i) => segments.seconds(first + i)))) {+ const index = k++;+ const i = picks[index]!;+ if (!wrapped) throw new DapiError("not-found", "A tracked frame could not be decoded.");+ const { canvas } = wrapped;+ const mask = masks[i];+ if (mask) drawMask(canvas.getContext("2d") as Ctx2D, mask, grid, canvas.width, canvas.height, canvas.width / PREVIEW_WIDTH);+ const at = (first + i) / fps;+ await sheets.add(index, { at, timecode: formatTimecode(at, asset.frameRate), image: canvas });+ }+ return sheets.result()[0]!.png;+}++/**+ * The mask as an agent judges it: the object left at its own pixels, what is+ * outside it washed with a colour footage rarely has, so edges read against+ * real detail and a hole shows as a speck of it, and the edge traced in white+ * over a dark halo, legible on any picture.+ */+function drawMask(ctx: Ctx2D, mask: MaskFrame, grid: number, width: number, height: number, scale: number): void {+ const cells = new Path2D();+ traceMask(mask.field, grid, grid, cells);+ const outline = new Path2D();+ outline.addPath(cells, new DOMMatrix().scale(width / grid, height / grid));+ // The frame and the mask's loops together, filled even-odd: everything but the object, holes included.+ const outside = new Path2D();+ outside.rect(0, 0, width, height);+ outside.addPath(outline);++ ctx.save();+ ctx.fillStyle = OUTSIDE;+ ctx.fill(outside, "evenodd");+ ctx.lineCap = "round";+ ctx.lineJoin = "round";+ ctx.lineWidth = Math.max(2, 4 * scale);+ ctx.strokeStyle = "rgba(0, 0, 0, 0.6)";+ ctx.stroke(outline);+ ctx.lineWidth = Math.max(1, 1.5 * scale);+ ctx.strokeStyle = "#FFFFFF";+ ctx.stroke(outline);+ ctx.restore();+}++/**+ * The prompts, told apart by fill rather than hue: points on the object white,+ * points off it black, each labelled `+n` or `−n` in the order it was given,+ * and the box as a dashed amber frame.+ */+function drawPrompts(ctx: Ctx2D, args: MediaSegmentRequest, width: number, height: number, scale: number): void {+ ctx.save();+ if (args.box) {+ const [x0, y0, x1, y1] = args.box;+ ctx.setLineDash([8 * scale, 5 * scale]);+ ctx.lineWidth = Math.max(1, 2 * scale);+ ctx.strokeStyle = BOX;+ ctx.strokeRect(x0 * width, y0 * height, (x1 - x0) * width, (y1 - y0) * height);+ ctx.setLineDash([]);+ }++ const marks = [+ ...(args.points ?? []).map((point, i) => ({ ...point, on: true, label: `+${i + 1}` })),+ ...(args.exclude ?? []).map((point, i) => ({ ...point, on: false, label: `\u2212${i + 1}` })),+ ];+ const radius = Math.max(4, 6 * scale);+ for (const { x, y, on, label } of marks) {+ const cx = x * width;+ const cy = y * height;+ ctx.beginPath();+ ctx.arc(cx, cy, radius, 0, Math.PI * 2);+ ctx.fillStyle = on ? "#FFFFFF" : "#000000";+ ctx.fill();+ ctx.lineWidth = Math.max(1, 1.5 * scale);+ ctx.strokeStyle = on ? "#000000" : "#FFFFFF";+ ctx.stroke();+ // Beside the point, on whichever side keeps it in the frame.+ const right = x < 0.9;+ drawLabel(ctx, label, cx + (right ? 1 : -1) * (radius + 3 * scale), cy, right ? "left" : "right", scale);+ }+ ctx.restore();+}++/** Faint lines at every tenth of the frame; `drawGridLabels` numbers them, over the prompts. */+function drawGrid(ctx: Ctx2D, width: number, height: number): void {+ ctx.save();+ ctx.lineWidth = 1;+ ctx.strokeStyle = "rgba(255, 255, 255, 0.3)";+ ctx.beginPath();+ for (let i = 1; i < 10; i++) {+ const x = Math.round((i / 10) * width) + 0.5;+ const y = Math.round((i / 10) * height) + 0.5;+ ctx.moveTo(x, 0);+ ctx.lineTo(x, height);+ ctx.moveTo(0, y);+ ctx.lineTo(width, y);+ }+ ctx.stroke();+ ctx.restore();+}++/** The grid's tenths along the top and left edges. */+function drawGridLabels(ctx: Ctx2D, width: number, height: number, scale: number): void {+ const inset = Math.max(10, 14 * scale);+ for (let i = 1; i < 10; i++) {+ const label = (i / 10).toFixed(1);+ drawLabel(ctx, label, (i / 10) * width, inset, "center", scale);+ drawLabel(ctx, label, 3 * scale, (i / 10) * height, "left", scale);+ }+}++/** White text on a dark pill, centred on `y` and aligned on `x`: readable over any picture. */+function drawLabel(ctx: Ctx2D, text: string, x: number, y: number, align: "left" | "center" | "right", scale: number): void {+ ctx.save();+ ctx.font = `600 ${Math.max(10, Math.round(12 * scale))}px sans-serif`;+ const padX = Math.max(3, 4 * scale);+ const h = Math.max(14, Math.round(17 * scale));+ const w = ctx.measureText(text).width + 2 * padX;+ const left = align === "left" ? x : align === "right" ? x - w : x - w / 2;+ ctx.beginPath();+ ctx.roundRect(left, y - h / 2, w, h, h / 2);+ ctx.fillStyle = "rgba(0, 0, 0, 0.62)";+ ctx.fill();+ ctx.fillStyle = "#FFFFFF";+ ctx.textAlign = "left";+ ctx.textBaseline = "middle";+ ctx.fillText(text, left + padX, y + 0.5);+ ctx.restore();+}++// ── Progress in the app ──────────────────────────────────────++/**+ * What the app shows while a call runs long: the model's download, and a+ * track's frames, in a toast with a Cancel button. A preview shows only a+ * download; it is over too quickly otherwise.+ */+class Progress {+ private readonly controller = new AbortController();+ private readonly id = `media-segment-${crypto.randomUUID()}`;+ private shown: string | null = null;++ constructor(+ private readonly asset: VideoAsset,+ private readonly preview: boolean,+ ) {}++ /** Fires when the Cancel button is pressed. */+ get signal(): AbortSignal {+ return this.controller.signal;+ }++ download(label: string, fraction: number | null): void {+ this.show(fraction === null ? `Loading ${label}…` : `Downloading ${label}… ${Math.floor(fraction * 100)}%`);+ }++ tracked(completed: number, total: number): void {+ if (this.preview) return;+ this.show(`Tracking an object in ${assetName(this.asset)}… ${Math.floor((completed / total) * 100)}%`);+ }++ done(): void {+ if (this.shown !== null) toast.dismiss(this.id);+ }++ // A percent at a time: downloads and tracks report far more often.+ private show(message: string): void {+ if (message === this.shown) return;+ this.shown = message;+ toast.loading(message, {+ id: this.id,+ duration: Infinity,+ cancel: { label: "Cancel", onClick: () => this.controller.abort() },+ });+ }+}diff --git a/apps/web/src/engine/object-mask/commit.tsx b/apps/web/src/engine/object-mask/commit.tsxindex 53a9d81b..f005b5bb 100644--- a/apps/web/src/engine/object-mask/commit.tsx+++ b/apps/web/src/engine/object-mask/commit.tsx@@ -12,7 +12,7 @@ import { getVideoRect } from './media';import { getTargetEffect } from './store';import type { Entity, World } from 'koota';-import type { AssetLibrary, MaskFrame, MaskRecipe } from '@diffusionstudio/assets';+import type { AssetLibrary, MaskFrame, MaskRecipe, VideoAsset } from '@diffusionstudio/assets';import type { ObjectTrack } from './store';/** Where masks go in the library: a folder per video under this one. */@@ -37,7 +37,7 @@ export async function commitObjectMask(world: World, track: ObjectTrack): Promisconst { signal } = track.controller;const fps = world.get(FrameRate)?.value ?? 30;- const folder = `${MASKS_FOLDER}/${assetName(rect.asset).replace(/\.[^.]+$/, '')}`;+ const folder = objectMaskFolder(rect.asset);const blob = encodeObjectMask(rect.asset, fps, geometryOf(model.imageSize).maskSize, track.masks, {model: model.repo,@@ -83,8 +83,13 @@ export function encodeObjectMask();}+/** Where masks tracked on `video` go in the library: `masks/<video>`. */+export function objectMaskFolder(video: VideoAsset): string {+ return `${MASKS_FOLDER}/${assetName(video).replace(/\.[^.]+$/, '')}`;+}+/** `Tracking <n>.mask`, `n` the first count not yet taken in `folder`. */-function nextTrackingName(library: AssetLibrary, folder: string): string {+export function nextTrackingName(library: AssetLibrary, folder: string): string {for (let n = 1; ; n++) {const name = `Tracking ${n}.${MASK_EXTENSION}`;if (!library.get(`${folder}/${name}`)) return name;diff --git a/apps/web/src/engine/object-mask/tracking.ts b/apps/web/src/engine/object-mask/tracking.tsindex 31345752..3a91bd4f 100644--- a/apps/web/src/engine/object-mask/tracking.ts+++ b/apps/web/src/engine/object-mask/tracking.ts@@ -20,8 +20,8 @@ import {import type { Entity, World } from 'koota';import type { InputVideoTrack } from 'mediabunny';import type { Sam2Mask } from '@diffusionstudio/sam2/mask';-import type { Sam2Video } from '@diffusionstudio/sam2';-import type { Sam2ModelId } from '@diffusionstudio/sam2/models';+import type { Sam2Point, Sam2Video } from '@diffusionstudio/sam2';+import type { Sam2Model, Sam2ModelId } from '@diffusionstudio/sam2/models';import type { AssetLibrary, MaskAsset, MaskFrame, VideoAsset } from '@diffusionstudio/assets';import type { MaskPoint, MaskRestore, MaskStroke, ObjectMaskModelLoad, ObjectTrack } from './store';@@ -41,8 +41,12 @@ type Session = {aspect: number;};-/** The frame the model holds encoded, by clip and source frame, so prompts on it cost the decoder alone. */-let held: { clip: Entity; frame: number } | null = null;+/**+ * The frame the model holds encoded, by what it was decoded for — a clip, or+ * footage on its own (see `segmentFootage`) — and its source frame, so+ * prompts on it cost the decoder alone.+ */+let held: { owner: Entity | string; frame: number } | null = null;/*** Prompts the object on `clip` with one more point and segments the current@@ -303,6 +307,94 @@ export function getMaskRestoreOf(asset: MaskAsset | null | undefined): MaskRestoreturn asset && restore?.asset.id === asset.id ? restore : null;}+// ── Footage ──────────────────────────────────────────────────++/**+ * An object to segment in footage on its own, rather than on a clip: what+ * the agent's `media_segment` asks. Frames are counted on a grid of `fps`,+ * the one the mask file will hold.+ */+export type FootageSegmentRequest = {+ asset: VideoAsset;+ fps: number;+ model: Sam2ModelId;+ points: Sam2Point[];+ /** The frame the points are placed on. */+ seedFrame: number;+ /** The frames to mask, `count` from `first`, the seed among them; ignored by a preview. */+ first: number;+ count: number;+ /** Segments the seed frame alone, without tracking: the frame stays encoded, so the next prompt on it is quick. */+ preview: boolean;+ signal: AbortSignal;+ /** The model's download, 0 to 1, while it is on its way. */+ onDownload?: (progress: number | null) => void;+ /** Frames masked so far, of `count`. */+ onProgress?: (completed: number, total: number) => void;+};++export type FootageSegments = {+ model: Sam2Model;+ /** A frame per frame asked for, from `first` (the seed alone for a preview); null where none came. */+ masks: (MaskFrame | null)[];+ /** The side of the masks' grid. */+ grid: number;+ /** Where a frame of the grid is in the file, in seconds: where its mask was decoded. */+ seconds: (frame: number) => number;+};++/**+ * Segments an object in `request.asset` and follows it through the frames+ * asked for, or segments the prompted frame alone for a preview. It waits its+ * turn behind the tool's own sessions and holds the model while it runs, so+ * hovers wait too. Rejects with an `AbortError` when `signal` fires.+ */+export function segmentFootage(request: FootageSegmentRequest): Promise<FootageSegments> {+ const { asset, fps, points, seedFrame, first, count, signal } = request;+ return new Promise((resolve, reject) => {+ queue = queue.then(async () => {+ busy = true;+ try {+ signal.throwIfAborted();+ const session = await openSession(asset, fps, request.model, request.onDownload);+ signal.throwIfAborted();+ const model = session.model.model;+ const grid = session.model.maskSize;++ if (request.preview) {+ await holdFrame(session, `${asset.id}@${fps}`, seedFrame);+ const mask = await session.model.preview(points);+ resolve({ model, masks: [maskFrame(mask)], grid, seconds: session.seconds });+ return;+ }++ const { trackObject } = await import('@diffusionstudio/sam2');+ const masks = new Array<MaskFrame | null>(count).fill(null);+ let completed = 0;+ releaseFrame(session);+ session.model.reset();+ await trackObject(session.model, {+ track: session.video,+ timestamps: masks.map((_, i) => session.seconds(first + i)),+ seedIndex: seedFrame - first,+ points,+ signal,+ onMask: (index, mask) => {+ masks[index] = maskFrame(mask);+ request.onProgress?.(++completed, count);+ },+ });+ signal.throwIfAborted();+ resolve({ model, masks, grid, seconds: session.seconds });+ } catch (error) {+ reject(error);+ } finally {+ busy = false;+ }+ });+ });+}+// ── The model ────────────────────────────────────────────────────/** Whether the tool has been opened, and so wants its model loaded. */@@ -419,12 +511,12 @@ function corrector(strokes: readonly MaskStroke[], aspect: number): ((mask: Sam2return strokes.length > 0 ? (mask) => paintMask(mask, strokes, aspect) : undefined;}-/** Has the model hold `frame` of `clip`, unless it already does. */-async function holdFrame(session: Session, clip: Entity, frame: number): Promise<void> {- if (held && held.clip === clip && held.frame === frame && session.model.holding) return;+/** Has the model hold `frame` of what `owner` plays, unless it already does. */+async function holdFrame(session: Session, owner: Entity | string, frame: number): Promise<void> {+ if (held && held.owner === owner && held.frame === frame && session.model.holding) return;const { holdFrame: hold } = await import('@diffusionstudio/sam2');await hold(session.model, { track: session.video, timestamp: session.seconds(frame) });- held = { clip, frame };+ held = { owner, frame };}function releaseFrame(session: Session): void {diff --git a/docs/reference/jsx/styles.md b/docs/reference/jsx/styles.mdindex 53d1963b..7bd4a446 100644--- a/docs/reference/jsx/styles.md+++ b/docs/reference/jsx/styles.md@@ -68,7 +68,7 @@ A filter over the parent's **rendered pixels** — its fills, strokes and childr## `<mask>`-A matte limiting the `<effect>` holding it: a **picture** whose alpha says where the effect applies, fitted into the node's box the way the node fits its footage. It is what the editor's object mask tool makes — pick an object in a video clip, track it, and the frames it found are written to the library and named by `src`. Several under one effect intersect. Without a `src` a `<mask>` does nothing.+A matte limiting the `<effect>` holding it: a **picture** whose alpha says where the effect applies, fitted into the node's box the way the node fits its footage. It is what the editor's object mask tool makes — pick an object in a video clip, track it, and the frames it found are written to the library and named by `src` — and what [`media_segment`](../tools/media/segment.md) makes from a tool call. Several under one effect intersect. Without a `src` a `<mask>` does nothing.Under an `"opacity"` effect the mask is the cut-out: the clip shows inside the mask and goes transparent outside, as a mask on Premiere's Opacity does. Under any other effect the effect shows through inside the mask and the picture is untouched outside: an inverted mask on a `"blur"` blurs the background behind a person.diff --git a/docs/reference/tools/README.md b/docs/reference/tools/README.mdindex 94dc9c5a..f3bba849 100644--- a/docs/reference/tools/README.md+++ b/docs/reference/tools/README.md@@ -43,6 +43,7 @@ Every tool runs inside the app, so the app has to be running. Over MCP that is a| [`media_filmstrip`](./media/filmstrip.md) | `diffusion media filmstrip` | Filmstrip preview || [`media_waveform`](./media/waveform.md) | `diffusion media waveform` | Waveform preview || [`media_listen`](./media/listen.md) | `diffusion media listen` | Listen to audio |+| [`media_segment`](./media/segment.md) | `diffusion media segment` | Segment object || [`models`](./models.md) | `diffusion models` | Generation models || [`voices`](./voices.md) | `diffusion voices` | Speech voices || [`logs`](./logs.md) | `diffusion logs` | App logs |@@ -54,6 +55,7 @@ How the surface is divided:- **The project loop.** [`open`](./open.md) a folder, edit its JSX, [`context`](./context.md) for what the source cannot say, [`capture`](./capture.md) and [`check`](./check.md) to verify, [`export`](./export.md) when asked.- **Media inspection** (`media_*`): a file by path, without adding it to the project. Absolute paths and URLs work with or without an open project; library paths (`b-roll/clip.mp4`) need one.+- **Masks.** [`media_segment`](./media/segment.md) segments and tracks an object in footage and writes the mask file a `<mask src>` names.- **What a declaration may name.** [`models`](./models.md), [`voices`](./voices.md), [`fonts`](./fonts.md). Generation itself is declared in the project module (`generate.*`, see [jsx/generate.md](../jsx/generate.md)); no tool generates.- **The app and the machine.** [`logs`](./logs.md), [`screenshot`](./screenshot.md), [`report`](./report.md).diff --git a/docs/reference/tools/media/segment.md b/docs/reference/tools/media/segment.mdnew file mode 100644index 00000000..f93e1c00--- /dev/null+++ b/docs/reference/tools/media/segment.md@@ -0,0 +1,81 @@+# media_segment++Segment an object in a video with SAM 2.1 and track it through the footage, writing a mask file (`.mask`) that a [`<mask src>`](../../jsx/styles.md#mask) names: the file the editor's object mask tool makes (local model on the GPU, no credits). Prompt the object on the frame at `time` with `points` on it, `exclude` points off it, and/or a `box` around it, all in 0..1 of the frame (x right, y down). `preview: true` segments that frame alone and writes only an image: the frame with the mask tinted and outlined, the prompts drawn, and a 0.1 grid to read coordinates off. Refine the prompts with previews (fast once the frame is encoded), then make the same call without `preview` to track. Tracking covers `start` to `end`, the whole file by default, at roughly 0.25 s a frame with the tiny model and several times that with the larger ones, so pass the span the clip actually plays. Returns the mask's path (and its library path when it went into the project), a contact sheet of tracked frames, and the spans where the object was lost or the mask is weak. The first use of a model downloads it (83 MB for tiny, up to 475 MB for large).++| | |+| --- | --- |+| MCP tool | `media_segment` |+| CLI | `diffusion media segment <path> --time <time> [options]` |+| CLI aliases | `diffusion media mask` |++## Input++| Field | Type | CLI | Description |+| --- | --- | --- | --- |+| `path` | `string`, required | `<path>` | absolute file path or URL (works with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project) |+| `time` | `Time`, required | `-t, --time <time>` | the frame the prompts are placed on, in source time — seconds ("1.5"), frames ("45f"), or "MM:SS" |+| `points` | `{ x, y }[]` | `-p, --point <x,y>` | points on the object, each { x, y } in 0..1 of the frame; repeat the option for more |+| `exclude` | `{ x, y }[]` | `-x, --exclude <x,y>` | points on what is not the object, to take it out of the mask; each { x, y } in 0..1 of the frame; repeat the option for more |+| `box` | `[x0, y0, x1, y1]` | `-b, --box <x0,y0,x1,y1>` | a box around the object, in 0..1 of the frame |+| `preview` | `boolean` | `--preview` | segment only the frame at `time` and write no mask: for checking a prompt before tracking |+| `start` | `Time` | `-s, --start <time>` | start of the span to track (default: 0) |+| `end` | `Time` | `-e, --end <time>` | end of the span to track (default: the file's end) |+| `model` | `"tiny" \| "small" \| "base-plus" \| "large"` | `-m, --model <size>` | SAM 2.1 size: tiny (fastest), small, base-plus, or large (slowest; finest edges, best on small objects) (default: tiny) |+| `output` | `string` | `-o, --output <path>` | where the mask file goes: a library path like `masks/skater.mask` (needs an open project; a mask already there is replaced, so every `<mask>` naming it follows) or an absolute file path (default: `masks/<video>/Tracking <n>.mask` in the library with a project open, else a fresh file under the system temp dir). On the CLI a path starting with `.` resolves against the working directory. |++## Prompting++A prompt needs at least one point on the object or a box around it; `exclude` points only take things away. Coordinates are fractions of the frame as it displays (rotation applied), the same space [`media_grab`](./grab.md) frames are drawn in. A box is usually the most reliable first prompt: draw it around the object as seen in a grabbed frame. When the mask spills onto something next to the object, add an `exclude` point on that thing; when it misses part of the object, add a point on the missing part.++The loop is preview, look, adjust:++```bash+diffusion media segment footage/skater.mp4 --time 2.4 --box 0.4,0.2,0.65,0.95 --preview+diffusion media segment footage/skater.mp4 --time 2.4 --box 0.4,0.2,0.65,0.95 --exclude 0.5,0.9 --preview+diffusion media segment footage/skater.mp4 --time 2.4 --box 0.4,0.2,0.65,0.95 --exclude 0.5,0.9 --start 2 --end 9 -o masks/skater.mask+```++The first preview of a frame encodes it, which takes a moment; further previews on the same frame, file and model run the mask decoder alone and return at once. A preview writes no mask, so it takes neither a span nor an `output`.++## Tracking++Without `preview`, the object is segmented on the prompted frame and followed forward and backward through every frame from `start` to `end`, one mask frame per frame at the project's frame rate (the file's own with no project open). `time` must lie in the span. The app shows the progress with a Cancel button, and the tool waits for the whole track: the CLI allows it an hour, and an MCP client should give the call a comparable timeout. Tracking shares the model with the editor's object mask tool, so a call waits for the tool's own work to finish first.++The mask carries its recipe (footage, prompt, span, model), so the editor can make it again if its file goes missing. Name it on the clip it was made from, with `start` as its `sourceIn`, so it stays on the footage whatever the trim:++```tsx+<video src="footage/skater.mp4" sourceIn={2} width={1920} height={1080}>+ <effect type="opacity" value={1}>+ <mask src="masks/skater.mask" sourceIn={2} />+ </effect>+</video>+```++## Output++One JSON object:++```ts+{+ image: string; // PNG: the prompted frame with its mask on a preview, a contact sheet of up to 6 tracked frames otherwise+ path?: string; // absolute path of the mask file; absent on a preview+ src?: string; // the mask's library path, for <mask src>; when it went into the project's library+ model: "tiny" | "small" | "base-plus" | "large";+ frameRate: number; // frames per second the mask holds+ start: number; // source time of the first masked frame, seconds: the <mask>'s sourceIn+ end: number; // source time just past the last masked frame, seconds+ frames: number; // frames masked (1 on a preview)+ bbox: [x0, y0, x1, y1] | null; // on the prompted frame, the object's bounds in 0..1 of the frame+ area: number; // on the prompted frame, the share of the frame the object covers, 0..1+ score: number; // on the prompted frame, the object score: at or below 0 the model sees no object+ iou: number; // on the prompted frame, the model's estimate of the mask's quality, 0..1+ lost: [number, number][]; // [start, end) spans, seconds, where no object was found+ weak: [number, number][]; // [start, end) spans, seconds, where the model rated its mask below 0.5+}+```++`lost` covers occlusion and the object leaving the frame as well as a track that slipped; look at those moments with [`media_grab`](./grab.md) before trusting or re-prompting them. Over MCP the image also arrives inline when it is at most a megabyte.++## Errors++Fails when the path can't be resolved or the asset is not a video, `time` or `start` is past the video's end, no point or box prompts the object, a box's bounds are out of order or outside 0..1, `time` lies outside `start`–`end`, a preview is given a span or an `output`, `output` is a library path with no project open or names something in the library that is not one of the project's masks, `output` has an extension other than `.mask`, the model cannot load (no WebGPU, or the download failed), or the call is canceled in the app.diff --git a/docs/skills/editor.md b/docs/skills/editor.mdindex 9ba80b44..57f9f46e 100644--- a/docs/skills/editor.md+++ b/docs/skills/editor.md@@ -46,6 +46,7 @@ How to confirm a change actually produced what you intended. A clean save does n- Use the built-in tags for the media a composition is made of (audio, video, images, captions).- Hoist the properties that define the composition's look — title copy, font family and size, accent colors, key padding — into top-level consts annotated with `@inspect`, so they become live controls in the app's inspector.- For anything 3D, use Three.js drawn into a `<surface>` tag.+- To cut an object out of footage, or confine an effect to it, track it with `media_segment` (preview the prompt first) and name the mask file in a `<mask>` under the clip's `<effect>`.- For motion graphics, overlays and UI-heavy graphics, the `<html>` tag driven by a paused [anime.js](https://animejs.com) timeline- Before animating anything, read the [easings reference](../guides/motion/easings.md) and choose easings deliberately — default or linear easing is what makes motion read as a slideshow.- Add auto captions last, after everything else is assembled, so they transcribe the finished audio at its final placement.diff --git a/packages/assets/src/mask.ts b/packages/assets/src/mask.tsindex b2826cfc..0fcbf5c5 100644--- a/packages/assets/src/mask.ts+++ b/packages/assets/src/mask.ts@@ -76,8 +76,11 @@ export function emptyField(cells: number): Int8Array {return new Int8Array(cells).fill(-MASK_FIELD_MAX);}-/** A prompt on the footage, in 0..1 of its frame; label 1 marks the object, 0 marks background. */-export type MaskPoint = { x: number; y: number; label: 0 | 1 };+/**+ * A prompt on the footage, in 0..1 of its frame; label 1 marks the object, 0+ * marks background, and 2 and 3 are the corners of a box around it (see `Sam2Point`).+ */+export type MaskPoint = { x: number; y: number; label: 0 | 1 | 2 | 3 };/*** A brush stroke over the prompted frame's mask: its points in 0..1 of thediff --git a/packages/dapi/src/catalog.ts b/packages/dapi/src/catalog.tsindex 5770e8bb..466941af 100644--- a/packages/dapi/src/catalog.ts+++ b/packages/dapi/src/catalog.ts@@ -20,6 +20,7 @@ import { mediaTranscribe } from "./tools/media-transcribe";import { mediaFilmstrip } from "./tools/media-filmstrip";import { mediaWaveform } from "./tools/media-waveform";import { mediaListen } from "./tools/media-listen";+import { mediaSegment } from "./tools/media-segment";import { fonts } from "./tools/fonts";import { report } from "./tools/report";@@ -40,6 +41,7 @@ export const catalog = [mediaFilmstrip,mediaWaveform,mediaListen,+ mediaSegment,models,voices,logs,diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.tsindex b996bfba..41ee3178 100644--- a/packages/dapi/src/index.ts+++ b/packages/dapi/src/index.ts@@ -25,6 +25,7 @@ export type { ToolJsonSchemas } from "./json-schema";export { DAPI_WIRE } from "./ipc";export type { DapiCall, DapiCancel, DapiReply } from "./ipc";export { FRAME_CAP } from "./tools/media-grab";+export { SEGMENT_MODELS, WEAK_IOU } from "./tools/media-segment";export { ISSUE_LOG_TAIL } from "./tools/report";export { LOG_TAIL, LOG_MESSAGE_MAX } from "./tools/logs";export { FONT_LIMIT } from "./tools/fonts";@@ -82,5 +83,7 @@ export type MediaWaveformRequest = ToolArgs<"media_waveform">;export type MediaWaveformResult = ToolResult<"media_waveform">;export type MediaListenRequest = ToolArgs<"media_listen">;export type MediaListenResult = ToolResult<"media_listen">;+export type MediaSegmentRequest = ToolArgs<"media_segment">;+export type MediaSegmentResult = ToolResult<"media_segment">;export type FontsRequest = ToolArgs<"fonts">;export type ReportRequest = ToolArgs<"report">;diff --git a/packages/dapi/src/tools/media-segment.ts b/packages/dapi/src/tools/media-segment.tsnew file mode 100644index 00000000..c23bbbef--- /dev/null+++ b/packages/dapi/src/tools/media-segment.ts@@ -0,0 +1,123 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { z } from "zod";+import { defineTool } from "../tool";+import { AssetPath, Bytes, checkWindow } from "../schemas";+import { NonNegativeTime, TIME_FORMS } from "../time";++/** The SAM 2.1 sizes the app offers, fastest first: the ids of its model catalogue (`SAM2_MODELS`). */+export const SEGMENT_MODELS = ["tiny", "small", "base-plus", "large"] as const;++/** A tracked frame the model rates below this is reported as weak. */+export const WEAK_IOU = 0.5;++const Unit = z.number().min(0).max(1);++const Point = z.object({ x: Unit, y: Unit });++/** A span of source time, `[start, end)` in seconds. */+const Range = z.tuple([z.number(), z.number()]);++const Bounds = z.tuple([z.number(), z.number(), z.number(), z.number()]);++/** What the call found, the same whether the server or the renderer writes the files. */+const found = {+ model: z.enum(SEGMENT_MODELS),+ frameRate: z.number().describe("frames per second the mask holds: the project's, or the file's own with no project open"),+ start: z.number().describe("source time of the first masked frame, in seconds: the `sourceIn` of a `<mask>` naming the file"),+ end: z.number().describe("source time just past the last masked frame, in seconds"),+ frames: z.int().describe("frames masked"),+ bbox: Bounds.nullable().describe(+ "on the prompted frame, the object's bounds as [x0, y0, x1, y1] in 0..1 of the frame; null when nothing was segmented",+ ),+ area: z.number().describe("on the prompted frame, the share of the frame the object covers, 0..1"),+ score: z.number().describe("on the prompted frame, the model's object score: at or below 0 it sees no object there"),+ iou: z.number().describe("on the prompted frame, the model's estimate of the mask's quality, 0..1"),+ lost: z+ .array(Range)+ .describe("source-time spans, [start, end) in seconds, where no object was found: occluded, out of frame, or lost by the track"),+ weak: z.array(Range).describe(`source-time spans, [start, end) in seconds, where the model rated its mask below ${WEAK_IOU}`),+};++export const mediaSegment = defineTool({+ name: "media_segment",+ title: "Segment object",+ description:+ "Segment an object in a video with SAM 2.1 and track it through the footage, writing a mask file (`.mask`) that a `<mask src>` names: the file the editor's object mask tool makes (local model on the GPU, no credits). " ++ "Prompt the object on the frame at `time` with `points` on it, `exclude` points off it, and/or a `box` around it, all in 0..1 of the frame (x right, y down). " ++ "`preview: true` segments that frame alone and writes only an image: the frame with the mask tinted and outlined, the prompts drawn, and a 0.1 grid to read coordinates off. Refine the prompts with previews (fast once the frame is encoded), then make the same call without `preview` to track. " ++ "Tracking covers `start` to `end`, the whole file by default, at roughly 0.25 s a frame with the tiny model and several times that with the larger ones, so pass the span the clip actually plays. " ++ "Returns the mask's path (and its library path when it went into the project), a contact sheet of tracked frames, and the spans where the object was lost or the mask is weak. The first use of a model downloads it (83 MB for tiny, up to 475 MB for large).",+ input: z+ .object({+ path: AssetPath,+ time: NonNegativeTime.describe(`the frame the prompts are placed on, in source time — ${TIME_FORMS}`),+ points: z.array(Point).optional().describe("points on the object, each { x, y } in 0..1 of the frame"),+ exclude: z+ .array(Point)+ .optional()+ .describe("points on what is not the object, to take it out of the mask; each { x, y } in 0..1 of the frame"),+ box: Bounds.optional().describe("a box around the object, [x0, y0, x1, y1] in 0..1 of the frame"),+ preview: z+ .boolean()+ .optional()+ .describe("segment only the frame at `time` and write no mask: for checking a prompt before tracking"),+ start: NonNegativeTime.optional().describe(`start of the span to track — ${TIME_FORMS} (default: 0)`),+ end: NonNegativeTime.optional().describe(`end of the span to track — ${TIME_FORMS} (default: the file's end)`),+ model: z+ .enum(SEGMENT_MODELS)+ .optional()+ .describe("SAM 2.1 size: tiny (fastest), small, base-plus, or large (slowest; finest edges, best on small objects) (default: tiny)"),+ output: z+ .string()+ .optional()+ .describe(+ "where the mask file goes: a library path like `masks/skater.mask` (needs an open project; a mask already there is replaced, so every `<mask>` naming it follows) or an absolute file path (default: `masks/<video>/Tracking <n>.mask` in the library with a project open, else a fresh file under the system temp dir)",+ ),+ })+ .superRefine((value, ctx) => {+ if (!value.points?.length && !value.box) {+ ctx.addIssue({ code: "custom", path: ["points"], message: "prompt the object with points on it or a box around it" });+ }+ if (value.box && (value.box[0] >= value.box[2] || value.box[1] >= value.box[3])) {+ ctx.addIssue({ code: "custom", path: ["box"], message: "a box is [x0, y0, x1, y1] in 0..1 of the frame, with x0 < x1 and y0 < y1" });+ }+ if (value.box?.some((bound) => bound < 0 || bound > 1)) {+ ctx.addIssue({ code: "custom", path: ["box"], message: "a box's bounds are in 0..1 of the frame" });+ }+ if (value.preview) {+ for (const key of ["start", "end", "output"] as const) {+ if (value[key] !== undefined) {+ ctx.addIssue({ code: "custom", path: [key], message: `a preview segments one frame and writes no mask; ${key} applies to tracking only` });+ }+ }+ return;+ }+ checkWindow(value, ctx);+ if (value.start !== undefined && value.time < value.start) {+ ctx.addIssue({ code: "custom", path: ["time"], message: `time (${value.time}s) is before start (${value.start}s); the prompted frame must be tracked` });+ }+ if (value.end !== undefined && value.time >= value.end) {+ ctx.addIssue({ code: "custom", path: ["time"], message: `time (${value.time}s) is not before end (${value.end}s); the prompted frame must be tracked` });+ }+ }),+ output: z.object({+ image: z+ .string()+ .describe("absolute path of the PNG: the prompted frame with its mask on a preview, a contact sheet of tracked frames otherwise"),+ path: z.string().optional().describe("absolute path of the mask file; absent on a preview"),+ src: z.string().optional().describe("the mask's library path, for `<mask src>`; present when it was written into the project's library"),+ ...found,+ }),+ result: z.object({+ png: Bytes,+ /** The mask file, for the server to write where the renderer cannot: an absolute path, or the temp dir. */+ mask: Bytes.optional(),+ path: z.string().optional(),+ src: z.string().optional(),+ ...found,+ }),+ environment: "renderer",+});diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.tsindex e0186ea8..abe62e95 100644--- a/packages/dapi/src/tools/tools.test.ts+++ b/packages/dapi/src/tools/tools.test.ts@@ -10,6 +10,7 @@ import { logs } from "./logs";import { mediaFilmstrip } from "./media-filmstrip";import { mediaGrab } from "./media-grab";import { mediaListen } from "./media-listen";+import { mediaSegment } from "./media-segment";import { mediaTranscribe } from "./media-transcribe";/** The messages of a failed parse, keyed by the path they point at. */@@ -130,3 +131,46 @@ describe("image tools", () => {expect(capture.output.safeParse({ images: [{ timecode: "0f", png: new Uint8Array(3) }] }).success).toBe(false);});});++describe("media_segment", () => {+ const input = mediaSegment.input;+ const at = { path: "/c.mp4", time: "1.5" };++ it("takes points, exclusions and a box, with the time in every form", () => {+ const args = input.parse({ ...at, points: [{ x: 0.5, y: 0.4 }], exclude: [{ x: 0.5, y: 0.9 }], box: [0.2, 0.1, 0.8, 0.95] });+ expect(args.time).toBe(1.5);+ expect(input.parse({ path: "/c.mp4", time: "45f", box: [0, 0, 1, 1] }).time).toBe(1.5);+ });++ it("needs the object prompted by points or a box", () => {+ expect(issues(input.safeParse(at))).toHaveProperty("points");+ expect(issues(input.safeParse({ ...at, exclude: [{ x: 0.1, y: 0.1 }] }))).toHaveProperty("points");+ expect(input.safeParse({ ...at, points: [{ x: 0, y: 1 }] }).success).toBe(true);+ });++ it("keeps prompts inside the frame and boxes the right way round", () => {+ expect(input.safeParse({ ...at, points: [{ x: 1.2, y: 0.5 }] }).success).toBe(false);+ expect(issues(input.safeParse({ ...at, box: [0.8, 0.1, 0.2, 0.9] }))).toHaveProperty("box");+ expect(issues(input.safeParse({ ...at, box: [-0.1, 0.1, 0.2, 0.9] }))).toHaveProperty("box");+ });++ it("rejects a span or an output on a preview", () => {+ const preview = { ...at, points: [{ x: 0.5, y: 0.5 }], preview: true };+ expect(input.safeParse(preview).success).toBe(true);+ expect(issues(input.safeParse({ ...preview, start: 1 }))).toHaveProperty("start");+ expect(issues(input.safeParse({ ...preview, output: "masks/a.mask" }))).toHaveProperty("output");+ });++ it("tracks a span that holds the prompted frame", () => {+ const track = { ...at, points: [{ x: 0.5, y: 0.5 }] };+ expect(input.safeParse({ ...track, start: 1, end: "0:03" }).success).toBe(true);+ expect(issues(input.safeParse({ ...track, start: 2 }))).toHaveProperty("time");+ expect(issues(input.safeParse({ ...track, end: 1.5 }))).toHaveProperty("time");+ expect(issues(input.safeParse({ ...track, start: 3, end: 2 }))).toHaveProperty("end");+ });++ it("names only the models the app offers", () => {+ expect(input.safeParse({ ...at, box: [0, 0, 1, 1], model: "base-plus" }).success).toBe(true);+ expect(input.safeParse({ ...at, box: [0, 0, 1, 1], model: "huge" }).success).toBe(false);+ });+});diff --git a/packages/sam2/src/tracker.ts b/packages/sam2/src/tracker.tsindex 7c604d3f..ac1710e4 100644--- a/packages/sam2/src/tracker.ts+++ b/packages/sam2/src/tracker.ts@@ -13,8 +13,12 @@ import type { Rotation } from './gpu';import type { Sam2Mask } from './mask';import type { Outputs, Session, Tensor } from './sessions';-/** A prompt on the displayed frame, in 0..1 of its width and height; label 1 is the object, 0 is background. */-export type Sam2Point = { x: number; y: number; label: 0 | 1 };+/**+ * A prompt on the displayed frame, in 0..1 of its width and height; label 1+ * is the object, 0 is background, and 2 and 3 are the top-left and+ * bottom-right corners of a box around it, as SAM 2 prompts a box.+ */+export type Sam2Point = { x: number; y: number; label: 0 | 1 | 2 | 3 };/*** The prompted frame's mask as the user corrected it, from the model's own: