Skip to content

The VS Code of video editing, built for agents.

TypeScript
5.3 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// SAM 2.1, the editor's object mask model: segments an object in a video and
// tracks it through the footage, making the mask file (`.mask`) a `<mask src>`
// names.

import { z } from "zod";
import { checkWindow } from "../schemas";
import { NonNegativeTime, TIME_FORMS } from "../time";
import { FileRef, requestFields } from "./fields";

/** The SAM 2.1 sizes the app offers, fastest first: the ids of its model catalogue (`SAM2_MODELS`). */
export const SAM_SIZES = ["tiny", "small", "base-plus", "large"] as const;

/** A tracked frame the model rates below this is reported as weak. */
export const WEAK_IOU = 0.5;

const Unit = z.number().min(0).max(1);

const Point = z.strictObject({ x: Unit, y: Unit });

const Bounds = z.tuple([z.number(), z.number(), z.number(), z.number()]);

/** A span of source time, `[start, end)` in seconds. */
const Range = z.tuple([z.number(), z.number()]);

export const samRequest = z
  .strictObject({
    model: z.literal("sam-2.1"),
    video: FileRef.describe("the footage, `{ path }`"),
    size: z
      .enum(SAM_SIZES)
      .default("tiny")
      .describe("tiny (fastest), small, base-plus, or large (slowest; finest edges, best on small objects) (default: tiny)"),
    time: NonNegativeTime.describe(`the frame the prompts are placed on, in source time — ${TIME_FORMS}`),
    points: z.array(Point).optional().describe("points on the object, each { x, y } in 0..1 of the frame"),
    exclude: z
      .array(Point)
      .optional()
      .describe("points on what is not the object, to take it out of the mask; each { x, y } in 0..1 of the frame"),
    box: Bounds.optional().describe("a box around the object, [x0, y0, x1, y1] in 0..1 of the frame"),
    preview: z
      .boolean()
      .optional()
      .describe("segment only the frame at `time` and write no mask: for checking a prompt before tracking"),
    start: NonNegativeTime.optional().describe(`start of the span to track — ${TIME_FORMS} (default: 0)`),
    end: NonNegativeTime.optional().describe(`end of the span to track — ${TIME_FORMS} (default: the file's end)`),
    ...requestFields,
  })
  .superRefine((value, ctx) => {
    if (!value.points?.length && !value.box) {
      ctx.addIssue({ code: "custom", path: ["points"], message: "prompt the object with points on it or a box around it" });
    }
    if (value.box && (value.box[0] >= value.box[2] || value.box[1] >= value.box[3])) {
      ctx.addIssue({ code: "custom", path: ["box"], message: "a box is [x0, y0, x1, y1] in 0..1 of the frame, with x0 < x1 and y0 < y1" });
    }
    if (value.box?.some((bound) => bound < 0 || bound > 1)) {
      ctx.addIssue({ code: "custom", path: ["box"], message: "a box's bounds are in 0..1 of the frame" });
    }
    if (value.preview) {
      for (const key of ["start", "end", "output"] as const) {
        if (value[key] !== undefined) {
          ctx.addIssue({ code: "custom", path: [key], message: `a preview segments one frame and writes no mask; ${key} applies to tracking only` });
        }
      }
      return;
    }
    checkWindow(value, ctx);
    if (value.start !== undefined && value.time < value.start) {
      ctx.addIssue({ code: "custom", path: ["time"], message: `time (${value.time}s) is before start (${value.start}s); the prompted frame must be tracked` });
    }
    if (value.end !== undefined && value.time >= value.end) {
      ctx.addIssue({ code: "custom", path: ["time"], message: `time (${value.time}s) is not before end (${value.end}s); the prompted frame must be tracked` });
    }
  });

/**
 * A job's `details`: the span, known when the job starts, and what was found,
 * once it has succeeded.
 */
export const samDetails = z.object({
  frameRate: z.number().describe("frames per second the mask holds: the project's, or the file's own with no project open"),
  start: z.number().describe("source time of the first masked frame, in seconds: the `sourceIn` of a `<mask>` naming the file"),
  end: z.number().describe("source time just past the last masked frame, in seconds"),
  frames: z.int().describe("frames masked"),
  image: z
    .string()
    .optional()
    .describe("absolute path of a PNG: the prompted frame with its mask on a preview, a contact sheet of tracked frames otherwise"),
  bbox: Bounds.nullable()
    .optional()
    .describe("on the prompted frame, the object's bounds as [x0, y0, x1, y1] in 0..1 of the frame; null when nothing was segmented"),
  area: z.number().optional().describe("on the prompted frame, the share of the frame the object covers, 0..1"),
  score: z.number().optional().describe("on the prompted frame, the model's object score: at or below 0 it sees no object there"),
  iou: z.number().optional().describe("on the prompted frame, the model's estimate of the mask's quality, 0..1"),
  lost: z
    .array(Range)
    .optional()
    .describe("source-time spans, [start, end) in seconds, where no object was found: occluded, out of frame, or lost by the track"),
  weak: z.array(Range).optional().describe(`source-time spans, [start, end) in seconds, where the model rated its mask below ${WEAK_IOU}`),
});

export type SamRequest = z.output<typeof samRequest>;
export type SamDetails = z.output<typeof samDetails>;