Skip to content

The VS Code of video editing, built for agents.

TypeScript
4.1 KB
Raw Blame
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { z } from "zod";
import { defineTool } from "../tool";
import { AssetPath, checkSheetOptions, checkWindow, ImageRef, outputDirField, sheetFields, TimecodedImage, windowFields } from "../schemas";
import { Time } from "../time";

export const FrameQuality = z.enum(["small", "medium", "large", "fullres"]);

/** Guardrail against accidentally decoding a huge number of frames; `uncapped` lifts it. */
export const FRAME_CAP = 100;

export const grab = defineTool({
  name: "grab",
  title: "Grab frames",
  description:
    "Decode frames of a video file and write them as PNGs (local render, no credits). By default the frames are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and drawn as large as fits, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one (separate: true writes a PNG per frame). Grabs the asset's own pixels, unlike capture which renders the composited node. The recommended tool for understanding a video at the frame level; past ~12 frames prefer filmstrip.",
  input: z
    .object({
      path: AssetPath,
      times: z
        .array(Time)
        .optional()
        .describe(
          'timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: [0])',
        ),
      count: z
        .int()
        .min(1)
        .optional()
        .describe("instead of times, grab this many frames evenly spaced across the clip (or across the start/end window)"),
      auto: z
        .boolean()
        .optional()
        .describe(
          "scan the clip at 2fps and keep a frame each time the footage settles into a new visual state (transitions are waited out, so picks stay sharp); returns at most count frames (default cap: 30), static footage like screen recordings returns far fewer; requires WebGPU",
        ),
      start: windowFields.start.describe("with count or auto, start of the window to sample (default: 0)"),
      end: windowFields.end.describe("with count or auto, end of the window to sample (default: asset duration)"),
      quality: FrameQuality.optional().describe(
        "frame resolution as a pixel budget, aspect ratio kept and never enlarged past the source: small (384² pixels, 512x288 for 16:9), medium (768², 1024x576), large (1536², 2048x1152), or fullres (native); default: as large as the sheet cell allows, or small with separate: true",
      ),
      ...sheetFields,
      uncapped: z
        .boolean()
        .optional()
        .describe(`lift the ${FRAME_CAP}-frame safety cap (grabbing many frames is slow and token-heavy)`),
      output: outputDirField,
    })
    .superRefine((value, ctx) => {
      if (value.times !== undefined && value.count !== undefined) {
        ctx.addIssue({ code: "custom", path: ["count"], message: "pass either times or count, not both" });
      }
      if (value.auto && value.times !== undefined) {
        ctx.addIssue({ code: "custom", path: ["times"], message: "auto picks its own timestamps; it cannot be combined with times" });
      }
      const windowed = value.start !== undefined || value.end !== undefined;
      if (windowed && value.count === undefined && !value.auto) {
        ctx.addIssue({ code: "custom", path: ["start"], message: "start and end only apply together with count or auto" });
      }
      checkWindow(value, ctx);
      const requested = value.count ?? value.times?.length ?? 1;
      if (!value.uncapped && requested > FRAME_CAP) {
        ctx.addIssue({
          code: "custom",
          path: [value.count !== undefined ? "count" : "times"],
          message: `grabbing ${requested} frames exceeds the ${FRAME_CAP}-frame cap; pass uncapped: true to override`,
        });
      }
      checkSheetOptions(value, ctx);
    }),
  output: z.object({ images: z.array(ImageRef) }),
  result: z.array(TimecodedImage),
  environment: "renderer",
});