The VS Code of video editing, built for agents.
TypeScript
Enhance media frame grabbing functionality and update documentation
konstantin-pauluse22e4bbc30e6
diff --git a/apps/cli/reference/media/grab.md b/apps/cli/reference/media/grab.mdindex bb0b843f..3a564586 100644--- a/apps/cli/reference/media/grab.md+++ b/apps/cli/reference/media/grab.md@@ -5,7 +5,13 @@ Decodes one or more frames of a video asset at the given times and writes each t## Input- `<id|path>`: a video asset id, or a local video file to grab frames from in place without adding it to the library (required).-- `-t, --time <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). Order is preserved in the output regardless of the order given.+- `-t, --time <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). A negative value is an offset back from the end of the clip, so `-1` is one second before the end and `-1f` one frame before it. Order is preserved in the output regardless of the order given. Mutually exclusive with `--count`.+- `-c, --count <n>`: instead of `--time`, grab `n` frames evenly spaced across the clip at a fixed interval of `window / n`, starting at the window start (optional; positive integer).+- `-s, --start <time>`: with `--count`, the start of the window to sample (optional; `Time` value; default `0`).+- `-e, --end <time>`: with `--count`, the end of the window to sample (optional; `Time` value; default the asset duration).+- `-q, --quality <preset>`: per-frame resolution preset (optional; default `small`). One of `small` (384x384), `medium` (768x768), `large` (1536x1536), or `fullres` (native). Each caps the total pixel count while preserving aspect ratio.+- `--no-timestamp`: don't stamp each frame with its `HH:MM:SS:FF` timestamp label (optional; the label is drawn by default).+- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--time`) is rejected.- `-o, --output <dir>`: directory to write the PNGs into (optional; default the system temp directory). Each frame is written as a uuid-named `.png`.## Output@@ -18,4 +24,4 @@ JSON Lines, one object per requested timestamp, in the requested order:## Errors-Exits non-zero if the id is unknown, the asset is not a video, any `--time` is past the asset's duration, or a PNG can't be written.+Exits non-zero if the id is unknown, the asset is not a video, any `--time` is past the asset's duration, the `--count` window is empty, `--time` and `--count` are combined, `--start`/`--end` are given without `--count`, more than 100 frames are requested without `--uncapped`, or a PNG can't be written.diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex 03be64b7..5f2412ae 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -121,7 +121,15 @@ export type AssetRef = { id: string } | { path: string };export type MediaProbeRequest = AssetRef;-export type MediaFrameRequest = AssetRef & { times?: number[]; resolution?: number };+export type FrameQuality = "small" | "medium" | "large" | "fullres";+export type MediaFrameRequest = AssetRef & {+ times?: number[];+ count?: number;+ start?: number;+ end?: number;+ quality?: FrameQuality;+ timestamp?: boolean;+};export type MediaFrameResult = Array<{ time: number; base64: string }>;export type MediaTranscribeRequest = AssetRef & { start?: number; end?: number };diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex b542f5cf..a3351438 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -16,7 +16,7 @@ import { compileProject } from "./compile-project";import { listLocalFonts } from "./fonts";import { openFolder } from "./open-folder";import { fetchVideo } from "./ytdlp";-import type { AssetRef, EncoderConfigInput, NodePatch } from "./protocol";+import type { AssetRef, EncoderConfigInput, FrameQuality, NodePatch } from "./protocol";// Long-running commands (renders, AI generation) override the default 60s.const GENERATE = { context: { timeoutMs: GENERATE_TIMEOUT_MS } };@@ -450,27 +450,72 @@ async function nodeScreenshot(id: string | undefined, opts: ScreenshotOptions):}}-type MediaFrameOptions = { time?: string[]; resolution?: string; output?: string };+const FRAME_QUALITIES: FrameQuality[] = ["small", "medium", "large", "fullres"];++// Guardrail against accidentally decoding a huge number of frames; --uncapped lifts it.+const FRAME_CAP = 100;++type MediaFrameOptions = {+ time?: string[];+ count?: string;+ start?: string;+ end?: string;+ quality?: string;+ timestamp?: boolean;+ uncapped?: boolean;+ output?: string;+};async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {+ if (opts.time !== undefined && opts.count !== undefined) {+ console.error("Pass either --time or --count, not both.");+ process.exit(1);+ }+let times: number[] | undefined;if (opts.time !== undefined) {- times = opts.time.map((t) => parseTimeArg(t, "--time"));+ times = opts.time.map((t) => parseTimeArg(t, "--time", true));+ }++ let count: number | undefined;+ if (opts.count !== undefined) {+ count = Number(opts.count);+ if (!Number.isInteger(count) || count < 1) {+ console.error(`--count must be a positive integer (got "${opts.count}")`);+ process.exit(1);+ }+ }++ const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;+ const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;+ if (start !== undefined && end !== undefined && start >= end) {+ console.error(`--start (${start}s) must be less than --end (${end}s).`);+ process.exit(1);+ }+ if ((start !== undefined || end !== undefined) && count === undefined) {+ console.error("--start and --end only apply together with --count.");+ process.exit(1);+ }++ const requested = count ?? times?.length ?? 1;+ if (!opts.uncapped && requested > FRAME_CAP) {+ console.error(`Grabbing ${requested} frames exceeds the ${FRAME_CAP}-frame cap; pass --uncapped to override.`);+ process.exit(1);}- let resolution: number | undefined;- if (opts.resolution !== undefined) {- resolution = Number(opts.resolution);- if (!Number.isInteger(resolution) || resolution < 0) {- console.error(`--resolution must be a non-negative integer pixel count, or 0 for native (got "${opts.resolution}")`);+ let quality: FrameQuality | undefined;+ if (opts.quality !== undefined) {+ if (!FRAME_QUALITIES.includes(opts.quality as FrameQuality)) {+ console.error(`--quality must be one of ${FRAME_QUALITIES.join(", ")} (got "${opts.quality}")`);process.exit(1);}+ quality = opts.quality as FrameQuality;}const target = resolveAssetRef(ref);const dir = opts.output ?? tmpdir();try {- const frames = await editor.media.frame.query({ ...target, times, resolution });+ const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, timestamp: opts.timestamp });for (const { time, base64 } of frames) {const path = join(dir, `${randomUUID()}.png`);writeFileSync(path, Buffer.from(base64, "base64"));@@ -558,11 +603,11 @@ async function mediaListen(ref: string, opts: MediaListenOptions): Promise<void>type MediaPreviewOptions = { start?: string; end?: string; scale?: string; output?: string };-function parseTimeArg(value: string, flag: string): number {+function parseTimeArg(value: string, flag: string, allowNegative = false): number {const seconds = parseTime(value);- if (seconds === undefined || seconds < 0) {+ if (seconds === undefined || (!allowNegative && seconds < 0)) {console.error(- `${flag} must be a non-negative Time — seconds ("1.5"), frames ("45f"), or "MM:SS" (got "${value}")`,+ `${flag} must be a ${allowNegative ? "" : "non-negative "}Time — seconds ("1.5"), frames ("45f"), or "MM:SS" (got "${value}")`,);process.exit(1);}@@ -1111,17 +1156,22 @@ mediamedia.command("grab").alias("sample")- .description(`Decode one or more frames of a video file and write them as high-res PNGs, each stamped in the top-left with its HH:MM:SS:FF timestamp. Best for inspecting individual frames in detail where a filmstrip is too coarse: seeks to the exact requested time with frame-level precision${docs("media/grab")}`)+ .description(`Decode frames of a video file and write them as PNGs, each stamped in the top-left with its HH:MM:SS:FF timestamp. Grab explicit moments with --time, or a fixed number of evenly-spaced frames with --count. The recommended tool for understanding a video at the frame level${docs("media/grab")}`).argument("<id|path>", "video asset id, or a local video file to grab frames from")- .option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`)- .option("-r, --resolution <pixels>", "cap each frame to this many total pixels, preserving aspect ratio; 0 for native (default: 147456, i.e. 384x384)")+ .option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`)+ .option("-c, --count <n>", "instead of --time, grab this many frames evenly spaced across the clip (or across the --start/--end window)")+ .option("-s, --start <time>", `with --count, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)+ .option("-e, --end <time>", `with --count, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`)+ .option("-q, --quality <preset>", "frame resolution: small (384x384, default), medium (768x768), large (1536x1536), or fullres (native)")+ .option("--no-timestamp", "don't stamp each frame with its HH:MM:SS:FF timestamp label")+ .option("--uncapped", "lift the 100-frame safety cap (grabbing many frames is slow and token-heavy)").option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)").action((ref: string, opts: MediaFrameOptions) => mediaFrame(ref, opts));media.command("filmstrip").alias("film")- .description(`Start here for any video: render a grid of thumbnails sampled across the timeline to a PNG, each row stamped with timestamps. Fast and token-efficient; narrow the window to zoom into a region of interest. Video only, no audio — use \`waveform\` to inspect the audio track${docs("media/filmstrip")}`)+ .description(`Render a grid of thumbnails sampled across the timeline to a PNG, each row stamped with timestamps. It's a fast and token-efficient video track preview; narrow the window to zoom into a region of interest${docs("media/filmstrip")}`).argument("<id|path>", "video asset id, or a local video file to preview").option("-s, --start <time>", `start of the window to preview — seconds, "45f" frames, or "MM:SS" (default: 0)`).option("-e, --end <time>", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)@@ -1132,7 +1182,7 @@ mediamedia.command("waveform").alias("wave")- .description(`Render the audio track of a video or audio file as a waveform PNG with a timestamp ruler. Shows loudness over time and highlights the silent stretches in red, which are also returned as [start, end] second ranges in the JSON. Fast and token-efficient; narrow the window to zoom into a region of interest${docs("media/waveform")}`)+ .description(`Render the audio track of a video or audio file as a waveform PNG with a timestamp ruler. It's a fast and token-efficient audio track preview; shows loudness over time and highlights the silent stretches in red, which are also returned as [start, end] second ranges in the JSON${docs("media/waveform")}`).argument("<id|path>", "video or audio asset id, or a local file to preview").option("-s, --start <time>", `start of the window to preview — seconds, "45f" frames, or "MM:SS" (default: 0)`).option("-e, --end <time>", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)@@ -1142,13 +1192,11 @@ mediamedia.command("listen")- .alias("watch").description(`Prompt a multimodal model for a semantic analysis of an audio track and print its answer. Shines on the semantics of audio (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video, by default only the audio track is analyzed${docs("media/listen")}`).argument("<id|path>", "video or audio asset id, or a local file to analyze").option("-p, --prompt <str>", "question or instruction to guide the analysis").option("-s, --start <time>", `start of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: 0); timestamps in the analysis are relative to this point`).option("-e, --end <time>", `end of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: media duration)`)- .option("--keep-video", "keep the video track instead of stripping it to audio (expensive: requires a full video upload)").action((ref: string, opts: MediaListenOptions) => mediaListen(ref, opts));const folder = programdiff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.tsindex c912d6e2..4ac2d427 100644--- a/apps/web/src/context/editor-api/media.ts+++ b/apps/web/src/context/editor-api/media.ts@@ -97,25 +97,49 @@ export function handleMediaProbe(engine: Accessor<Engine>) {};}-// Default cap on the pixel count of each decoded frame (384x384). Keeps frames-// small enough for vision models without the caller having to know the aspect ratio.-export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;+// Named quality presets mapped to a per-frame total-pixel budget (aspect ratio+// preserved). A budget of 0 means native resolution. `small` keeps frames small+// enough for vision models and is the default.+const FRAME_QUALITY_BUDGETS = {+ small: 384 * 384, // 147,456+ medium: 768 * 768, // 589,824+ large: 1536 * 1536, // 2,359,296+ fullres: 0, // native+} as const;export function handleMediaFrame(engine: Accessor<Engine>) {return async (req: MediaFrameRequest) => {- const { times, resolution } = req;+ const { times, count, start, end, quality, timestamp } = req;const { world } = engine();const asset = await resolveAssetRef(world, req);const id = asset.id;assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);- const requested = times && times.length ? times : [0];- for (const t of requested) {- assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);+ // `count` samples evenly across a window (default the whole clip); otherwise+ // grab the explicit `times` (falling back to a single frame at 0).+ let requested: number[];+ if (count !== undefined) {+ const from = Math.min(Math.max(start ?? 0, 0), asset.duration);+ const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration);+ assert(to > from, `The requested window is empty; start (${from.toFixed(2)}s) is at or past end (${to.toFixed(2)}s).`);+ const interval = (to - from) / count;+ requested = Array.from({ length: count }, (_, i) => from + i * interval);+ } else {+ const raw = times && times.length ? times : [0];+ // A negative time is an offset back from the end of the clip: -1 is one+ // second before the end, -1f one frame before it.+ requested = raw.map((t) => {+ if (t >= 0) {+ assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);+ return t;+ }+ const resolved = asset.duration + t;+ assert(resolved >= 0, `--time ${t} counts past the start of the clip (duration ${asset.duration.toFixed(2)}s).`);+ return resolved;+ });}- // A budget of 0 means "native resolution"; anything else caps the pixel count.- const budget = resolution ?? DEFAULT_FRAME_PIXEL_BUDGET;+ const budget = FRAME_QUALITY_BUDGETS[quality ?? "small"];const blob = await getAssetFile(asset);const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });@@ -152,7 +176,7 @@ export function handleMediaFrame(engine: Accessor<Engine>) {const ctx = wrapped.canvas.getContext("2d");- if (ctx) {+ if (ctx && timestamp !== false) {const label = formatTimestamp(time, asset.frameRate)const bandHeight = Math.max(20, Math.round(wrapped.canvas.height * 0.06));const fontSize = Math.round(bandHeight * 0.72);