Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance media frame grabbing functionality and update documentation

konstantin-pauluse22e4bbc30e6
diff --git a/apps/cli/reference/media/grab.md b/apps/cli/reference/media/grab.md
index bb0b843f..3a564586 100644
--- a/apps/cli/reference/media/grab.md
+++ b/apps/cli/reference/media/grab.md
@@ -5,7 +5,13 @@ Decodes one or more frames of a video asset at the given times and writes each t
## Input
- `<id|path>`: a video asset id, or a local video file to grab frames from in place without adding it to the library (required).
-- `-t, --time <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). Order is preserved in the output regardless of the order given.
+- `-t, --time <time...>`: one or more timestamps to grab, `Time` values in source/content time (optional; default `0`). A negative value is an offset back from the end of the clip, so `-1` is one second before the end and `-1f` one frame before it. Order is preserved in the output regardless of the order given. Mutually exclusive with `--count`.
+- `-c, --count <n>`: instead of `--time`, grab `n` frames evenly spaced across the clip at a fixed interval of `window / n`, starting at the window start (optional; positive integer).
+- `-s, --start <time>`: with `--count`, the start of the window to sample (optional; `Time` value; default `0`).
+- `-e, --end <time>`: with `--count`, the end of the window to sample (optional; `Time` value; default the asset duration).
+- `-q, --quality <preset>`: per-frame resolution preset (optional; default `small`). One of `small` (384x384), `medium` (768x768), `large` (1536x1536), or `fullres` (native). Each caps the total pixel count while preserving aspect ratio.
+- `--no-timestamp`: don't stamp each frame with its `HH:MM:SS:FF` timestamp label (optional; the label is drawn by default).
+- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--time`) is rejected.
- `-o, --output <dir>`: directory to write the PNGs into (optional; default the system temp directory). Each frame is written as a uuid-named `.png`.
## Output
@@ -18,4 +24,4 @@ JSON Lines, one object per requested timestamp, in the requested order:
## Errors
-Exits non-zero if the id is unknown, the asset is not a video, any `--time` is past the asset's duration, or a PNG can't be written.
+Exits non-zero if the id is unknown, the asset is not a video, any `--time` is past the asset's duration, the `--count` window is empty, `--time` and `--count` are combined, `--start`/`--end` are given without `--count`, more than 100 frames are requested without `--uncapped`, or a PNG can't be written.
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index 03be64b7..5f2412ae 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -121,7 +121,15 @@ export type AssetRef = { id: string } | { path: string };
export type MediaProbeRequest = AssetRef;
-export type MediaFrameRequest = AssetRef & { times?: number[]; resolution?: number };
+export type FrameQuality = "small" | "medium" | "large" | "fullres";
+export type MediaFrameRequest = AssetRef & {
+ times?: number[];
+ count?: number;
+ start?: number;
+ end?: number;
+ quality?: FrameQuality;
+ timestamp?: boolean;
+};
export type MediaFrameResult = Array<{ time: number; base64: string }>;
export type MediaTranscribeRequest = AssetRef & { start?: number; end?: number };
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index b542f5cf..a3351438 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -16,7 +16,7 @@ import { compileProject } from "./compile-project";
import { listLocalFonts } from "./fonts";
import { openFolder } from "./open-folder";
import { fetchVideo } from "./ytdlp";
-import type { AssetRef, EncoderConfigInput, NodePatch } from "./protocol";
+import type { AssetRef, EncoderConfigInput, FrameQuality, NodePatch } from "./protocol";
// Long-running commands (renders, AI generation) override the default 60s.
const GENERATE = { context: { timeoutMs: GENERATE_TIMEOUT_MS } };
@@ -450,27 +450,72 @@ async function nodeScreenshot(id: string | undefined, opts: ScreenshotOptions):
}
}
-type MediaFrameOptions = { time?: string[]; resolution?: string; output?: string };
+const FRAME_QUALITIES: FrameQuality[] = ["small", "medium", "large", "fullres"];
+
+// Guardrail against accidentally decoding a huge number of frames; --uncapped lifts it.
+const FRAME_CAP = 100;
+
+type MediaFrameOptions = {
+ time?: string[];
+ count?: string;
+ start?: string;
+ end?: string;
+ quality?: string;
+ timestamp?: boolean;
+ uncapped?: boolean;
+ output?: string;
+};
async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
+ if (opts.time !== undefined && opts.count !== undefined) {
+ console.error("Pass either --time or --count, not both.");
+ process.exit(1);
+ }
+
let times: number[] | undefined;
if (opts.time !== undefined) {
- times = opts.time.map((t) => parseTimeArg(t, "--time"));
+ times = opts.time.map((t) => parseTimeArg(t, "--time", true));
+ }
+
+ let count: number | undefined;
+ if (opts.count !== undefined) {
+ count = Number(opts.count);
+ if (!Number.isInteger(count) || count < 1) {
+ console.error(`--count must be a positive integer (got "${opts.count}")`);
+ process.exit(1);
+ }
+ }
+
+ const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined;
+ const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined;
+ if (start !== undefined && end !== undefined && start >= end) {
+ console.error(`--start (${start}s) must be less than --end (${end}s).`);
+ process.exit(1);
+ }
+ if ((start !== undefined || end !== undefined) && count === undefined) {
+ console.error("--start and --end only apply together with --count.");
+ process.exit(1);
+ }
+
+ const requested = count ?? times?.length ?? 1;
+ if (!opts.uncapped && requested > FRAME_CAP) {
+ console.error(`Grabbing ${requested} frames exceeds the ${FRAME_CAP}-frame cap; pass --uncapped to override.`);
+ process.exit(1);
}
- let resolution: number | undefined;
- if (opts.resolution !== undefined) {
- resolution = Number(opts.resolution);
- if (!Number.isInteger(resolution) || resolution < 0) {
- console.error(`--resolution must be a non-negative integer pixel count, or 0 for native (got "${opts.resolution}")`);
+ let quality: FrameQuality | undefined;
+ if (opts.quality !== undefined) {
+ if (!FRAME_QUALITIES.includes(opts.quality as FrameQuality)) {
+ console.error(`--quality must be one of ${FRAME_QUALITIES.join(", ")} (got "${opts.quality}")`);
process.exit(1);
}
+ quality = opts.quality as FrameQuality;
}
const target = resolveAssetRef(ref);
const dir = opts.output ?? tmpdir();
try {
- const frames = await editor.media.frame.query({ ...target, times, resolution });
+ const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, timestamp: opts.timestamp });
for (const { time, base64 } of frames) {
const path = join(dir, `${randomUUID()}.png`);
writeFileSync(path, Buffer.from(base64, "base64"));
@@ -558,11 +603,11 @@ async function mediaListen(ref: string, opts: MediaListenOptions): Promise<void>
type MediaPreviewOptions = { start?: string; end?: string; scale?: string; output?: string };
-function parseTimeArg(value: string, flag: string): number {
+function parseTimeArg(value: string, flag: string, allowNegative = false): number {
const seconds = parseTime(value);
- if (seconds === undefined || seconds < 0) {
+ if (seconds === undefined || (!allowNegative && seconds < 0)) {
console.error(
- `${flag} must be a non-negative Time — seconds ("1.5"), frames ("45f"), or "MM:SS" (got "${value}")`,
+ `${flag} must be a ${allowNegative ? "" : "non-negative "}Time — seconds ("1.5"), frames ("45f"), or "MM:SS" (got "${value}")`,
);
process.exit(1);
}
@@ -1111,17 +1156,22 @@ media
media
.command("grab")
.alias("sample")
- .description(`Decode one or more frames of a video file and write them as high-res PNGs, each stamped in the top-left with its HH:MM:SS:FF timestamp. Best for inspecting individual frames in detail where a filmstrip is too coarse: seeks to the exact requested time with frame-level precision${docs("media/grab")}`)
+ .description(`Decode frames of a video file and write them as PNGs, each stamped in the top-left with its HH:MM:SS:FF timestamp. Grab explicit moments with --time, or a fixed number of evenly-spaced frames with --count. The recommended tool for understanding a video at the frame level${docs("media/grab")}`)
.argument("<id|path>", "video asset id, or a local video file to grab frames from")
- .option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`)
- .option("-r, --resolution <pixels>", "cap each frame to this many total pixels, preserving aspect ratio; 0 for native (default: 147456, i.e. 384x384)")
+ .option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`)
+ .option("-c, --count <n>", "instead of --time, grab this many frames evenly spaced across the clip (or across the --start/--end window)")
+ .option("-s, --start <time>", `with --count, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)
+ .option("-e, --end <time>", `with --count, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`)
+ .option("-q, --quality <preset>", "frame resolution: small (384x384, default), medium (768x768), large (1536x1536), or fullres (native)")
+ .option("--no-timestamp", "don't stamp each frame with its HH:MM:SS:FF timestamp label")
+ .option("--uncapped", "lift the 100-frame safety cap (grabbing many frames is slow and token-heavy)")
.option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")
.action((ref: string, opts: MediaFrameOptions) => mediaFrame(ref, opts));
media
.command("filmstrip")
.alias("film")
- .description(`Start here for any video: render a grid of thumbnails sampled across the timeline to a PNG, each row stamped with timestamps. Fast and token-efficient; narrow the window to zoom into a region of interest. Video only, no audio — use \`waveform\` to inspect the audio track${docs("media/filmstrip")}`)
+ .description(`Render a grid of thumbnails sampled across the timeline to a PNG, each row stamped with timestamps. It's a fast and token-efficient video track preview; narrow the window to zoom into a region of interest${docs("media/filmstrip")}`)
.argument("<id|path>", "video asset id, or a local video file to preview")
.option("-s, --start <time>", `start of the window to preview — seconds, "45f" frames, or "MM:SS" (default: 0)`)
.option("-e, --end <time>", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
@@ -1132,7 +1182,7 @@ media
media
.command("waveform")
.alias("wave")
- .description(`Render the audio track of a video or audio file as a waveform PNG with a timestamp ruler. Shows loudness over time and highlights the silent stretches in red, which are also returned as [start, end] second ranges in the JSON. Fast and token-efficient; narrow the window to zoom into a region of interest${docs("media/waveform")}`)
+ .description(`Render the audio track of a video or audio file as a waveform PNG with a timestamp ruler. It's a fast and token-efficient audio track preview; shows loudness over time and highlights the silent stretches in red, which are also returned as [start, end] second ranges in the JSON${docs("media/waveform")}`)
.argument("<id|path>", "video or audio asset id, or a local file to preview")
.option("-s, --start <time>", `start of the window to preview — seconds, "45f" frames, or "MM:SS" (default: 0)`)
.option("-e, --end <time>", `end of the window to preview — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
@@ -1142,13 +1192,11 @@ media
media
.command("listen")
- .alias("watch")
.description(`Prompt a multimodal model for a semantic analysis of an audio track and print its answer. Shines on the semantics of audio (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video, by default only the audio track is analyzed${docs("media/listen")}`)
.argument("<id|path>", "video or audio asset id, or a local file to analyze")
.option("-p, --prompt <str>", "question or instruction to guide the analysis")
.option("-s, --start <time>", `start of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: 0); timestamps in the analysis are relative to this point`)
.option("-e, --end <time>", `end of the segment to analyze — seconds, "45f" frames, or "MM:SS" (default: media duration)`)
- .option("--keep-video", "keep the video track instead of stripping it to audio (expensive: requires a full video upload)")
.action((ref: string, opts: MediaListenOptions) => mediaListen(ref, opts));
const folder = program
diff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.ts
index c912d6e2..4ac2d427 100644
--- a/apps/web/src/context/editor-api/media.ts
+++ b/apps/web/src/context/editor-api/media.ts
@@ -97,25 +97,49 @@ export function handleMediaProbe(engine: Accessor<Engine>) {
};
}
-// Default cap on the pixel count of each decoded frame (384x384). Keeps frames
-// small enough for vision models without the caller having to know the aspect ratio.
-export const DEFAULT_FRAME_PIXEL_BUDGET = 384 * 384;
+// Named quality presets mapped to a per-frame total-pixel budget (aspect ratio
+// preserved). A budget of 0 means native resolution. `small` keeps frames small
+// enough for vision models and is the default.
+const FRAME_QUALITY_BUDGETS = {
+ small: 384 * 384, // 147,456
+ medium: 768 * 768, // 589,824
+ large: 1536 * 1536, // 2,359,296
+ fullres: 0, // native
+} as const;
export function handleMediaFrame(engine: Accessor<Engine>) {
return async (req: MediaFrameRequest) => {
- const { times, resolution } = req;
+ const { times, count, start, end, quality, timestamp } = req;
const { world } = engine();
const asset = await resolveAssetRef(world, req);
const id = asset.id;
assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);
- const requested = times && times.length ? times : [0];
- for (const t of requested) {
- assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);
+ // `count` samples evenly across a window (default the whole clip); otherwise
+ // grab the explicit `times` (falling back to a single frame at 0).
+ let requested: number[];
+ if (count !== undefined) {
+ const from = Math.min(Math.max(start ?? 0, 0), asset.duration);
+ const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration);
+ assert(to > from, `The requested window is empty; start (${from.toFixed(2)}s) is at or past end (${to.toFixed(2)}s).`);
+ const interval = (to - from) / count;
+ requested = Array.from({ length: count }, (_, i) => from + i * interval);
+ } else {
+ const raw = times && times.length ? times : [0];
+ // A negative time is an offset back from the end of the clip: -1 is one
+ // second before the end, -1f one frame before it.
+ requested = raw.map((t) => {
+ if (t >= 0) {
+ assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`);
+ return t;
+ }
+ const resolved = asset.duration + t;
+ assert(resolved >= 0, `--time ${t} counts past the start of the clip (duration ${asset.duration.toFixed(2)}s).`);
+ return resolved;
+ });
}
- // A budget of 0 means "native resolution"; anything else caps the pixel count.
- const budget = resolution ?? DEFAULT_FRAME_PIXEL_BUDGET;
+ const budget = FRAME_QUALITY_BUDGETS[quality ?? "small"];
const blob = await getAssetFile(asset);
const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
@@ -152,7 +176,7 @@ export function handleMediaFrame(engine: Accessor<Engine>) {
const ctx = wrapped.canvas.getContext("2d");
- if (ctx) {
+ if (ctx && timestamp !== false) {
const label = formatTimestamp(time, asset.frameRate)
const bandHeight = Math.max(20, Math.round(wrapped.canvas.height * 0.06));
const fontSize = Math.round(bandHeight * 0.72);