The VS Code of video editing, built for agents.
TypeScript
Refactor media frame and node capture to include timecode in output
konstantin-paulus877259150271
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.tsindex 44fa2809..8bbc9b87 100644--- a/apps/cli/src/cli-channels.ts+++ b/apps/cli/src/cli-channels.ts@@ -115,10 +115,10 @@ export type MediaFrameRequest = AssetRef & {start?: number;end?: number;quality?: FrameQuality;- timestamp?: boolean;auto?: boolean;};-export type MediaFrameResult = Array<{ time: number; base64: string }>;++export type MediaFrameResult = Array<{ time: number; timecode: string; base64: string }>;export type MediaTranscribeRequest = AssetRef;export type TranscriptWord = { text: string; start: number; end: number };diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 459bb25c..de1ad13e 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -432,7 +432,7 @@ async function grepNodes(pattern: string, id: string | undefined, opts: NodeGrep}}-type CaptureOptions = { time?: string[]; timestamp?: boolean; output?: string };+type CaptureOptions = { time?: string[]; output?: string };async function nodeCapture(id: string, opts: CaptureOptions): Promise<void> {const eid = parseNodeIds([id])[0];@@ -440,12 +440,12 @@ async function nodeCapture(id: string, opts: CaptureOptions): Promise<void> {const times = (opts.time ?? ["0"]).map((t) => parseTimeArg(t, "--time"));const frames = times.map((t) => Math.round(t * TIME_FPS));- const dir = opts.output ?? tmpdir();+ const dir = opts.output ?? join(tmpdir(), `dapi-capture-${randomUUID().slice(0, 8)}`);mkdirSync(dir, { recursive: true });try {- const shots = await editor.node.capture.query({ id: eid, frames, timestamp: opts.timestamp }, GENERATE);- for (const [i, { base64 }] of shots.entries()) {- const path = join(dir, `${randomUUID()}.png`);+ const shots = await editor.node.capture.query({ id: eid, frames }, GENERATE);+ for (const [i, { base64, timecode }] of shots.entries()) {+ const path = join(dir, `${timecode.replaceAll(":", "-")}.png`);writeFileSync(path, Buffer.from(base64, "base64"));console.log(JSON.stringify({ time: times[i], path }));}@@ -465,7 +465,6 @@ type MediaFrameOptions = {start?: string;end?: string;quality?: string;- timestamp?: boolean;uncapped?: boolean;output?: string;auto?: boolean;@@ -522,12 +521,12 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {}const target = resolveAssetRef(ref);- const dir = opts.output ?? tmpdir();+ const dir = opts.output ?? join(tmpdir(), `dapi-grab-${randomUUID().slice(0, 8)}`);mkdirSync(dir, { recursive: true });try {- const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, timestamp: opts.timestamp, auto: opts.auto });- for (const { time, base64 } of frames) {- const path = join(dir, `${randomUUID()}.png`);+ const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, auto: opts.auto });+ for (const { time, timecode, base64 } of frames) {+ const path = join(dir, `${timecode.replaceAll(":", "-")}.png`);writeFileSync(path, Buffer.from(base64, "base64"));console.log(JSON.stringify({ time, path }));}@@ -1305,7 +1304,7 @@ media.command("grab").alias("sample").description(- `Decode frames of a video file and write them as PNGs (local render, no credits), each stamped in the top-left with its HH:MM:SS:FF timestamp. Grabs the asset's own pixels at full resolution, unlike \`node capture\` which renders the composited node. The recommended tool for understanding a video at the frame level.`,+ `Decode frames of a video file and write them as PNGs (local render, no credits), each named after its HH-MM-SS-FF timecode. Grabs the asset's own pixels at full resolution, unlike \`node capture\` which renders the composited node. The recommended tool for understanding a video at the frame level.`,).argument("<id|path>", "video asset id, or a local video file to grab frames from").option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`)@@ -1314,9 +1313,8 @@ media.option("-s, --start <time>", `with --count or --auto, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`).option("-e, --end <time>", `with --count or --auto, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`).option("-q, --quality <preset>", "frame resolution: small (384x384, default), medium (768x768), large (1536x1536), or fullres (native)")- .option("--no-timestamp", "don't stamp each frame with its HH:MM:SS:FF timestamp label").option("--uncapped", "lift the 100-frame safety cap (grabbing many frames is slow and token-heavy)")- .option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")+ .option("-o, --output <dir>", "directory to write the PNGs into (default: a fresh dir in the system temp dir)").action((ref: string, opts: MediaFrameOptions) => mediaFrame(ref, opts));media@@ -1468,12 +1466,11 @@ nodenode.command("capture").description(- `Render a node in isolation to PNGs, one per position, each stamped in the top-left with its HH:MM:SS:FF timestamp. The node is drawn offscreen at 720p height, tightly framed to its own bounds on a transparent background — siblings and overlapping scene content are not included, so capture a scene id to check composition ("what plays at time T": layout, overlaps, text, timing). For a video asset's own full-resolution pixels use \`media grab\`.`,+ `Render a node in isolation to PNGs, one per position, each named after the HH-MM-SS-FF timecode of the frame rendered. The node is drawn offscreen at 720p height, tightly framed to its own bounds on a transparent background — siblings and overlapping scene content are not included, so capture a scene id to check composition ("what plays at time T": layout, overlaps, text, timing). For a video asset's own full-resolution pixels use \`media grab\`.`,).argument("<id>", "node id to capture").option("-t, --time <time...>", `one or more positions to capture, relative to the node's start (0 = its first visible frame) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0, the node's first visible frame)`)- .option("--no-timestamp", "don't stamp each capture with its HH:MM:SS:FF timestamp label")- .option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")+ .option("-o, --output <dir>", "directory to write the PNGs into (default: a fresh dir in the system temp dir)").action((id: string, opts: CaptureOptions) => nodeCapture(id, opts));nodediff --git a/apps/web/src/components/engine/encode/image-encoder.ts b/apps/web/src/components/engine/encode/image-encoder.tsindex 538a7049..3dd782fd 100644--- a/apps/web/src/components/engine/encode/image-encoder.ts+++ b/apps/web/src/components/engine/encode/image-encoder.ts@@ -17,7 +17,7 @@ import { renderSystem } from '../systems/render';import { cloneFromRecords, serializeEntity } from '../api/serialize';import { getEntityTree } from '../api/query';import { realizeMounts } from '@/utils/mount';-import { framesToSeconds, formatTimestamp, stampTimestampLabel } from '../utils';+import { framesToSeconds, formatTimestamp } from '../utils';import { assert } from '@/utils';import type { EngineWorld } from '../api/world';@@ -27,14 +27,15 @@ type ImageEncoderConfig = {eid: number;/** Frames to capture, relative to the entity's first visible frame. */frames: number[];- /** Stamp each capture with the HH:MM:SS:FF timecode of the rendered frame. */- timestamp?: boolean;/** Target output height in px (default: the entity's native size). */resolution?: number;};+/** One capture: the PNG plus the HH:MM:SS:FF timecode of the frame rendered. */+export type CapturedImage = { base64: string; timecode: string };+export type ImageExportResult =- | { type: 'success'; data: string[] }+ | { type: 'success'; data: CapturedImage[] }| { type: 'canceled' }| { type: 'error'; error: Error };@@ -187,7 +188,7 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: Imageconst render = async (): Promise<ImageExportResult> => {try {- const images: string[] = [];+ const images: CapturedImage[] = [];for (const frame of config.frames) {if (canceled) {@@ -200,16 +201,11 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: ImagetransformSystem(world);renderSystem(world);- if (config.timestamp) {- // The render system leaves the camera transform on the context;- // the label is drawn in device space.- offscreenCtx.setTransform(1, 0, 0, 1, 0, 0);- const stampFrame = startFrame + frame;- const label = formatTimestamp(framesToSeconds(stampFrame, world.frameRate), world.frameRate);- stampTimestampLabel(offscreenCtx, offscreenCanvas.height, label);- }-- images.push(await toBase64Png(offscreenCanvas));+ const stampFrame = startFrame + frame;+ images.push({+ base64: await toBase64Png(offscreenCanvas),+ timecode: formatTimestamp(framesToSeconds(stampFrame, world.frameRate), world.frameRate),+ });}return { type: 'success', data: images };diff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.tsindex 35bcc4c1..64577b28 100644--- a/apps/web/src/context/editor-api/media.ts+++ b/apps/web/src/context/editor-api/media.ts@@ -6,7 +6,7 @@ import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';import { ElectronFileHandle } from '@/lib/electron-file-handle';import { pickInformativeTimes } from './frame-triage';import { trpc } from '@/lib/trpc';-import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimestamp, stampTimestampLabel } from '@/components/engine';+import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimestamp } from '@/components/engine';import { assert } from '@/utils';import {transcodeForTranscription,@@ -113,7 +113,7 @@ const AUTO_MAX_FRAMES = 30;export function handleMediaFrame(engine: Accessor<Engine>) {return async (req: MediaFrameRequest) => {- const { times, count, start, end, quality, timestamp, auto } = req;+ const { times, count, start, end, quality, auto } = req;const { world } = engine();const asset = await resolveAssetRef(world, req);const id = asset.id;@@ -185,20 +185,17 @@ export function handleMediaFrame(engine: Accessor<Engine>) {const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);const timestamps = ordered.map(({ time }) => firstTimestamp + time);- const result: Array<{ time: number; base64: string }> = new Array(requested.length);+ const result: Array<{ time: number; timecode: string; base64: string }> = new Array(requested.length);let i = 0;for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {const { time, index } = ordered[i++];assert(wrapped, `No frame found at ${time}s.`);- // The webgpu getContext overload muddies inference on the canvas union.- const ctx = wrapped.canvas.getContext("2d") as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null;-- if (ctx && timestamp !== false) {- stampTimestampLabel(ctx, wrapped.canvas.height, formatTimestamp(time, asset.frameRate));- }-- result[index] = { time, base64: await canvasToPngBase64(wrapped.canvas) };+ result[index] = {+ time,+ timecode: formatTimestamp(time, asset.frameRate),+ base64: await canvasToPngBase64(wrapped.canvas),+ };}return result;} finally {diff --git a/apps/web/src/context/editor-api/node.ts b/apps/web/src/context/editor-api/node.tsindex e30c0b2d..26ce7af9 100644--- a/apps/web/src/context/editor-api/node.ts+++ b/apps/web/src/context/editor-api/node.ts@@ -37,6 +37,7 @@ import type {NodeRenderRequest,NodeRenderResult,} from "@diffusionstudio/cli/channels";+import type { CapturedImage } from "@/components/engine/encode/image-encoder";import type { EncoderConfig } from "@/components/engine/encode/interfaces";import type { Accessor } from "solid-js";@@ -289,7 +290,7 @@ export function handleNodeGrep(engine: Accessor<Engine>) {}export function handleNodeCapture(engine: Accessor<Engine>) {- return async ({ id, frames, timestamp }: { id: number; frames?: number[]; timestamp?: boolean }): Promise<{ base64: string }[]> => {+ return async ({ id, frames }: { id: number; frames?: number[] }): Promise<CapturedImage[]> => {const e = engine();const w = e.world;@@ -312,7 +313,6 @@ export function handleNodeCapture(engine: Accessor<Engine>) {const encoder = await createImageEncoder(w, {eid,frames: shots,- timestamp: timestamp !== false,resolution: 720,});const result = await encoder.render();@@ -320,7 +320,7 @@ export function handleNodeCapture(engine: Accessor<Engine>) {if (result.type === "canceled") throw new Error("Capture canceled");if (result.type === "error") throw result.error;- return result.data.map((base64) => ({ base64 }));+ return result.data;};}diff --git a/reference/media/grab.md b/reference/media/grab.mdindex 08a17403..e30fbbdb 100644--- a/reference/media/grab.md+++ b/reference/media/grab.md@@ -1,6 +1,6 @@# `dapi media grab <id|path>`-Decodes one or more frames of a video asset at the given times and writes each to a PNG file; like [`node capture`](../node/capture.md), but grabs the asset's own pixels at full resolution (unlike `node capture`, which captures the composited canvas). Renders locally; no credits.+Decodes one or more frames of a video asset at the given times and writes each to a PNG file named after its `HH-MM-SS-FF` timecode (e.g. `00-00-01-12.png`); like [`node capture`](../node/capture.md), but grabs the asset's own pixels at full resolution (unlike `node capture`, which captures the composited canvas). Renders locally; no credits.## Input@@ -10,16 +10,15 @@ Decodes one or more frames of a video asset at the given times and writes each t- `-s, --start <time>`: with `--count`, the start of the window to sample (optional; `Time` value; default `0`).- `-e, --end <time>`: with `--count`, the end of the window to sample (optional; `Time` value; default the asset duration).- `-q, --quality <preset>`: per-frame resolution preset (optional; default `small`). One of `small` (384x384), `medium` (768x768), `large` (1536x1536), or `fullres` (native). Each caps the total pixel count while preserving aspect ratio.-- `--no-timestamp`: don't stamp each frame with its `HH:MM:SS:FF` timestamp label (optional; the label is drawn by default).- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--time`) is rejected.-- `-o, --output <dir>`: directory to write the PNGs into (optional; default the system temp directory). Each frame is written as a uuid-named `.png`.+- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-grab-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites frames whose timecode matches; requested times that land on the same frame share one file.## OutputJSON Lines, one object per requested timestamp, in the requested order:```ts-{ time: number; path: string } // time = the requested timestamp in seconds; path = the written PNG+{ time: number; path: string } // time = the requested timestamp in seconds; path = the written PNG, named HH-MM-SS-FF.png```## Errorsdiff --git a/reference/node/capture.md b/reference/node/capture.mdindex f3902407..c751e3bd 100644--- a/reference/node/capture.md+++ b/reference/node/capture.md@@ -1,17 +1,18 @@-# `dapi node capture [id] [-t, --time <time...>]`+# `dapi node capture <id> [-t, --time <time...>]`++Renders a node in isolation and writes one PNG per timeline position, each named after the `HH-MM-SS-FF` timecode of the frame actually rendered (e.g. `00-00-01-12.png`). The node is drawn offscreen at 720p height, tightly framed to its own bounds on a transparent background; siblings and overlapping scene content are not included, so capture a scene id to check composition. To grab a video asset's own pixels at full resolution instead of the composited node, use [`media grab`](../media/grab.md).-Focuses the node on the canvas and captures it, writing one PNG to the system temp directory per timeline position. Each capture is stamped in the top-left with its `HH:MM:SS:FF` timestamp. If `[id]` is omitted, captures the canvas (the active scene). To grab a video asset's own pixels at full resolution instead of the composited canvas, use [`media grab`](../media/grab.md).## Input-- `[id]`: node id (optional; defaults to the canvas)-- `-t, --time <time...>`: one or more timeline positions at which to record the node, each a `Time` value (optional; defaults to the current playhead position)-- `--no-timestamp`: don't stamp each capture with its `HH:MM:SS:FF` timestamp label+- `<id>`: node id to capture (required)+- `-t, --time <time...>`: one or more positions to capture, relative to the node's start (`0` = its first visible frame), each a `Time` value (optional; default `0`)+- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-capture-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites captures whose timecode matches; requested times that land on the same frame share one file.## Output-One JSON object per line (JSON Lines), in the order the times were given: each has the requested time in seconds (`null` for the current playhead) and the absolute path to a freshly written PNG in the system temp directory with a uuid filename.+One JSON object per line (JSON Lines), in the order the times were given: the requested time in seconds and the absolute path to the written PNG.```ts-{ time: number | null, path: string } // e.g. { "time": 1.5, "path": "/tmp/3f2c1a8e-....png" }+{ time: number, path: string } // e.g. { "time": 1.5, "path": "/tmp/dapi-capture-3f2c1a8e/00-00-01-15.png" }```