Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor media frame and node capture to include timecode in output

konstantin-paulus877259150271
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index 44fa2809..8bbc9b87 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -115,10 +115,10 @@ export type MediaFrameRequest = AssetRef & {
start?: number;
end?: number;
quality?: FrameQuality;
- timestamp?: boolean;
auto?: boolean;
};
-export type MediaFrameResult = Array<{ time: number; base64: string }>;
+
+export type MediaFrameResult = Array<{ time: number; timecode: string; base64: string }>;
export type MediaTranscribeRequest = AssetRef;
export type TranscriptWord = { text: string; start: number; end: number };
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 459bb25c..de1ad13e 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -432,7 +432,7 @@ async function grepNodes(pattern: string, id: string | undefined, opts: NodeGrep
}
}
-type CaptureOptions = { time?: string[]; timestamp?: boolean; output?: string };
+type CaptureOptions = { time?: string[]; output?: string };
async function nodeCapture(id: string, opts: CaptureOptions): Promise<void> {
const eid = parseNodeIds([id])[0];
@@ -440,12 +440,12 @@ async function nodeCapture(id: string, opts: CaptureOptions): Promise<void> {
const times = (opts.time ?? ["0"]).map((t) => parseTimeArg(t, "--time"));
const frames = times.map((t) => Math.round(t * TIME_FPS));
- const dir = opts.output ?? tmpdir();
+ const dir = opts.output ?? join(tmpdir(), `dapi-capture-${randomUUID().slice(0, 8)}`);
mkdirSync(dir, { recursive: true });
try {
- const shots = await editor.node.capture.query({ id: eid, frames, timestamp: opts.timestamp }, GENERATE);
- for (const [i, { base64 }] of shots.entries()) {
- const path = join(dir, `${randomUUID()}.png`);
+ const shots = await editor.node.capture.query({ id: eid, frames }, GENERATE);
+ for (const [i, { base64, timecode }] of shots.entries()) {
+ const path = join(dir, `${timecode.replaceAll(":", "-")}.png`);
writeFileSync(path, Buffer.from(base64, "base64"));
console.log(JSON.stringify({ time: times[i], path }));
}
@@ -465,7 +465,6 @@ type MediaFrameOptions = {
start?: string;
end?: string;
quality?: string;
- timestamp?: boolean;
uncapped?: boolean;
output?: string;
auto?: boolean;
@@ -522,12 +521,12 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
}
const target = resolveAssetRef(ref);
- const dir = opts.output ?? tmpdir();
+ const dir = opts.output ?? join(tmpdir(), `dapi-grab-${randomUUID().slice(0, 8)}`);
mkdirSync(dir, { recursive: true });
try {
- const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, timestamp: opts.timestamp, auto: opts.auto });
- for (const { time, base64 } of frames) {
- const path = join(dir, `${randomUUID()}.png`);
+ const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, auto: opts.auto });
+ for (const { time, timecode, base64 } of frames) {
+ const path = join(dir, `${timecode.replaceAll(":", "-")}.png`);
writeFileSync(path, Buffer.from(base64, "base64"));
console.log(JSON.stringify({ time, path }));
}
@@ -1305,7 +1304,7 @@ media
.command("grab")
.alias("sample")
.description(
- `Decode frames of a video file and write them as PNGs (local render, no credits), each stamped in the top-left with its HH:MM:SS:FF timestamp. Grabs the asset's own pixels at full resolution, unlike \`node capture\` which renders the composited node. The recommended tool for understanding a video at the frame level.`,
+ `Decode frames of a video file and write them as PNGs (local render, no credits), each named after its HH-MM-SS-FF timecode. Grabs the asset's own pixels at full resolution, unlike \`node capture\` which renders the composited node. The recommended tool for understanding a video at the frame level.`,
)
.argument("<id|path>", "video asset id, or a local video file to grab frames from")
.option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`)
@@ -1314,9 +1313,8 @@ media
.option("-s, --start <time>", `with --count or --auto, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)
.option("-e, --end <time>", `with --count or --auto, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`)
.option("-q, --quality <preset>", "frame resolution: small (384x384, default), medium (768x768), large (1536x1536), or fullres (native)")
- .option("--no-timestamp", "don't stamp each frame with its HH:MM:SS:FF timestamp label")
.option("--uncapped", "lift the 100-frame safety cap (grabbing many frames is slow and token-heavy)")
- .option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")
+ .option("-o, --output <dir>", "directory to write the PNGs into (default: a fresh dir in the system temp dir)")
.action((ref: string, opts: MediaFrameOptions) => mediaFrame(ref, opts));
media
@@ -1468,12 +1466,11 @@ node
node
.command("capture")
.description(
- `Render a node in isolation to PNGs, one per position, each stamped in the top-left with its HH:MM:SS:FF timestamp. The node is drawn offscreen at 720p height, tightly framed to its own bounds on a transparent background — siblings and overlapping scene content are not included, so capture a scene id to check composition ("what plays at time T": layout, overlaps, text, timing). For a video asset's own full-resolution pixels use \`media grab\`.`,
+ `Render a node in isolation to PNGs, one per position, each named after the HH-MM-SS-FF timecode of the frame rendered. The node is drawn offscreen at 720p height, tightly framed to its own bounds on a transparent background — siblings and overlapping scene content are not included, so capture a scene id to check composition ("what plays at time T": layout, overlaps, text, timing). For a video asset's own full-resolution pixels use \`media grab\`.`,
)
.argument("<id>", "node id to capture")
.option("-t, --time <time...>", `one or more positions to capture, relative to the node's start (0 = its first visible frame) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0, the node's first visible frame)`)
- .option("--no-timestamp", "don't stamp each capture with its HH:MM:SS:FF timestamp label")
- .option("-o, --output <dir>", "directory to write the PNGs into (default: system temp dir)")
+ .option("-o, --output <dir>", "directory to write the PNGs into (default: a fresh dir in the system temp dir)")
.action((id: string, opts: CaptureOptions) => nodeCapture(id, opts));
node
diff --git a/apps/web/src/components/engine/encode/image-encoder.ts b/apps/web/src/components/engine/encode/image-encoder.ts
index 538a7049..3dd782fd 100644
--- a/apps/web/src/components/engine/encode/image-encoder.ts
+++ b/apps/web/src/components/engine/encode/image-encoder.ts
@@ -17,7 +17,7 @@ import { renderSystem } from '../systems/render';
import { cloneFromRecords, serializeEntity } from '../api/serialize';
import { getEntityTree } from '../api/query';
import { realizeMounts } from '@/utils/mount';
-import { framesToSeconds, formatTimestamp, stampTimestampLabel } from '../utils';
+import { framesToSeconds, formatTimestamp } from '../utils';
import { assert } from '@/utils';
import type { EngineWorld } from '../api/world';
@@ -27,14 +27,15 @@ type ImageEncoderConfig = {
eid: number;
/** Frames to capture, relative to the entity's first visible frame. */
frames: number[];
- /** Stamp each capture with the HH:MM:SS:FF timecode of the rendered frame. */
- timestamp?: boolean;
/** Target output height in px (default: the entity's native size). */
resolution?: number;
};
+/** One capture: the PNG plus the HH:MM:SS:FF timecode of the frame rendered. */
+export type CapturedImage = { base64: string; timecode: string };
+
export type ImageExportResult =
- | { type: 'success'; data: string[] }
+ | { type: 'success'; data: CapturedImage[] }
| { type: 'canceled' }
| { type: 'error'; error: Error };
@@ -187,7 +188,7 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: Image
const render = async (): Promise<ImageExportResult> => {
try {
- const images: string[] = [];
+ const images: CapturedImage[] = [];
for (const frame of config.frames) {
if (canceled) {
@@ -200,16 +201,11 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: Image
transformSystem(world);
renderSystem(world);
- if (config.timestamp) {
- // The render system leaves the camera transform on the context;
- // the label is drawn in device space.
- offscreenCtx.setTransform(1, 0, 0, 1, 0, 0);
- const stampFrame = startFrame + frame;
- const label = formatTimestamp(framesToSeconds(stampFrame, world.frameRate), world.frameRate);
- stampTimestampLabel(offscreenCtx, offscreenCanvas.height, label);
- }
-
- images.push(await toBase64Png(offscreenCanvas));
+ const stampFrame = startFrame + frame;
+ images.push({
+ base64: await toBase64Png(offscreenCanvas),
+ timecode: formatTimestamp(framesToSeconds(stampFrame, world.frameRate), world.frameRate),
+ });
}
return { type: 'success', data: images };
diff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.ts
index 35bcc4c1..64577b28 100644
--- a/apps/web/src/context/editor-api/media.ts
+++ b/apps/web/src/context/editor-api/media.ts
@@ -6,7 +6,7 @@ import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
import { ElectronFileHandle } from '@/lib/electron-file-handle';
import { pickInformativeTimes } from './frame-triage';
import { trpc } from '@/lib/trpc';
-import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimestamp, stampTimestampLabel } from '@/components/engine';
+import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimestamp } from '@/components/engine';
import { assert } from '@/utils';
import {
transcodeForTranscription,
@@ -113,7 +113,7 @@ const AUTO_MAX_FRAMES = 30;
export function handleMediaFrame(engine: Accessor<Engine>) {
return async (req: MediaFrameRequest) => {
- const { times, count, start, end, quality, timestamp, auto } = req;
+ const { times, count, start, end, quality, auto } = req;
const { world } = engine();
const asset = await resolveAssetRef(world, req);
const id = asset.id;
@@ -185,20 +185,17 @@ export function handleMediaFrame(engine: Accessor<Engine>) {
const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);
const timestamps = ordered.map(({ time }) => firstTimestamp + time);
- const result: Array<{ time: number; base64: string }> = new Array(requested.length);
+ const result: Array<{ time: number; timecode: string; base64: string }> = new Array(requested.length);
let i = 0;
for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {
const { time, index } = ordered[i++];
assert(wrapped, `No frame found at ${time}s.`);
- // The webgpu getContext overload muddies inference on the canvas union.
- const ctx = wrapped.canvas.getContext("2d") as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null;
-
- if (ctx && timestamp !== false) {
- stampTimestampLabel(ctx, wrapped.canvas.height, formatTimestamp(time, asset.frameRate));
- }
-
- result[index] = { time, base64: await canvasToPngBase64(wrapped.canvas) };
+ result[index] = {
+ time,
+ timecode: formatTimestamp(time, asset.frameRate),
+ base64: await canvasToPngBase64(wrapped.canvas),
+ };
}
return result;
} finally {
diff --git a/apps/web/src/context/editor-api/node.ts b/apps/web/src/context/editor-api/node.ts
index e30c0b2d..26ce7af9 100644
--- a/apps/web/src/context/editor-api/node.ts
+++ b/apps/web/src/context/editor-api/node.ts
@@ -37,6 +37,7 @@ import type {
NodeRenderRequest,
NodeRenderResult,
} from "@diffusionstudio/cli/channels";
+import type { CapturedImage } from "@/components/engine/encode/image-encoder";
import type { EncoderConfig } from "@/components/engine/encode/interfaces";
import type { Accessor } from "solid-js";
@@ -289,7 +290,7 @@ export function handleNodeGrep(engine: Accessor<Engine>) {
}
export function handleNodeCapture(engine: Accessor<Engine>) {
- return async ({ id, frames, timestamp }: { id: number; frames?: number[]; timestamp?: boolean }): Promise<{ base64: string }[]> => {
+ return async ({ id, frames }: { id: number; frames?: number[] }): Promise<CapturedImage[]> => {
const e = engine();
const w = e.world;
@@ -312,7 +313,6 @@ export function handleNodeCapture(engine: Accessor<Engine>) {
const encoder = await createImageEncoder(w, {
eid,
frames: shots,
- timestamp: timestamp !== false,
resolution: 720,
});
const result = await encoder.render();
@@ -320,7 +320,7 @@ export function handleNodeCapture(engine: Accessor<Engine>) {
if (result.type === "canceled") throw new Error("Capture canceled");
if (result.type === "error") throw result.error;
- return result.data.map((base64) => ({ base64 }));
+ return result.data;
};
}
diff --git a/reference/media/grab.md b/reference/media/grab.md
index 08a17403..e30fbbdb 100644
--- a/reference/media/grab.md
+++ b/reference/media/grab.md
@@ -1,6 +1,6 @@
# `dapi media grab <id|path>`
-Decodes one or more frames of a video asset at the given times and writes each to a PNG file; like [`node capture`](../node/capture.md), but grabs the asset's own pixels at full resolution (unlike `node capture`, which captures the composited canvas). Renders locally; no credits.
+Decodes one or more frames of a video asset at the given times and writes each to a PNG file named after its `HH-MM-SS-FF` timecode (e.g. `00-00-01-12.png`); like [`node capture`](../node/capture.md), but grabs the asset's own pixels at full resolution (unlike `node capture`, which captures the composited canvas). Renders locally; no credits.
## Input
@@ -10,16 +10,15 @@ Decodes one or more frames of a video asset at the given times and writes each t
- `-s, --start <time>`: with `--count`, the start of the window to sample (optional; `Time` value; default `0`).
- `-e, --end <time>`: with `--count`, the end of the window to sample (optional; `Time` value; default the asset duration).
- `-q, --quality <preset>`: per-frame resolution preset (optional; default `small`). One of `small` (384x384), `medium` (768x768), `large` (1536x1536), or `fullres` (native). Each caps the total pixel count while preserving aspect ratio.
-- `--no-timestamp`: don't stamp each frame with its `HH:MM:SS:FF` timestamp label (optional; the label is drawn by default).
- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--time`) is rejected.
-- `-o, --output <dir>`: directory to write the PNGs into (optional; default the system temp directory). Each frame is written as a uuid-named `.png`.
+- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-grab-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites frames whose timecode matches; requested times that land on the same frame share one file.
## Output
JSON Lines, one object per requested timestamp, in the requested order:
```ts
-{ time: number; path: string } // time = the requested timestamp in seconds; path = the written PNG
+{ time: number; path: string } // time = the requested timestamp in seconds; path = the written PNG, named HH-MM-SS-FF.png
```
## Errors
diff --git a/reference/node/capture.md b/reference/node/capture.md
index f3902407..c751e3bd 100644
--- a/reference/node/capture.md
+++ b/reference/node/capture.md
@@ -1,17 +1,18 @@
-# `dapi node capture [id] [-t, --time <time...>]`
+# `dapi node capture <id> [-t, --time <time...>]`
+
+Renders a node in isolation and writes one PNG per timeline position, each named after the `HH-MM-SS-FF` timecode of the frame actually rendered (e.g. `00-00-01-12.png`). The node is drawn offscreen at 720p height, tightly framed to its own bounds on a transparent background; siblings and overlapping scene content are not included, so capture a scene id to check composition. To grab a video asset's own pixels at full resolution instead of the composited node, use [`media grab`](../media/grab.md).
-Focuses the node on the canvas and captures it, writing one PNG to the system temp directory per timeline position. Each capture is stamped in the top-left with its `HH:MM:SS:FF` timestamp. If `[id]` is omitted, captures the canvas (the active scene). To grab a video asset's own pixels at full resolution instead of the composited canvas, use [`media grab`](../media/grab.md).
## Input
-- `[id]`: node id (optional; defaults to the canvas)
-- `-t, --time <time...>`: one or more timeline positions at which to record the node, each a `Time` value (optional; defaults to the current playhead position)
-- `--no-timestamp`: don't stamp each capture with its `HH:MM:SS:FF` timestamp label
+- `<id>`: node id to capture (required)
+- `-t, --time <time...>`: one or more positions to capture, relative to the node's start (`0` = its first visible frame), each a `Time` value (optional; default `0`)
+- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-capture-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites captures whose timecode matches; requested times that land on the same frame share one file.
## Output
-One JSON object per line (JSON Lines), in the order the times were given: each has the requested time in seconds (`null` for the current playhead) and the absolute path to a freshly written PNG in the system temp directory with a uuid filename.
+One JSON object per line (JSON Lines), in the order the times were given: the requested time in seconds and the absolute path to the written PNG.
```ts
-{ time: number | null, path: string } // e.g. { "time": 1.5, "path": "/tmp/3f2c1a8e-....png" }
+{ time: number, path: string } // e.g. { "time": 1.5, "path": "/tmp/dapi-capture-3f2c1a8e/00-00-01-15.png" }
```