Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance media frame and node capture functionality with timecoded images

konstantin-paulus4e47a7d00f82
diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts
index 8bbc9b87..82f3d29e 100644
--- a/apps/cli/src/cli-channels.ts
+++ b/apps/cli/src/cli-channels.ts
@@ -116,9 +116,29 @@ export type MediaFrameRequest = AssetRef & {
end?: number;
quality?: FrameQuality;
auto?: boolean;
+ combine?: boolean;
+ perSheet?: number;
};
-export type MediaFrameResult = Array<{ time: number; timecode: string; base64: string }>;
+/** Beyond this the cells get too small to be worth the tokens; use `filmstrip`. */
+export const MAX_FRAMES_PER_SHEET = 12;
+
+/**
+ * One written image: a single frame stamped with its timecode, or a contact
+ * sheet stamped with the span it covers (`0f-08s10f`).
+ */
+export type TimecodedImage = { timecode: string; base64: string };
+
+export type MediaFrameResult = TimecodedImage[];
+
+export type NodeCaptureRequest = {
+ id: number;
+ frames?: number[];
+ combine?: boolean;
+ perSheet?: number;
+};
+
+export type NodeCaptureResult = TimecodedImage[];
export type MediaTranscribeRequest = AssetRef;
export type TranscriptWord = { text: string; start: number; end: number };
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index bfcebd4c..16b3d5c8 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -17,7 +17,8 @@ import { listLocalFonts } from "./fonts";
import { buildIssueReport } from "./report";
import { openFolder } from "./open-folder";
import { fetchVideo } from "./ytdlp";
-import type { AssetRef, EncoderConfigInput, FrameQuality, LogEntry, LogLevel, NodePatch } from "./protocol";
+import { MAX_FRAMES_PER_SHEET } from "./protocol";
+import type { AssetRef, EncoderConfigInput, FrameQuality, LogEntry, LogLevel, NodePatch, TimecodedImage } from "./protocol";
// Long-running commands (renders, AI generation) override the default 60s.
const GENERATE = { context: { timeoutMs: GENERATE_TIMEOUT_MS } };
@@ -432,23 +433,23 @@ async function grepNodes(pattern: string, id: string | undefined, opts: NodeGrep
}
}
-type CaptureOptions = { time?: string[]; output?: string };
+type CaptureOptions = { time?: string[]; output?: string; separate?: boolean; perSheet?: string };
async function nodeCapture(id: string, opts: CaptureOptions): Promise<void> {
const eid = parseNodeIds([id])[0];
const times = (opts.time ?? ["0"]).map((t) => parseTimeArg(t, "--time"));
const frames = times.map((t) => Math.round(t * TIME_FPS));
+ const perSheet = parsePerSheet(opts.perSheet, opts.separate);
const dir = opts.output ?? join(tmpdir(), `dapi-capture-${randomUUID().slice(0, 8)}`);
mkdirSync(dir, { recursive: true });
try {
- const shots = await editor.node.capture.query({ id: eid, frames }, GENERATE);
- for (const [i, { base64, timecode }] of shots.entries()) {
- const path = join(dir, timecodeFilename(timecode));
- writeFileSync(path, Buffer.from(base64, "base64"));
- console.log(JSON.stringify({ time: times[i], path }));
- }
+ const images = await editor.node.capture.query(
+ { id: eid, frames, combine: !opts.separate, perSheet },
+ GENERATE,
+ );
+ writeImages(images, dir);
} catch (e) {
handleSocketError(e);
}
@@ -468,6 +469,8 @@ type MediaFrameOptions = {
uncapped?: boolean;
output?: string;
auto?: boolean;
+ separate?: boolean;
+ perSheet?: string;
};
async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
@@ -520,16 +523,23 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
quality = opts.quality as FrameQuality;
}
+ const perSheet = parsePerSheet(opts.perSheet, opts.separate);
const target = resolveAssetRef(ref);
const dir = opts.output ?? join(tmpdir(), `dapi-grab-${randomUUID().slice(0, 8)}`);
mkdirSync(dir, { recursive: true });
try {
- const frames = await editor.media.frame.query({ ...target, times, count, start, end, quality, auto: opts.auto });
- for (const { time, timecode, base64 } of frames) {
- const path = join(dir, timecodeFilename(timecode));
- writeFileSync(path, Buffer.from(base64, "base64"));
- console.log(JSON.stringify({ time, path }));
- }
+ const images = await editor.media.frame.query({
+ ...target,
+ times,
+ count,
+ start,
+ end,
+ quality,
+ auto: opts.auto,
+ combine: !opts.separate,
+ perSheet,
+ });
+ writeImages(images, dir);
} catch (e) {
handleSocketError(e);
}
@@ -614,9 +624,29 @@ function parseTimeArg(value: string, flag: string, allowNegative = false): numbe
return seconds;
}
-function timecodeFilename(timecode: string): string {
- const [hh, mm, ss, ff] = timecode.split(":");
- return `${hh}h${mm}m${ss}s${ff}f.png`;
+// Frames and contact sheets arrive in the same shape: the app stamps each
+// image with its timecode (`08s10f`, or `0f-08s10f` for a sheet), which is the
+// filename too.
+function writeImages(images: TimecodedImage[], dir: string): void {
+ for (const { timecode, base64 } of images) {
+ const path = join(dir, `${timecode}.png`);
+ writeFileSync(path, Buffer.from(base64, "base64"));
+ console.log(JSON.stringify({ timecode, path }));
+ }
+}
+
+function parsePerSheet(value: string | undefined, separate?: boolean): number | undefined {
+ if (value === undefined) return undefined;
+ if (separate) {
+ console.error("--per-sheet lays out contact sheets; it cannot be combined with --separate.");
+ process.exit(1);
+ }
+ const n = Number(value);
+ if (!Number.isInteger(n) || n < 1 || n > MAX_FRAMES_PER_SHEET) {
+ console.error(`--per-sheet must be an integer between 1 and ${MAX_FRAMES_PER_SHEET} (got "${value}")`);
+ process.exit(1);
+ }
+ return n;
}
// Parse the window/scale flags shared by `filmstrip` and `waveform`.
@@ -997,12 +1027,26 @@ function formatLogEntry(entry: LogEntry): string {
type ScreenshotOptions = { output?: string };
+// `diffusion-studio_2026-07-31_08-55-12.png`
+function screenshotFilename(taken: Date, attempt: number): string {
+ const pad = (value: number) => String(value).padStart(2, "0");
+ const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-");
+ const time = [pad(taken.getHours()), pad(taken.getMinutes()), pad(taken.getSeconds())].join("-");
+ const slug = APP_NAME.toLowerCase().replace(/[^a-z0-9]+/g, "-");
+ return `${slug}_${date}_${time}${attempt > 1 ? `-${attempt}` : ""}.png`;
+}
+
async function appScreenshot(opts: ScreenshotOptions): Promise<void> {
const dir = opts.output ?? tmpdir();
mkdirSync(dir, { recursive: true });
try {
const { base64, width, height } = await editor.screenshot.query();
- const path = join(dir, `${randomUUID()}.png`);
+ const taken = new Date();
+ let attempt = 1;
+ let path = join(dir, screenshotFilename(taken, attempt));
+ while (existsSync(path)) {
+ path = join(dir, screenshotFilename(taken, ++attempt));
+ }
writeFileSync(path, Buffer.from(base64, "base64"));
console.log(JSON.stringify({ path, width, height }));
} catch (e) {
@@ -1309,7 +1353,7 @@ media
.command("grab")
.alias("sample")
.description(
- `Decode frames of a video file and write them as PNGs (local render, no credits), each named after its timecode (e.g. 00h00m01s15f.png). Grabs the asset's own pixels at full resolution, unlike \`node capture\` which renders the composited node. The recommended tool for understanding a video at the frame level.`,
+ `Decode frames of a video file and write them as PNGs (local render, no credits). By default the frames are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (\`08s10f\`, zero segments dropped) and drawn as large as fits, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one (\`--separate\` writes a PNG per frame). Grabs the asset's own pixels, unlike \`node capture\` which renders the composited node. The recommended tool for understanding a video at the frame level; past ~12 frames prefer \`media filmstrip\`.`,
)
.argument("<id|path>", "video asset id, or a local video file to grab frames from")
.option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`)
@@ -1317,7 +1361,9 @@ media
.option("-a, --auto", "scan the clip at 2fps and keep a frame each time the footage settles into a new visual state (transitions are waited out, so picks stay sharp); returns at most --count frames (default cap: 30), static footage like screen recordings returns far fewer; requires WebGPU")
.option("-s, --start <time>", `with --count or --auto, start of the window to sample (seconds, "45f" frames, or "MM:SS"; default: 0)`)
.option("-e, --end <time>", `with --count or --auto, end of the window to sample (seconds, "45f" frames, or "MM:SS"; default: asset duration)`)
- .option("-q, --quality <preset>", "frame resolution: small (384x384, default), medium (768x768), large (1536x1536), or fullres (native)")
+ .option("-q, --quality <preset>", "frame resolution: small (384x384), medium (768x768), large (1536x1536), or fullres (native); default: as large as the sheet cell allows, or small with --separate")
+ .option("-S, --separate", "write one PNG per frame instead of merging them into contact sheets")
+ .option("--per-sheet <n>", "frames per contact sheet, 1-12; fewer frames means a larger cell each (default: as many as fit)")
.option("--uncapped", "lift the 100-frame safety cap (grabbing many frames is slow and token-heavy)")
.option("-o, --output <dir>", "directory to write the PNGs into (default: a fresh dir in the system temp dir)")
.action((ref: string, opts: MediaFrameOptions) => mediaFrame(ref, opts));
@@ -1471,10 +1517,12 @@ node
node
.command("capture")
.description(
- `Render a node in isolation to PNGs, one per position, each named after the timecode of the frame rendered (e.g. 00h00m01s15f.png). The node is drawn offscreen at 720p height, tightly framed to its own bounds on a transparent background — siblings and overlapping scene content are not included, so capture a scene id to check composition ("what plays at time T": layout, overlaps, text, timing). For a video asset's own full-resolution pixels use \`media grab\`.`,
+ `Render a node in isolation to PNGs. By default the positions are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (\`08s10f\`, zero segments dropped) and rendered as large as fits, so a few positions arrive as one high-resolution picture instead of a directory to open one by one (\`--separate\` writes a PNG per position, at 720p height, keeping the alpha channel). The node is drawn offscreen, tightly framed to its own bounds on a transparent background; siblings and overlapping scene content are not included, so capture a scene id to check composition ("what plays at time T": layout, overlaps, text, timing). For a video asset's own full-resolution pixels use \`media grab\`.`,
)
.argument("<id>", "node id to capture")
.option("-t, --time <time...>", `one or more positions to capture, relative to the node's start (0 = its first visible frame) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0, the node's first visible frame)`)
+ .option("-S, --separate", "write one PNG per position instead of merging them into contact sheets")
+ .option("--per-sheet <n>", "positions per contact sheet, 1-12; fewer means a larger cell each (default: as many as fit)")
.option("-o, --output <dir>", "directory to write the PNGs into (default: a fresh dir in the system temp dir)")
.action((id: string, opts: CaptureOptions) => nodeCapture(id, opts));
diff --git a/apps/web/src/components/engine/encode/image-encoder.ts b/apps/web/src/components/engine/encode/image-encoder.ts
index 3dd782fd..38d251f9 100644
--- a/apps/web/src/components/engine/encode/image-encoder.ts
+++ b/apps/web/src/components/engine/encode/image-encoder.ts
@@ -17,7 +17,7 @@ import { renderSystem } from '../systems/render';
import { cloneFromRecords, serializeEntity } from '../api/serialize';
import { getEntityTree } from '../api/query';
import { realizeMounts } from '@/utils/mount';
-import { framesToSeconds, formatTimestamp } from '../utils';
+import { framesToSeconds, formatTimecode } from '../utils';
import { assert } from '@/utils';
import type { EngineWorld } from '../api/world';
@@ -31,7 +31,7 @@ type ImageEncoderConfig = {
resolution?: number;
};
-/** One capture: the PNG plus the HH:MM:SS:FF timecode of the frame rendered. */
+/** One capture: the PNG plus the timecode of the frame rendered, e.g. `01s15f`. */
export type CapturedImage = { base64: string; timecode: string };
export type ImageExportResult =
@@ -173,13 +173,18 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: Image
const boundsWidth = Math.max(1, bounds.maxX - bounds.minX);
const boundsHeight = Math.max(1, bounds.maxY - bounds.minY);
- const scale = config.resolution ? config.resolution / boundsHeight : 1;
- offscreenCanvas.width = Math.max(1, Math.round(boundsWidth * scale));
- offscreenCanvas.height = Math.max(1, Math.round(boundsHeight * scale));
+ /** Re-target the output height; callers that lay frames out do this once measured. */
+ const resize = (height?: number) => {
+ const scale = height ? height / boundsHeight : 1;
+ offscreenCanvas.width = Math.max(1, Math.round(boundsWidth * scale));
+ offscreenCanvas.height = Math.max(1, Math.round(boundsHeight * scale));
+ world.resolution = scale;
+ };
+
+ resize(config.resolution);
// Shift the measured box onto the canvas so the node is centered in view;
// the camera translation is scaled by world.resolution at render time.
- world.resolution = scale;
world.camera.e = -bounds.minX;
world.camera.f = -bounds.minY;
@@ -204,7 +209,7 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: Image
const stampFrame = startFrame + frame;
images.push({
base64: await toBase64Png(offscreenCanvas),
- timecode: formatTimestamp(framesToSeconds(stampFrame, world.frameRate), world.frameRate),
+ timecode: formatTimecode(framesToSeconds(stampFrame, world.frameRate), world.frameRate),
});
}
@@ -224,6 +229,8 @@ export async function createImageEncoder(sourceWorld: EngineWorld, config: Image
return {
render,
cancel,
+ resize,
+ bounds: { width: boundsWidth, height: boundsHeight },
};
}
diff --git a/apps/web/src/components/engine/utils/contact-sheet.ts b/apps/web/src/components/engine/utils/contact-sheet.ts
new file mode 100644
index 00000000..483cc504
--- /dev/null
+++ b/apps/web/src/components/engine/utils/contact-sheet.ts
@@ -0,0 +1,142 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { MAX_FRAMES_PER_SHEET } from '@diffusionstudio/cli/channels';
+import { stampTimestampLabel } from './transcode';
+
+/**
+ * Contact sheets: a handful of frames merged into one labelled image, so a
+ * grab or a capture reads as a single picture instead of a directory of PNGs.
+ * The sheet never exceeds the largest image a vision model reads at full
+ * detail (92x52 patches of 28px), and frames are never upscaled past their
+ * source resolution.
+ */
+export const SHEET_MAX_WIDTH = 2576;
+export const SHEET_MAX_HEIGHT = 1456;
+
+const MARGIN = 4;
+const GUTTER = 8;
+const BACKGROUND = '#444';
+
+export type SheetPlan = {
+ columns: number;
+ rows: number;
+ /** Size each frame is drawn at, in px. */
+ cellWidth: number;
+ cellHeight: number;
+ width: number;
+ height: number;
+};
+
+/**
+ * Pick the grid for `count` frames of the given source size: the layout that
+ * draws each frame as large as possible within the sheet budget, preferring
+ * fewer blank cells and then wider rows when the sizes tie. The last row may be
+ * partially filled, and frames are never enlarged past `source`.
+ */
+export function planSheet(count: number, source: { width: number; height: number }): SheetPlan {
+ const cells = Math.max(1, Math.min(count, MAX_FRAMES_PER_SHEET));
+ let best: SheetPlan | undefined;
+
+ for (let columns = 1; columns <= cells; columns++) {
+ const rows = Math.ceil(cells / columns);
+ const availableWidth = SHEET_MAX_WIDTH - 2 * MARGIN - GUTTER * (columns - 1);
+ const availableHeight = SHEET_MAX_HEIGHT - 2 * MARGIN - GUTTER * (rows - 1);
+ if (availableWidth < columns || availableHeight < rows) continue;
+
+ // Fit the frame into its cell, never enlarging it past its own pixels.
+ const scale = Math.min(availableWidth / columns / source.width, availableHeight / rows / source.height, 1);
+ const cellWidth = Math.max(1, Math.floor(source.width * scale));
+ const cellHeight = Math.max(1, Math.floor(source.height * scale));
+
+ const candidate: SheetPlan = {
+ columns,
+ rows,
+ cellWidth,
+ cellHeight,
+ width: 2 * MARGIN + columns * cellWidth + GUTTER * (columns - 1),
+ height: 2 * MARGIN + rows * cellHeight + GUTTER * (rows - 1),
+ };
+ if (best === undefined || isBetterPlan(candidate, best, cells)) best = candidate;
+ }
+
+ // Every source fits at 1x1, so the loop always yields a plan.
+ return best!;
+}
+
+function isBetterPlan(candidate: SheetPlan, best: SheetPlan, cells: number): boolean {
+ const candidateArea = candidate.cellWidth * candidate.cellHeight;
+ const bestArea = best.cellWidth * best.cellHeight;
+ if (candidateArea !== bestArea) return candidateArea > bestArea;
+
+ const candidateBlanks = candidate.columns * candidate.rows - cells;
+ const bestBlanks = best.columns * best.rows - cells;
+ if (candidateBlanks !== bestBlanks) return candidateBlanks < bestBlanks;
+
+ // Same cells either way: fill rows before starting new ones.
+ return candidate.columns > best.columns;
+}
+
+/**
+ * A sheet is stamped with the span it covers, earliest to latest: `0f-08s10f`,
+ * or just the one timecode when every cell lands on the same frame. The cells
+ * themselves stay in the order they were requested in, hence `at`.
+ */
+export function sheetTimecode(cells: Array<{ at: number; timecode: string }>): string {
+ const sorted = [...cells].sort((a, b) => a.at - b.at);
+ const first = sorted[0].timecode;
+ const last = sorted[sorted.length - 1].timecode;
+ return first === last ? first : `${first}-${last}`;
+}
+
+/**
+ * Split `total` frames into balanced sheets of at most `perSheet`, so a run of
+ * 13 becomes 7 + 6 rather than 12 + 1. Returns the size of each sheet.
+ */
+export function planSheetSizes(total: number, perSheet?: number): number[] {
+ const max = Math.max(1, Math.min(Math.round(perSheet ?? MAX_FRAMES_PER_SHEET), MAX_FRAMES_PER_SHEET));
+ const sheets = Math.max(1, Math.ceil(total / max));
+ const base = Math.floor(total / sheets);
+ const extra = total % sheets;
+ return Array.from({ length: sheets }, (_, i) => base + (i < extra ? 1 : 0));
+}
+
+/**
+ * Draw frames into a sheet in reading order, each stamped with its timecode.
+ * The gutters are opaque, so transparent captures composite onto flat grey.
+ */
+export function composeSheet(
+ frames: Array<{ image: CanvasImageSource; label: string }>,
+ plan: SheetPlan,
+): string {
+ const canvas = document.createElement('canvas');
+ canvas.width = plan.width;
+ canvas.height = plan.height;
+ const ctx = canvas.getContext('2d')!;
+
+ ctx.fillStyle = BACKGROUND;
+ ctx.fillRect(0, 0, canvas.width, canvas.height);
+
+ for (const [i, frame] of frames.entries()) {
+ const column = i % plan.columns;
+ const row = Math.floor(i / plan.columns);
+ const x = MARGIN + column * (plan.cellWidth + GUTTER);
+ const y = MARGIN + row * (plan.cellHeight + GUTTER);
+
+ ctx.drawImage(frame.image, x, y, plan.cellWidth, plan.cellHeight);
+
+ ctx.save();
+ ctx.translate(x, y);
+ stampTimestampLabel(ctx, plan.cellHeight, frame.label, { minBandHeight: 30 });
+ ctx.restore();
+ }
+
+ const dataUrl = canvas.toDataURL('image/png');
+ return dataUrl.slice(dataUrl.indexOf(',') + 1);
+}
+
+export async function decodePngBase64(base64: string): Promise<ImageBitmap> {
+ const bytes = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0));
+ return createImageBitmap(new Blob([bytes], { type: 'image/png' }));
+}
diff --git a/apps/web/src/components/engine/utils/index.ts b/apps/web/src/components/engine/utils/index.ts
index 4d4d9ab3..a3124219 100644
--- a/apps/web/src/components/engine/utils/index.ts
+++ b/apps/web/src/components/engine/utils/index.ts
@@ -9,5 +9,6 @@ export * from './time';
export * from './sort';
export * from './cursor';
export * from './transcode';
+export * from './contact-sheet';
export * from './audio-sync';
export * from './captions';
diff --git a/apps/web/src/components/engine/utils/transcode.ts b/apps/web/src/components/engine/utils/transcode.ts
index ba136f16..22db4fbf 100644
--- a/apps/web/src/components/engine/utils/transcode.ts
+++ b/apps/web/src/components/engine/utils/transcode.ts
@@ -87,15 +87,34 @@ const SILENCE_THRESHOLD = 12; // peak value (0-255) at/below which audio counts
const SILENCE_MIN_SECONDS = 0.4;
const AUDIO_RULER_FRAME_RATE = 30;
-export function formatTimestamp(seconds: number, frameRate: number): string {
+function splitTimecode(seconds: number, frameRate: number): [hours: number, minutes: number, seconds: number, frames: number] {
const fps = Math.max(1, Math.round(frameRate));
const totalFrames = Math.round(seconds * fps);
- const frames = totalFrames % fps;
const totalSeconds = Math.floor(totalFrames / fps);
- const hh = Math.floor(totalSeconds / 3600);
- const mm = Math.floor((totalSeconds % 3600) / 60);
- const ss = totalSeconds % 60;
- return [hh, mm, ss, frames].map((v) => String(v).padStart(2, '0')).join(':');
+ return [
+ Math.floor(totalSeconds / 3600),
+ Math.floor((totalSeconds % 3600) / 60),
+ totalSeconds % 60,
+ totalFrames % fps,
+ ];
+}
+
+/** `HH:MM:SS:FF`, fixed width so ruler ticks line up. */
+export function formatTimestamp(seconds: number, frameRate: number): string {
+ return splitTimecode(seconds, frameRate).map((v) => String(v).padStart(2, '0')).join(':');
+}
+
+/**
+ * The same timecode with its zero segments dropped, for labels and filenames:
+ * `08s10f`, `01m05s`, `0f`. Each segment carries its unit, so nothing is
+ * ambiguous once the empty ones are gone.
+ */
+export function formatTimecode(seconds: number, frameRate: number): string {
+ const units = ['h', 'm', 's', 'f'];
+ const stamp = splitTimecode(seconds, frameRate)
+ .map((value, i) => (value === 0 ? '' : `${String(value).padStart(2, '0')}${units[i]}`))
+ .join('');
+ return stamp || '0f';
}
// Draws a timestamp label into the top-left corner of a canvas: white text with
@@ -104,8 +123,9 @@ export function stampTimestampLabel(
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
height: number,
label: string,
+ options?: { minBandHeight?: number },
): void {
- const bandHeight = Math.max(20, Math.round(height * 0.06));
+ const bandHeight = Math.max(options?.minBandHeight ?? 20, Math.round(height * 0.06));
const fontSize = Math.round(bandHeight * 0.72);
const paddingLeft = Math.round(bandHeight * 0.4);
const y = paddingLeft + bandHeight / 2;
diff --git a/apps/web/src/context/editor-api/media.ts b/apps/web/src/context/editor-api/media.ts
index 64577b28..1c094275 100644
--- a/apps/web/src/context/editor-api/media.ts
+++ b/apps/web/src/context/editor-api/media.ts
@@ -6,7 +6,7 @@ import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
import { ElectronFileHandle } from '@/lib/electron-file-handle';
import { pickInformativeTimes } from './frame-triage';
import { trpc } from '@/lib/trpc';
-import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimestamp } from '@/components/engine';
+import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimecode, composeSheet, planSheet, planSheetSizes, sheetTimecode } from '@/components/engine';
import { assert } from '@/utils';
import {
transcodeForTranscription,
@@ -17,7 +17,7 @@ import {
import type { Engine } from "@/components/engine";
import type { Asset } from "@/components/engine/db";
-import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaProbeRequest, AssetRef, MediaTranscribeRequest, MediaTranscribeResult, MediaFilmstripRequest, MediaFilmstripResult, MediaWaveformRequest, MediaWaveformResult, TranscriptSegment } from "@diffusionstudio/cli/channels";
+import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaFrameResult, TimecodedImage, MediaProbeRequest, AssetRef, MediaTranscribeRequest, MediaTranscribeResult, MediaFilmstripRequest, MediaFilmstripResult, MediaWaveformRequest, MediaWaveformResult, TranscriptSegment } from "@diffusionstudio/cli/channels";
import type { Accessor } from "solid-js";
/**
@@ -112,8 +112,9 @@ const FRAME_QUALITY_BUDGETS = {
const AUTO_MAX_FRAMES = 30;
export function handleMediaFrame(engine: Accessor<Engine>) {
- return async (req: MediaFrameRequest) => {
+ return async (req: MediaFrameRequest): Promise<MediaFrameResult> => {
const { times, count, start, end, quality, auto } = req;
+ const combine = req.combine ?? true;
const { world } = engine();
const asset = await resolveAssetRef(world, req);
const id = asset.id;
@@ -147,7 +148,7 @@ export function handleMediaFrame(engine: Accessor<Engine>) {
});
}
- const budget = FRAME_QUALITY_BUDGETS[quality ?? "small"];
+ const budget = FRAME_QUALITY_BUDGETS[quality ?? (combine ? "fullres" : "small")];
const blob = await getAssetFile(asset);
const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });
@@ -171,32 +172,74 @@ export function handleMediaFrame(engine: Accessor<Engine>) {
// only the width lets the sink derive a matching height.
const displayWidth = await track.getDisplayWidth();
const displayHeight = await track.getDisplayHeight();
- let width: number | undefined;
+ let sourceWidth = displayWidth;
+ let sourceHeight = displayHeight;
if (budget > 0 && displayWidth * displayHeight > budget) {
- width = Math.max(1, Math.round(displayWidth * Math.sqrt(budget / (displayWidth * displayHeight))));
+ const scale = Math.sqrt(budget / (displayWidth * displayHeight));
+ sourceWidth = Math.max(1, Math.round(displayWidth * scale));
+ sourceHeight = Math.max(1, Math.round(displayHeight * scale));
}
+ // Lay the sheets out up front: the largest cell across them sets the
+ // decode size, so no frame is decoded bigger than it will be drawn.
+ const sizes = combine ? planSheetSizes(requested.length, req.perSheet) : [];
+ const plans = sizes.map((n) => planSheet(n, { width: sourceWidth, height: sourceHeight }));
+ const width = combine
+ ? Math.min(sourceWidth, Math.max(...plans.map((plan) => plan.cellWidth)))
+ : sourceWidth;
+
// Decode in ascending order (the sink's fast path), remember each
// entry's original slot so output mirrors the requested order.
const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time);
// No pool: each yielded canvas is fresh, so converting to PNG can't race
// the generator's read-ahead reusing a pooled canvas.
- const sink = new CanvasSink(track, width !== undefined ? { width } : undefined);
+ const sink = new CanvasSink(track, width < displayWidth ? { width } : undefined);
const timestamps = ordered.map(({ time }) => firstTimestamp + time);
- const result: Array<{ time: number; timecode: string; base64: string }> = new Array(requested.length);
+ // Which sheet each frame belongs to, and where that sheet starts.
+ const sheetOf: number[] = [];
+ const sheetStart: number[] = [];
+ for (const [sheet, size] of sizes.entries()) {
+ sheetStart.push(sheetOf.length);
+ for (let k = 0; k < size; k++) sheetOf.push(sheet);
+ }
+ const missing = [...sizes];
+
+ const cells: Array<{ at: number; timecode: string }> = new Array(requested.length);
+ const canvases: Array<CanvasImageSource | undefined> = new Array(requested.length);
+ const result: TimecodedImage[] = new Array(combine ? sizes.length : requested.length);
+
let i = 0;
for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {
const { time, index } = ordered[i++];
assert(wrapped, `No frame found at ${time}s.`);
+ const timecode = formatTimecode(time, asset.frameRate);
+ cells[index] = { at: time, timecode };
- result[index] = {
- time,
- timecode: formatTimestamp(time, asset.frameRate),
- base64: await canvasToPngBase64(wrapped.canvas),
+ if (!combine) {
+ result[index] = { timecode, base64: await canvasToPngBase64(wrapped.canvas) };
+ continue;
+ }
+
+ // Compose a sheet as soon as its last frame lands and drop the
+ // canvases, so a long run never holds every decoded frame at once.
+ canvases[index] = wrapped.canvas;
+ const sheet = sheetOf[index];
+ if (--missing[sheet] > 0) continue;
+
+ const from = sheetStart[sheet];
+ const to = from + sizes[sheet];
+ result[sheet] = {
+ timecode: sheetTimecode(cells.slice(from, to)),
+ base64: composeSheet(
+ canvases.slice(from, to).map((canvas, k) => ({ image: canvas!, label: cells[from + k].timecode })),
+ plans[sheet],
+ ),
};
+ for (let k = from; k < to; k++) canvases[k] = undefined;
}
+
return result;
} finally {
input.dispose();
diff --git a/apps/web/src/context/editor-api/node.ts b/apps/web/src/context/editor-api/node.ts
index 26ce7af9..f015cba6 100644
--- a/apps/web/src/context/editor-api/node.ts
+++ b/apps/web/src/context/editor-api/node.ts
@@ -17,6 +17,11 @@ import {
getEntityTree,
cloneSubtree,
serializeEntity,
+ composeSheet,
+ decodePngBase64,
+ planSheet,
+ planSheetSizes,
+ sheetTimecode,
} from "@/components/engine";
import { ChildOf } from "@/components/engine/components";
import { ANIMATION_TYPE_MAP, CAPTION_ALIGN_MAP, WorldDocument } from "@/utils/jsx";
@@ -36,8 +41,10 @@ import type {
NodeTree,
NodeRenderRequest,
NodeRenderResult,
+ NodeCaptureRequest,
+ NodeCaptureResult,
+ TimecodedImage,
} from "@diffusionstudio/cli/channels";
-import type { CapturedImage } from "@/components/engine/encode/image-encoder";
import type { EncoderConfig } from "@/components/engine/encode/interfaces";
import type { Accessor } from "solid-js";
@@ -289,8 +296,11 @@ export function handleNodeGrep(engine: Accessor<Engine>) {
};
}
+// Ceiling on the height a sheet cell renders a node at.
+const SHEET_CAPTURE_HEIGHT = 1080;
+
export function handleNodeCapture(engine: Accessor<Engine>) {
- return async ({ id, frames }: { id: number; frames?: number[] }): Promise<CapturedImage[]> => {
+ return async ({ id, frames, combine = true, perSheet }: NodeCaptureRequest): Promise<NodeCaptureResult> => {
const e = engine();
const w = e.world;
@@ -315,12 +325,43 @@ export function handleNodeCapture(engine: Accessor<Engine>) {
frames: shots,
resolution: 720,
});
+
+ // Sheets render at their cell size instead of the flat 720p: with a few
+ // frames that is sharper than a standalone capture, never coarser. A node
+ // is drawn, not decoded, so a small one is worth rendering past its own
+ // bounds; beyond SHEET_CAPTURE_HEIGHT that only costs tokens.
+ const aspect = encoder.bounds.width / encoder.bounds.height;
+ const height = Math.max(encoder.bounds.height, SHEET_CAPTURE_HEIGHT);
+ const sizes = combine ? planSheetSizes(shots.length, perSheet) : [];
+ const plans = sizes.map((n) => planSheet(n, { width: height * aspect, height }));
+ if (combine) {
+ encoder.resize(Math.max(...plans.map((plan) => plan.cellHeight)));
+ }
+
const result = await encoder.render();
if (result.type === "canceled") throw new Error("Capture canceled");
if (result.type === "error") throw result.error;
- return result.data;
+ if (!combine) return result.data;
+
+ const sheets: TimecodedImage[] = [];
+ let offset = 0;
+ for (const [sheet, size] of sizes.entries()) {
+ const group = result.data.slice(offset, offset + size);
+ const cells = group.map(({ timecode }, k) => ({ at: shots[offset + k], timecode }));
+ offset += size;
+ const images = await Promise.all(group.map((image) => decodePngBase64(image.base64)));
+ sheets.push({
+ timecode: sheetTimecode(cells),
+ base64: composeSheet(
+ images.map((image, k) => ({ image, label: group[k].timecode })),
+ plans[sheet],
+ ),
+ });
+ for (const image of images) image.close();
+ }
+ return sheets;
};
}
diff --git a/reference/README.md b/reference/README.md
index 9c1b78ed..914e5bb3 100644
--- a/reference/README.md
+++ b/reference/README.md
@@ -50,7 +50,7 @@ How the surface is divided:
- [`dapi node ls`](./node/ls.md): raw entity records
- [`dapi node tree`](./node/tree.md): an entity's subtree as nested JSON
- [`dapi node grep`](./node/grep.md): search entity records with a regex
-- [`dapi node capture`](./node/capture.md): capture a node as a PNG
+- [`dapi node capture`](./node/capture.md): capture a node to a labelled contact sheet, or one PNG per position
- [`dapi node insert`](./node/insert.md): insert JSX tags into an existing parent
- [`dapi node rm`](./node/rm.md): delete entities
- [`dapi node cp`](./node/cp.md): deep-clone nodes
@@ -78,7 +78,7 @@ How the surface is divided:
- [`dapi media probe`](./media/probe.md): container and track metadata
- [`dapi media transcribe`](./media/transcribe.md): timed speech transcript
-- [`dapi media grab`](./media/grab.md): decode video frames to PNGs
+- [`dapi media grab`](./media/grab.md): decode video frames to a labelled contact sheet, or one PNG per frame
- [`dapi media filmstrip`](./media/filmstrip.md): grid of video frames as a PNG
- [`dapi media waveform`](./media/waveform.md): audio waveform PNG with silence highlighting
- [`dapi media listen`](./media/listen.md): AI description of an audio track
diff --git a/reference/media/grab.md b/reference/media/grab.md
index d2244b93..cfbe5ad6 100644
--- a/reference/media/grab.md
+++ b/reference/media/grab.md
@@ -1,6 +1,6 @@
# `dapi media grab <id|path>`
-Decodes one or more frames of a video asset at the given times and writes each to a PNG file named after its timecode, hours through frames (e.g. `00h00m01s12f.png`); like [`node capture`](../node/capture.md), but grabs the asset's own pixels at full resolution (unlike `node capture`, which captures the composited canvas). Renders locally; no credits.
+Decodes one or more frames of a video asset at the given times and merges them into **contact sheets**: up to 12 frames per PNG, each cell labelled with its timecode and drawn as large as the sheet allows, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one. `--separate` writes a PNG per frame instead. Like [`node capture`](../node/capture.md), but grabs the asset's own pixels (unlike `node capture`, which renders the composited node). Renders locally; no credits. Past ~12 frames, [`media filmstrip`](./filmstrip.md) is the cheaper way to scan a clip.
## Input
@@ -9,18 +9,37 @@ Decodes one or more frames of a video asset at the given times and writes each t
- `-c, --count <n>`: instead of `--time`, grab `n` frames evenly spaced across the clip at a fixed interval of `window / n`, starting at the window start (optional; positive integer).
- `-s, --start <time>`: with `--count`, the start of the window to sample (optional; `Time` value; default `0`).
- `-e, --end <time>`: with `--count`, the end of the window to sample (optional; `Time` value; default the asset duration).
-- `-q, --quality <preset>`: per-frame resolution preset (optional; default `small`). One of `small` (384x384), `medium` (768x768), `large` (1536x1536), or `fullres` (native). Each caps the total pixel count while preserving aspect ratio.
+- `-q, --quality <preset>`: per-frame resolution preset (optional). One of `small` (384x384), `medium` (768x768), `large` (1536x1536), or `fullres` (native); each caps the total pixel count while preserving aspect ratio. Defaults to as much detail as the sheet cell can show, or `small` with `--separate`.
+- `-S, --separate`: write one PNG per frame instead of merging them into contact sheets (optional). The frames keep their own resolution and alpha, and each file is named after its timecode (e.g. `01s12f.png`).
+- `--per-sheet <n>`: frames per contact sheet, 1 to 12 (optional; default as many as fit). Fewer frames per sheet means a larger cell each. Sheets are balanced, so 13 frames become 7 + 6 rather than 12 + 1.
- `--uncapped`: lift the 100-frame safety cap (optional). Without it, requesting more than 100 frames (via `--count` or `--time`) is rejected.
-- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-grab-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites frames whose timecode matches; requested times that land on the same frame share one file.
+- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-grab-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites images whose name matches; with `--separate`, requested times that land on the same frame share one file.
+
+## Timecodes
+
+Cell labels, the `timecode` field, and the filenames all use the same stamp, which drops its zero segments: `08s10f` is 8 seconds and 10 frames, `01m05s` is 65 seconds, and the first frame is `0f`. Each segment carries its unit, so nothing is ambiguous once the empty ones are gone. (The rulers `filmstrip` and `waveform` draw stay on fixed-width `HH:MM:SS:FF`, so their ticks line up.)
+
+## Layout
+
+A sheet never exceeds 2576x1456, the largest image a vision model reads at full detail. Within that budget the grid is the one that draws each frame largest, frames are never enlarged past their source resolution, and the last row may be partially filled. Sheets are named after the span they cover, e.g. `0f-08s10f.png`. For 16:9 footage the cell sizes are:
+
+| Frames | Grid | Per-frame |
+|---|---|---|
+| 1 | 1x1 | 2568x1444 |
+| 2-4 | 2x1, 2x2 | 1280x720 |
+| 5-9 | 3x2, 3x3 | 850x478 |
+| 10-12 | 4x3 | 636x358 |
## Output
-JSON Lines, one object per requested timestamp, in the requested order:
+JSON Lines, one object per written image: a contact sheet by default, a frame with `--separate`.
```ts
-{ time: number; path: string } // time = the requested timestamp in seconds; path = the written PNG, e.g. …/00h00m01s12f.png
+{ timecode: string; path: string } // e.g. { "timecode": "0f-08s10f", "path": "…/0f-08s10f.png" }
```
+A sheet's timecode is the span it covers; a frame's is its own. Sheets come in time order, and their cells in the order the times were requested in.
+
## Errors
-Exits non-zero if the id is unknown, the asset is not a video, any `--time` is past the asset's duration, the `--count` window is empty, `--time` and `--count` are combined, `--start`/`--end` are given without `--count`, more than 100 frames are requested without `--uncapped`, or a PNG can't be written.
+Exits non-zero if the id is unknown, the asset is not a video, any `--time` is past the asset's duration, the `--count` window is empty, `--time` and `--count` are combined, `--start`/`--end` are given without `--count`, `--per-sheet` is outside 1 to 12, more than 100 frames are requested without `--uncapped`, or a PNG can't be written.
diff --git a/reference/node/capture.md b/reference/node/capture.md
index 81438790..3304e9e3 100644
--- a/reference/node/capture.md
+++ b/reference/node/capture.md
@@ -1,18 +1,42 @@
# `dapi node capture <id> [-t, --time <time...>]`
-Renders a node in isolation and writes one PNG per timeline position, each named after the timecode of the frame actually rendered, hours through frames (e.g. `00h00m01s12f.png`). The node is drawn offscreen at 720p height, tightly framed to its own bounds on a transparent background; siblings and overlapping scene content are not included, so capture a scene id to check composition. To grab a video asset's own pixels at full resolution instead of the composited node, use [`media grab`](../media/grab.md).
-
+Renders a node in isolation at one or more timeline positions and merges them into **contact sheets**: up to 12 positions per PNG, each cell labelled with the timecode of the frame actually rendered and drawn as large as the sheet allows, so a few positions arrive as one high-resolution picture instead of a directory to open one by one. `--separate` writes a PNG per position instead. The node is drawn offscreen, tightly framed to its own bounds; siblings and overlapping scene content are not included, so capture a scene id to check composition. To grab a video asset's own pixels instead of the composited node, use [`media grab`](../media/grab.md).
## Input
- `<id>`: node id to capture (required)
- `-t, --time <time...>`: one or more positions to capture, relative to the node's start (`0` = its first visible frame), each a `Time` value (optional; default `0`)
-- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-capture-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites captures whose timecode matches; requested times that land on the same frame share one file.
+- `-S, --separate`: write one PNG per position instead of merging them into contact sheets (optional). Each is rendered at 720p height on a transparent background and named after its timecode (e.g. `01s12f.png`). The way to keep the alpha channel.
+- `--per-sheet <n>`: positions per contact sheet, 1 to 12 (optional; default as many as fit). Fewer positions per sheet means a larger cell each. Sheets are balanced, so 13 positions become 7 + 6 rather than 12 + 1.
+- `-o, --output <dir>`: directory to write the PNGs into (optional; default a fresh `dapi-capture-*` directory in the system temp directory, so runs never overwrite each other). Writing into the same directory twice overwrites images whose name matches; with `--separate`, requested times that land on the same frame share one file.
+
+## Timecodes
+
+Cell labels, the `timecode` field, and the filenames all use the same stamp, which drops its zero segments: `08s10f` is 8 seconds and 10 frames, `01m05s` is 65 seconds, and the node's first visible frame is `0f`. Each segment carries its unit, so nothing is ambiguous once the empty ones are gone.
+
+## Layout
+
+A sheet never exceeds 2576x1456, the largest image a vision model reads at full detail. Within that budget the grid is the one that draws each frame largest, and the last row may be partially filled. Cells render at their own size rather than the flat 720p of `--separate`, so a few positions are sharper than a standalone capture and never coarser; a node smaller than 1080p tall is rendered up to that height, and nothing is rendered beyond it. Sheets are named after the span they cover, e.g. `0f-11s.png`. For a 16:9 scene the cell sizes are:
+
+| Positions | Grid | Per-frame |
+|---|---|---|
+| 1 | 1x1 | 1920x1080 |
+| 2-4 | 2x1, 2x2 | 1280x720 |
+| 5-9 | 3x2, 3x3 | 850x478 |
+| 10-12 | 4x3 | 636x357 |
+
+Sheets are opaque: the node's transparent background composites onto flat grey, and the gutters between cells use the same grey. Use `--separate` when the alpha channel itself matters.
## Output
-One JSON object per line (JSON Lines), in the order the times were given: the requested time in seconds and the absolute path to the written PNG.
+JSON Lines, one object per written image: a contact sheet by default, a position with `--separate`.
```ts
-{ time: number, path: string } // e.g. { "time": 1.5, "path": "/tmp/dapi-capture-3f2c1a8e/00h00m01s15f.png" }
+{ timecode: string; path: string } // e.g. { "timecode": "0f-01s15f", "path": "/tmp/dapi-capture-3f2c1a8e/0f-01s15f.png" }
```
+
+A sheet's timecode is the span it covers; a single position's is its own. Sheets come in timeline order, and their cells in the order the positions were requested in.
+
+## Errors
+
+Exits non-zero if the id is unknown, the node has no timeline clock of its own (capture its scene instead), the node is not visible at any requested position, `--per-sheet` is outside 1 to 12, or a PNG can't be written.
diff --git a/reference/screenshot.md b/reference/screenshot.md
index a3be05e4..f59fb877 100644
--- a/reference/screenshot.md
+++ b/reference/screenshot.md
@@ -10,7 +10,7 @@ The shot is taken from the app's own window buffer, so it works even when the wi
## Output
-One JSON object: the absolute path to a freshly written PNG with a uuid filename, plus the image's pixel dimensions.
+One JSON object: the absolute path to a freshly written PNG, plus the image's pixel dimensions.
```ts
{ path: string, width: number, height: number }