Skip to content

The VS Code of video editing, built for agents.

TypeScript

Implement image encoding for entity frames in node capture

konstantin-paulus35534e31891a
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 8dc72a53..fb4ecbd6 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -1178,7 +1178,7 @@ media
.command("grab")
.alias("sample")
.description(
- `Decode frames of a video file and write them as PNGs (local render, no credits), each stamped in the top-left with its HH:MM:SS:FF timestamp. Grabs the asset's own pixels at full resolution, unlike \`node capture\` which captures the composited canvas. The recommended tool for understanding a video at the frame level.`,
+ `Decode frames of a video file and write them as PNGs (local render, no credits), each stamped in the top-left with its HH:MM:SS:FF timestamp. Grabs the asset's own pixels at full resolution, unlike \`node capture\` which renders the composited node. The recommended tool for understanding a video at the frame level.`,
)
.argument("<id|path>", "video asset id, or a local video file to grab frames from")
.option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`)
@@ -1341,10 +1341,10 @@ node
node
.command("capture")
.description(
- `Focus a node on the canvas and capture it as a PNG, one per timeline position, each stamped in the top-left with its HH:MM:SS:FF timestamp. Captures the composited canvas (the truest "what plays at time T" check: layout, overlaps, text, timing); for a video asset's own full-resolution pixels use \`media grab\`.`,
+ `Render a node in isolation to PNGs, one per position, each stamped in the top-left with its HH:MM:SS:FF timestamp. The node is drawn offscreen, tightly framed to its own bounds on a transparent background — siblings and overlapping scene content are not included, so capture a scene id to check composition ("what plays at time T": layout, overlaps, text, timing). For a video asset's own full-resolution pixels use \`media grab\`.`,
)
.argument("<id>", "node id to capture")
- .option("-t, --time <time...>", `one or more timeline positions to record at — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: the current playhead)`)
+ .option("-t, --time <time...>", `one or more positions to capture, relative to the node's start (0 = its first visible frame) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: the node's frame under the current playhead)`)
.option("--no-timestamp", "don't stamp each capture with its HH:MM:SS:FF timestamp label")
.action((id: string, opts: CaptureOptions) => nodeCapture(id, opts));
diff --git a/apps/web/src/components/engine/encode/image-encoder.ts b/apps/web/src/components/engine/encode/image-encoder.ts
new file mode 100644
index 00000000..5f723773
--- /dev/null
+++ b/apps/web/src/components/engine/encode/image-encoder.ts
@@ -0,0 +1,239 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { hasComponent, Hierarchy, Not, Or, query } from 'bitecs';
+
+import { ChildOf } from '../components';
+import { createEngineWorld } from '../api';
+import { createEntity } from '../api/entities';
+import { appendChild } from '../api/hierarchy';
+import { recomputeEntityTimeRange } from '../api/utils';
+import { addComponent, removeComponent } from '../api/events';
+import { playbackSystem } from '../systems/playback';
+import { motionSystem } from '../systems/motion';
+import { transformSystem } from '../systems/transform';
+import { renderSystem } from '../systems/render';
+import { cloneFromRecords, serializeEntity } from '../api/serialize';
+import { getEntityTree } from '../api/query';
+import { framesToSeconds, formatTimestamp, stampTimestampLabel } from '../utils';
+import { assert } from '@/utils';
+
+import type { EngineWorld } from '../api/world';
+import type { AABB } from '../math';
+
+type ImageEncoderConfig = {
+ eid: number;
+ /** Frames to capture, relative to the entity's first visible frame. */
+ frames: number[];
+ /** Stamp each capture with its HH:MM:SS:FF label (frame-relative). */
+ timestamp?: boolean;
+};
+
+export type ImageExportResult =
+ | { type: 'success'; data: string[] }
+ | { type: 'canceled' }
+ | { type: 'error'; error: Error };
+
+/**
+ * Renders single frames of one entity into standalone PNGs (base64, no
+ * data-url prefix). The entity's subtree is cloned into a fresh offline
+ * world and drawn at the top level: the canvas is sized to the union of the
+ * entity's world-space AABBs across the requested frames and the camera
+ * shifts that box onto the canvas, so the node is fully visible regardless
+ * of where it sat in its source scene.
+ */
+export async function createImageEncoder(sourceWorld: EngineWorld, config: ImageEncoderConfig) {
+ assert(config.frames.length > 0, 'No frames requested');
+
+ const subtree = getEntityTree(sourceWorld, config.eid);
+ const records = subtree.map(eid => serializeEntity(sourceWorld, eid));
+
+ // Re-root the node. Its parent isn't part of the clone, so a dangling
+ // ChildOf would attach to an arbitrary entity in the new world; Delay and
+ // Hidden only mean something on the timeline it was lifted out of.
+ const rootRecord = records.find(record => record.eid === config.eid);
+ assert(rootRecord !== undefined, `No such entity: ${config.eid}`);
+ delete rootRecord.ChildOf;
+ delete rootRecord.Delay;
+ delete rootRecord.Hidden;
+
+ const offscreenCanvas = new OffscreenCanvas(2, 2);
+ const offscreenCtx = offscreenCanvas.getContext('2d')!;
+
+ // Never started — image capture is video-only; the context just satisfies
+ // world construction and lazy audio-bus wiring.
+ const offlineAudioCtx = new OfflineAudioContext(2, 1, 48000);
+
+ const world = createEngineWorld(sourceWorld.projectId, offlineAudioCtx);
+ world.mode = 'offline-video';
+ world.assets = sourceWorld.assets;
+ world.fonts = [...sourceWorld.fonts];
+ world.ctx = offscreenCtx;
+ world.canvas = offscreenCanvas;
+ world.resolution = 1;
+ world.camera = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 };
+ world.frameRate = sourceWorld.frameRate;
+ world.promises = [];
+
+ const eidMap = cloneFromRecords(world, records);
+ const rootEid = eidMap.get(config.eid);
+ assert(rootEid !== undefined, 'Failed to clone entity subtree');
+
+ const clonedEids = [...eidMap.values()];
+ const c = world.components;
+
+ // Mute everything so the playback system never initializes audio decoders.
+ for (const eid of clonedEids) {
+ addComponent(world, eid, c.Muted, false);
+ }
+
+ // The motion system only samples parented entities, so the node's own
+ // keyframes would be skipped as a top-level root. A synthetic group above
+ // it owns the playhead instead and keeps the node an ordinary child.
+ const stageEid = createEntity(world);
+ addComponent(world, stageEid, c.Group, false);
+ appendChild(world, rootEid, stageEid);
+
+ recomputeAllTimeRanges(world);
+
+ // Map "frame 0 = first visible frame": with Delay stripped the node's
+ // computed start is 0 unless it carries a Trim, whose start offsets it.
+ const startFrame = c.Computed.start[rootEid] ?? 0;
+
+ const seek = (frame: number) => {
+ const stageFrame = startFrame + frame;
+ c.Computed.localTime[stageEid] = stageFrame;
+ c.Computed.localTimeInSeconds[stageEid] = framesToSeconds(stageFrame, world.frameRate);
+ world.timestamp.delta = 1000 / world.frameRate;
+ world.timestamp.now += world.timestamp.delta;
+
+ // Cull flags derived from a stale canvas/camera would make the motion
+ // system skip entities and keep decoders idle — clear them every tick.
+ for (const eid of clonedEids) {
+ removeComponent(world, eid, c.Culled, false);
+ }
+
+ playbackSystem(world);
+ };
+
+ // Union an entity's world-space AABB into `bounds`, descending so children
+ // that overflow a leaf's own rect still count — except where content is
+ // clipped. Group bounds already aggregate their children.
+ const measure = (eid: number, bounds: AABB): void => {
+ if (hasComponent(world, eid, c.Hidden) || hasComponent(world, eid, c.IsMask)) return;
+ if (c.Computed.visibility[eid] === 0) return;
+
+ bounds.minX = Math.min(bounds.minX, c.WorldBounds.minX[eid]);
+ bounds.minY = Math.min(bounds.minY, c.WorldBounds.minY[eid]);
+ bounds.maxX = Math.max(bounds.maxX, c.WorldBounds.maxX[eid]);
+ bounds.maxY = Math.max(bounds.maxY, c.WorldBounds.maxY[eid]);
+
+ if (hasComponent(world, eid, c.ClipsContent)) return;
+ for (const childEid of query(world, [Or(c.Geometry, c.Group), ChildOf(eid), Not(c.Deleted)])) {
+ measure(childEid, bounds);
+ }
+ };
+
+ // Warm-up tick: group bounds aggregate from their children's local
+ // matrices, which don't exist until the first transform walk has visited
+ // them — measuring on that walk would union stale boxes.
+ seek(config.frames[0]);
+ motionSystem(world);
+ transformSystem(world);
+
+ // Measure pass: run motion + transforms over every requested frame with an
+ // identity camera and union the bounds, so an animated node stays fully
+ // visible in a fixed-size output across all frames.
+ const bounds: AABB = { minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity };
+ for (const frame of config.frames) {
+ seek(frame);
+ motionSystem(world);
+ transformSystem(world);
+ measure(rootEid, bounds);
+ }
+ assert(Number.isFinite(bounds.minX), 'Entity is not visible at any of the requested frames');
+
+ offscreenCanvas.width = Math.max(1, Math.ceil(bounds.maxX - bounds.minX));
+ offscreenCanvas.height = Math.max(1, Math.ceil(bounds.maxY - bounds.minY));
+ // Shift the measured box onto the canvas so the node is centered in view.
+ world.camera.e = -bounds.minX;
+ world.camera.f = -bounds.minY;
+
+ let canceled = false;
+ const cancel = () => (canceled = true);
+
+ const render = async (): Promise<ImageExportResult> => {
+ try {
+ const images: string[] = [];
+
+ for (const frame of config.frames) {
+ if (canceled) {
+ return { type: 'canceled' };
+ }
+
+ seek(frame);
+ await resolverSystem(world);
+ motionSystem(world);
+ transformSystem(world);
+ renderSystem(world);
+
+ if (config.timestamp) {
+ // The render system leaves the camera transform on the context;
+ // the label is drawn in device space.
+ offscreenCtx.setTransform(1, 0, 0, 1, 0, 0);
+ const label = formatTimestamp(framesToSeconds(frame, world.frameRate), world.frameRate);
+ stampTimestampLabel(offscreenCtx, offscreenCanvas.height, label);
+ }
+
+ images.push(await toBase64Png(offscreenCanvas));
+ }
+
+ return { type: 'success', data: images };
+ } catch (e) {
+ return {
+ type: 'error',
+ error: e instanceof Error ? e : new Error('Unknown error'),
+ };
+ }
+ };
+
+ return {
+ render,
+ cancel,
+ };
+}
+
+async function toBase64Png(canvas: OffscreenCanvas): Promise<string> {
+ const blob = await canvas.convertToBlob({ type: 'image/png' });
+ const dataUrl = await new Promise<string>((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(reader.result as string);
+ reader.onerror = () => reject(reader.error);
+ reader.readAsDataURL(blob);
+ });
+ return dataUrl.split(',')[1] ?? '';
+}
+
+async function resolverSystem(world: EngineWorld) {
+ if (world.promises?.length) {
+ await Promise.all(world.promises.filter(promise => promise !== null));
+ world.promises = [];
+ }
+}
+
+function recomputeAllTimeRanges(world: EngineWorld): void {
+ const c = world.components;
+ // Hierarchy(ChildOf) yields parents before children (top-down). A single pass
+ // is wrong for group-like nodes: their bounds aggregate from children, but the
+ // children haven't been recomputed yet when the parent is visited. Do two passes:
+ // 1. top-down — refresh accumulated delay + leaf/Trim bounds (parents first)
+ // 2. bottom-up — groups now aggregate from already-recomputed children
+ const entities = query(world, [Or(c.Geometry, c.Group), Hierarchy(ChildOf), Not(c.Deleted)]);
+ for (let i = 0; i < entities.length; i++) {
+ recomputeEntityTimeRange(world, entities[i]);
+ }
+ for (let i = entities.length - 1; i >= 0; i--) {
+ recomputeEntityTimeRange(world, entities[i]);
+ }
+}
diff --git a/apps/web/src/context/editor-api/node.ts b/apps/web/src/context/editor-api/node.ts
index 56f70e19..a8d14aa0 100644
--- a/apps/web/src/context/editor-api/node.ts
+++ b/apps/web/src/context/editor-api/node.ts
@@ -5,27 +5,17 @@
import { entityExists, hasComponent, query, Not } from "bitecs";
import { createEncoder } from "@/components/engine/encode/encoder";
-import { playbackSystem } from "@/components/engine/systems/playback";
-import { motionSystem } from "@/components/engine/systems/motion";
-import { transformSystem } from "@/components/engine/systems/transform";
-import { renderSystem } from "@/components/engine/systems/render";
-import { hudSystem } from "@/components/engine/systems/hud";
+import { createImageEncoder } from "@/components/engine/encode/image-encoder";
import { ElectronWritableFileHandle } from "@/lib/electron-file-writable";
import {
deleteEntity,
- addComponent,
- removeComponent,
getParentEntity,
isText,
isScene,
- switchActiveScene,
getEntityTree,
cloneSubtree,
serializeEntity,
- formatTimestamp,
- stampTimestampLabel,
- clearComponent,
} from "@/components/engine";
import { ChildOf } from "@/components/engine/components";
import { WorldDocument } from "@/utils/jsx";
@@ -47,7 +37,7 @@ import type {
NodeRenderRequest,
NodeRenderResult,
} from "@diffusionstudio/cli/channels";
-import { assert } from "@/utils";
+import { clamp } from "@/utils";
import type { EncoderConfig } from "@/components/engine/encode/interfaces";
import type { Accessor } from "solid-js";
@@ -298,75 +288,34 @@ export function handleNodeCapture(engine: Accessor<Engine>) {
const e = engine();
const w = e.world;
const c = w.components;
- const canvas = w.canvas;
-
- assert(canvas instanceof HTMLCanvasElement, "Canvas is not ready");
const eid = resolveNodeEid(w, id);
- const sceneEid = sceneOfNode(w, eid);
- assert(sceneEid !== null, "Node is not in a scene");
- switchActiveScene(w, sceneEid);
- clearComponent(w, c.Selected, false);
- addComponent(w, eid, c.Selected, false);
-
- for (const treeEid of getEntityTree(w, eid)) {
- removeComponent(w, treeEid, c.Culled, false);
- }
-
- // Renders one pipeline pass onto the on-screen canvas. We drive the systems
- // directly rather than relying on the rAF loop: it's throttled or paused
- // whenever the app isn't the foreground window (i.e. every CLI capture), so
- // the canvas would otherwise stay frozen on a stale frame with no selection.
- const renderPass = () => {
- playbackSystem(w);
- motionSystem(w);
- transformSystem(w);
- renderSystem(w);
- hudSystem(w);
- };
- e.camera.focusEntities([eid]);
+ // `undefined` means the current playhead, mapped into the node's local
+ // timeline (the encoder's frame 0 is the node's first visible frame).
+ let shots = frames;
+ if (shots === undefined || shots.length === 0) {
+ const sceneEid = sceneOfNode(w, eid);
+ const sceneFrame = sceneEid !== null ? c.Computed.localTime[sceneEid] ?? 0 : 0;
+ const start = c.Computed.start[eid] ?? 0;
+ const duration = c.Computed.duration[eid] ?? 0;
+ shots = [clamp(sceneFrame - start, 0, Math.max(0, duration - 1))];
+ }
- // `undefined` means the current playhead; each explicit frame is captured in order.
- const shots = frames !== undefined && frames.length > 0 ? frames : [undefined];
- const results: { base64: string }[] = [];
- for (const frame of shots) {
- if (frame !== undefined && sceneEid !== null) {
- c.Playback.playing[sceneEid] = 0;
- c.Computed.localTime[sceneEid] = frame;
- c.Computed.localTimeInSeconds[sceneEid] = frame / w.frameRate;
- }
+ const encoder = await createImageEncoder(w, {
+ eid,
+ frames: shots,
+ timestamp: timestamp !== false,
+ });
+ const result = await encoder.render();
- renderPass();
- await new Promise((resolve) => setTimeout(resolve, 2000));
- renderPass();
+ if (result.type === "canceled") throw new Error("Capture canceled");
+ if (result.type === "error") throw result.error;
- // The effective frame is the one we just seeked to, or the live playhead.
- const shotFrame = frame ?? c.Computed.localTime[sceneEid ?? 0] ?? 0;
- const label = formatTimestamp(shotFrame / w.frameRate, w.frameRate);
- results.push({ base64: stampedPng(canvas, timestamp !== false ? label : undefined) });
- }
- return results;
+ return result.data.map((base64) => ({ base64 }));
};
}
-// Composites the live canvas onto a fresh one so the timestamp label can't leak
-// into the on-screen editor, and returns it as base64 PNG (no data-url prefix).
-function stampedPng(source: HTMLCanvasElement, label: string | undefined): string {
- const toBase64 = (c: HTMLCanvasElement) => c.toDataURL("image/png").split(",")[1] ?? "";
- if (label === undefined) return toBase64(source);
-
- const out = document.createElement("canvas");
- out.width = source.width;
- out.height = source.height;
- const ctx = out.getContext("2d");
- if (!ctx) return toBase64(source);
-
- ctx.drawImage(source, 0, 0);
- stampTimestampLabel(ctx, out.height, label);
- return toBase64(out);
-}
-
export function handleNodeRender(engine: Accessor<Engine>) {
return async ({ id, output, config }: NodeRenderRequest): Promise<NodeRenderResult> => {
const e = engine();