The VS Code of video editing, built for agents.
TypeScript
Implement image encoding for entity frames in node capture
konstantin-paulus35534e31891a
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 8dc72a53..fb4ecbd6 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -1178,7 +1178,7 @@ media.command("grab").alias("sample").description(- `Decode frames of a video file and write them as PNGs (local render, no credits), each stamped in the top-left with its HH:MM:SS:FF timestamp. Grabs the asset's own pixels at full resolution, unlike \`node capture\` which captures the composited canvas. The recommended tool for understanding a video at the frame level.`,+ `Decode frames of a video file and write them as PNGs (local render, no credits), each stamped in the top-left with its HH:MM:SS:FF timestamp. Grabs the asset's own pixels at full resolution, unlike \`node capture\` which renders the composited node. The recommended tool for understanding a video at the frame level.`,).argument("<id|path>", "video asset id, or a local video file to grab frames from").option("-t, --time <time...>", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`)@@ -1341,10 +1341,10 @@ nodenode.command("capture").description(- `Focus a node on the canvas and capture it as a PNG, one per timeline position, each stamped in the top-left with its HH:MM:SS:FF timestamp. Captures the composited canvas (the truest "what plays at time T" check: layout, overlaps, text, timing); for a video asset's own full-resolution pixels use \`media grab\`.`,+ `Render a node in isolation to PNGs, one per position, each stamped in the top-left with its HH:MM:SS:FF timestamp. The node is drawn offscreen, tightly framed to its own bounds on a transparent background — siblings and overlapping scene content are not included, so capture a scene id to check composition ("what plays at time T": layout, overlaps, text, timing). For a video asset's own full-resolution pixels use \`media grab\`.`,).argument("<id>", "node id to capture")- .option("-t, --time <time...>", `one or more timeline positions to record at — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: the current playhead)`)+ .option("-t, --time <time...>", `one or more positions to capture, relative to the node's start (0 = its first visible frame) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: the node's frame under the current playhead)`).option("--no-timestamp", "don't stamp each capture with its HH:MM:SS:FF timestamp label").action((id: string, opts: CaptureOptions) => nodeCapture(id, opts));diff --git a/apps/web/src/components/engine/encode/image-encoder.ts b/apps/web/src/components/engine/encode/image-encoder.tsnew file mode 100644index 00000000..5f723773--- /dev/null+++ b/apps/web/src/components/engine/encode/image-encoder.ts@@ -0,0 +1,239 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { hasComponent, Hierarchy, Not, Or, query } from 'bitecs';++import { ChildOf } from '../components';+import { createEngineWorld } from '../api';+import { createEntity } from '../api/entities';+import { appendChild } from '../api/hierarchy';+import { recomputeEntityTimeRange } from '../api/utils';+import { addComponent, removeComponent } from '../api/events';+import { playbackSystem } from '../systems/playback';+import { motionSystem } from '../systems/motion';+import { transformSystem } from '../systems/transform';+import { renderSystem } from '../systems/render';+import { cloneFromRecords, serializeEntity } from '../api/serialize';+import { getEntityTree } from '../api/query';+import { framesToSeconds, formatTimestamp, stampTimestampLabel } from '../utils';+import { assert } from '@/utils';++import type { EngineWorld } from '../api/world';+import type { AABB } from '../math';++type ImageEncoderConfig = {+ eid: number;+ /** Frames to capture, relative to the entity's first visible frame. */+ frames: number[];+ /** Stamp each capture with its HH:MM:SS:FF label (frame-relative). */+ timestamp?: boolean;+};++export type ImageExportResult =+ | { type: 'success'; data: string[] }+ | { type: 'canceled' }+ | { type: 'error'; error: Error };++/**+ * Renders single frames of one entity into standalone PNGs (base64, no+ * data-url prefix). The entity's subtree is cloned into a fresh offline+ * world and drawn at the top level: the canvas is sized to the union of the+ * entity's world-space AABBs across the requested frames and the camera+ * shifts that box onto the canvas, so the node is fully visible regardless+ * of where it sat in its source scene.+ */+export async function createImageEncoder(sourceWorld: EngineWorld, config: ImageEncoderConfig) {+ assert(config.frames.length > 0, 'No frames requested');++ const subtree = getEntityTree(sourceWorld, config.eid);+ const records = subtree.map(eid => serializeEntity(sourceWorld, eid));++ // Re-root the node. Its parent isn't part of the clone, so a dangling+ // ChildOf would attach to an arbitrary entity in the new world; Delay and+ // Hidden only mean something on the timeline it was lifted out of.+ const rootRecord = records.find(record => record.eid === config.eid);+ assert(rootRecord !== undefined, `No such entity: ${config.eid}`);+ delete rootRecord.ChildOf;+ delete rootRecord.Delay;+ delete rootRecord.Hidden;++ const offscreenCanvas = new OffscreenCanvas(2, 2);+ const offscreenCtx = offscreenCanvas.getContext('2d')!;++ // Never started — image capture is video-only; the context just satisfies+ // world construction and lazy audio-bus wiring.+ const offlineAudioCtx = new OfflineAudioContext(2, 1, 48000);++ const world = createEngineWorld(sourceWorld.projectId, offlineAudioCtx);+ world.mode = 'offline-video';+ world.assets = sourceWorld.assets;+ world.fonts = [...sourceWorld.fonts];+ world.ctx = offscreenCtx;+ world.canvas = offscreenCanvas;+ world.resolution = 1;+ world.camera = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 };+ world.frameRate = sourceWorld.frameRate;+ world.promises = [];++ const eidMap = cloneFromRecords(world, records);+ const rootEid = eidMap.get(config.eid);+ assert(rootEid !== undefined, 'Failed to clone entity subtree');++ const clonedEids = [...eidMap.values()];+ const c = world.components;++ // Mute everything so the playback system never initializes audio decoders.+ for (const eid of clonedEids) {+ addComponent(world, eid, c.Muted, false);+ }++ // The motion system only samples parented entities, so the node's own+ // keyframes would be skipped as a top-level root. A synthetic group above+ // it owns the playhead instead and keeps the node an ordinary child.+ const stageEid = createEntity(world);+ addComponent(world, stageEid, c.Group, false);+ appendChild(world, rootEid, stageEid);++ recomputeAllTimeRanges(world);++ // Map "frame 0 = first visible frame": with Delay stripped the node's+ // computed start is 0 unless it carries a Trim, whose start offsets it.+ const startFrame = c.Computed.start[rootEid] ?? 0;++ const seek = (frame: number) => {+ const stageFrame = startFrame + frame;+ c.Computed.localTime[stageEid] = stageFrame;+ c.Computed.localTimeInSeconds[stageEid] = framesToSeconds(stageFrame, world.frameRate);+ world.timestamp.delta = 1000 / world.frameRate;+ world.timestamp.now += world.timestamp.delta;++ // Cull flags derived from a stale canvas/camera would make the motion+ // system skip entities and keep decoders idle — clear them every tick.+ for (const eid of clonedEids) {+ removeComponent(world, eid, c.Culled, false);+ }++ playbackSystem(world);+ };++ // Union an entity's world-space AABB into `bounds`, descending so children+ // that overflow a leaf's own rect still count — except where content is+ // clipped. Group bounds already aggregate their children.+ const measure = (eid: number, bounds: AABB): void => {+ if (hasComponent(world, eid, c.Hidden) || hasComponent(world, eid, c.IsMask)) return;+ if (c.Computed.visibility[eid] === 0) return;++ bounds.minX = Math.min(bounds.minX, c.WorldBounds.minX[eid]);+ bounds.minY = Math.min(bounds.minY, c.WorldBounds.minY[eid]);+ bounds.maxX = Math.max(bounds.maxX, c.WorldBounds.maxX[eid]);+ bounds.maxY = Math.max(bounds.maxY, c.WorldBounds.maxY[eid]);++ if (hasComponent(world, eid, c.ClipsContent)) return;+ for (const childEid of query(world, [Or(c.Geometry, c.Group), ChildOf(eid), Not(c.Deleted)])) {+ measure(childEid, bounds);+ }+ };++ // Warm-up tick: group bounds aggregate from their children's local+ // matrices, which don't exist until the first transform walk has visited+ // them — measuring on that walk would union stale boxes.+ seek(config.frames[0]);+ motionSystem(world);+ transformSystem(world);++ // Measure pass: run motion + transforms over every requested frame with an+ // identity camera and union the bounds, so an animated node stays fully+ // visible in a fixed-size output across all frames.+ const bounds: AABB = { minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity };+ for (const frame of config.frames) {+ seek(frame);+ motionSystem(world);+ transformSystem(world);+ measure(rootEid, bounds);+ }+ assert(Number.isFinite(bounds.minX), 'Entity is not visible at any of the requested frames');++ offscreenCanvas.width = Math.max(1, Math.ceil(bounds.maxX - bounds.minX));+ offscreenCanvas.height = Math.max(1, Math.ceil(bounds.maxY - bounds.minY));+ // Shift the measured box onto the canvas so the node is centered in view.+ world.camera.e = -bounds.minX;+ world.camera.f = -bounds.minY;++ let canceled = false;+ const cancel = () => (canceled = true);++ const render = async (): Promise<ImageExportResult> => {+ try {+ const images: string[] = [];++ for (const frame of config.frames) {+ if (canceled) {+ return { type: 'canceled' };+ }++ seek(frame);+ await resolverSystem(world);+ motionSystem(world);+ transformSystem(world);+ renderSystem(world);++ if (config.timestamp) {+ // The render system leaves the camera transform on the context;+ // the label is drawn in device space.+ offscreenCtx.setTransform(1, 0, 0, 1, 0, 0);+ const label = formatTimestamp(framesToSeconds(frame, world.frameRate), world.frameRate);+ stampTimestampLabel(offscreenCtx, offscreenCanvas.height, label);+ }++ images.push(await toBase64Png(offscreenCanvas));+ }++ return { type: 'success', data: images };+ } catch (e) {+ return {+ type: 'error',+ error: e instanceof Error ? e : new Error('Unknown error'),+ };+ }+ };++ return {+ render,+ cancel,+ };+}++async function toBase64Png(canvas: OffscreenCanvas): Promise<string> {+ const blob = await canvas.convertToBlob({ type: 'image/png' });+ const dataUrl = await new Promise<string>((resolve, reject) => {+ const reader = new FileReader();+ reader.onload = () => resolve(reader.result as string);+ reader.onerror = () => reject(reader.error);+ reader.readAsDataURL(blob);+ });+ return dataUrl.split(',')[1] ?? '';+}++async function resolverSystem(world: EngineWorld) {+ if (world.promises?.length) {+ await Promise.all(world.promises.filter(promise => promise !== null));+ world.promises = [];+ }+}++function recomputeAllTimeRanges(world: EngineWorld): void {+ const c = world.components;+ // Hierarchy(ChildOf) yields parents before children (top-down). A single pass+ // is wrong for group-like nodes: their bounds aggregate from children, but the+ // children haven't been recomputed yet when the parent is visited. Do two passes:+ // 1. top-down — refresh accumulated delay + leaf/Trim bounds (parents first)+ // 2. bottom-up — groups now aggregate from already-recomputed children+ const entities = query(world, [Or(c.Geometry, c.Group), Hierarchy(ChildOf), Not(c.Deleted)]);+ for (let i = 0; i < entities.length; i++) {+ recomputeEntityTimeRange(world, entities[i]);+ }+ for (let i = entities.length - 1; i >= 0; i--) {+ recomputeEntityTimeRange(world, entities[i]);+ }+}diff --git a/apps/web/src/context/editor-api/node.ts b/apps/web/src/context/editor-api/node.tsindex 56f70e19..a8d14aa0 100644--- a/apps/web/src/context/editor-api/node.ts+++ b/apps/web/src/context/editor-api/node.ts@@ -5,27 +5,17 @@import { entityExists, hasComponent, query, Not } from "bitecs";import { createEncoder } from "@/components/engine/encode/encoder";-import { playbackSystem } from "@/components/engine/systems/playback";-import { motionSystem } from "@/components/engine/systems/motion";-import { transformSystem } from "@/components/engine/systems/transform";-import { renderSystem } from "@/components/engine/systems/render";-import { hudSystem } from "@/components/engine/systems/hud";+import { createImageEncoder } from "@/components/engine/encode/image-encoder";import { ElectronWritableFileHandle } from "@/lib/electron-file-writable";import {deleteEntity,- addComponent,- removeComponent,getParentEntity,isText,isScene,- switchActiveScene,getEntityTree,cloneSubtree,serializeEntity,- formatTimestamp,- stampTimestampLabel,- clearComponent,} from "@/components/engine";import { ChildOf } from "@/components/engine/components";import { WorldDocument } from "@/utils/jsx";@@ -47,7 +37,7 @@ import type {NodeRenderRequest,NodeRenderResult,} from "@diffusionstudio/cli/channels";-import { assert } from "@/utils";+import { clamp } from "@/utils";import type { EncoderConfig } from "@/components/engine/encode/interfaces";import type { Accessor } from "solid-js";@@ -298,75 +288,34 @@ export function handleNodeCapture(engine: Accessor<Engine>) {const e = engine();const w = e.world;const c = w.components;- const canvas = w.canvas;-- assert(canvas instanceof HTMLCanvasElement, "Canvas is not ready");const eid = resolveNodeEid(w, id);- const sceneEid = sceneOfNode(w, eid);- assert(sceneEid !== null, "Node is not in a scene");- switchActiveScene(w, sceneEid);- clearComponent(w, c.Selected, false);- addComponent(w, eid, c.Selected, false);-- for (const treeEid of getEntityTree(w, eid)) {- removeComponent(w, treeEid, c.Culled, false);- }-- // Renders one pipeline pass onto the on-screen canvas. We drive the systems- // directly rather than relying on the rAF loop: it's throttled or paused- // whenever the app isn't the foreground window (i.e. every CLI capture), so- // the canvas would otherwise stay frozen on a stale frame with no selection.- const renderPass = () => {- playbackSystem(w);- motionSystem(w);- transformSystem(w);- renderSystem(w);- hudSystem(w);- };- e.camera.focusEntities([eid]);+ // `undefined` means the current playhead, mapped into the node's local+ // timeline (the encoder's frame 0 is the node's first visible frame).+ let shots = frames;+ if (shots === undefined || shots.length === 0) {+ const sceneEid = sceneOfNode(w, eid);+ const sceneFrame = sceneEid !== null ? c.Computed.localTime[sceneEid] ?? 0 : 0;+ const start = c.Computed.start[eid] ?? 0;+ const duration = c.Computed.duration[eid] ?? 0;+ shots = [clamp(sceneFrame - start, 0, Math.max(0, duration - 1))];+ }- // `undefined` means the current playhead; each explicit frame is captured in order.- const shots = frames !== undefined && frames.length > 0 ? frames : [undefined];- const results: { base64: string }[] = [];- for (const frame of shots) {- if (frame !== undefined && sceneEid !== null) {- c.Playback.playing[sceneEid] = 0;- c.Computed.localTime[sceneEid] = frame;- c.Computed.localTimeInSeconds[sceneEid] = frame / w.frameRate;- }+ const encoder = await createImageEncoder(w, {+ eid,+ frames: shots,+ timestamp: timestamp !== false,+ });+ const result = await encoder.render();- renderPass();- await new Promise((resolve) => setTimeout(resolve, 2000));- renderPass();+ if (result.type === "canceled") throw new Error("Capture canceled");+ if (result.type === "error") throw result.error;- // The effective frame is the one we just seeked to, or the live playhead.- const shotFrame = frame ?? c.Computed.localTime[sceneEid ?? 0] ?? 0;- const label = formatTimestamp(shotFrame / w.frameRate, w.frameRate);- results.push({ base64: stampedPng(canvas, timestamp !== false ? label : undefined) });- }- return results;+ return result.data.map((base64) => ({ base64 }));};}-// Composites the live canvas onto a fresh one so the timestamp label can't leak-// into the on-screen editor, and returns it as base64 PNG (no data-url prefix).-function stampedPng(source: HTMLCanvasElement, label: string | undefined): string {- const toBase64 = (c: HTMLCanvasElement) => c.toDataURL("image/png").split(",")[1] ?? "";- if (label === undefined) return toBase64(source);-- const out = document.createElement("canvas");- out.width = source.width;- out.height = source.height;- const ctx = out.getContext("2d");- if (!ctx) return toBase64(source);-- ctx.drawImage(source, 0, 0);- stampTimestampLabel(ctx, out.height, label);- return toBase64(out);-}-export function handleNodeRender(engine: Accessor<Engine>) {return async ({ id, output, config }: NodeRenderRequest): Promise<NodeRenderResult> => {const e = engine();