Code-first video & image, rendered natively on WebGPU.
/**
* The ONE constructor from document to render tree (spec: SKILL.md).
*
* A validated v2 program (`CompositorProgramConfig`) becomes the
* `VirtualMediaData` tree the compositor renderer consumes. Every layout node
* becomes a `CompositorLayer` op carrying its document fields verbatim
* (id/kind/style/timing/animation); containers carry nested node ops as
* children; media nodes bind their graph source as the content child when
* resolved.
*
* Pure + deterministic: same doc → same tree. No width/height resolution
* here — geometry comes from `computeLayout` at render time.
*/
import {
createVirtualMedia,
type DataType,
type FrameContext,
getActiveMediaMetadata,
MediaTimeRemapInputSchema,
type MediaTimeSignalRemap,
parseCssColor,
type ResolvedMediaTimeRemap,
type VirtualMediaData,
validateMediaTimeRemap,
} from "@framefields/core";
import { collectInlineSignals } from "./inline-signals.js";
import { hoistMattes } from "./mattes.js";
import {
type MediaSignalLayer,
sampleMediaTimeSignal,
} from "./media-signal.js";
import type {
CompositorProgramConfig,
LayoutNode,
ProgramEffect,
} from "./schema.js";
import { computeStaggerDelay } from "./stagger.js";
export interface CompositorToProgramOptions {
isVideoMode: boolean;
fps: number;
/** Comp duration in ms (metadata only; derived upstream). */
durationMs?: number;
/** Bind a media node's `inputHandleId` to its graph source item. */
resolveMedia?: (inputHandleId?: string) => VirtualMediaData | undefined;
/** External signals mapped by inputHandleId. */
signals?: Record<string, unknown>;
/** Per-frame notification hook. */
onRequestFrame?: ((ctx: FrameContext) => void) | unknown;
/** Composition-wide effect chain (`comp.apply(effect)`). */
effects?: ProgramEffect[];
/** Composition-wide vision config or bundle. */
vision?: unknown;
/**
* Resolve a signal `timeRemap` to the history its consumers read. By
* default the signal is sampled up front on the composition clock.
*/
resolveMediaSignal?: MediaSignalResolver;
}
export type MediaSignalResolver = (
binding: MediaTimeSignalRemap,
layer: MediaSignalLayer,
) => ResolvedMediaTimeRemap;
/**
* A container holding a camera and the layers it films is a 3D scene of its
* own: its camera, lights and environment see only its layers. It needs a
* fixed size to render at. A container holding only a camera (and lights) is
* a shot in a camera cut list over the shared world, and stays one.
*/
export function isScene3D(node: LayoutNode): boolean {
if (
(node.kind !== "box" && node.kind !== "flex" && node.kind !== "block") ||
typeof node.width !== "number" ||
typeof node.height !== "number" ||
!Array.isArray(node.children)
)
return false;
const kids = node.children;
return (
kids.some((c) => c.kind === "camera") &&
kids.some((c) => c.kind !== "camera" && c.kind !== "light")
);
}
/**
* `media` with `node`'s effects applied in order, each a pass over the
* picture below it: a clip's frames, or a 3D scene's finished composition.
*/
function wrapEffects(
media: VirtualMediaData,
node: { id?: string; effects?: unknown },
): VirtualMediaData {
if (!Array.isArray(node.effects)) return media;
let current = media;
for (const eff of node.effects) {
const effOp =
eff &&
typeof (eff as { toOperation?: unknown }).toOperation === "function"
? (eff as { toOperation: () => Record<string, unknown> }).toOperation()
: (eff as Record<string, unknown>);
current = {
metadata: { ...current.metadata },
operation: {
op: String(effOp.op ?? "effect"),
id: node.id ? `${node.id}-${effOp.op ?? "effect"}` : undefined,
effect: eff,
...effOp,
dataType: current.operation.dataType,
},
children: [current],
} as VirtualMediaData;
}
return current;
}
function nodeToVirtualMedia(
node: LayoutNode,
resolveMedia: CompositorToProgramOptions["resolveMedia"],
fps: number,
inheritedOffset = 0,
compDurationMs?: number,
inheritedDurationFrames?: number,
resolveSignal?: MediaSignalResolver,
): VirtualMediaData {
if (node.kind === "media" && node.timeRemap !== undefined) {
const parsed = MediaTimeRemapInputSchema.safeParse(node.timeRemap);
if (!parsed.success)
throw new Error(
`Invalid timeRemap on '${node.id}': ${parsed.error.message}`,
);
const problem = validateMediaTimeRemap(
parsed.data,
node.durationFrames ?? 0,
);
if (problem)
throw new Error(`Invalid timeRemap on '${node.id}': ${problem}`);
if (
node.trimStartSec !== undefined ||
(node.dataType !== "Video" && node.dataType !== "Audio")
)
throw new Error(
`Invalid timeRemap on '${node.id}': use audio or video without trimStartSec`,
);
}
let children: VirtualMediaData[] = [];
const sourceRef =
node.kind === "media"
? (node.src ?? node.inputHandleId)
: "inputHandleId" in node
? (node as { inputHandleId?: string }).inputHandleId
: undefined;
const isDirectMediaSource = (s: unknown) =>
typeof s === "string" &&
(s.startsWith("http://") ||
s.startsWith("https://") ||
s.startsWith("file://") ||
s.startsWith("/") ||
s.startsWith("./") ||
s.startsWith("blob:") ||
s.startsWith("data:") ||
/\.(png|jpe?g|webp|gif|svg|mp4|webm|mov|mkv|mp3|wav|ogg|m4a|srt|vtt|json|lottie)$/i.test(
s,
));
let media: VirtualMediaData | undefined;
if (node.kind === "media") {
if (resolveMedia && node.inputHandleId) {
media = resolveMedia(node.inputHandleId);
}
if (
!media &&
sourceRef &&
(isDirectMediaSource(sourceRef) || Boolean(node.src))
) {
media = createVirtualMedia(
sourceRef,
(node.dataType as DataType) ?? "Video",
);
}
if (
!media &&
!resolveMedia &&
sourceRef &&
(sourceRef.includes(".") || sourceRef.includes("/"))
) {
console.warn(
`[Compositor] Media layer '${node.id ?? "?"}' source '${sourceRef}' could not be resolved.`,
);
}
if (media && node.crop) {
const cropRaw = node.crop as
| { toOperation?: () => Record<string, unknown> }
| Record<string, unknown>;
const cropOp =
typeof cropRaw.toOperation === "function"
? cropRaw.toOperation()
: cropRaw;
media = {
metadata: { ...media.metadata },
operation: {
op: "Crop",
...cropOp,
dataType: media.operation.dataType,
},
children: [media],
};
}
if (media) media = wrapEffects(media, node);
if (media && node.vision) {
const visionRaw = node.vision as
| { toOperation?: () => Record<string, unknown> }
| Record<string, unknown>;
const visionOp =
typeof visionRaw.toOperation === "function"
? visionRaw.toOperation()
: visionRaw;
media = {
metadata: { ...media.metadata },
operation: {
op: "Vision",
id: node.id,
effect: visionRaw,
...visionOp,
dataType: media.operation.dataType,
},
children: [media],
};
}
}
if (node.kind === "media") {
children = media ? [media] : [];
} else if ("children" in node && node.children) {
const staggerFrames =
node.kind === "flex" && typeof node.staggerFrames === "number"
? node.staggerFrames
: 0;
const staggerDirection =
node.kind === "flex" && typeof node.staggerDirection === "string"
? node.staggerDirection
: "forward";
const nodeStart =
typeof node.startFrame === "number" && Number.isFinite(node.startFrame)
? Math.round(node.startFrame)
: 0;
const nodeDuration =
typeof node.durationFrames === "number" &&
Number.isFinite(node.durationFrames)
? Math.round(node.durationFrames)
: inheritedDurationFrames;
const nodeChildren = node.children;
// A 3D scene renders as a composition of its own on its own clock, which
// this layer then places like a picture: transitions, opacity and mattes
// on the layer blend whole scenes, each through its own camera.
const scene = isScene3D(node);
children = nodeChildren.map((c, idx) => {
const delay = computeStaggerDelay(
idx,
nodeChildren.length,
staggerFrames,
staggerDirection,
);
const childOffset = scene ? delay : inheritedOffset + nodeStart + delay;
return nodeToVirtualMedia(
c,
resolveMedia,
fps,
childOffset,
compDurationMs,
c.durationFrames === undefined ? nodeDuration : undefined,
resolveSignal,
);
});
if (scene) {
const frames =
nodeDuration ?? Math.round(((compDurationMs ?? 10000) / 1000) * fps);
const durationMs = (frames / fps) * 1000;
// A plain color clears the scene; any other background (a gradient,
// an image) is the scene's first layer, so its 3D passes see it.
const background =
typeof node.background === "string" ? node.background : undefined;
const clear =
background && parseCssColor(background) ? background : undefined;
if (background && !clear) {
children.unshift(
nodeToVirtualMedia(
{
kind: "box",
id: `${node.id ?? "scene"}-background`,
position: "absolute",
x: 0,
y: 0,
width: node.width as number,
height: node.height as number,
background,
} as LayoutNode,
resolveMedia,
fps,
0,
compDurationMs,
frames,
resolveSignal,
),
);
}
media = {
metadata: {
width: node.width as number,
height: node.height as number,
fps,
durationMs,
},
operation: {
op: "Compositor",
width: node.width as number,
height: node.height as number,
fps,
mode: "Video",
dataType: "Video",
volume: 1,
...(clear && { backgroundColor: clear }),
},
children,
} as unknown as VirtualMediaData;
// Its effects run over the finished scene, as over a clip's frames.
media = wrapEffects(media, node);
children = [media];
}
}
let dataType: DataType = "Image";
let limit: number | undefined;
if (media) {
dataType = media.operation?.dataType ?? "Image";
const isStaticAsset = dataType === "Image" || dataType === "SVG";
const durationMs = isStaticAsset
? undefined
: getActiveMediaMetadata(media)?.durationMs;
if (durationMs !== undefined && durationMs !== null) {
limit = Math.max(1, Math.round((durationMs / 1000) * fps));
}
}
const isCaption = media?.operation?.dataType === "Caption";
const isContainer =
node.kind === "flex" || node.kind === "block" || node.kind === "box";
const startFrame =
(typeof node.startFrame === "number" &&
Number.isFinite(node.startFrame) &&
node.startFrame >= 0
? Math.round(node.startFrame)
: 0) + inheritedOffset;
let dur =
typeof node.durationFrames === "number" &&
Number.isFinite(node.durationFrames) &&
node.durationFrames > 0
? Math.round(node.durationFrames)
: undefined;
const defaultCompDuration =
inheritedDurationFrames ??
Math.round(((compDurationMs ?? 10000) / 1000) * fps);
if (isContainer && children.length > 0) {
// A container without its own timing keeps its parent's clock — its keyframes
// are authored parent-relative and its background shows before its first child
// arrives — and stays active until its last child ends.
// Also when its own durationFrames is unusable (0, NaN): `dur` is unset then.
// A 3D scene's one child is its own composition, on the scene's clock
// from 0: it lasts the scene's inherited duration, not until an
// absolute frame (inside a parent that starts late, that would end it
// early).
if (dur === undefined && isScene3D(node)) {
dur = Math.max(1, defaultCompDuration);
} else if (dur === undefined) {
const childEnds = children.map(
(c) =>
(c.operation.startFrame ?? 0) +
(c.operation.durationFrames ?? defaultCompDuration),
);
dur = Math.max(1, Math.max(...childEnds) - startFrame);
}
} else {
if (isCaption && limit !== undefined && limit > 0) {
dur = limit;
} else if (dur !== undefined) {
if (
(dataType === "Video" || dataType === "Audio") &&
!(node.kind === "media" && node.timeRemap) &&
limit !== undefined &&
limit > 0
) {
dur = Math.min(dur, limit);
}
} else {
dur =
limit ??
inheritedDurationFrames ??
Math.round(((compDurationMs ?? 10000) / 1000) * fps);
}
}
const durationMs = (dur / fps) * 1000;
if (node.kind === "media" && media) {
let curr: VirtualMediaData | undefined = media;
while (curr) {
if (!curr.metadata?.durationMs || curr.metadata.durationMs <= 0) {
if (!curr.metadata) {
(curr as unknown as { metadata: Record<string, unknown> }).metadata =
{};
}
curr.metadata.durationMs = durationMs;
}
if (
curr.operation?.sourceMeta &&
!(curr.operation.sourceMeta as Record<string, unknown>).durationMs
) {
(curr.operation.sourceMeta as Record<string, unknown>).durationMs =
durationMs;
}
curr = curr.children?.[0];
}
}
const signalRemap =
node.kind === "media" && node.timeRemap?.kind === "signal"
? node.timeRemap
: undefined;
if (signalRemap && !resolveSignal)
throw new Error(`Signal timeRemap on '${node.id}' has no resolver`);
return {
metadata: {
durationMs,
},
operation: {
op: "CompositorLayer",
...node,
// A scene is placed as media: its nested composition paints its
// background, and its effects already wrap that composition.
...(isScene3D(node) && {
kind: "media",
background: undefined,
children: undefined,
effects: undefined,
}),
startFrame,
durationFrames: dur,
dataType,
...(signalRemap &&
resolveSignal && {
timeRemap: resolveSignal(signalRemap, {
id: node.id,
startFrame,
durationFrames: dur,
}),
}),
},
children,
} as unknown as VirtualMediaData;
}
export function compositorToProgram(
config: CompositorProgramConfig,
options: CompositorToProgramOptions,
): VirtualMediaData {
const { isVideoMode, fps, durationMs, resolveMedia, signals } = options;
const programSignals = {
...collectInlineSignals(config.layout),
...(signals ?? config.signals ?? {}),
};
const resolveMediaSignal: MediaSignalResolver =
options.resolveMediaSignal ??
((binding, layer) =>
sampleMediaTimeSignal(binding, layer, {
fps,
compositionDurationFrames: Math.round(((durationMs ?? 0) / 1000) * fps),
signals: programSignals,
}));
let rootMedia: VirtualMediaData = {
metadata: {
width: config.width,
height: config.height,
fps,
...(durationMs !== undefined && { durationMs }),
},
operation: {
op: "Compositor",
width: config.width,
height: config.height,
fps,
mode: config.mode ?? "Video",
backgroundColor: config.backgroundColor,
...(config.antialias3d !== undefined && {
antialias3d: config.antialias3d,
}),
// Master gain — consumed by the compositor's audio processor when
// the extractor dispatches the root op (audio-processor.ts).
volume: config.volume ?? 1,
dataType: isVideoMode ? "Video" : "Image",
// Debug-only snapshot of the document (L7): the render tree is
// `children`; this copy exists for tree inspection in devtools and
// is NOT read by the renderer — do not treat it as the source of
// truth (the config is).
layout: config.layout,
// Signal objects passed straight to `LayerAnimation.signal` ride on
// their tracks; fold them in under their generated handle ids.
signals: programSignals,
...(config.onRequestFrame || options.onRequestFrame
? {
onRequestFrame: config.onRequestFrame ?? options.onRequestFrame,
}
: {}),
},
children: hoistMattes(config.layout, {
width: config.width,
height: config.height,
}).map((n) =>
nodeToVirtualMedia(
n,
resolveMedia,
fps,
0,
durationMs,
undefined,
resolveMediaSignal,
),
),
} as unknown as VirtualMediaData;
shareSignals(rootMedia.children ?? [], programSignals);
const effectsList: ProgramEffect[] = Array.isArray(options.effects)
? options.effects
: Array.isArray(config.effects)
? config.effects
: [];
for (const eff of effectsList) {
const effOp =
eff &&
typeof (eff as { toOperation?: unknown }).toOperation === "function"
? (eff as { toOperation: () => Record<string, unknown> }).toOperation()
: (eff as Record<string, unknown>);
rootMedia = {
metadata: { ...rootMedia.metadata },
operation: {
...effOp,
dataType: rootMedia.operation.dataType,
},
children: [rootMedia],
} as unknown as VirtualMediaData;
}
const visionRaw = config.vision ?? options.vision;
if (visionRaw) {
const visionOp =
typeof (visionRaw as { toOperation?: unknown }).toOperation === "function"
? (
visionRaw as { toOperation: () => Record<string, unknown> }
).toOperation()
: (visionRaw as Record<string, unknown>);
rootMedia = {
metadata: { ...rootMedia.metadata },
operation: {
op: "Vision",
...visionOp,
dataType: rootMedia.operation.dataType,
},
children: [rootMedia],
} as unknown as VirtualMediaData;
}
return rootMedia;
}
/** Hands every nested scene composition the program's signals, which its tracks read. */
function shareSignals(
nodes: readonly VirtualMediaData[],
signals: Record<string, unknown>,
) {
for (const vm of nodes) {
if (vm?.operation?.op === "Compositor")
(vm.operation as { signals?: unknown }).signals = signals;
shareSignals(vm?.children ?? [], signals);
}
}
/**
* Recursively gather every node op from a render tree (in order), stopping at
* nested scene compositions: their layers belong to their own render.
*/
/**
* Every layer, 3D scenes' layers included, with its start on the
* composition's clock. A scene's layers run on the scene's own clock, which
* starts where the scene's layer does, so their starts are offset by it.
*/
export function collectLayersOnClock(
nodes: VirtualMediaData[],
): Array<{ vm: VirtualMediaData; startFrame: number }> {
const out: Array<{ vm: VirtualMediaData; startFrame: number }> = [];
const walk = (vm: VirtualMediaData, offset: number, layerStart: number) => {
const op = vm?.operation as
| { op?: string; startFrame?: number }
| undefined;
if (op?.op === "Compositor") {
for (const c of vm.children ?? []) walk(c, layerStart, layerStart);
return;
}
let start = layerStart;
if (op?.op === "CompositorLayer") {
start = offset + (op.startFrame ?? 0);
out.push({ vm, startFrame: start });
}
for (const c of vm?.children ?? []) walk(c, offset, start);
};
for (const n of nodes) walk(n, 0, 0);
return out;
}
export function collectNodeOps(nodes: VirtualMediaData[]): VirtualMediaData[] {
const out: VirtualMediaData[] = [];
const walk = (vm: VirtualMediaData) => {
if (vm?.operation?.op === "Compositor") return;
if (vm?.operation?.op === "CompositorLayer") {
out.push(vm);
}
for (const c of vm?.children ?? []) walk(c);
};
for (const n of nodes) walk(n);
return out;
}