Skip to content

Code-first video & image, rendered natively on WebGPU.

16.4 KB
Raw Blame
import {
	mediaSourceTime,
	type ResolvedMediaTimeRemap,
	Signal,
	type VirtualMediaData,
} from "@framefields/core";
import { describe, expect, it } from "vitest";
import type { CompositorProgramConfig, LayoutNode } from "./schema.js";
import {
	collectLayersOnClock,
	compositorToProgram,
	isScene3D,
} from "./to-virtual-media.js";

const doc = {
	width: 1920,
	height: 1080,
	fps: 30,
	backgroundColor: "#101014",
	mode: "Video",
	layout: [
		{
			id: "hero",
			kind: "flex",
			dir: "column",
			gap: 24,
			children: [
				{ id: "title", kind: "text", text: "Big Title", fontSize: 96 },
				{
					id: "badges",
					kind: "flex",
					dir: "row",
					gap: 16,
					children: [
						{ id: "avatarImg", kind: "media", inputHandleId: "h_avatar" },
						{ id: "heroImg", kind: "media", inputHandleId: "h_hero" },
					],
				},
			],
		},
	],
} as unknown as CompositorProgramConfig;

const mediaVM = (id: string): VirtualMediaData =>
	({
		metadata: { width: 100, height: 100, durationMs: 2000 },
		operation: { op: "source", dataType: "Image", source: { entity: { id } } },
		children: [],
	}) as unknown as VirtualMediaData;

describe("compositorToProgram", () => {
	it("preserves the remapped layer duration when the resolved source is shorter", () => {
		const video = {
			metadata: { durationMs: 1000 },
			operation: { op: "source", dataType: "Video" },
			children: [],
		} as unknown as VirtualMediaData;
		const config = {
			width: 64,
			height: 64,
			fps: 30,
			mode: "Video",
			layout: [
				{
					id: "slow",
					kind: "media",
					inputHandleId: "source",
					dataType: "Video",
					durationFrames: 60,
					timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },
				},
			],
		} as unknown as CompositorProgramConfig;
		const vm = compositorToProgram(config, {
			isVideoMode: true,
			fps: 30,
			resolveMedia: () => video,
		});
		expect(vm.children[0].operation.durationFrames).toBe(60);
		expect(vm.children[0].metadata.durationMs).toBe(2000);
	});
	it("resolves signal time remaps on the composition clock", () => {
		const video = {
			metadata: { durationMs: 10000 },
			operation: { op: "source", dataType: "Video" },
			children: [],
		} as unknown as VirtualMediaData;
		const layer = (timeRemap: unknown) => ({
			width: 64,
			height: 64,
			fps: 4,
			mode: "Video",
			layout: [
				{
					id: "wrap",
					kind: "box",
					startFrame: 2,
					children: [
						{
							id: "clip",
							kind: "media",
							inputHandleId: "source",
							dataType: "Video",
							startFrame: 1,
							durationFrames: 4,
							timeRemap,
						},
					],
				},
			],
		});
		const remapOf = (vm: VirtualMediaData) =>
			(
				vm.children[0].children[0].operation as unknown as {
					timeRemap: ResolvedMediaTimeRemap;
				}
			).timeRemap;
		const options = {
			isVideoMode: true,
			fps: 4,
			durationMs: 2500,
			resolveMedia: () => video,
		};
		const instance = compositorToProgram(
			layer({
				kind: "signal",
				mode: "source-time",
				signal: Signal.builder({ type: "custom", fn: (ctx) => ctx.time }),
			}) as unknown as CompositorProgramConfig,
			options,
		);
		// The layer starts at composition frame 3: its frame 1 reads frame 4.
		expect(mediaSourceTime(remapOf(instance), 1, 4)).toBe(1);
		const named = compositorToProgram(
			layer({
				kind: "signal",
				mode: "speed",
				signal: "pace",
				sourceStartSec: 2,
			}) as unknown as CompositorProgramConfig,
			{ ...options, signals: { pace: [0, 0, 0, 2, 2, 2, 2, 2] } },
		);
		expect(mediaSourceTime(remapOf(named), 4, 4)).toBe(4);
		expect(() =>
			compositorToProgram(
				layer({
					kind: "signal",
					mode: "speed",
					signal: "missing",
				}) as unknown as CompositorProgramConfig,
				options,
			),
		).toThrow(/not a composition signal/);
		expect(() =>
			compositorToProgram(
				layer({
					kind: "signal",
					mode: "source-time",
					signal: "pace",
					sourceStartSec: 1,
				}) as unknown as CompositorProgramConfig,
				{ ...options, signals: { pace: [0] } },
			),
		).toThrow(/only to speed signals/);
	});
	it("builds the render tree 1:1 from the document", () => {
		const vm = compositorToProgram(doc, { isVideoMode: true, fps: 30 });
		const op = vm.operation as any;
		expect(op.op).toBe("Compositor");
		expect(op.width).toBe(1920);
		expect(op.layout).toEqual(doc.layout); // doc embedded verbatim (debug-only snapshot, L7)
		expect(op.volume).toBe(1); // master gain always carried on the root op (L1)
		expect(vm.children).toHaveLength(1);

		const hero = vm.children![0];
		expect((hero.operation as any).op).toBe("CompositorLayer");
		expect((hero.operation as any).kind).toBe("flex");
		expect(hero.children).toHaveLength(2);
		expect((hero.children![1].operation as any).kind).toBe("flex");
	});

	it("binds media nodes to resolved graph sources", () => {
		const vm = compositorToProgram(doc, {
			isVideoMode: true,
			fps: 30,
			resolveMedia: (handleId) =>
				handleId === "h_avatar" ? mediaVM("avatar") : undefined,
		});
		const badges = vm.children![0].children![1];
		const avatar = badges.children![0];
		expect(avatar.children).toHaveLength(1);
		expect((avatar.children![0].operation as any).source.entity.id).toBe(
			"avatar",
		);
		// unresolved media keeps no content child — renderer skips it
		const heroImg = badges.children![1];
		expect(heroImg.children).toHaveLength(0);
	});

	it("carries animation tracks and timing on the node ops", () => {
		const heroNode: LayoutNode = {
			...doc.layout[0],
			animation: {
				tracks: [
					{
						id: "t1",
						prop: "opacity",
						keyframes: [
							{ id: "k1", frame: 0, value: 0 },
							{ id: "k2", frame: 15, value: 1 },
						],
					},
				],
			},
		};
		const vm = compositorToProgram(
			{ ...doc, layout: [heroNode] },
			{ isVideoMode: true, fps: 30 },
		);
		const heroOp = vm.children![0].operation as any;
		expect(heroOp.animation.tracks[0].prop).toBe("opacity");
	});

	it("converts kind: 'shape' nodes into CompositorLayer operations with all vector properties", () => {
		const shapeNode: LayoutNode = {
			id: "star-1",
			kind: "shape",
			shapeType: "star",
			starPoints: 5,
			starInnerRadiusRatio: 0.4,
			width: 250,
			height: 250,
			fillType: "solid",
			fillColor: "#e11d48",
			strokeColor: "#ffffff",
			strokeWidth: 6,
			trimStart: 0.1,
			trimEnd: 0.85,
			trimOffset: 90,
			startFrame: 0,
			durationFrames: 60,
		};

		const vm = compositorToProgram(
			{ ...doc, layout: [shapeNode] },
			{ isVideoMode: true, fps: 30 },
		);

		expect(vm.children).toHaveLength(1);
		const layer = vm.children![0];
		const op = layer.operation as any;
		expect(op.op).toBe("CompositorLayer");
		expect(op.kind).toBe("shape");
		expect(op.shapeType).toBe("star");
		expect(op.starPoints).toBe(5);
		expect(op.starInnerRadiusRatio).toBe(0.4);
		expect(op.fillColor).toBe("#e11d48");
		expect(op.strokeWidth).toBe(6);
		expect(op.trimStart).toBe(0.1);
		expect(op.trimEnd).toBe(0.85);
		expect(op.trimOffset).toBe(90);
		expect(op.dataType).toBe("Image");
		expect(layer.children).toHaveLength(0);
	});

	it("automatically resolves direct URL sources without resolveMedia callback", () => {
		const docWithUrls: CompositorProgramConfig = {
			width: 1920,
			height: 1080,
			fps: 60,
			backgroundColor: "#000",
			mode: "Video",
			layout: [
				{
					id: "clip-1",
					kind: "media",
					inputHandleId: "https://example.com/footage.mp4",
					dataType: "Video",
				},
			],
		} as any;

		const vm = compositorToProgram(docWithUrls, { isVideoMode: true, fps: 60 });
		expect(vm.children).toHaveLength(1);
		const layer = vm.children![0];
		expect(layer.children).toHaveLength(1);
		const sourceNode = layer.children![0];
		expect(sourceNode.operation.op).toBe("source");
		expect((sourceNode.operation as any).url).toBe(
			"https://example.com/footage.mp4",
		);
	});

	it("inherits parent container startFrame for child nodes with explicit startFrame", () => {
		const sceneDoc: CompositorProgramConfig = {
			width: 1280,
			height: 720,
			fps: 30,
			mode: "Video",
			layout: [
				{
					id: "scene-2",
					kind: "box",
					startFrame: 105,
					durationFrames: 120,
					children: [
						{
							id: "callout-1",
							kind: "box",
							startFrame: 15,
							durationFrames: 105,
						},
					],
				},
			],
		} as any;

		const vm = compositorToProgram(sceneDoc, { isVideoMode: true, fps: 30 });
		const scene = vm.children![0];
		expect((scene.operation as any).startFrame).toBe(105);
		const callout = scene.children![0];
		// Child startFrame must be 105 + 15 = 120, NOT 15!
		expect((callout.operation as any).startFrame).toBe(120);
		expect((callout.operation as any).durationFrames).toBe(105);
	});

	it("keeps an untimed container on its parent's clock when its children start later", () => {
		const doc: CompositorProgramConfig = {
			width: 1280,
			height: 720,
			fps: 30,
			mode: "Video",
			layout: [
				{
					id: "scene",
					kind: "box",
					startFrame: 450,
					durationFrames: 144,
					children: [
						{
							id: "card",
							kind: "box",
							children: [
								{
									id: "clip",
									kind: "box",
									startFrame: 40,
									durationFrames: 104,
								},
							],
						},
					],
				},
			],
		} as any;

		const vm = compositorToProgram(doc, { isVideoMode: true, fps: 30 });
		const card = vm.children![0].children![0];
		// The card's keyframes are authored against the scene, so its clock must not
		// jump to its first child's start (490).
		expect((card.operation as any).startFrame).toBe(450);
		expect((card.operation as any).durationFrames).toBe(144);
		expect((card.children![0].operation as any).startFrame).toBe(490);
	});

	it("resolves .json and .lottie files as direct media sources with Lottie dataType", () => {
		const doc: CompositorProgramConfig = {
			width: 1920,
			height: 1080,
			fps: 30,
			mode: "Video",
			layout: [
				{
					id: "lottie1",
					kind: "media",
					inputHandleId: "animation.json",
					dataType: "Lottie",
				},
				{
					id: "lottie2",
					kind: "media",
					inputHandleId: "animation.lottie",
					dataType: "Lottie",
				},
			],
		} as any;

		const vm = compositorToProgram(doc, { isVideoMode: true, fps: 30 });
		expect(vm.children).toHaveLength(2);

		const layer1 = vm.children![0];
		expect(layer1.children).toHaveLength(1);
		expect((layer1.children![0].operation as any).op).toBe("source");
		expect((layer1.children![0].operation as any).dataType).toBe("Lottie");
		expect((layer1.children![0].operation as any).url).toBe("animation.json");

		const layer2 = vm.children![1];
		expect(layer2.children).toHaveLength(1);
		expect((layer2.children![0].operation as any).op).toBe("source");
		expect((layer2.children![0].operation as any).dataType).toBe("Lottie");
		expect((layer2.children![0].operation as any).url).toBe("animation.lottie");
	});

	it("preserves effects on non-media layers as CompositorLayer lop.effects", () => {
		const docWithEffects = {
			width: 1920,
			height: 1080,
			fps: 30,
			mode: "Video" as const,
			layout: [
				{
					id: "box-fx",
					kind: "box" as const,
					width: 200,
					height: 200,
					background: "#ff0000",
					effects: [{ op: "Blur", radius: 10 }],
				},
				{
					id: "text-fx",
					kind: "text" as const,
					text: "Hello World",
					fontSize: 48,
					effects: [{ op: "FilmGrain", strength: 0.1 }],
				},
			],
		};

		const vm = compositorToProgram(docWithEffects, {
			isVideoMode: true,
			fps: 30,
		});
		expect(vm.children).toHaveLength(2);

		const boxLayer = vm.children?.[0];
		const boxOp = boxLayer?.operation as Record<string, unknown> | undefined;
		expect(boxOp?.op).toBe("CompositorLayer");
		expect(boxOp?.kind).toBe("box");
		expect(boxOp?.effects).toEqual([{ op: "Blur", radius: 10 }]);

		const textLayer = vm.children?.[1];
		const textOp = textLayer?.operation as Record<string, unknown> | undefined;
		expect(textOp?.op).toBe("CompositorLayer");
		expect(textOp?.kind).toBe("text");
		expect(textOp?.effects).toEqual([{ op: "FilmGrain", strength: 0.1 }]);
	});
});

describe("compositorToProgram: 3D scenes", () => {
	const scene = (id: string, startFrame: number): LayoutNode =>
		({
			id,
			kind: "box",
			position: "absolute",
			width: 1080,
			height: 1920,
			background: "#202020",
			startFrame,
			durationFrames: 60,
			children: [
				{ id: `${id}-cam`, kind: "camera", z: -800 },
				{
					id: `${id}-cube`,
					kind: "box",
					is3D: true,
					width: 100,
					height: 100,
					startFrame: 10,
				},
			],
		}) as unknown as LayoutNode;
	const ramp = [0, 1, 2];
	const program = compositorToProgram(
		{
			width: 1080,
			height: 1920,
			fps: 30,
			mode: "Video",
			layout: [scene("a", 0), scene("b", 50)],
			signals: { ramp },
		} as unknown as CompositorProgramConfig,
		{ isVideoMode: true, fps: 30, durationMs: 4000 },
	);
	const [a, b] = program.children as VirtualMediaData[];

	it("places a container holding a camera as media whose content is its own composition", () => {
		const op = b.operation as Record<string, unknown>;
		expect(op.kind).toBe("media");
		expect(op.startFrame).toBe(50);
		expect(op.background).toBeUndefined();
		const nested = b.children?.[0];
		expect(nested?.operation.op).toBe("Compositor");
		expect((nested?.operation as Record<string, unknown>).backgroundColor).toBe(
			"#202020",
		);
		expect((nested?.operation as Record<string, unknown>).signals).toEqual({
			ramp,
		});
	});
	it("runs a scene's effects over its finished composition, once", () => {
		const fx = compositorToProgram(
			{
				width: 1080,
				height: 1920,
				fps: 30,
				mode: "Video",
				layout: [
					{
						...(scene("fx", 0) as unknown as Record<string, unknown>),
						effects: [{ op: "Vignette", strength: 80 }],
					} as unknown as LayoutNode,
				],
			} as unknown as CompositorProgramConfig,
			{ isVideoMode: true, fps: 30, durationMs: 4000 },
		);
		const layer = (fx.children as VirtualMediaData[])[0];
		// The placed picture carries no effects of its own...
		expect(
			(layer?.operation as Record<string, unknown>).effects,
		).toBeUndefined();
		// ...they wrap the nested composition, once.
		const pass = layer?.children?.[0];
		expect(pass?.operation.op).toBe("Vignette");
		expect(pass?.children?.[0]?.operation.op).toBe("Compositor");
	});
	it("keeps a windowless scene for its parent's whole window", () => {
		const fx = compositorToProgram(
			{
				width: 1080,
				height: 1920,
				fps: 30,
				mode: "Video",
				layout: [
					{
						kind: "box",
						id: "act",
						width: 1080,
						height: 1920,
						startFrame: 282,
						durationFrames: 341,
						children: [
							{
								...(scene("inner", 0) as unknown as Record<string, unknown>),
								startFrame: undefined,
								durationFrames: undefined,
							} as unknown as LayoutNode,
						],
					} as unknown as LayoutNode,
				],
			} as unknown as CompositorProgramConfig,
			{ isVideoMode: true, fps: 30, durationMs: 30000 },
		);
		const act = (fx.children as VirtualMediaData[])[0];
		const inner = act?.children?.[0]?.operation as Record<string, unknown>;
		expect(inner.startFrame).toBe(282);
		expect(inner.durationFrames).toBe(341);
	});
	it("puts a scene's layers on the composition's clock", () => {
		const fx = compositorToProgram(
			{
				width: 1080,
				height: 1920,
				fps: 30,
				mode: "Video",
				layout: [
					{
						kind: "box",
						id: "act",
						width: 1080,
						height: 1920,
						startFrame: 200,
						durationFrames: 300,
						children: [
							{
								...(scene("inner", 40) as unknown as Record<string, unknown>),
								durationFrames: 200,
							} as unknown as LayoutNode,
						],
					} as unknown as LayoutNode,
				],
			} as unknown as CompositorProgramConfig,
			{ isVideoMode: true, fps: 30, durationMs: 30000 },
		);
		const starts = new Map(
			collectLayersOnClock(fx.children as VirtualMediaData[]).map(
				({ vm, startFrame }) => [
					(vm.operation as { id?: string }).id,
					startFrame,
				],
			),
		);
		expect(starts.get("act")).toBe(200);
		expect(starts.get("inner")).toBe(240);
		// The cube starts 10 frames into the scene, on the scene's clock.
		expect(starts.get("inner-cube")).toBe(250);
	});
	it("leaves a box holding only a camera as a shot over the shared world", () => {
		expect(
			isScene3D({
				id: "shot",
				kind: "box",
				width: 1080,
				height: 1920,
				children: [{ id: "shot-cam", kind: "camera" }],
			} as unknown as LayoutNode),
		).toBe(false);
	});
	it("keeps the scene's layers on the scene's own clock", () => {
		const layers = b.children?.[0].children ?? [];
		const cube = layers.find(
			(l) => (l.operation as { id?: string }).id === "b-cube",
		);
		expect(cube?.operation.startFrame).toBe(10);
		expect(a.children?.[0].children?.length).toBe(2);
	});
});