Skip to content

Code-first video & image, rendered natively on WebGPU.

8.3 KB
Raw Blame
import fs from "node:fs/promises";
import type { VirtualMediaData } from "@framefields/core";
import { describe, expect, it } from "vitest";
import { mixAudioTracks } from "./audio/index.js";
import { Composition, Layer, Media } from "./index.js";
import { HeadlessMediaRenderer } from "./renderer/index.js";
import { buildWGSLSignalFn, SignalRegistry } from "./signals/index.js";

// Helper to verify PNG header and IHDR dimensions
function assertValidPng(
	buffer: Buffer,
	expectedWidth?: number,
	expectedHeight?: number,
) {
	expect(Buffer.isBuffer(buffer)).toBe(true);
	expect(buffer.length).toBeGreaterThan(100);

	// PNG Signature: 0x89 0x50 0x4E 0x47 0x0D 0x0A 0x1A 0x0A
	expect(buffer[0]).toBe(0x89);
	expect(buffer[1]).toBe(0x50);
	expect(buffer[2]).toBe(0x4e);
	expect(buffer[3]).toBe(0x47);
	expect(buffer[4]).toBe(0x0d);
	expect(buffer[5]).toBe(0x0a);
	expect(buffer[6]).toBe(0x1a);
	expect(buffer[7]).toBe(0x0a);

	// IHDR chunk: width at byte 16, height at byte 20
	if (expectedWidth !== undefined) {
		const width = buffer.readUInt32BE(16);
		expect(width).toBe(expectedWidth);
	}
	if (expectedHeight !== undefined) {
		const height = buffer.readUInt32BE(20);
		expect(height).toBe(expectedHeight);
	}
}

describe("Framefields Rendering Engine", () => {
	const renderer = new HeadlessMediaRenderer();

	describe("1. Composition Rendering", () => {
		it("should render a multi-layer composition with background, shapes, and text to PNG", async () => {
			const comp = new Composition({
				width: 640,
				height: 360,
				fps: 30,
				durationMs: 2000,
				backgroundColor: "#0d1117",
			});

			comp.add(
				Layer.shape("rect", {
					fillColor: "#238636",
					width: 300,
					height: 180,
					x: 40,
					y: 40,
					borderRadius: 12,
				}),
			);

			comp.add(
				Layer.shape("circle", {
					fillColor: "#1f6feb",
					width: 100,
					height: 100,
					x: 450,
					y: 150,
				}),
			);

			comp.add(
				Layer.text("Framefields WebGPU Engine", {
					fontSize: 28,
					fill: "#ffffff",
					x: 60,
					y: 280,
				}),
			);

			const vm = comp.toVirtualMedia();
			expect(vm.operation.op).toBe("Compositor");
			expect(vm.metadata.width).toBe(640);
			expect(vm.metadata.height).toBe(360);

			const pngBuffer = await renderer.renderImage(vm, 0, 30);
			assertValidPng(pngBuffer, 640, 360);
		});

		it("should render nested flex and box layouts deterministically", async () => {
			const comp = new Composition({
				width: 800,
				height: 450,
				fps: 30,
				durationMs: 1000,
				backgroundColor: "#161b22",
			});

			comp.add(
				Layer.flex({
					dir: "row",
					gap: 20,
					children: [
						Layer.box({
							width: 200,
							height: 150,
							background: "#8957e5",
							children: [
								Layer.text("Box A", { fontSize: 20, fill: "#ffffff" }),
							],
						}),
						Layer.box({
							width: 200,
							height: 150,
							background: "#da3633",
							children: [
								Layer.text("Box B", { fontSize: 20, fill: "#ffffff" }),
							],
						}),
					],
				}),
			);

			const vm = comp.toVirtualMedia();
			const pngBuffer = await renderer.renderImage(vm, 0, 30);
			assertValidPng(pngBuffer, 800, 450);
		});
	});

	describe("2. Chained 2D VFX Shader Pipeline", () => {
		it("should apply Blur, Vignette, Levels, and Flip shader nodes in sequence", async () => {
			const baseVM: VirtualMediaData = {
				metadata: { width: 480, height: 270, fps: 30, durationMs: 1000 },
				operation: {
					op: "Paint",
					dataType: "Video",
					backgroundColor: "#1f6feb",
					timeline: { startFrame: 0, segments: [{ startSec: 0, endSec: 1 }] },
				},
				children: [],
			};

			const media = new Media(baseVM)
				.apply("Blur", { strength: 10 })
				.apply("Vignette", { strength: 70, radius: 0.5 })
				.apply("Levels", {})
				.apply("Flip", { horizontal: true, vertical: false })
				.apply("FilmGrain", { strength: 25 });

			const pipelineVM = media.toVirtualMedia();
			expect(pipelineVM.operation.op).toBe("FilmGrain");

			const pngBuffer = await renderer.renderImage(pipelineVM, 0, 30);
			assertValidPng(pngBuffer, 480, 270);
		});

		it("should render color grading with ColorBalance and HalftoneScreen", async () => {
			const baseVM: VirtualMediaData = {
				metadata: { width: 320, height: 240, fps: 24, durationMs: 1000 },
				operation: {
					op: "Paint",
					dataType: "Video",
					backgroundColor: "#f0883e",
					timeline: { startFrame: 0, segments: [{ startSec: 0, endSec: 1 }] },
				},
				children: [],
			};

			const media = new Media(baseVM)
				.apply("ColorBalance", {
					shadows: { cyanRed: 10, magentaGreen: 0, yellowBlue: -10 },
					midtones: { cyanRed: 0, magentaGreen: 5, yellowBlue: 0 },
				})
				.apply("HalftoneScreen", { frequency: 30 });

			const pipelineVM = media.toVirtualMedia();
			const pngBuffer = await renderer.renderImage(pipelineVM, 0, 24);
			assertValidPng(pngBuffer, 320, 240);
		});
	});

	describe("3. Video Export Rendering", () => {
		it("should render and multiplex an animated video composition to MP4", async () => {
			const comp = new Composition({
				width: 320,
				height: 240,
				fps: 24,
				durationMs: 1000,
				backgroundColor: "#0366d6",
			});

			comp.add(
				Layer.shape("rect", {
					fillColor: "#ffffff",
					width: 100,
					height: 100,
					x: 50,
					y: 50,
				}),
			);

			const vm = comp.toVirtualMedia();

			const { filePath, cleanup } = await renderer.renderVideo(vm, {
				codec: "h264",
				quality: "low",
				concurrency: 2,
			});

			try {
				const stat = await fs.stat(filePath);
				expect(stat.size).toBeGreaterThan(500);

				const videoHeader = Buffer.alloc(16);
				const fh = await fs.open(filePath, "r");
				await fh.read(videoHeader, 0, 16, 0);
				await fh.close();

				const ftyp = videoHeader.toString("ascii", 4, 8);
				expect(ftyp).toBe("ftyp");
			} finally {
				await cleanup();
			}

			await expect(fs.stat(filePath)).rejects.toThrow();
		});

		it("fails fast when a local video file does not exist", async () => {
			const comp = new Composition({
				width: 320,
				height: 240,
				fps: 30,
				durationMs: 500,
			});
			comp.add(
				Layer.video("../media/does-not-exist.mp4", {
					width: 320,
					height: 240,
				}),
			);
			await expect(comp.renderVideo()).rejects.toThrow(
				/Cannot open local media file.*does-not-exist\.mp4/,
			);
		});

		it("fails fast when a local audio file does not exist", async () => {
			const comp = new Composition({
				width: 320,
				height: 240,
				fps: 30,
				durationMs: 500,
			});
			comp.add(
				Layer.audio("../media/does-not-exist.wav", {
					volume: 1,
					startFrame: 0,
					durationFrames: 15,
				}),
			);
			await expect(comp.renderVideo()).rejects.toThrow(
				/Cannot open local media file.*does-not-exist\.wav/,
			);
		});
	});

	describe("4. Audio DSP & Mixing Pipeline", () => {
		it("should extract and mix multi-channel audio tracks from composition", async () => {
			const comp = new Composition({
				width: 640,
				height: 360,
				fps: 30,
				durationMs: 1000,
			});
			comp.addAudio(
				Layer.audio("https://example.com/soundtrack.mp3", { volume: 0.8 }),
			);

			const vm = comp.toVirtualMedia();
			const result = await mixAudioTracks(vm, 30, 48000);

			expect(result).toBeDefined();
			expect(result.sampleRate).toBe(48000);
			expect(Array.isArray(result.channels)).toBe(true);
			expect(result.channels.length).toBeGreaterThanOrEqual(1);

			// Verify samples are allocated and valid floats
			const channel0 = result.channels[0];
			expect(channel0.length).toBe(48000);
			expect(channel0 instanceof Float32Array).toBe(true);
			for (let i = 0; i < Math.min(channel0.length, 1000); i++) {
				expect(Number.isNaN(channel0[i])).toBe(false);
			}
		});
	});

	describe("5. Reactive Signals Pipeline", () => {
		it("should manage signal statistics and generate deterministic WGSL code", () => {
			const registry = new SignalRegistry();
			registry.setStats("audio_bass", { min: 0.1, max: 0.95 });

			const stats = registry.getStats("audio_bass");
			expect(stats).toBeDefined();
			expect(stats?.min).toBe(0.1);
			expect(stats?.max).toBe(0.95);

			const fnResult = buildWGSLSignalFn({ baseType: "sine" }, "sample_bass");
			expect(fnResult).toBeDefined();
			expect(fnResult.name).toBe("signal_sample_bass");
			expect(typeof fnResult.wgsl).toBe("string");
			expect(fnResult.wgsl.length).toBeGreaterThan(0);
			expect(fnResult.wgsl).toContain("fn signal_sample_bass");
		});
	});
});