Code-first video & image, rendered natively on WebGPU.
/**
* glTF beyond static meshes: material textures, node hierarchies and skinned
* meshes posed by animation clips (rotation tracks through a joint chain).
* Each scene is a GLB built in the test and checked by its pixels.
*/
import fs from "node:fs/promises";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import { GlbBuilder } from "@framefields/gltf-fixtures";
import { Canvas, loadImage } from "skia-canvas";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { Composition, Layer } from "./index.js";
const SIZE = 400;
const BG = "#000000";
/** An axis-aligned rectangle in the XY plane as two triangles. */
function rect(x0: number, y0: number, x1: number, y1: number) {
return {
positions: [x0, y0, 0, x1, y0, 0, x1, y1, 0, x0, y1, 0],
uvs: [0, 1, 1, 1, 1, 0, 0, 0],
indices: [0, 1, 2, 0, 2, 3],
};
}
async function solidPng(color: string): Promise<Buffer> {
const canvas = new Canvas(8, 8);
const ctx = canvas.getContext("2d");
ctx.fillStyle = color;
ctx.fillRect(0, 0, 8, 8);
return canvas.toBuffer("png");
}
async function pixels(png: Buffer) {
const img = await loadImage(png);
const canvas = new Canvas(img.width, img.height);
const ctx = canvas.getContext("2d");
ctx.drawImage(img, 0, 0);
const data = ctx.getImageData(0, 0, img.width, img.height).data;
return (x: number, y: number) => {
const i = (Math.round(y) * img.width + Math.round(x)) * 4;
return [data[i] ?? 0, data[i + 1] ?? 0, data[i + 2] ?? 0];
};
}
/** Renders a GLB centred in a SIZE² frame, model units = pixels, Y up flipped to the screen's Y down. */
async function render(file: string, frame = 0, fps = 30, lit = false) {
const comp = new Composition({
width: SIZE,
height: SIZE,
fps,
durationFrames: 120,
backgroundColor: BG,
});
comp.add(
Layer.camera({
id: "cam",
x: SIZE / 2,
y: SIZE / 2,
// At this distance a 50° field of view maps one model unit to one pixel.
z: -SIZE / 2 / Math.tan((25 * Math.PI) / 180),
targetX: SIZE / 2,
targetY: SIZE / 2,
targetZ: 0,
}),
);
comp.add(
Layer.glb(file, {
id: "model",
x: SIZE / 2,
y: SIZE / 2,
z: 0,
rotateX: 180,
material: lit ? "lit" : "unlit",
loop: false,
}),
);
return pixels(await comp.renderFrame({ frame }));
}
let dir = "";
beforeAll(async () => {
dir = await fs.mkdtemp(path.join(os.tmpdir(), "framefields-gltf-scene-"));
});
afterAll(async () => {
await fs.rm(dir, { recursive: true, force: true });
});
describe("glTF scene rendering", () => {
it("samples a material's embedded base-color texture", async () => {
const g = new GlbBuilder();
const q = rect(-100, -100, 100, 100);
const file = path.join(dir, "textured.glb");
await fs.writeFile(
file,
g.build({
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
TEXCOORD_0: g.accessor(new Float32Array(q.uvs), "VEC2"),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
material: 0,
},
],
},
],
materials: [
{ pbrMetallicRoughness: { baseColorTexture: { index: 0 } } },
],
textures: [{ source: 0 }],
images: [
{
bufferView: g.image(await solidPng("#ff2000")),
mimeType: "image/png",
},
],
}),
);
const px = await render(file);
const [r, gr, b] = px(SIZE / 2, SIZE / 2);
expect(r).toBeGreaterThan(220);
expect(gr).toBeLessThan(60);
expect(b).toBeLessThan(40);
});
it("reports unsupported required extensions instead of rendering a blank model", async () => {
const file = path.join(dir, "unsupported.gltf");
await fs.writeFile(
file,
JSON.stringify({
asset: { version: "2.0" },
extensionsRequired: ["VENDOR_unknown"],
}),
);
const comp = new Composition({
width: SIZE,
height: SIZE,
fps: 30,
durationMs: 1000,
backgroundColor: BG,
});
comp.add(
Layer.camera({
x: 200,
y: 200,
z: -600,
targetX: 200,
targetY: 200,
targetZ: 0,
}),
);
comp.add(Layer.glb(file, { id: "unsupported", x: 200, y: 200 }));
await expect(comp.renderFrame({ frame: 0 })).rejects.toThrow(
/VENDOR_unknown/,
);
});
it.each([
"etc1s",
"uastc-ldr",
])("renders required KTX2/Basis %s textures offline", async (format) => {
const g = new GlbBuilder();
const q = rect(-100, -100, 100, 100);
const image = await fs.readFile(
new URL(
`../../webgpu-renderers/src/renderer3d/fixtures/${format}.ktx2`,
import.meta.url,
),
);
const file = path.join(dir, `${format}.glb`);
await fs.writeFile(
file,
g.build({
extensionsUsed: ["KHR_texture_basisu"],
extensionsRequired: ["KHR_texture_basisu"],
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
TEXCOORD_0: g.accessor(new Float32Array(q.uvs), "VEC2"),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
material: 0,
},
],
},
],
materials: [
{ pbrMetallicRoughness: { baseColorTexture: { index: 0 } } },
],
textures: [{ extensions: { KHR_texture_basisu: { source: 0 } } }],
images: [{ bufferView: g.image(image), mimeType: "image/ktx2" }],
}),
);
const color = (await render(file))(200, 200);
expect(color[0]).toBeGreaterThan(180);
expect(color[1]).toBeLessThan(60);
expect(color[2]).toBeLessThan(50);
});
it("blends an ETC1S KTX2 texture with alpha", async () => {
const g = new GlbBuilder();
const q = rect(-100, -100, 100, 100);
const image = await fs.readFile(
new URL(
"../../webgpu-renderers/src/renderer3d/fixtures/etc1s-alpha.ktx2",
import.meta.url,
),
);
const file = path.join(dir, "ktx-alpha.glb");
await fs.writeFile(
file,
g.build({
extensionsUsed: ["KHR_texture_basisu"],
extensionsRequired: ["KHR_texture_basisu"],
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
TEXCOORD_0: g.accessor(new Float32Array(q.uvs), "VEC2"),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
material: 0,
},
],
},
],
materials: [
{
alphaMode: "BLEND",
pbrMetallicRoughness: { baseColorTexture: { index: 0 } },
},
],
textures: [{ extensions: { KHR_texture_basisu: { source: 0 } } }],
images: [{ bufferView: g.image(image), mimeType: "image/ktx2" }],
}),
);
const color = (await render(file))(200, 200);
expect(color[0]).toBeGreaterThan(100);
expect(color[0]).toBeLessThan(135);
});
it("renders meshopt-compressed glTF through the compositor", async () => {
const encoded = Buffer.from(
"oAAAAAEzAAAA//8AAAABDAAAAP8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMjCAADIwgAAAAA=",
"base64",
);
const file = path.join(dir, "meshopt.gltf");
await fs.writeFile(
file,
JSON.stringify({
asset: { version: "2.0" },
extensionsUsed: ["EXT_meshopt_compression"],
extensionsRequired: ["EXT_meshopt_compression"],
buffers: [
{
byteLength: encoded.length,
uri: `data:application/octet-stream;base64,${encoded.toString("base64")}`,
},
],
bufferViews: [
{
buffer: 0,
byteLength: 48,
byteStride: 12,
extensions: {
EXT_meshopt_compression: {
buffer: 0,
byteOffset: 0,
byteLength: encoded.length,
byteStride: 12,
count: 4,
mode: "ATTRIBUTES",
},
},
},
],
accessors: [
{ bufferView: 0, componentType: 5126, count: 4, type: "VEC3" },
],
meshes: [{ primitives: [{ attributes: { POSITION: 0 }, mode: 6 }] }],
}),
);
expect((await render(file))(200, 200)[0]).toBeGreaterThan(180);
});
it("renders Draco-compressed glTF through the compositor", async () => {
const requireFromRenderer = createRequire(
import.meta.resolve("@framefields/webgpu-renderers"),
);
const dracoRoot = path.dirname(requireFromRenderer.resolve("draco3dgltf"));
const encoded = await fs.readFile(path.join(dracoRoot, "bunny.drc"));
const file = path.join(dir, "draco.gltf");
await fs.writeFile(
file,
JSON.stringify({
asset: { version: "2.0" },
extensionsUsed: ["KHR_draco_mesh_compression"],
extensionsRequired: ["KHR_draco_mesh_compression"],
buffers: [
{
byteLength: encoded.length,
uri: `data:application/octet-stream;base64,${encoded.toString("base64")}`,
},
],
bufferViews: [{ buffer: 0, byteLength: encoded.length }],
accessors: [
{ componentType: 5126, count: 34834, type: "VEC3" },
{ componentType: 5125, count: 69451 * 3, type: "SCALAR" },
],
meshes: [
{
primitives: [
{
attributes: { POSITION: 0 },
indices: 1,
extensions: {
KHR_draco_mesh_compression: {
bufferView: 0,
attributes: { POSITION: 0 },
},
},
},
],
},
],
}),
);
const comp = new Composition({
width: SIZE,
height: SIZE,
fps: 30,
durationMs: 1000,
backgroundColor: BG,
});
comp.add(
Layer.camera({
x: 200,
y: 200,
z: -600,
targetX: 200,
targetY: 200,
targetZ: 0,
}),
);
comp.add(
Layer.glb(file, {
x: 200,
y: 200,
center: true,
normalizeSize: 240,
material: "unlit",
}),
);
const px = await pixels(await comp.renderFrame({ frame: 0 }));
let lit = 0;
for (let y = 100; y < 300; y += 4)
for (let x = 100; x < 300; x += 4) if ((px(x, y)[0] ?? 0) > 50) lit++;
expect(lit).toBeGreaterThan(100);
});
it.each([
0, 1, 2, 3, 5, 6,
])("renders glTF primitive mode %i", async (mode) => {
const g = new GlbBuilder();
const file = path.join(dir, `primitive-${mode}.glb`);
const q = rect(-100, -100, 100, 100);
await fs.writeFile(
file,
g.build({
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
},
mode,
},
],
},
],
}),
);
const px = await render(file);
const litNear = (x: number, y: number) => {
for (let dy = -2; dy <= 2; dy++)
for (let dx = -2; dx <= 2; dx++) {
if ((px(x + dx, y + dy)[0] ?? 0) > 50) return true;
}
return false;
};
if (mode === 0) expect(litNear(100, 300)).toBe(true);
else if (mode < 4) expect(litNear(200, 300)).toBe(true);
else expect(litNear(200, 200)).toBe(true);
});
it("samples an emissive map using its own UV set and transform", async () => {
const canvas = new Canvas(8, 8);
const ctx = canvas.getContext("2d");
ctx.fillStyle = "#ff0000";
ctx.fillRect(0, 0, 4, 8);
ctx.fillStyle = "#00ff00";
ctx.fillRect(4, 0, 4, 8);
const image = await canvas.toBuffer("png");
const renderUv = async (offset: number) => {
const g = new GlbBuilder();
const q = rect(-100, -100, 100, 100);
const file = path.join(dir, `map-uv-${offset}.glb`);
await fs.writeFile(
file,
g.build({
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
TEXCOORD_0: g.accessor(
new Float32Array([
0.25, 0.5, 0.25, 0.5, 0.25, 0.5, 0.25, 0.5,
]),
"VEC2",
),
TEXCOORD_1: g.accessor(
new Float32Array([
0.75, 0.5, 0.75, 0.5, 0.75, 0.5, 0.75, 0.5,
]),
"VEC2",
),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
material: 0,
},
],
},
],
materials: [
{
pbrMetallicRoughness: {
baseColorFactor: [0, 0, 0, 1],
baseColorTexture: { index: 0, texCoord: 0 },
},
emissiveFactor: [1, 1, 1],
emissiveTexture: {
index: 0,
texCoord: 1,
extensions: { KHR_texture_transform: { offset: [offset, 0] } },
},
},
],
textures: [{ source: 0 }],
images: [{ bufferView: g.image(image), mimeType: "image/png" }],
}),
);
return (await render(file, 0, 30, true))(SIZE / 2, SIZE / 2);
};
const green = await renderUv(0);
const red = await renderUv(-0.5);
expect(green[1]).toBeGreaterThan(200);
expect(green[0]).toBeLessThan(40);
expect(red[0]).toBeGreaterThan(200);
expect(red[1]).toBeLessThan(40);
});
it("applies animated morph weights to mesh positions", async () => {
const g = new GlbBuilder();
const q = rect(-100, -30, -40, 30);
const file = path.join(dir, "morph.glb");
const delta = new Float32Array([
120, 0, 0, 120, 0, 0, 120, 0, 0, 120, 0, 0,
]);
await fs.writeFile(
file,
g.build({
nodes: [{ mesh: 0 }],
scenes: [{ nodes: [0] }],
meshes: [
{
weights: [0],
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
targets: [{ POSITION: g.accessor(delta, "VEC3") }],
},
],
},
],
animations: [
{
channels: [{ sampler: 0, target: { node: 0, path: "weights" } }],
samplers: [
{
input: g.accessor(new Float32Array([0, 1]), "SCALAR"),
output: g.accessor(new Float32Array([0, 1]), "SCALAR"),
interpolation: "LINEAR",
},
],
},
],
}),
);
const rest = await render(file, 0);
const halfway = await render(file, 15);
const end = await render(file, 30);
expect(rest(130, 200)[0]).toBeGreaterThan(200);
expect(rest(250, 200)[0]).toBeLessThan(10);
expect(halfway(190, 200)[0]).toBeGreaterThan(200);
expect(end(130, 200)[0]).toBeLessThan(10);
expect(end(250, 200)[0]).toBeGreaterThan(200);
});
it("renders a skin whose vertices reference joint 129", async () => {
const g = new GlbBuilder();
const q = rect(-100, -20, -60, 20);
const file = path.join(dir, "large-skin.glb");
const joints = new Uint16Array([
128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0, 128, 0, 0, 0,
]);
const weights = new Float32Array([
1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0,
]);
const nodes = [
{ mesh: 0, skin: 0 },
...Array.from({ length: 129 }, (_, i) => ({
translation: [i === 128 ? 90 : 0, 0, 0],
})),
];
await fs.writeFile(
file,
g.build({
nodes,
scenes: [{ nodes: nodes.map((_, i) => i) }],
skins: [{ joints: Array.from({ length: 129 }, (_, i) => i + 1) }],
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
JOINTS_0: g.accessor(joints, "VEC4"),
WEIGHTS_0: g.accessor(weights, "VEC4"),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
},
],
},
],
}),
);
const px = await render(file);
expect(px(210, 200)[0]).toBeGreaterThan(200);
expect(px(120, 200)[0]).toBeLessThan(10);
});
it("applies morph deltas before the skin's joint transform", async () => {
const g = new GlbBuilder();
const q = rect(-100, -20, -60, 20);
const file = path.join(dir, "morph-skin.glb");
const delta = new Float32Array([50, 0, 0, 50, 0, 0, 50, 0, 0, 50, 0, 0]);
const joints = new Uint16Array(16);
const weights = new Float32Array([
1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0,
]);
await fs.writeFile(
file,
g.build({
nodes: [
{ mesh: 0, skin: 0, weights: [1] },
{ translation: [40, 0, 0] },
],
scenes: [{ nodes: [0, 1] }],
skins: [{ joints: [1] }],
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
JOINTS_0: g.accessor(joints, "VEC4"),
WEIGHTS_0: g.accessor(weights, "VEC4"),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
targets: [{ POSITION: g.accessor(delta, "VEC3") }],
},
],
},
],
}),
);
const px = await render(file);
expect(px(210, 200)[0]).toBeGreaterThan(200);
expect(px(120, 200)[0]).toBeLessThan(10);
});
it("keeps independent default weights on instances of the same morph mesh", async () => {
const g = new GlbBuilder();
const q = rect(-20, -20, 20, 20);
const file = path.join(dir, "morph-instances.glb");
const delta = new Float32Array([
100, 0, 0, 100, 0, 0, 100, 0, 0, 100, 0, 0,
]);
await fs.writeFile(
file,
g.build({
nodes: [
{ mesh: 0, translation: [-100, 70, 0], weights: [0] },
{ mesh: 0, translation: [-100, -70, 0], weights: [1] },
],
scenes: [{ nodes: [0, 1] }],
meshes: [
{
weights: [0],
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
targets: [{ POSITION: g.accessor(delta, "VEC3") }],
},
],
},
],
}),
);
const px = await render(file);
expect(px(100, 130)[0]).toBeGreaterThan(200);
expect(px(200, 270)[0]).toBeGreaterThan(200);
expect(px(100, 270)[0]).toBeLessThan(10);
});
it("poses a skinned mesh through its joint chain from a rotation track", async () => {
// A 20×200 strip: the lower half on joint 0 (origin), the upper on
// joint 1 (100 up, child of joint 0). The clip turns joint 1 by 90°
// about Z over one second, folding the upper half to the left.
const g = new GlbBuilder();
const positions = [
-10, 0, 0, 10, 0, 0, -10, 100, 0, 10, 100, 0, -10, 200, 0, 10, 200, 0,
];
const joints = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0,
];
const weights = [
1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0,
];
const ibm = [
...[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1],
...[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, -100, 0, 1],
];
const s45 = Math.SQRT1_2;
const file = path.join(dir, "skinned.glb");
await fs.writeFile(
file,
g.build({
scenes: [{ nodes: [0, 1] }],
nodes: [
{ mesh: 0, skin: 0 },
{ name: "root", children: [2] },
{ name: "tip", translation: [0, 100, 0] },
],
skins: [
{
joints: [1, 2],
inverseBindMatrices: g.accessor(new Float32Array(ibm), "MAT4"),
},
],
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(positions), "VEC3"),
JOINTS_0: g.accessor(new Uint16Array(joints), "VEC4"),
WEIGHTS_0: g.accessor(new Float32Array(weights), "VEC4"),
},
indices: g.accessor(
new Uint16Array([0, 1, 3, 0, 3, 2, 2, 3, 5, 2, 5, 4]),
"SCALAR",
),
material: 0,
},
],
},
],
materials: [
{ pbrMetallicRoughness: { baseColorFactor: [1, 1, 1, 1] } },
],
animations: [
{
channels: [{ sampler: 0, target: { node: 2, path: "rotation" } }],
samplers: [
{
input: g.accessor(new Float32Array([0, 1]), "SCALAR"),
output: g.accessor(
new Float32Array([0, 0, 0, 1, 0, 0, s45, s45]),
"VEC4",
),
},
],
},
],
}),
);
// Model (x, y) lands on screen (SIZE/2 + x, SIZE/2 - y).
const lit = (
px: (x: number, y: number) => number[],
x: number,
y: number,
) => (px(SIZE / 2 + x, SIZE / 2 - y)[0] ?? 0) > 128;
const rest = await render(file, 0);
expect(lit(rest, 0, 150)).toBe(true);
expect(lit(rest, -60, 100)).toBe(false);
const bent = await render(file, 30);
expect(lit(bent, 0, 150)).toBe(false);
expect(lit(bent, -60, 100)).toBe(true);
expect(lit(bent, 0, 50)).toBe(true);
});
it("places an unskinned mesh by its node's parent chain", async () => {
const g = new GlbBuilder();
const q = rect(-20, -20, 20, 20);
const file = path.join(dir, "nested.glb");
await fs.writeFile(
file,
g.build({
scenes: [{ nodes: [0] }],
nodes: [
// Turned 90° about Z, so the child's +X offset points up.
{ children: [1], rotation: [0, 0, Math.SQRT1_2, Math.SQRT1_2] },
{ mesh: 0, translation: [120, 0, 0] },
],
meshes: [
{
primitives: [
{
attributes: {
POSITION: g.accessor(new Float32Array(q.positions), "VEC3"),
},
indices: g.accessor(new Uint16Array(q.indices), "SCALAR"),
},
],
},
],
}),
);
const px = await render(file);
expect((px(SIZE / 2, SIZE / 2 - 120)[0] ?? 0) > 128).toBe(true);
expect((px(SIZE / 2 + 120, SIZE / 2)[0] ?? 0) > 128).toBe(false);
});
});