Skip to content

Code-first video & image, rendered natively on WebGPU.

Automatically instance repeated opaque meshes

okanasl1da767da81bd
diff --git a/.changeset/smart-birds-draw.md b/.changeset/smart-birds-draw.md
new file mode 100644
index 00000000..3aebb07b
--- /dev/null
+++ b/.changeset/smart-birds-draw.md
@@ -0,0 +1,6 @@
+---
+"@framefields/node-compositor": patch
+"@framefields/webgpu-renderers": patch
+---
+
+Batch compatible repeated opaque meshes into hardware-instanced draws while preserving individual transforms.
diff --git a/nodes/node-compositor/SKILL.md b/nodes/node-compositor/SKILL.md
index 45502a05..80e5c0ef 100644
--- a/nodes/node-compositor/SKILL.md
+++ b/nodes/node-compositor/SKILL.md
@@ -212,6 +212,7 @@ Each track represents animatable property modifications:
- **Mesh wireframe:** `wireframe: true` on a `model3d` draws triangle edges as one-pixel lines, including triangulation and back edges; it also applies to skinned and audio-deformed meshes and their shadows.
- **glTF scenes:** a glTF/GLB `model3d` keeps its node hierarchy, every skin and its animation clips (translation, rotation, scale and morph weights; linear, step and cubic-spline). Draco and meshopt compressed geometry and all seven core primitive modes are supported; per-map UV sets and texture transforms are independent. Basis KTX2 (ETC1S/UASTC) textures transcode offline to RGBA8 with their mip levels. Unsupported required extensions fail clearly. Morph targets run on the GPU before skinning and audio deformation. Skinned meshes are posed per frame from the clip (`animationName`, `animationTime`/`animationProgress`, `loop`); base-colour textures (with mipmaps), alpha `MASK`/`BLEND` and VRM MToon materials (`material: "toon"`, picked automatically when `material` is unset) render as authored. `center`/`normalizeSize` apply above the scene graph, so skinning stays intact.
- **Procedural Shards (`Layer3D.shard`):** Procedural polyhedron primitive generated by Sutherland-Hodgman convex half-space clipping against Fibonacci lattice normals with PCG jitter. Faceted flat normals, automatic planar UVs, Rapier 3D convex hull collision.
+- **Automatic mesh instancing:** consecutive opaque draws of the same shared mesh with matching material state use one GPU draw with per-object model and normal matrices. Transparent, skinned, morphed, audio-deformed and glass meshes retain separate draws; shadows retain their individual draw submissions.
- **Instanced 3D Meshes (`instances3d`):** `kind: "model3d"` with `instances3d` renders hundreds or thousands of mesh instances in 1 GPU draw call (`drawIndexed(indexCount, instanceCount)`). Supports single or multi-mesh variants with round-robin or variant indexing, dynamic per-instance transforms, and per-instance emissive / color modulation. In `framefields`, created with `Layer3D.instances({ count, mesh, instanceAt, bodies, emissiveBySpeed })`.
- **GPU 3D Swarms (`swarm3d`):** `kind: "model3d"` with `swarm3d` runs continuous 3D particle simulation in a WebGPU compute shader with 3D curl noise, 3D vortex fields (`swirl`, `pull`, `axial`), ground plane collisions with Coulomb sliding friction and rotational roll torque, and kinetic heat accumulation/cooling, packing directly into GPU instance buffers.
- **Planar Reflective Floor (`floor` in `environment`):** `kind: "light"` with `lightType: "environment"` and `floor: { y, color, reflectivity, roughness, fadeDistance, opacity }` renders a mirrored-camera reflection pass, subterranean fragment clipping, 2-pass anisotropic roughness blur ($\sigma_y \approx 3.5 \sigma_x$), Schlick Fresnel grazing reflections, and analytical depth unprojection.
diff --git a/nodes/node-compositor/src/renderers/mesh-instancing.test.ts b/nodes/node-compositor/src/renderers/mesh-instancing.test.ts
new file mode 100644
index 00000000..68b8f94b
--- /dev/null
+++ b/nodes/node-compositor/src/renderers/mesh-instancing.test.ts
@@ -0,0 +1,35 @@
+import { getPrimitiveGeometry } from "@framefields/webgpu-renderers";
+import { describe, expect, it } from "vitest";
+import { compatibleMeshDraws, type MeshDraw } from "./mesh-instancing.js";
+
+const mesh = getPrimitiveGeometry({ type: "sphere", radius: 20 }).meshes[0];
+function draw(opts: MeshDraw["opts"] = {}): MeshDraw {
+ if (!mesh) throw new Error("Missing test mesh");
+ return { kind: "mesh", mesh, opts };
+}
+
+describe("automatic mesh instancing eligibility", () => {
+ it("accepts different transforms with equal material state", () => {
+ expect(
+ compatibleMeshDraws(
+ draw({ position: [1, 2, 3], colorTint: [1, 0, 0, 1] }),
+ draw({ position: [5, 6, 7], colorTint: [1, 0, 0, 1] }),
+ ),
+ ).toBe(true);
+ });
+ it("keeps incompatible materials, transparent meshes and per-object deformation separate", () => {
+ expect(
+ compatibleMeshDraws(
+ draw({ colorTint: [1, 0, 0, 1] }),
+ draw({ colorTint: [0, 0, 1, 1] }),
+ ),
+ ).toBe(false);
+ expect(compatibleMeshDraws(draw(), draw({ transparent: true }))).toBe(
+ false,
+ );
+ expect(compatibleMeshDraws(draw(), draw({ morphWeights: [0.5] }))).toBe(
+ false,
+ );
+ expect(compatibleMeshDraws(draw(), draw({ isSkinned: true }))).toBe(false);
+ });
+});
diff --git a/nodes/node-compositor/src/renderers/mesh-instancing.ts b/nodes/node-compositor/src/renderers/mesh-instancing.ts
new file mode 100644
index 00000000..5dcf193a
--- /dev/null
+++ b/nodes/node-compositor/src/renderers/mesh-instancing.ts
@@ -0,0 +1,85 @@
+import type {
+ DrawMesh3DOpts,
+ GPUMeshBuffers,
+ Mesh3DData,
+} from "@framefields/webgpu-renderers";
+
+export interface MeshDraw {
+ kind: "mesh";
+ mesh: Mesh3DData;
+ texture?: GPUTexture;
+ opts: DrawMesh3DOpts;
+ meshBuffers?: GPUMeshBuffers;
+}
+
+const TRANSFORMS = new Set([
+ "position",
+ "rotation",
+ "scale",
+ "modelMatrix",
+ "normalMatrix",
+ "bounds",
+]);
+
+/** Compare material state without treating different GPU resources as equal. */
+function sameValue(a: unknown, b: unknown): boolean {
+ if (a === b) return true;
+ if (Array.isArray(a) && Array.isArray(b)) {
+ return (
+ a.length === b.length && a.every((value, i) => sameValue(value, b[i]))
+ );
+ }
+ if (!a || !b || typeof a !== "object" || typeof b !== "object") return false;
+ if (
+ Object.getPrototypeOf(a) !== Object.prototype ||
+ Object.getPrototypeOf(b) !== Object.prototype
+ )
+ return false;
+ const left = a as Record<string, unknown>;
+ const right = b as Record<string, unknown>;
+ const keys = Object.keys(left);
+ return (
+ keys.length === Object.keys(right).length &&
+ keys.every(
+ (key) => Object.hasOwn(right, key) && sameValue(left[key], right[key]),
+ )
+ );
+}
+
+export function canInstanceMesh(item: MeshDraw): boolean {
+ const o = item.opts;
+ return (
+ !item.meshBuffers &&
+ !o.transparent &&
+ !o.instanceBuffer &&
+ (o.instanceCount ?? 1) === 1 &&
+ !o.firstInstance &&
+ !o.matrixInstances &&
+ !o.isSkinned &&
+ !o.skinningMatrices &&
+ !o.audioDeform &&
+ !o.morphWeights?.length &&
+ o.material !== "glass" &&
+ o.material !== "acrylic" &&
+ o.material !== "frosted" &&
+ (o.opacity ?? 1) >= 1 &&
+ (o.colorTint?.[3] ?? 1) >= 1
+ );
+}
+
+export function compatibleMeshDraws(a: MeshDraw, b: MeshDraw): boolean {
+ if (
+ !canInstanceMesh(a) ||
+ !canInstanceMesh(b) ||
+ a.mesh !== b.mesh ||
+ a.texture !== b.texture
+ )
+ return false;
+ const left = a.opts as Record<string, unknown>;
+ const right = b.opts as Record<string, unknown>;
+ const keys = new Set([...Object.keys(left), ...Object.keys(right)]);
+ for (const key of keys) {
+ if (!TRANSFORMS.has(key) && !sameValue(left[key], right[key])) return false;
+ }
+ return true;
+}
diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.ts
index 208fc313..c48101cb 100644
--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts
+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts
@@ -78,6 +78,7 @@ import {
import { compileTimeline, seekTimeline } from "../shared/compiler.js";
import type { CompositorOperation } from "../shared/config.js";
import { shouldPaintContainerBackground } from "./container-bg.js";
+import { compatibleMeshDraws } from "./mesh-instancing.js";
import {
buildLayerMatrix,
resolveLayer2DPosition,
@@ -1492,8 +1493,56 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {
} else opaque.push(item);
}
transparent.sort((a, b) => b.depth - a.depth);
+ // Batch consecutive compatible opaque meshes without changing draw order.
+ // Each pair of storage records holds the full model and normal matrices.
+ const batchedOpaque: Pending3DItem[] = [];
+ const maxInstances = Math.max(
+ 1,
+ Math.floor(ctx.device.limits.maxStorageBufferBindingSize / 128),
+ );
+ for (let i = 0; i < opaque.length; ) {
+ const first = opaque[i];
+ if (first.kind !== "mesh") {
+ batchedOpaque.push(first);
+ i++;
+ continue;
+ }
+ let end = i + 1;
+ while (end < opaque.length && end - i < maxInstances) {
+ const next = opaque[end];
+ if (next.kind !== "mesh" || !compatibleMeshDraws(first, next)) break;
+ end++;
+ }
+ if (end - i === 1) {
+ batchedOpaque.push(first);
+ } else {
+ const data = new Float32Array((end - i) * 32);
+ for (let j = i; j < end; j++) {
+ const item = opaque[j];
+ if (item.kind !== "mesh") throw new Error("Invalid mesh batch");
+ const matrix = r3d.meshModelMatrix(item.opts);
+ data.set(matrix, (j - i) * 32);
+ data.set(
+ item.opts.normalMatrix ?? Light3D.computeNormalMatrix(matrix),
+ (j - i) * 32 + 16,
+ );
+ }
+ batchedOpaque.push({
+ ...first,
+ opts: {
+ ...first.opts,
+ modelMatrix: Matrix4Math.identity(),
+ normalMatrix: Matrix4Math.identity(),
+ instanceBuffer: r3d.uploadInstanceData(data),
+ instanceCount: end - i,
+ matrixInstances: true,
+ },
+ });
+ }
+ i = end;
+ }
const ordered = [
- ...opaque,
+ ...batchedOpaque,
...transparent.map(({ item }) => item),
...glass,
];
diff --git a/packages/framefields/src/automatic-instancing-render.test.ts b/packages/framefields/src/automatic-instancing-render.test.ts
new file mode 100644
index 00000000..8e6afcdc
--- /dev/null
+++ b/packages/framefields/src/automatic-instancing-render.test.ts
@@ -0,0 +1,152 @@
+import { Renderer3D } from "@framefields/webgpu-renderers";
+import { Canvas, loadImage } from "skia-canvas";
+import { describe, expect, it, vi } from "vitest";
+import { Composition, Layer, Layer3D } from "./index.js";
+
+describe("automatic repeated mesh instancing", () => {
+ it("renders repeated opaque meshes with one instanced draw and separates different materials", async () => {
+ const calls = vi.spyOn(Renderer3D.prototype, "drawMesh3D");
+ try {
+ const comp = new Composition({
+ width: 600,
+ height: 300,
+ fps: 30,
+ durationMs: 1000,
+ backgroundColor: "#000000",
+ });
+ comp.add(
+ Layer.camera({
+ x: 300,
+ y: 150,
+ z: -600,
+ targetX: 300,
+ targetY: 150,
+ targetZ: 0,
+ }),
+ );
+ for (const x of [150, 300, 450])
+ comp.add(
+ Layer3D.roundedBox({
+ width: 70,
+ height: 70,
+ depth: 70,
+ radius: 0,
+ x,
+ y: 150,
+ color: "#ff0000",
+ material: "unlit",
+ }),
+ );
+ comp.add(
+ Layer3D.roundedBox({
+ width: 70,
+ height: 70,
+ depth: 70,
+ radius: 0,
+ x: 300,
+ y: 245,
+ color: "#0000ff",
+ material: "unlit",
+ }),
+ );
+ const png = await comp.renderFrame({ frame: 0 });
+ const image = await loadImage(png);
+ const canvas = new Canvas(image.width, image.height);
+ const ctx = canvas.getContext("2d");
+ ctx.drawImage(image, 0, 0);
+ const pixels = ctx.getImageData(0, 0, image.width, image.height).data;
+ const redAt = (x: number, y: number) =>
+ pixels[(y * image.width + x) * 4] ?? 0;
+ for (const x of [220, 300, 380])
+ expect(redAt(x, 150)).toBeGreaterThan(100);
+ expect(
+ calls.mock.calls.some(
+ (call) => call[5]?.matrixInstances && call[5]?.instanceCount === 3,
+ ),
+ ).toBe(true);
+ expect(calls.mock.calls.some((call) => !call[5]?.matrixInstances)).toBe(
+ true,
+ );
+ } finally {
+ calls.mockRestore();
+ }
+ });
+ it("matches separate draws for rotated, non-uniformly scaled lit meshes", async () => {
+ const calls = vi.spyOn(Renderer3D.prototype, "drawMesh3D");
+ try {
+ const render = async (separate: boolean) => {
+ const comp = new Composition({
+ width: 600,
+ height: 300,
+ fps: 30,
+ durationMs: 1000,
+ backgroundColor: "#000000",
+ });
+ comp.add(
+ Layer.camera({
+ x: 300,
+ y: 150,
+ z: -600,
+ targetX: 300,
+ targetY: 150,
+ targetZ: 0,
+ }),
+ );
+ comp.add(Layer.ambientLight("#ffffff", 0.25));
+ comp.add(
+ Layer.directionalLight({
+ x: 0,
+ y: -200,
+ z: -400,
+ targetX: 300,
+ targetY: 150,
+ targetZ: 0,
+ color: "#ffffff",
+ intensity: 1,
+ }),
+ );
+ for (const [i, x] of [150, 300, 450].entries())
+ comp.add(
+ Layer3D.roundedBox({
+ width: 70,
+ height: 70,
+ depth: 70,
+ radius: 0,
+ x,
+ y: 150,
+ rotateY: i * 18,
+ rotateZ: i * 11,
+ scaleX: 0.6 + i * 0.3,
+ scaleY: 1.2,
+ color: "#cc9966",
+ material: "lit",
+ specularIntensity: 0,
+ shininess: separate ? 20 + i : 20,
+ }),
+ );
+ const img = await loadImage(await comp.renderFrame({ frame: 0 }));
+ const canvas = new Canvas(img.width, img.height);
+ const ctx = canvas.getContext("2d");
+ ctx.drawImage(img, 0, 0);
+ return ctx.getImageData(0, 0, img.width, img.height).data;
+ };
+ const batched = await render(false);
+ expect(
+ calls.mock.calls.some(
+ (call) => call[5]?.matrixInstances && call[5]?.instanceCount === 3,
+ ),
+ ).toBe(true);
+ calls.mockClear();
+ const separate = await render(true);
+ expect(calls.mock.calls.some((call) => call[5]?.matrixInstances)).toBe(
+ false,
+ );
+ let difference = 0;
+ for (let i = 0; i < batched.length; i++)
+ difference += Math.abs((batched[i] ?? 0) - (separate[i] ?? 0));
+ expect(difference / batched.length).toBeLessThan(0.1);
+ } finally {
+ calls.mockRestore();
+ }
+ });
+});
diff --git a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts
index a823e4a3..f93af933 100644
--- a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts
+++ b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts
@@ -55,6 +55,8 @@ export interface MeshUniformData extends SurfaceParams {
skinningMatrices?: Float32Array; // 128 * 16 floats
/** Hardware-instanced draw path. */
isInstanced?: boolean;
+ /** Instance storage contains model and normal matrices in pairs. */
+ matrixInstances?: boolean;
/** Storage buffer containing `array<InstanceData>` (64 bytes per instance). */
instanceBuffer?: GPUBuffer;
material?: "lit" | "unlit" | "toon";
@@ -459,7 +461,8 @@ export class Mesh3DPipeline {
// Bit 0 selects instance data; bit 1 selects the extra map UVs.
this.modelData[39] =
(data.isInstanced || data.instanceBuffer ? 1 : 0) +
- (data.hasMapUvs ? 2 : 0);
+ (data.hasMapUvs ? 2 : 0) +
+ (data.matrixInstances ? 4 : 0);
const shininess =
data.roughness !== undefined
diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
index 81bd0753..fc0e6265 100644
--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
@@ -210,6 +210,8 @@ export interface DrawMesh3DOpts extends SurfaceParams {
firstInstance?: number;
/** Storage buffer holding `InstanceData` structs (64 bytes per instance). */
instanceBuffer?: GPUBuffer;
+ /** Each instance stores a model matrix followed by its normal matrix. */
+ matrixInstances?: boolean;
/** Optional overall bounding box (e.g. for instanced clusters or swarms) for shadow frustum fitting. */
bounds?: Aabb;
}
@@ -1341,7 +1343,7 @@ export class Renderer3D {
}
/** T(pos) * Rz * Ry * Rx * S, or the draw's precomputed model matrix. */
- private meshModelMatrix(opts: DrawMesh3DOpts): Mat4 {
+ meshModelMatrix(opts: DrawMesh3DOpts): Mat4 {
if (opts.modelMatrix) return opts.modelMatrix;
const {
position = [0, 0, 0],
@@ -1686,6 +1688,7 @@ export class Renderer3D {
opts.instanceBuffer || (opts.instanceCount && opts.instanceCount > 1),
),
instanceBuffer: opts.instanceBuffer,
+ matrixInstances: opts.matrixInstances,
},
useMrt,
this.passWritesHdr,
diff --git a/packages/webgpu-renderers/src/shaders/mesh3d.ts b/packages/webgpu-renderers/src/shaders/mesh3d.ts
index 7b02d92a..862ffe27 100644
--- a/packages/webgpu-renderers/src/shaders/mesh3d.ts
+++ b/packages/webgpu-renderers/src/shaders/mesh3d.ts
@@ -17,7 +17,7 @@ struct ModelUniforms {
modelMatrix : mat4x4<f32>,
normalMatrix : mat4x4<f32>,
colorTint : vec4<f32>,
- params : vec4<f32>, // x = opacity, y = twoSided, z = isSkinned, w = flags (bit 0 instanced, bit 1 map UVs)
+ params : vec4<f32>, // x = opacity, y = twoSided, z = isSkinned, w = flags (bit 0 instanced, bit 1 map UVs, bit 2 matrix instances)
materialParams : vec4<f32>, // x = shininess, y = specularIntensity, z = ambientIntensity, w = materialMode (0=unlit, 1=lit, 2=toon)
shade : vec4<f32>, // rgb = toon shadow-side color multiplier, a = alpha cutoff (0 = none)
pbr : vec4<f32>, // x = roughness (<0 = default), y = metallic
@@ -124,7 +124,14 @@ fn vs_main(
var instColor = vec4<f32>(1.0);
var instEmissive = vec4<f32>(0.0);
- if ((u32(model.params.w) & 1u) != 0u) {
+ if ((u32(model.params.w) & 4u) != 0u) {
+ let a = instances[instanceIndex * 2u];
+ let b = instances[instanceIndex * 2u + 1u];
+ let instanceModel = mat4x4<f32>(a.posScale, a.quat, a.color, a.emissive);
+ let instanceNormal = mat4x4<f32>(b.posScale, b.quat, b.color, b.emissive);
+ localPos = (instanceModel * vec4<f32>(localPos, 1.0)).xyz;
+ localNorm = (instanceNormal * vec4<f32>(localNorm, 0.0)).xyz;
+ } else if ((u32(model.params.w) & 1u) != 0u) {
let inst = instances[instanceIndex];
let scaledPos = localPos * inst.posScale.w;
localPos = rotateVecByQuat(scaledPos, inst.quat) + inst.posScale.xyz;