Skip to content

Code-first video & image, rendered natively on WebGPU.

Fix 3D mesh wireframe rendering (#60)

Okan Aslankane96a32f18d81
diff --git a/.changeset/fix-mesh-wireframe.md b/.changeset/fix-mesh-wireframe.md
new file mode 100644
index 00000000..19213c5f
--- /dev/null
+++ b/.changeset/fix-mesh-wireframe.md
@@ -0,0 +1,7 @@
+---
+"framefields": patch
+"@framefields/webgpu-renderers": patch
+"@framefields/node-compositor": patch
+---
+
+Render model and procedural mesh wireframes as triangle edges, including animated and shadow-casting meshes.
diff --git a/nodes/node-compositor/SKILL.md b/nodes/node-compositor/SKILL.md
index d41825a5..7d997706 100644
--- a/nodes/node-compositor/SKILL.md
+++ b/nodes/node-compositor/SKILL.md
@@ -208,6 +208,7 @@ Each track represents animatable property modifications:
- **Watermarking a Video:** a `flex` row (`align: "end"`, `justify: "end"`, full canvas) containing a `media` node bound to the PNG input; low `opacity`.
- **Picture-in-Picture:** a `flex` row with two `media` nodes (`fit: "cover"`, each `grow: 1`).
- **Audio-Reactive 3D Models:** a `kind: "model3d"` node loading an OBJ/FBX/GLTF/STL/PLY mesh with `audioDeform: { audioTrackId, mode: "normal_extrusion" | "radial_pulse" | "harmonic_wave" | "twist" | "ripple", amplitudeMultiplier, damping }`, deformed directly in VRAM via WebGPU compute shaders driven by audio frequency spectra and transient energies. `twist` turns each slice about Y by its height (harder on the bass); `ripple` runs concentric waves across XY, struck by the drums. `audioTrackId` may be an audio layer's id or an audio file path.
+- **Mesh wireframe:** `wireframe: true` on a `model3d` draws triangle edges as one-pixel lines, including triangulation and back edges; it also applies to skinned and audio-deformed meshes and their shadows.
- **glTF scenes:** a glTF/GLB `model3d` keeps its node hierarchy, every skin and its animation clips (translation, rotation, scale; linear, step and cubic-spline). Skinned meshes are posed per frame from the clip (`animationName`, `animationTime`/`animationProgress`, `loop`); base-colour textures (with mipmaps), alpha `MASK`/`BLEND` and VRM MToon materials (`material: "toon"`, picked automatically when `material` is unset) render as authored. `center`/`normalizeSize` apply above the scene graph, so skinning stays intact.
## Don't Forget
diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.ts
index 36fd97b4..99fbb140 100644
--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts
+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts
@@ -1309,6 +1309,7 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {
encoder,
item.mesh,
item.opts.audioDeform,
+ item.opts.wireframe,
);
}
}
diff --git a/packages/framefields/src/primitives-3d-render.test.ts b/packages/framefields/src/primitives-3d-render.test.ts
index 43dc2a16..470fa89b 100644
--- a/packages/framefields/src/primitives-3d-render.test.ts
+++ b/packages/framefields/src/primitives-3d-render.test.ts
@@ -41,6 +41,49 @@ function stage() {
}
describe("Layer3D mesh primitives", () => {
+ it("renders mesh triangle edges when wireframe is enabled", async () => {
+ const render = async (wireframe: boolean) => {
+ const comp = stage();
+ comp.add(
+ Layer.directionalLight({
+ color: "#ffffff",
+ intensity: 1,
+ x: 200,
+ y: -300,
+ z: -400,
+ targetX: 400,
+ targetY: 300,
+ targetZ: 0,
+ castShadow: true,
+ }),
+ );
+ comp.add(
+ Layer3D.roundedBox({
+ width: 220,
+ height: 180,
+ depth: 120,
+ x: 400,
+ y: 300,
+ color: "#ffffff",
+ material: "lit",
+ wireframe,
+ }),
+ );
+ return decode(await comp.renderFrame({ frame: 0 }));
+ };
+ const solid = await render(false);
+ const wire = await render(true);
+ const lit = (pixels: Uint8ClampedArray) => {
+ let count = 0;
+ for (let i = 0; i < pixels.length; i += 4) {
+ if (pixels[i]! > 50) count++;
+ }
+ return count;
+ };
+ expect(lit(wire.data)).toBeGreaterThan(100);
+ expect(lit(wire.data)).toBeLessThan(lit(solid.data) / 4);
+ });
+
it("draws a rounded box's top face from above, single-sided", async () => {
const comp = stage();
comp.add(
diff --git a/packages/framefields/src/shapes3d.ts b/packages/framefields/src/shapes3d.ts
index 6e479fca..a64677d2 100644
--- a/packages/framefields/src/shapes3d.ts
+++ b/packages/framefields/src/shapes3d.ts
@@ -277,6 +277,8 @@ export interface Primitive3DNodeOptions extends Surface3DOptions {
scaleX?: number;
scaleY?: number;
scaleZ?: number;
+ /** Draw triangle edges instead of filled surfaces. */
+ wireframe?: boolean;
/** Surface color baked into the mesh (default white). */
color?: string;
/**
@@ -358,6 +360,7 @@ function primitiveNode(
castShadow: o.castShadow,
}),
twoSided: false,
+ wireframe: o.wireframe,
startFrame: o.startFrame,
durationFrames: o.durationFrames,
});
diff --git a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts
index cabeb82c..7c1e5d7a 100644
--- a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts
+++ b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts
@@ -79,6 +79,8 @@ export interface GPUMeshBuffers {
indexFormat: GPUIndexFormat;
/** Per-vertex RGBA (display / sRGB), 4 floats each; absent = white. */
colorBuffer?: GPUBuffer;
+ wireIndexBuffer?: GPUBuffer;
+ wireIndexCount?: number;
}
export class Mesh3DPipeline {
@@ -86,6 +88,10 @@ export class Mesh3DPipeline {
public pipelineSingle: GPURenderPipeline;
public pipelineMrt: GPURenderPipeline;
private bloomVariants: BloomPipelineVariants;
+ private wireSingle: GPURenderPipeline;
+ private wireMrt: GPURenderPipeline;
+ private wireBloom: BloomPipelineVariants;
+ private wireShadow: GPURenderPipeline;
/** Depth only (skinning included): draws a caster into a shadow map. */
public pipelineShadow: GPURenderPipeline;
public cameraLayout: GPUBindGroupLayout;
@@ -231,6 +237,29 @@ export class Mesh3DPipeline {
this.pipelineSingle = single;
this.pipelineMrt = mrt;
this.bloomVariants = new BloomPipelineVariants(device, base);
+ const wireBase: PipelineVariantBase = {
+ ...base,
+ label: "Mesh3DWireframe",
+ primitive: { topology: "line-list" },
+ };
+ const wire = createBasePipelines(device, wireBase);
+ this.wireSingle = wire.single;
+ this.wireMrt = wire.mrt;
+ this.wireBloom = new BloomPipelineVariants(device, wireBase);
+ this.wireShadow = device.createRenderPipeline({
+ label: "Mesh3DWireframeShadow",
+ layout: device.createPipelineLayout({
+ bindGroupLayouts: [this.cameraLayout, this.modelLayout],
+ }),
+ vertex: vertexState,
+ // WebGPU prohibits depth bias on line topology.
+ depthStencil: {
+ format: SHADOW_DEPTH_STENCIL.format,
+ depthWriteEnabled: true,
+ depthCompare: "less-equal",
+ },
+ primitive: { topology: "line-list" },
+ });
this.pipelineShadow = device.createRenderPipeline({
label: "Mesh3DPipelineShadow",
@@ -304,7 +333,7 @@ export class Mesh3DPipeline {
this.modelData[36] = data.opacity ?? 1.0;
this.modelData[37] = data.twoSided ? 1.0 : 0.0;
this.modelData[38] = data.isSkinned ? 1.0 : 0.0;
- this.modelData[39] = data.wireframe ? 1.0 : 0.0;
+ this.modelData[39] = 0;
const shininess =
data.roughness !== undefined
@@ -402,12 +431,20 @@ export class Mesh3DPipeline {
],
});
+ const wireframe = data.wireframe === true;
+ if (wireframe && !meshBuffers.wireIndexBuffer) {
+ throw new Error("Wireframe requires mesh buffers with edge indices");
+ }
pass.setPipeline(
useBloom
- ? this.bloomVariants.get(useMrt)
- : useMrt
- ? this.pipelineMrt
- : this.pipelineSingle,
+ ? (wireframe ? this.wireBloom : this.bloomVariants).get(useMrt)
+ : wireframe
+ ? useMrt
+ ? this.wireMrt
+ : this.wireSingle
+ : useMrt
+ ? this.pipelineMrt
+ : this.pipelineSingle,
);
pass.setBindGroup(0, cameraBindGroup);
pass.setBindGroup(1, modelBindGroup);
@@ -415,8 +452,13 @@ export class Mesh3DPipeline {
pass.setBindGroup(3, lightsBindGroup);
pass.setVertexBuffer(0, meshBuffers.vertexBuffer);
pass.setVertexBuffer(1, this.colorsFor(meshBuffers));
- pass.setIndexBuffer(meshBuffers.indexBuffer, meshBuffers.indexFormat);
- pass.drawIndexed(meshBuffers.indexCount);
+ pass.setIndexBuffer(
+ wireframe ? meshBuffers.wireIndexBuffer! : meshBuffers.indexBuffer,
+ meshBuffers.indexFormat,
+ );
+ pass.drawIndexed(
+ wireframe ? meshBuffers.wireIndexCount! : meshBuffers.indexCount,
+ );
}
/** Draws the mesh into a shadow map (`lightCameraBindGroup` holds the light's view-projection). */
@@ -426,13 +468,22 @@ export class Mesh3DPipeline {
meshBuffers: GPUMeshBuffers,
data: MeshUniformData,
): void {
- pass.setPipeline(this.pipelineShadow);
+ const wireframe = data.wireframe === true;
+ if (wireframe && !meshBuffers.wireIndexBuffer) {
+ throw new Error("Wireframe requires mesh buffers with edge indices");
+ }
+ pass.setPipeline(wireframe ? this.wireShadow : this.pipelineShadow);
pass.setBindGroup(0, lightCameraBindGroup);
pass.setBindGroup(1, this.packModel(data));
pass.setVertexBuffer(0, meshBuffers.vertexBuffer);
pass.setVertexBuffer(1, this.colorsFor(meshBuffers));
- pass.setIndexBuffer(meshBuffers.indexBuffer, meshBuffers.indexFormat);
- pass.drawIndexed(meshBuffers.indexCount);
+ pass.setIndexBuffer(
+ wireframe ? meshBuffers.wireIndexBuffer! : meshBuffers.indexBuffer,
+ meshBuffers.indexFormat,
+ );
+ pass.drawIndexed(
+ wireframe ? meshBuffers.wireIndexCount! : meshBuffers.indexCount,
+ );
}
private colorsFor(meshBuffers: GPUMeshBuffers): GPUBuffer {
diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts
index 1a9408b7..c28ad134 100644
--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts
+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts
@@ -1,6 +1,5 @@
import { beforeAll, beforeEach, describe, expect, it } from "vitest";
import { Camera3D } from "../math3d/index.js";
-import { Renderer3D } from "./renderer3d.js";
import {
createMockCommandEncoder,
createMockDevice,
@@ -8,6 +7,7 @@ import {
createMockTexture,
ensureDOMGlobals,
} from "../renderer2d/test-helpers.js";
+import { Renderer3D } from "./renderer3d.js";
describe("Renderer3D Subsystem", () => {
beforeAll(() => {
@@ -94,14 +94,20 @@ describe("Renderer3D Subsystem", () => {
const lightsBindGroup = {} as GPUBindGroup;
const texture = createMockTexture(500, 300) as any;
- renderer3d.drawTextureQuad(pass as any, cameraBindGroup, lightsBindGroup, texture, {
- position: [960, 540, 200],
- width: 600,
- height: 400,
- rotation: [15, -20, 0],
- opacity: 0.9,
- twoSided: true,
- });
+ renderer3d.drawTextureQuad(
+ pass as any,
+ cameraBindGroup,
+ lightsBindGroup,
+ texture,
+ {
+ position: [960, 540, 200],
+ width: 600,
+ height: 400,
+ rotation: [15, -20, 0],
+ opacity: 0.9,
+ twoSided: true,
+ },
+ );
expect(pass.setPipeline).toHaveBeenCalled();
expect(pass.setBindGroup).toHaveBeenCalledWith(0, cameraBindGroup);
@@ -148,6 +154,41 @@ describe("Renderer3D Subsystem", () => {
glass.destroy();
});
+ for (const IndexArray of [Uint16Array, Uint32Array]) {
+ it(`deduplicates shared wireframe edges and draws them with ${IndexArray.name}`, () => {
+ const mesh = {
+ positions: new Float32Array([0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0]),
+ normals: new Float32Array(12),
+ uvs: new Float32Array(8),
+ indices: new IndexArray([0, 1, 2, 0, 2, 3]),
+ };
+ const solid = renderer3d.getOrCreateMeshBuffers(mesh);
+ expect(solid.wireIndexBuffer).toBeUndefined();
+ const wire = renderer3d.getOrCreateMeshBuffers(mesh, true);
+ expect(wire).toBe(solid);
+ expect(wire.wireIndexCount).toBe(10);
+ if (!wire.wireIndexBuffer)
+ throw new Error("Missing wireframe index buffer");
+ expect(
+ Array.from(new IndexArray(wire.wireIndexBuffer.getMappedRange())),
+ ).toEqual([0, 1, 1, 2, 2, 0, 2, 3, 3, 0]);
+ const pass = createMockRenderPassEncoder();
+ renderer3d.drawMesh3D(
+ pass,
+ {} as GPUBindGroup,
+ {} as GPUBindGroup,
+ mesh,
+ undefined,
+ { wireframe: true },
+ );
+ expect(pass.setIndexBuffer).toHaveBeenCalledWith(
+ wire.wireIndexBuffer,
+ wire.indexFormat,
+ );
+ expect(pass.drawIndexed).toHaveBeenCalledWith(10);
+ });
+ }
+
it("deforms 3D mesh via deformMesh before draw dispatch", () => {
const encoder = createMockCommandEncoder();
const meshData = {
@@ -162,19 +203,25 @@ describe("Renderer3D Subsystem", () => {
usage: GPUBufferUsage.STORAGE,
});
- const deformedBuffers = renderer3d.deformMesh(encoder as any, meshData, {
- config: {
- audioTrackId: "beat",
- mode: "normal_extrusion",
- amplitudeMultiplier: 1.5,
+ const deformedBuffers = renderer3d.deformMesh(
+ encoder as any,
+ meshData,
+ {
+ config: {
+ audioTrackId: "beat",
+ mode: "normal_extrusion",
+ amplitudeMultiplier: 1.5,
+ },
+ latentBuffer: latentBuffer as any,
+ timeMs: 250,
},
- latentBuffer: latentBuffer as any,
- timeMs: 250,
- });
+ true,
+ );
+ expect(deformedBuffers.wireIndexCount).toBe(6);
+ expect(deformedBuffers.wireIndexBuffer).toBeDefined();
expect(deformedBuffers).toBeDefined();
expect(deformedBuffers.vertexBuffer).toBeDefined();
expect(encoder.beginComputePass).toHaveBeenCalled();
});
});
-
diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
index 57a05252..04232f7b 100644
--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts
@@ -705,10 +705,11 @@ export class Renderer3D {
this.mesh3dPipeline.drawShadow(
pass,
lightCamBindGroup,
- c.buffers ?? this.getOrCreateMeshBuffers(c.mesh),
+ c.buffers ?? this.getOrCreateMeshBuffers(c.mesh, c.opts.wireframe),
{
modelMatrix: m,
normalMatrix: c.opts.normalMatrix ?? Light3D.computeNormalMatrix(m),
+ wireframe: c.opts.wireframe,
isSkinned: c.opts.isSkinned,
skinningMatrices: c.opts.skinningMatrices,
},
@@ -1093,9 +1094,12 @@ export class Renderer3D {
);
}
- getOrCreateMeshBuffers(mesh: Mesh3DData): GPUMeshBuffers {
+ getOrCreateMeshBuffers(mesh: Mesh3DData, wireframe = false): GPUMeshBuffers {
let cached = this.meshBuffersCache.get(mesh);
- if (cached) return cached;
+ if (cached) {
+ if (wireframe) this.ensureWireIndices(mesh, cached);
+ return cached;
+ }
const numVerts = mesh.positions.length / 3;
// 16 floats per vertex: 3 (pos) + 2 (uv) + 3 (norm) + 4 (joints) + 4 (weights)
@@ -1193,9 +1197,48 @@ export class Renderer3D {
colorBuffer,
};
this.meshBuffersCache.set(mesh, cached);
+ if (wireframe) this.ensureWireIndices(mesh, cached);
return cached;
}
+ private ensureWireIndices(mesh: Mesh3DData, buffers: GPUMeshBuffers): void {
+ if (buffers.wireIndexBuffer) return;
+ const edges = new Set<string>();
+ const indices: number[] = [];
+ for (let i = 0; i + 2 < mesh.indices.length; i += 3) {
+ const triangle = [
+ mesh.indices[i]!,
+ mesh.indices[i + 1]!,
+ mesh.indices[i + 2]!,
+ ];
+ for (let j = 0; j < 3; j++) {
+ const a = triangle[j]!;
+ const b = triangle[(j + 1) % 3]!;
+ if (a === b) continue;
+ const key = a < b ? `${a},${b}` : `${b},${a}`;
+ if (edges.has(key)) continue;
+ edges.add(key);
+ indices.push(a, b);
+ }
+ }
+ const data =
+ buffers.indexFormat === "uint32"
+ ? new Uint32Array(indices)
+ : new Uint16Array(indices);
+ const buffer = this.device.createBuffer({
+ label: "Mesh3DWireIndices",
+ size: Math.max(4, Math.ceil(data.byteLength / 4) * 4),
+ usage: GPUBufferUsage.INDEX,
+ mappedAtCreation: true,
+ });
+ if (data instanceof Uint32Array)
+ new Uint32Array(buffer.getMappedRange()).set(data);
+ else new Uint16Array(buffer.getMappedRange()).set(data);
+ buffer.unmap();
+ buffers.wireIndexBuffer = buffer;
+ buffers.wireIndexCount = data.length;
+ }
+
deformMesh(
encoder: GPUCommandEncoder,
mesh: Mesh3DData,
@@ -1205,8 +1248,9 @@ export class Renderer3D {
timeMs?: number;
sampleRate?: number;
},
+ wireframe = false,
): GPUMeshBuffers {
- const meshBuffers = this.getOrCreateMeshBuffers(mesh);
+ const meshBuffers = this.getOrCreateMeshBuffers(mesh, wireframe);
const numVerts = mesh.positions.length / 3;
const deformedVertexBuffer = this.audioMeshDeformPipeline.execute(
encoder,
@@ -1225,6 +1269,8 @@ export class Renderer3D {
indexCount: meshBuffers.indexCount,
indexFormat: meshBuffers.indexFormat,
colorBuffer: meshBuffers.colorBuffer,
+ wireIndexBuffer: meshBuffers.wireIndexBuffer,
+ wireIndexCount: meshBuffers.wireIndexCount,
};
}
@@ -1240,7 +1286,7 @@ export class Renderer3D {
const meshBuffers =
"vertexBuffer" in meshOrBuffers
? meshOrBuffers
- : this.getOrCreateMeshBuffers(meshOrBuffers);
+ : this.getOrCreateMeshBuffers(meshOrBuffers, opts.wireframe);
const {
opacity = 1.0,
diff --git a/packages/webgpu-renderers/src/shaders/mesh3d.ts b/packages/webgpu-renderers/src/shaders/mesh3d.ts
index b8dc3a95..5b63a458 100644
--- a/packages/webgpu-renderers/src/shaders/mesh3d.ts
+++ b/packages/webgpu-renderers/src/shaders/mesh3d.ts
@@ -17,7 +17,7 @@ struct ModelUniforms {
modelMatrix : mat4x4<f32>,
normalMatrix : mat4x4<f32>,
colorTint : vec4<f32>,
- params : vec4<f32>, // x = opacity, y = twoSided (1.0 = yes), z = isSkinned (1.0 = yes), w = wireframe
+ params : vec4<f32>, // x = opacity, y = twoSided (1.0 = yes), z = isSkinned (1.0 = yes), w = reserved
materialParams : vec4<f32>, // x = shininess, y = specularIntensity, z = ambientIntensity, w = materialMode (0=unlit, 1=lit, 2=toon)
shade : vec4<f32>, // rgb = toon shadow-side color multiplier, a = alpha cutoff (0 = none)
pbr : vec4<f32>, // x = roughness (<0 = default), y = metallic
diff --git a/plugins/framefields/skills/framefields-compose/SKILL.md b/plugins/framefields/skills/framefields-compose/SKILL.md
index be1ddec2..febca0f8 100644
--- a/plugins/framefields/skills/framefields-compose/SKILL.md
+++ b/plugins/framefields/skills/framefields-compose/SKILL.md
@@ -104,7 +104,7 @@ All `Layer.*` builders return `AnimatableNode<T>` instances with chainable `.ani
| `Layer.environment(preset, options)` | `"light"` | The 3D scene's world: studio image-based light, exposure, tone mapping, fog, bloom. **Adding one switches the scene to physically based shading.** See *3D Look* below. |
| `Layer.sequence(entries, options)` | `"box"` | Scenes back to back with `Transition`s between them; returns a box whose `totalFrames` is the length. See *Scenes & Transitions*. |
| `Layer.model(options)` | `"model3d"` | 3D mesh loader (`.gltf`, `.glb`, `.obj`, `.fbx`, `.stl`, `.ply`). Supports skinning and audio deformation compute shaders. |
-| `Layer3D.roundedBox / sphere / cylinder(options)` | `"model3d"` | Procedural solid meshes, centered on `x`/`y`/`z` (y is down, so "top" is -y). `roundedBox`: `width`, `height`, `depth`, `radius`, `segments`, `taper` (top face scale, <1 for a keycap). `sphere`: `radius`. `cylinder`: `radius` or `radiusTop`/`radiusBottom`, `height`, `openEnded`. `color` is baked in; `tint` multiplies it and animates as the `color` prop. Surface: `roughness`, `metallic`, `emissive`, `emissiveIntensity`, `castShadow`. Equal shapes share one GPU buffer, so hundreds of copies are cheap. |
+| `Layer3D.roundedBox / sphere / cylinder(options)` | `"model3d"` | Procedural solid meshes, centered on `x`/`y`/`z` (y is down, so "top" is -y). `roundedBox`: `width`, `height`, `depth`, `radius`, `segments`, `taper` (top face scale, <1 for a keycap). `sphere`: `radius`. `cylinder`: `radius` or `radiusTop`/`radiusBottom`, `height`, `openEnded`. `color` is baked in; `tint` multiplies it and animates as the `color` prop. Surface: `roughness`, `metallic`, `emissive`, `emissiveIntensity`, `castShadow`. `wireframe: true` draws the mesh triangle edges as one-pixel lines, including triangulation and back edges. Equal shapes share one GPU buffer, so hundreds of copies are cheap. |
---