Code-first video & image, rendered natively on WebGPU.
Fix 3D mesh wireframe rendering (#60)
Okan Aslankane96a32f18d81
diff --git a/.changeset/fix-mesh-wireframe.md b/.changeset/fix-mesh-wireframe.mdnew file mode 100644index 00000000..19213c5f--- /dev/null+++ b/.changeset/fix-mesh-wireframe.md@@ -0,0 +1,7 @@+---+"framefields": patch+"@framefields/webgpu-renderers": patch+"@framefields/node-compositor": patch+---++Render model and procedural mesh wireframes as triangle edges, including animated and shadow-casting meshes.diff --git a/nodes/node-compositor/SKILL.md b/nodes/node-compositor/SKILL.mdindex d41825a5..7d997706 100644--- a/nodes/node-compositor/SKILL.md+++ b/nodes/node-compositor/SKILL.md@@ -208,6 +208,7 @@ Each track represents animatable property modifications:- **Watermarking a Video:** a `flex` row (`align: "end"`, `justify: "end"`, full canvas) containing a `media` node bound to the PNG input; low `opacity`.- **Picture-in-Picture:** a `flex` row with two `media` nodes (`fit: "cover"`, each `grow: 1`).- **Audio-Reactive 3D Models:** a `kind: "model3d"` node loading an OBJ/FBX/GLTF/STL/PLY mesh with `audioDeform: { audioTrackId, mode: "normal_extrusion" | "radial_pulse" | "harmonic_wave" | "twist" | "ripple", amplitudeMultiplier, damping }`, deformed directly in VRAM via WebGPU compute shaders driven by audio frequency spectra and transient energies. `twist` turns each slice about Y by its height (harder on the bass); `ripple` runs concentric waves across XY, struck by the drums. `audioTrackId` may be an audio layer's id or an audio file path.+- **Mesh wireframe:** `wireframe: true` on a `model3d` draws triangle edges as one-pixel lines, including triangulation and back edges; it also applies to skinned and audio-deformed meshes and their shadows.- **glTF scenes:** a glTF/GLB `model3d` keeps its node hierarchy, every skin and its animation clips (translation, rotation, scale; linear, step and cubic-spline). Skinned meshes are posed per frame from the clip (`animationName`, `animationTime`/`animationProgress`, `loop`); base-colour textures (with mipmaps), alpha `MASK`/`BLEND` and VRM MToon materials (`material: "toon"`, picked automatically when `material` is unset) render as authored. `center`/`normalizeSize` apply above the scene graph, so skinning stays intact.## Don't Forgetdiff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.tsindex 36fd97b4..99fbb140 100644--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts@@ -1309,6 +1309,7 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {encoder,item.mesh,item.opts.audioDeform,+ item.opts.wireframe,);}}diff --git a/packages/framefields/src/primitives-3d-render.test.ts b/packages/framefields/src/primitives-3d-render.test.tsindex 43dc2a16..470fa89b 100644--- a/packages/framefields/src/primitives-3d-render.test.ts+++ b/packages/framefields/src/primitives-3d-render.test.ts@@ -41,6 +41,49 @@ function stage() {}describe("Layer3D mesh primitives", () => {+ it("renders mesh triangle edges when wireframe is enabled", async () => {+ const render = async (wireframe: boolean) => {+ const comp = stage();+ comp.add(+ Layer.directionalLight({+ color: "#ffffff",+ intensity: 1,+ x: 200,+ y: -300,+ z: -400,+ targetX: 400,+ targetY: 300,+ targetZ: 0,+ castShadow: true,+ }),+ );+ comp.add(+ Layer3D.roundedBox({+ width: 220,+ height: 180,+ depth: 120,+ x: 400,+ y: 300,+ color: "#ffffff",+ material: "lit",+ wireframe,+ }),+ );+ return decode(await comp.renderFrame({ frame: 0 }));+ };+ const solid = await render(false);+ const wire = await render(true);+ const lit = (pixels: Uint8ClampedArray) => {+ let count = 0;+ for (let i = 0; i < pixels.length; i += 4) {+ if (pixels[i]! > 50) count++;+ }+ return count;+ };+ expect(lit(wire.data)).toBeGreaterThan(100);+ expect(lit(wire.data)).toBeLessThan(lit(solid.data) / 4);+ });+it("draws a rounded box's top face from above, single-sided", async () => {const comp = stage();comp.add(diff --git a/packages/framefields/src/shapes3d.ts b/packages/framefields/src/shapes3d.tsindex 6e479fca..a64677d2 100644--- a/packages/framefields/src/shapes3d.ts+++ b/packages/framefields/src/shapes3d.ts@@ -277,6 +277,8 @@ export interface Primitive3DNodeOptions extends Surface3DOptions {scaleX?: number;scaleY?: number;scaleZ?: number;+ /** Draw triangle edges instead of filled surfaces. */+ wireframe?: boolean;/** Surface color baked into the mesh (default white). */color?: string;/**@@ -358,6 +360,7 @@ function primitiveNode(castShadow: o.castShadow,}),twoSided: false,+ wireframe: o.wireframe,startFrame: o.startFrame,durationFrames: o.durationFrames,});diff --git a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.tsindex cabeb82c..7c1e5d7a 100644--- a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts+++ b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts@@ -79,6 +79,8 @@ export interface GPUMeshBuffers {indexFormat: GPUIndexFormat;/** Per-vertex RGBA (display / sRGB), 4 floats each; absent = white. */colorBuffer?: GPUBuffer;+ wireIndexBuffer?: GPUBuffer;+ wireIndexCount?: number;}export class Mesh3DPipeline {@@ -86,6 +88,10 @@ export class Mesh3DPipeline {public pipelineSingle: GPURenderPipeline;public pipelineMrt: GPURenderPipeline;private bloomVariants: BloomPipelineVariants;+ private wireSingle: GPURenderPipeline;+ private wireMrt: GPURenderPipeline;+ private wireBloom: BloomPipelineVariants;+ private wireShadow: GPURenderPipeline;/** Depth only (skinning included): draws a caster into a shadow map. */public pipelineShadow: GPURenderPipeline;public cameraLayout: GPUBindGroupLayout;@@ -231,6 +237,29 @@ export class Mesh3DPipeline {this.pipelineSingle = single;this.pipelineMrt = mrt;this.bloomVariants = new BloomPipelineVariants(device, base);+ const wireBase: PipelineVariantBase = {+ ...base,+ label: "Mesh3DWireframe",+ primitive: { topology: "line-list" },+ };+ const wire = createBasePipelines(device, wireBase);+ this.wireSingle = wire.single;+ this.wireMrt = wire.mrt;+ this.wireBloom = new BloomPipelineVariants(device, wireBase);+ this.wireShadow = device.createRenderPipeline({+ label: "Mesh3DWireframeShadow",+ layout: device.createPipelineLayout({+ bindGroupLayouts: [this.cameraLayout, this.modelLayout],+ }),+ vertex: vertexState,+ // WebGPU prohibits depth bias on line topology.+ depthStencil: {+ format: SHADOW_DEPTH_STENCIL.format,+ depthWriteEnabled: true,+ depthCompare: "less-equal",+ },+ primitive: { topology: "line-list" },+ });this.pipelineShadow = device.createRenderPipeline({label: "Mesh3DPipelineShadow",@@ -304,7 +333,7 @@ export class Mesh3DPipeline {this.modelData[36] = data.opacity ?? 1.0;this.modelData[37] = data.twoSided ? 1.0 : 0.0;this.modelData[38] = data.isSkinned ? 1.0 : 0.0;- this.modelData[39] = data.wireframe ? 1.0 : 0.0;+ this.modelData[39] = 0;const shininess =data.roughness !== undefined@@ -402,12 +431,20 @@ export class Mesh3DPipeline {],});+ const wireframe = data.wireframe === true;+ if (wireframe && !meshBuffers.wireIndexBuffer) {+ throw new Error("Wireframe requires mesh buffers with edge indices");+ }pass.setPipeline(useBloom- ? this.bloomVariants.get(useMrt)- : useMrt- ? this.pipelineMrt- : this.pipelineSingle,+ ? (wireframe ? this.wireBloom : this.bloomVariants).get(useMrt)+ : wireframe+ ? useMrt+ ? this.wireMrt+ : this.wireSingle+ : useMrt+ ? this.pipelineMrt+ : this.pipelineSingle,);pass.setBindGroup(0, cameraBindGroup);pass.setBindGroup(1, modelBindGroup);@@ -415,8 +452,13 @@ export class Mesh3DPipeline {pass.setBindGroup(3, lightsBindGroup);pass.setVertexBuffer(0, meshBuffers.vertexBuffer);pass.setVertexBuffer(1, this.colorsFor(meshBuffers));- pass.setIndexBuffer(meshBuffers.indexBuffer, meshBuffers.indexFormat);- pass.drawIndexed(meshBuffers.indexCount);+ pass.setIndexBuffer(+ wireframe ? meshBuffers.wireIndexBuffer! : meshBuffers.indexBuffer,+ meshBuffers.indexFormat,+ );+ pass.drawIndexed(+ wireframe ? meshBuffers.wireIndexCount! : meshBuffers.indexCount,+ );}/** Draws the mesh into a shadow map (`lightCameraBindGroup` holds the light's view-projection). */@@ -426,13 +468,22 @@ export class Mesh3DPipeline {meshBuffers: GPUMeshBuffers,data: MeshUniformData,): void {- pass.setPipeline(this.pipelineShadow);+ const wireframe = data.wireframe === true;+ if (wireframe && !meshBuffers.wireIndexBuffer) {+ throw new Error("Wireframe requires mesh buffers with edge indices");+ }+ pass.setPipeline(wireframe ? this.wireShadow : this.pipelineShadow);pass.setBindGroup(0, lightCameraBindGroup);pass.setBindGroup(1, this.packModel(data));pass.setVertexBuffer(0, meshBuffers.vertexBuffer);pass.setVertexBuffer(1, this.colorsFor(meshBuffers));- pass.setIndexBuffer(meshBuffers.indexBuffer, meshBuffers.indexFormat);- pass.drawIndexed(meshBuffers.indexCount);+ pass.setIndexBuffer(+ wireframe ? meshBuffers.wireIndexBuffer! : meshBuffers.indexBuffer,+ meshBuffers.indexFormat,+ );+ pass.drawIndexed(+ wireframe ? meshBuffers.wireIndexCount! : meshBuffers.indexCount,+ );}private colorsFor(meshBuffers: GPUMeshBuffers): GPUBuffer {diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.test.tsindex 1a9408b7..c28ad134 100644--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts@@ -1,6 +1,5 @@import { beforeAll, beforeEach, describe, expect, it } from "vitest";import { Camera3D } from "../math3d/index.js";-import { Renderer3D } from "./renderer3d.js";import {createMockCommandEncoder,createMockDevice,@@ -8,6 +7,7 @@ import {createMockTexture,ensureDOMGlobals,} from "../renderer2d/test-helpers.js";+import { Renderer3D } from "./renderer3d.js";describe("Renderer3D Subsystem", () => {beforeAll(() => {@@ -94,14 +94,20 @@ describe("Renderer3D Subsystem", () => {const lightsBindGroup = {} as GPUBindGroup;const texture = createMockTexture(500, 300) as any;- renderer3d.drawTextureQuad(pass as any, cameraBindGroup, lightsBindGroup, texture, {- position: [960, 540, 200],- width: 600,- height: 400,- rotation: [15, -20, 0],- opacity: 0.9,- twoSided: true,- });+ renderer3d.drawTextureQuad(+ pass as any,+ cameraBindGroup,+ lightsBindGroup,+ texture,+ {+ position: [960, 540, 200],+ width: 600,+ height: 400,+ rotation: [15, -20, 0],+ opacity: 0.9,+ twoSided: true,+ },+ );expect(pass.setPipeline).toHaveBeenCalled();expect(pass.setBindGroup).toHaveBeenCalledWith(0, cameraBindGroup);@@ -148,6 +154,41 @@ describe("Renderer3D Subsystem", () => {glass.destroy();});+ for (const IndexArray of [Uint16Array, Uint32Array]) {+ it(`deduplicates shared wireframe edges and draws them with ${IndexArray.name}`, () => {+ const mesh = {+ positions: new Float32Array([0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0]),+ normals: new Float32Array(12),+ uvs: new Float32Array(8),+ indices: new IndexArray([0, 1, 2, 0, 2, 3]),+ };+ const solid = renderer3d.getOrCreateMeshBuffers(mesh);+ expect(solid.wireIndexBuffer).toBeUndefined();+ const wire = renderer3d.getOrCreateMeshBuffers(mesh, true);+ expect(wire).toBe(solid);+ expect(wire.wireIndexCount).toBe(10);+ if (!wire.wireIndexBuffer)+ throw new Error("Missing wireframe index buffer");+ expect(+ Array.from(new IndexArray(wire.wireIndexBuffer.getMappedRange())),+ ).toEqual([0, 1, 1, 2, 2, 0, 2, 3, 3, 0]);+ const pass = createMockRenderPassEncoder();+ renderer3d.drawMesh3D(+ pass,+ {} as GPUBindGroup,+ {} as GPUBindGroup,+ mesh,+ undefined,+ { wireframe: true },+ );+ expect(pass.setIndexBuffer).toHaveBeenCalledWith(+ wire.wireIndexBuffer,+ wire.indexFormat,+ );+ expect(pass.drawIndexed).toHaveBeenCalledWith(10);+ });+ }+it("deforms 3D mesh via deformMesh before draw dispatch", () => {const encoder = createMockCommandEncoder();const meshData = {@@ -162,19 +203,25 @@ describe("Renderer3D Subsystem", () => {usage: GPUBufferUsage.STORAGE,});- const deformedBuffers = renderer3d.deformMesh(encoder as any, meshData, {- config: {- audioTrackId: "beat",- mode: "normal_extrusion",- amplitudeMultiplier: 1.5,+ const deformedBuffers = renderer3d.deformMesh(+ encoder as any,+ meshData,+ {+ config: {+ audioTrackId: "beat",+ mode: "normal_extrusion",+ amplitudeMultiplier: 1.5,+ },+ latentBuffer: latentBuffer as any,+ timeMs: 250,},- latentBuffer: latentBuffer as any,- timeMs: 250,- });+ true,+ );+ expect(deformedBuffers.wireIndexCount).toBe(6);+ expect(deformedBuffers.wireIndexBuffer).toBeDefined();expect(deformedBuffers).toBeDefined();expect(deformedBuffers.vertexBuffer).toBeDefined();expect(encoder.beginComputePass).toHaveBeenCalled();});});-diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.tsindex 57a05252..04232f7b 100644--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts@@ -705,10 +705,11 @@ export class Renderer3D {this.mesh3dPipeline.drawShadow(pass,lightCamBindGroup,- c.buffers ?? this.getOrCreateMeshBuffers(c.mesh),+ c.buffers ?? this.getOrCreateMeshBuffers(c.mesh, c.opts.wireframe),{modelMatrix: m,normalMatrix: c.opts.normalMatrix ?? Light3D.computeNormalMatrix(m),+ wireframe: c.opts.wireframe,isSkinned: c.opts.isSkinned,skinningMatrices: c.opts.skinningMatrices,},@@ -1093,9 +1094,12 @@ export class Renderer3D {);}- getOrCreateMeshBuffers(mesh: Mesh3DData): GPUMeshBuffers {+ getOrCreateMeshBuffers(mesh: Mesh3DData, wireframe = false): GPUMeshBuffers {let cached = this.meshBuffersCache.get(mesh);- if (cached) return cached;+ if (cached) {+ if (wireframe) this.ensureWireIndices(mesh, cached);+ return cached;+ }const numVerts = mesh.positions.length / 3;// 16 floats per vertex: 3 (pos) + 2 (uv) + 3 (norm) + 4 (joints) + 4 (weights)@@ -1193,9 +1197,48 @@ export class Renderer3D {colorBuffer,};this.meshBuffersCache.set(mesh, cached);+ if (wireframe) this.ensureWireIndices(mesh, cached);return cached;}+ private ensureWireIndices(mesh: Mesh3DData, buffers: GPUMeshBuffers): void {+ if (buffers.wireIndexBuffer) return;+ const edges = new Set<string>();+ const indices: number[] = [];+ for (let i = 0; i + 2 < mesh.indices.length; i += 3) {+ const triangle = [+ mesh.indices[i]!,+ mesh.indices[i + 1]!,+ mesh.indices[i + 2]!,+ ];+ for (let j = 0; j < 3; j++) {+ const a = triangle[j]!;+ const b = triangle[(j + 1) % 3]!;+ if (a === b) continue;+ const key = a < b ? `${a},${b}` : `${b},${a}`;+ if (edges.has(key)) continue;+ edges.add(key);+ indices.push(a, b);+ }+ }+ const data =+ buffers.indexFormat === "uint32"+ ? new Uint32Array(indices)+ : new Uint16Array(indices);+ const buffer = this.device.createBuffer({+ label: "Mesh3DWireIndices",+ size: Math.max(4, Math.ceil(data.byteLength / 4) * 4),+ usage: GPUBufferUsage.INDEX,+ mappedAtCreation: true,+ });+ if (data instanceof Uint32Array)+ new Uint32Array(buffer.getMappedRange()).set(data);+ else new Uint16Array(buffer.getMappedRange()).set(data);+ buffer.unmap();+ buffers.wireIndexBuffer = buffer;+ buffers.wireIndexCount = data.length;+ }+deformMesh(encoder: GPUCommandEncoder,mesh: Mesh3DData,@@ -1205,8 +1248,9 @@ export class Renderer3D {timeMs?: number;sampleRate?: number;},+ wireframe = false,): GPUMeshBuffers {- const meshBuffers = this.getOrCreateMeshBuffers(mesh);+ const meshBuffers = this.getOrCreateMeshBuffers(mesh, wireframe);const numVerts = mesh.positions.length / 3;const deformedVertexBuffer = this.audioMeshDeformPipeline.execute(encoder,@@ -1225,6 +1269,8 @@ export class Renderer3D {indexCount: meshBuffers.indexCount,indexFormat: meshBuffers.indexFormat,colorBuffer: meshBuffers.colorBuffer,+ wireIndexBuffer: meshBuffers.wireIndexBuffer,+ wireIndexCount: meshBuffers.wireIndexCount,};}@@ -1240,7 +1286,7 @@ export class Renderer3D {const meshBuffers ="vertexBuffer" in meshOrBuffers? meshOrBuffers- : this.getOrCreateMeshBuffers(meshOrBuffers);+ : this.getOrCreateMeshBuffers(meshOrBuffers, opts.wireframe);const {opacity = 1.0,diff --git a/packages/webgpu-renderers/src/shaders/mesh3d.ts b/packages/webgpu-renderers/src/shaders/mesh3d.tsindex b8dc3a95..5b63a458 100644--- a/packages/webgpu-renderers/src/shaders/mesh3d.ts+++ b/packages/webgpu-renderers/src/shaders/mesh3d.ts@@ -17,7 +17,7 @@ struct ModelUniforms {modelMatrix : mat4x4<f32>,normalMatrix : mat4x4<f32>,colorTint : vec4<f32>,- params : vec4<f32>, // x = opacity, y = twoSided (1.0 = yes), z = isSkinned (1.0 = yes), w = wireframe+ params : vec4<f32>, // x = opacity, y = twoSided (1.0 = yes), z = isSkinned (1.0 = yes), w = reservedmaterialParams : vec4<f32>, // x = shininess, y = specularIntensity, z = ambientIntensity, w = materialMode (0=unlit, 1=lit, 2=toon)shade : vec4<f32>, // rgb = toon shadow-side color multiplier, a = alpha cutoff (0 = none)pbr : vec4<f32>, // x = roughness (<0 = default), y = metallicdiff --git a/plugins/framefields/skills/framefields-compose/SKILL.md b/plugins/framefields/skills/framefields-compose/SKILL.mdindex be1ddec2..febca0f8 100644--- a/plugins/framefields/skills/framefields-compose/SKILL.md+++ b/plugins/framefields/skills/framefields-compose/SKILL.md@@ -104,7 +104,7 @@ All `Layer.*` builders return `AnimatableNode<T>` instances with chainable `.ani| `Layer.environment(preset, options)` | `"light"` | The 3D scene's world: studio image-based light, exposure, tone mapping, fog, bloom. **Adding one switches the scene to physically based shading.** See *3D Look* below. || `Layer.sequence(entries, options)` | `"box"` | Scenes back to back with `Transition`s between them; returns a box whose `totalFrames` is the length. See *Scenes & Transitions*. || `Layer.model(options)` | `"model3d"` | 3D mesh loader (`.gltf`, `.glb`, `.obj`, `.fbx`, `.stl`, `.ply`). Supports skinning and audio deformation compute shaders. |-| `Layer3D.roundedBox / sphere / cylinder(options)` | `"model3d"` | Procedural solid meshes, centered on `x`/`y`/`z` (y is down, so "top" is -y). `roundedBox`: `width`, `height`, `depth`, `radius`, `segments`, `taper` (top face scale, <1 for a keycap). `sphere`: `radius`. `cylinder`: `radius` or `radiusTop`/`radiusBottom`, `height`, `openEnded`. `color` is baked in; `tint` multiplies it and animates as the `color` prop. Surface: `roughness`, `metallic`, `emissive`, `emissiveIntensity`, `castShadow`. Equal shapes share one GPU buffer, so hundreds of copies are cheap. |+| `Layer3D.roundedBox / sphere / cylinder(options)` | `"model3d"` | Procedural solid meshes, centered on `x`/`y`/`z` (y is down, so "top" is -y). `roundedBox`: `width`, `height`, `depth`, `radius`, `segments`, `taper` (top face scale, <1 for a keycap). `sphere`: `radius`. `cylinder`: `radius` or `radiusTop`/`radiusBottom`, `height`, `openEnded`. `color` is baked in; `tint` multiplies it and animates as the `color` prop. Surface: `roughness`, `metallic`, `emissive`, `emissiveIntensity`, `castShadow`. `wireframe: true` draws the mesh triangle edges as one-pixel lines, including triangulation and back edges. Equal shapes share one GPU buffer, so hundreds of copies are cheap. |---