Code-first video & image, rendered natively on WebGPU.
Fix 3D transparent mesh ordering (#61)
Okan Aslankanb5c6470507dc
diff --git a/.changeset/fix-3d-transparency-ordering.md b/.changeset/fix-3d-transparency-ordering.mdnew file mode 100644index 00000000..d3962a97--- /dev/null+++ b/.changeset/fix-3d-transparency-ordering.md@@ -0,0 +1,7 @@+---+"framefields": patch+"@framefields/webgpu-renderers": patch+"@framefields/node-compositor": patch+---++Sort blended 3D meshes across the scene and render them without depth writes. Apply glTF base-color alpha once.diff --git a/nodes/node-compositor/SKILL.md b/nodes/node-compositor/SKILL.mdindex 7d997706..62fa7fba 100644--- a/nodes/node-compositor/SKILL.md+++ b/nodes/node-compositor/SKILL.md@@ -208,6 +208,7 @@ Each track represents animatable property modifications:- **Watermarking a Video:** a `flex` row (`align: "end"`, `justify: "end"`, full canvas) containing a `media` node bound to the PNG input; low `opacity`.- **Picture-in-Picture:** a `flex` row with two `media` nodes (`fit: "cover"`, each `grow: 1`).- **Audio-Reactive 3D Models:** a `kind: "model3d"` node loading an OBJ/FBX/GLTF/STL/PLY mesh with `audioDeform: { audioTrackId, mode: "normal_extrusion" | "radial_pulse" | "harmonic_wave" | "twist" | "ripple", amplitudeMultiplier, damping }`, deformed directly in VRAM via WebGPU compute shaders driven by audio frequency spectra and transient energies. `twist` turns each slice about Y by its height (harder on the bass); `ripple` runs concentric waves across XY, struck by the drums. `audioTrackId` may be an audio layer's id or an audio file path.+- **Transparent meshes:** glTF `BLEND` materials and meshes with reduced layer, material, tint, or vertex alpha draw after opaque items, back to front by the camera-space center of each mesh’s transformed bounds, without writing depth. This is object-level sorting: intersecting meshes, self-overlap, skinned/deformed bounds, translucent flat layers, glass, and transparent shadows have separate limitations.- **Mesh wireframe:** `wireframe: true` on a `model3d` draws triangle edges as one-pixel lines, including triangulation and back edges; it also applies to skinned and audio-deformed meshes and their shadows.- **glTF scenes:** a glTF/GLB `model3d` keeps its node hierarchy, every skin and its animation clips (translation, rotation, scale; linear, step and cubic-spline). Skinned meshes are posed per frame from the clip (`animationName`, `animationTime`/`animationProgress`, `loop`); base-colour textures (with mipmaps), alpha `MASK`/`BLEND` and VRM MToon materials (`material: "toon"`, picked automatically when `material` is unset) render as authored. `center`/`normalizeSize` apply above the scene graph, so skinning stays intact.diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.tsindex 99fbb140..0a7f92bf 100644--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts@@ -81,6 +81,26 @@ import {} from "./transform.js";import { has3DTransform, type Quad2D, Transform3DMath } from "./transform3d.js";+// Mesh vertex buffers are cached by identity, so their alpha classification can+// also be computed once instead of scanning large models on every frame.+const meshHasAlpha = new WeakMap<Mesh3DData, boolean>();+function hasVertexAlpha(mesh: Mesh3DData): boolean {+ const cached = meshHasAlpha.get(mesh);+ if (cached !== undefined) return cached;+ const colors = mesh.colors;+ let transparent = false;+ if (colors) {+ for (let i = 3; i < colors.length; i += 4) {+ if (colors[i] < 1) {+ transparent = true;+ break;+ }+ }+ }+ meshHasAlpha.set(mesh, transparent);+ return transparent;+}+// Layout depends on the document tree (whose `children` array keeps its// identity between frames), the viewport, media sizes and the few layout// props a timeline can animate. A film's tree holds every scene, so@@ -1400,16 +1420,38 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {Boolean(sceneBloom),);- // Glass reads what is behind the batch (the canvas so far), and goes- // last so opaque meshes are already in the depth buffer.+ // Opaque objects establish depth first. Sort blended meshes across the+ // whole batch by camera-space depth, then draw glass as before.const isGlass = (item: Pending3DItem) =>item.kind === "mesh" &&(item.opts.material === "glass" ||item.opts.material === "acrylic" ||item.opts.material === "frosted");+ const opaque: Pending3DItem[] = [];+ const glass: Pending3DItem[] = [];+ const transparent: { item: Pending3DItem; depth: number }[] = [];+ const forward = Vector3Math.subtract(camera.target, camera.eye);+ for (const item of pending3DItems) {+ if (isGlass(item)) glass.push(item);+ else if (item.kind === "mesh" && item.opts.transparent) {+ const bounds = r3d.worldBounds({+ kind: "mesh",+ mesh: item.mesh,+ opts: item.opts,+ });+ const center = bounds.min.map((v, i) => (v + bounds.max[i]) / 2);+ const depth =+ center[0] * forward[0] ++ center[1] * forward[1] ++ center[2] * forward[2];+ transparent.push({ item, depth });+ } else opaque.push(item);+ }+ transparent.sort((a, b) => b.depth - a.depth);const ordered = [- ...pending3DItems.filter((i) => !isGlass(i)),- ...pending3DItems.filter(isGlass),+ ...opaque,+ ...transparent.map(({ item }) => item),+ ...glass,];for (const item of ordered) {if (item.kind === "quad") {@@ -2066,15 +2108,7 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {}}- // Blended materials draw after everything opaque behind them.- const isBlend = (m: Mesh3DData) =>- m.materialName- ? model.materials[m.materialName]?.alphaMode === "blend"- : false;- const drawOrder = [...model.meshes].sort(- (a, b) => Number(isBlend(a)) - Number(isBlend(b)),- );- for (const mesh of drawOrder) {+ for (const mesh of model.meshes) {const mat = mesh.materialName? model.materials[mesh.materialName]: undefined;@@ -2265,6 +2299,11 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {modelMatrix: placed.matrix,opacity: modelOpacity * (mat?.opacity ?? 1.0),colorTint: meshColorTint,+ transparent:+ mat?.alphaMode === "blend" ||+ modelOpacity * (mat?.opacity ?? 1) < 1 ||+ meshColorTint[3] < 1 ||+ hasVertexAlpha(mesh),twoSided: target?.twoSided ?? lop.twoSided ?? mat?.twoSided ?? true,wireframe: Boolean(target?.wireframe ?? lop.wireframe),isSkinned: Boolean(diff --git a/packages/framefields/src/transparency-3d-render.test.ts b/packages/framefields/src/transparency-3d-render.test.tsnew file mode 100644index 00000000..ad37030d--- /dev/null+++ b/packages/framefields/src/transparency-3d-render.test.ts@@ -0,0 +1,170 @@+import { GltfLoader } from "@framefields/webgpu-renderers";+import { Canvas, loadImage } from "skia-canvas";+import { describe, expect, it } from "vitest";+import { Composition, Layer } from "./index.js";++/** Two overlapping glTF meshes. The front red and back blue each have 50% alpha. */+function model(order: ("front" | "back")[]) {+ const positions = new Float32Array([+ -150, -150, 0, 150, -150, 0, 150, 150, 0, -150, 150, 0,+ ]);+ const indices = new Uint16Array([0, 1, 2, 0, 2, 3]);+ const bin = Buffer.concat([+ Buffer.from(positions.buffer),+ Buffer.from(indices.buffer),+ ]);+ return GltfLoader.parse(+ JSON.stringify({+ asset: { version: "2.0" },+ buffers: [+ {+ byteLength: bin.length,+ uri: `data:application/octet-stream;base64,${bin.toString("base64")}`,+ },+ ],+ bufferViews: [+ { buffer: 0, byteOffset: 0, byteLength: 48 },+ { buffer: 0, byteOffset: 48, byteLength: 12 },+ ],+ accessors: [+ { bufferView: 0, componentType: 5126, count: 4, type: "VEC3" },+ { bufferView: 1, componentType: 5123, count: 6, type: "SCALAR" },+ ],+ materials: [+ {+ name: "red",+ alphaMode: "BLEND",+ doubleSided: true,+ pbrMetallicRoughness: { baseColorFactor: [1, 0, 0, 0.5] },+ extensions: { KHR_materials_unlit: {} },+ },+ {+ name: "blue",+ alphaMode: "BLEND",+ doubleSided: true,+ pbrMetallicRoughness: { baseColorFactor: [0, 0, 1, 0.5] },+ extensions: { KHR_materials_unlit: {} },+ },+ ],+ meshes: [0, 1].map((material) => ({+ primitives: [{ attributes: { POSITION: 0 }, indices: 1, material }],+ })),+ nodes: order.map((name) =>+ name === "front"+ ? { mesh: 0, translation: [0, 0, 0] }+ : { mesh: 1, translation: [0, 0, 100] },+ ),+ scenes: [{ nodes: order.map((_, i) => i) }],+ scene: 0,+ }),+ );+}++async function center(+ order: ("front" | "back")[],+ separate = false,+ reverseCamera = false,+) {+ const comp = new Composition({+ width: 400,+ height: 400,+ fps: 30,+ durationMs: 1000,+ backgroundColor: "#000000",+ });+ comp.add(+ Layer.camera({+ x: 200,+ y: 200,+ z: reverseCamera ? 500 : -500,+ targetX: 200,+ targetY: 200,+ targetZ: 0,+ }),+ );+ if (separate) {+ for (const name of order)+ comp.add(+ Layer.model({ id: name, x: 200, y: 200, modelData: model([name]) }),+ );+ } else+ comp.add(+ Layer.model({ id: "model", x: 200, y: 200, modelData: model(order) }),+ );+ const png = await comp.renderFrame({ frame: 0 });+ const canvas = new Canvas(400, 400);+ const ctx = canvas.getContext("2d");+ ctx.drawImage(await loadImage(png), 0, 0);+ return Array.from(ctx.getImageData(200, 200, 1, 1).data);+}++describe("3D transparent meshes", () => {+ it("sorts by camera direction when viewed from behind", async () => {+ const frontFirst = await center(["front", "back"], true, true);+ const backFirst = await center(["back", "front"], true, true);+ for (let i = 0; i < 4; i++)+ expect(frontFirst[i]).toBeCloseTo(backFirst[i], 0);+ expect(frontFirst[2]).toBeGreaterThan(115);+ expect(frontFirst[2]).toBeLessThan(140);+ expect(frontFirst[0]).toBeGreaterThan(55);+ expect(frontFirst[0]).toBeLessThan(75);+ });++ it("keeps opaque meshes in the depth buffer before blending", async () => {+ const render = async (blueZ: number) => {+ const comp = new Composition({+ width: 400,+ height: 400,+ fps: 30,+ durationMs: 1000,+ backgroundColor: "#000000",+ });+ comp.add(+ Layer.camera({+ x: 200,+ y: 200,+ z: -500,+ targetX: 200,+ targetY: 200,+ targetZ: 0,+ }),+ );+ const blue = model(["back"]);+ if (!blue.materials.blue) throw new Error("Missing blue material");+ blue.materials.blue.alphaMode = "opaque";+ blue.materials.blue.diffuseColor = [0, 0, 1, 1];+ comp.add(+ Layer.model({ id: "red", x: 200, y: 200, modelData: model(["front"]) }),+ );+ comp.add(+ Layer.model({ id: "blue", x: 200, y: 200, z: blueZ, modelData: blue }),+ );+ const canvas = new Canvas(400, 400);+ const ctx = canvas.getContext("2d");+ ctx.drawImage(+ await loadImage(await comp.renderFrame({ frame: 0 })),+ 0,+ 0,+ );+ return Array.from(ctx.getImageData(200, 200, 1, 1).data);+ };+ const behind = await render(0);+ const inFront = await render(-200);+ expect(behind[0]).toBeGreaterThan(115);+ expect(behind[2]).toBeGreaterThan(115);+ expect(inFront[0]).toBeLessThan(5);+ expect(inFront[2]).toBeGreaterThan(245);+ });+ for (const separate of [false, true]) {+ it(`blends glTF meshes back to front regardless of authoring order (${separate ? "separate models" : "one model"})`, async () => {+ const frontFirst = await center(["front", "back"], separate);+ const backFirst = await center(["back", "front"], separate);+ for (let i = 0; i < 4; i++)+ expect(frontFirst[i]).toBeCloseTo(backFirst[i], 0);+ expect(frontFirst[0]).toBeGreaterThan(115);+ expect(frontFirst[0]).toBeLessThan(140);+ expect(frontFirst[2]).toBeGreaterThan(55);+ expect(frontFirst[2]).toBeLessThan(75);+ });+ }+});diff --git a/packages/webgpu-renderers/src/loaders/gltf-loader.ts b/packages/webgpu-renderers/src/loaders/gltf-loader.tsindex c86ebc0f..4d04e9e8 100644--- a/packages/webgpu-renderers/src/loaders/gltf-loader.ts+++ b/packages/webgpu-renderers/src/loaders/gltf-loader.ts@@ -542,7 +542,6 @@ export class GltfLoader {roughness: pbr?.roughnessFactor ?? 0.5,metallic: pbr?.metallicFactor ?? 0.0,shininess: 32.0 * (1.0 - (pbr?.roughnessFactor ?? 0.5)),- opacity: baseColor[3] ?? 1.0,twoSided: mat.doubleSided ?? true,alphaMode:mat.alphaMode === "MASK"diff --git a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.tsindex 7c1e5d7a..2f9d57c2 100644--- a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts+++ b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts@@ -47,6 +47,8 @@ export interface MeshUniformData extends SurfaceParams {opacity?: number;twoSided?: boolean;wireframe?: boolean;+ /** Blend without writing depth; the caller draws transparent meshes back to front. */+ transparent?: boolean;isSkinned?: boolean;skinningMatrices?: Float32Array; // 128 * 16 floatsmaterial?: "lit" | "unlit" | "toon";@@ -92,6 +94,16 @@ export class Mesh3DPipeline {private wireMrt: GPURenderPipeline;private wireBloom: BloomPipelineVariants;private wireShadow: GPURenderPipeline;+ private transparentPipelines: {+ single: GPURenderPipeline;+ mrt: GPURenderPipeline;+ bloom: BloomPipelineVariants;+ };+ private transparentWirePipelines: {+ single: GPURenderPipeline;+ mrt: GPURenderPipeline;+ bloom: BloomPipelineVariants;+ };/** Depth only (skinning included): draws a caster into a shadow map. */public pipelineShadow: GPURenderPipeline;public cameraLayout: GPUBindGroupLayout;@@ -246,6 +258,24 @@ export class Mesh3DPipeline {this.wireSingle = wire.single;this.wireMrt = wire.mrt;this.wireBloom = new BloomPipelineVariants(device, wireBase);+ const transparentBase: PipelineVariantBase = {+ ...base,+ label: "Mesh3DTransparent",+ depthStencil: { ...base.depthStencil, depthWriteEnabled: false },+ };+ const transparentWireBase: PipelineVariantBase = {+ ...transparentBase,+ label: "Mesh3DTransparentWireframe",+ primitive: wireBase.primitive,+ };+ this.transparentPipelines = {+ ...createBasePipelines(device, transparentBase),+ bloom: new BloomPipelineVariants(device, transparentBase),+ };+ this.transparentWirePipelines = {+ ...createBasePipelines(device, transparentWireBase),+ bloom: new BloomPipelineVariants(device, transparentWireBase),+ };this.wireShadow = device.createRenderPipeline({label: "Mesh3DWireframeShadow",layout: device.createPipelineLayout({@@ -435,16 +465,23 @@ export class Mesh3DPipeline {if (wireframe && !meshBuffers.wireIndexBuffer) {throw new Error("Wireframe requires mesh buffers with edge indices");}+ const variants = data.transparent+ ? wireframe+ ? this.transparentWirePipelines+ : this.transparentPipelines+ : wireframe+ ? { single: this.wireSingle, mrt: this.wireMrt, bloom: this.wireBloom }+ : {+ single: this.pipelineSingle,+ mrt: this.pipelineMrt,+ bloom: this.bloomVariants,+ };pass.setPipeline(useBloom- ? (wireframe ? this.wireBloom : this.bloomVariants).get(useMrt)- : wireframe- ? useMrt- ? this.wireMrt- : this.wireSingle- : useMrt- ? this.pipelineMrt- : this.pipelineSingle,+ ? variants.bloom.get(useMrt)+ : useMrt+ ? variants.mrt+ : variants.single,);pass.setBindGroup(0, cameraBindGroup);pass.setBindGroup(1, modelBindGroup);diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.test.tsindex c28ad134..56d23915 100644--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.test.ts@@ -45,6 +45,25 @@ describe("Renderer3D Subsystem", () => {expect(mockDevice.createRenderPipeline).toHaveBeenCalled();});+ it("creates blended mesh and wireframe pipelines without depth writes", () => {+ for (const label of [+ "Mesh3DTransparentSingle",+ "Mesh3DTransparentMrt",+ "Mesh3DTransparentWireframeSingle",+ "Mesh3DTransparentWireframeMrt",+ ]) {+ expect(mockDevice.createRenderPipeline).toHaveBeenCalledWith(+ expect.objectContaining({+ label,+ depthStencil: expect.objectContaining({+ depthWriteEnabled: false,+ depthCompare: "less-equal",+ }),+ }),+ );+ }+ });+it("creates and caches depth textures for surface dimensions", () => {const depth1 = renderer3d.getOrCreateDepthTexture(1920, 1080);expect(depth1).toBeDefined();diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.tsindex 04232f7b..0fb8ae79 100644--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts@@ -150,6 +150,8 @@ export interface DrawMesh3DOpts extends SurfaceParams {twoSided?: boolean;/** Wireframe mode */wireframe?: boolean;+ /** Blend without writing depth; draw transparent meshes back to front. */+ transparent?: boolean;/** Whether vertex skinning is enabled */isSkinned?: boolean;/** Skinning matrices (128 * 16 floats) */@@ -1346,6 +1348,7 @@ export class Renderer3D {opacity,twoSided,wireframe,+ transparent: opts.transparent ?? (opacity < 1 || colorTint[3] < 1),isSkinned,skinningMatrices,material:diff --git a/plugins/framefields/skills/framefields-compose/SKILL.md b/plugins/framefields/skills/framefields-compose/SKILL.mdindex febca0f8..d26f739d 100644--- a/plugins/framefields/skills/framefields-compose/SKILL.md+++ b/plugins/framefields/skills/framefields-compose/SKILL.md@@ -446,6 +446,7 @@ Layer3D.cylinder({ ..., color: "#111", emissive: "#4da3ff", emissiveIntensity: 3- **Models:** glTF materials bring base color, normal, metallic-roughness, occlusion and emissive maps; OBJ (`map_Kd`, `norm` in the `.mtl`) and FBX bring diffuse and normal maps, looked up next to the model. Textures must be PNG, JPEG, WebP, GIF, TIFF or AVIF: TGA, PSD, DDS and KTX2 are skipped with a warning (convert them). Draco / meshopt-compressed glTF files fail to load: re-export without mesh compression. Vertex colors (VOX, PLY, OFF, glTF `COLOR_0`) multiply the base color.- **Model orientation:** the world is y-down, so a y-up model (most glTF, OBJ and FBX files) needs `rotateX: 180` to stand upright.- `emissive` / `emissiveIntensity` / `exposure` animate like any prop (`.keys("emissiveIntensity", …)`).+- **Transparent 3D meshes:** glTF `BLEND` materials and meshes with reduced opacity draw after opaque scene items, sorted back to front by each mesh's bounds center without writing depth. This is an object-level approximation: intersecting or self-overlapping meshes cannot always sort correctly; translucent flat layers and glass use separate rendering paths.- **Glass:** `material: "glass" | "acrylic" | "frosted"` on any Layer3D primitive, extruded text (`Layer3D.extrudedText`) or model refracts whatever is behind it on screen — the video, 2D layers, particles — with a rainbow fringe (`dispersion`, default 0.04), bend strength `thickness` (px, default 80) and `ior` (1.5). `color` tints the glass (deeper toward the edges); `frosted` blurs by `roughness`. Glass needs no environment: it draws its own studio reflections, and against a dark grade it reads best — skip `Lighting.*` rigs for glass-only scenes (their environment washes glass out). Glass shows only 2D content and opaque 3D drawn before it, not other glass.---