Code-first video & image, rendered natively on WebGPU.
Render per-vertex colors on 3D meshes
Clauded455f78526ed
diff --git a/packages/framefields/src/extended-models-render.test.ts b/packages/framefields/src/extended-models-render.test.tsindex 9292a3d8..ef71fbc2 100644--- a/packages/framefields/src/extended-models-render.test.ts+++ b/packages/framefields/src/extended-models-render.test.ts@@ -272,7 +272,17 @@ endsolid tetrahedron`;xyziChunk.writeUInt8(0, 18); // zxyziChunk.writeUInt8(1, 19); // color index 1- const mainChildren = Buffer.concat([sizeChunk, xyziChunk]);+ // Palette: color index 1 (the first RGBA entry) is red.+ const rgbaChunk = Buffer.alloc(12 + 256 * 4);+ rgbaChunk.write("RGBA", 0);+ rgbaChunk.writeUInt32LE(256 * 4, 4);+ rgbaChunk.writeUInt32LE(0, 8);+ rgbaChunk.writeUInt8(230, 12);+ rgbaChunk.writeUInt8(30, 13);+ rgbaChunk.writeUInt8(30, 14);+ rgbaChunk.writeUInt8(255, 15);++ const mainChildren = Buffer.concat([sizeChunk, xyziChunk, rgbaChunk]);const mainChunk = Buffer.alloc(12);mainChunk.write("MAIN", 0);mainChunk.writeUInt32LE(0, 4);@@ -340,6 +350,86 @@ endsolid tetrahedron`;expect(img.width).toBe(800);expect(img.height).toBe(600);+ // The voxel's palette color reaches the screen (vertex colors). The one+ // voxel sits in a corner of its 2x2x2 grid, up and right of center.+ const center = getPixel(img, 425, 265);+ expect(center.r).toBeGreaterThan(80);+ expect(center.r).toBeGreaterThan(center.g * 2);+ expect(center.r).toBeGreaterThan(center.b * 2);++ await fs.rm(tmpDir, { recursive: true, force: true });+ });++ it("renders PLY vertex colors", async () => {+ const tmpDir = await fs.mkdtemp(+ path.join(os.tmpdir(), "framefields-ply-colors-"),+ );+ const plyPath = path.join(tmpDir, "quad.ply");+ // Left edge red, right edge blue.+ await fs.writeFile(+ plyPath,+ `ply+format ascii 1.0+element vertex 4+property float x+property float y+property float z+property uchar red+property uchar green+property uchar blue+element face 2+property list uchar int vertex_indices+end_header+-100.0 -100.0 0.0 255 0 0+ 100.0 -100.0 0.0 0 0 255+ 100.0 100.0 0.0 0 0 255+-100.0 100.0 0.0 255 0 0+3 0 1 2+3 0 2 3+`,+ );++ const comp = new Composition({+ width: 400,+ height: 300,+ fps: 30,+ durationMs: 1000,+ backgroundColor: "#000000",+ });+ comp.add(+ Layer.camera({+ id: "cam3d",+ x: 200,+ y: 150,+ z: -600,+ targetX: 200,+ targetY: 150,+ targetZ: 0,+ }),+ );+ comp.add(+ Layer3D.ply(plyPath, {+ id: "ply-colored",+ x: 200,+ y: 150,+ z: 0,+ scale: 1.0,+ material: "unlit",+ }),+ );++ const img = await decodePngPixels(await comp.renderFrame({ frame: 0 }));+ // Sample inside the quad, near each edge (the quad may appear mirrored).+ const a = getPixel(img, 160, 150);+ const b = getPixel(img, 240, 150);+ const [left, right] = a.r > a.b ? [a, b] : [b, a];+ expect(left.r).toBeGreaterThan(150);+ expect(left.b).toBeLessThan(100);+ expect(right.b).toBeGreaterThan(150);+ expect(right.r).toBeLessThan(100);+ expect(left.g).toBeLessThan(40);+ expect(right.g).toBeLessThan(40);+await fs.rm(tmpDir, { recursive: true, force: true });});diff --git a/packages/webgpu-renderers/src/loaders/gltf-loader.ts b/packages/webgpu-renderers/src/loaders/gltf-loader.tsindex fc2463af..bbc71d11 100644--- a/packages/webgpu-renderers/src/loaders/gltf-loader.ts+++ b/packages/webgpu-renderers/src/loaders/gltf-loader.ts@@ -661,10 +661,13 @@ export class GltfLoader {const colors = read(prim.attributes.COLOR_0);let rgba: Float32Array | undefined;if (colors) {+ // COLOR_0 is linear; Mesh3DData colors are display (sRGB), like base colors.rgba = new Float32Array(numVerts * 4).fill(1);for (let v = 0; v < numVerts; v++)- for (let c = 0; c < colors.itemSize; c++)- rgba[v * 4 + c] = colors.data[v * colors.itemSize + c] ?? 1;+ for (let c = 0; c < colors.itemSize; c++) {+ const value = colors.data[v * colors.itemSize + c] ?? 1;+ rgba[v * 4 + c] = c < 3 ? linearToSrgb(value) : value;+ }}let indices: Uint16Array | Uint32Array;@@ -840,3 +843,8 @@ export function loadGLB(): Model3DData {return GltfLoader.load(srcOrData, options);}++function linearToSrgb(c: number): number {+ const x = Math.min(Math.max(c, 0), 1);+ return x <= 0.0031308 ? x * 12.92 : 1.055 * x ** (1 / 2.4) - 0.055;+}diff --git a/packages/webgpu-renderers/src/loaders/off-loader.ts b/packages/webgpu-renderers/src/loaders/off-loader.tsindex 8e3993df..f0b1cf6f 100644--- a/packages/webgpu-renderers/src/loaders/off-loader.ts+++ b/packages/webgpu-renderers/src/loaders/off-loader.ts@@ -1,6 +1,18 @@import fs from "node:fs";-import type { LoadModelOptions, Material3D, Mesh3DData, Model3DData } from "./types.js";-import { accumulateVertexNormals, computeBounds, createIndexArray, resolveModelTransform, transformModelBounds, transformPositions } from "./mesh-utils.js";+import {+ accumulateVertexNormals,+ computeBounds,+ createIndexArray,+ resolveModelTransform,+ transformModelBounds,+ transformPositions,+} from "./mesh-utils.js";+import type {+ LoadModelOptions,+ Material3D,+ Mesh3DData,+ Model3DData,+} from "./types.js";export class OffLoader {/**@@ -13,13 +25,18 @@ export class OffLoader {const text =typeof data === "string"? data- : new TextDecoder("utf-8").decode(data instanceof Uint8Array ? data : new Uint8Array(data));+ : new TextDecoder("utf-8").decode(+ data instanceof Uint8Array ? data : new Uint8Array(data),+ );const lines = text.split(/\r?\n/);let lineIdx = 0;// 1. Header line- while (lineIdx < lines.length && (!lines[lineIdx]!.trim() || lines[lineIdx]!.trim().startsWith("#"))) {+ while (+ lineIdx < lines.length &&+ (!lines[lineIdx]!.trim() || lines[lineIdx]!.trim().startsWith("#"))+ ) {lineIdx++;}@@ -91,10 +108,19 @@ export class OffLoader {}if (hasColors && parts.length > propOffset) {- const r = (Number.parseFloat(parts[propOffset] ?? "255") || 0) / (Number.parseFloat(parts[propOffset] ?? "255") > 1 ? 255 : 1);- const g = (Number.parseFloat(parts[propOffset + 1] ?? "255") || 0) / (Number.parseFloat(parts[propOffset + 1] ?? "255") > 1 ? 255 : 1);- const b = (Number.parseFloat(parts[propOffset + 2] ?? "255") || 0) / (Number.parseFloat(parts[propOffset + 2] ?? "255") > 1 ? 255 : 1);- const a = parts.length > propOffset + 3 ? (Number.parseFloat(parts[propOffset + 3] ?? "1") || 1) : 1.0;+ const r =+ (Number.parseFloat(parts[propOffset] ?? "255") || 0) /+ (Number.parseFloat(parts[propOffset] ?? "255") > 1 ? 255 : 1);+ const g =+ (Number.parseFloat(parts[propOffset + 1] ?? "255") || 0) /+ (Number.parseFloat(parts[propOffset + 1] ?? "255") > 1 ? 255 : 1);+ const b =+ (Number.parseFloat(parts[propOffset + 2] ?? "255") || 0) /+ (Number.parseFloat(parts[propOffset + 2] ?? "255") > 1 ? 255 : 1);+ const a =+ parts.length > propOffset + 3+ ? Number.parseFloat(parts[propOffset + 3] ?? "1") || 1+ : 1.0;rawColors.push(r, g, b, a);}@@ -134,7 +160,8 @@ export class OffLoader {const defaultMaterial: Material3D = {name: "off_default",- diffuseColor: [0.8, 0.8, 0.85, 1.0],+ // Vertex colors carry the color; the neutral gray would only dim them.+ diffuseColor: hasColors ? [1, 1, 1, 1] : [0.8, 0.8, 0.85, 1.0],shininess: 32.0,specularIntensity: 0.5,...options.defaultMaterial,@@ -153,7 +180,6 @@ export class OffLoader {},];-return {name: "OFF_Model",meshes,diff --git a/packages/webgpu-renderers/src/loaders/ply-loader.ts b/packages/webgpu-renderers/src/loaders/ply-loader.tsindex 9145a85f..1a11f6e5 100644--- a/packages/webgpu-renderers/src/loaders/ply-loader.ts+++ b/packages/webgpu-renderers/src/loaders/ply-loader.ts@@ -1,6 +1,18 @@import fs from "node:fs";-import type { LoadModelOptions, Material3D, Mesh3DData, Model3DData } from "./types.js";-import { accumulateVertexNormals, computeBounds, createIndexArray, resolveModelTransform, transformModelBounds, transformPositions } from "./mesh-utils.js";+import {+ accumulateVertexNormals,+ computeBounds,+ createIndexArray,+ resolveModelTransform,+ transformModelBounds,+ transformPositions,+} from "./mesh-utils.js";+import type {+ LoadModelOptions,+ Material3D,+ Mesh3DData,+ Model3DData,+} from "./types.js";interface PlyProperty {name: string;@@ -36,7 +48,9 @@ export class PlyLoader {const header = PlyLoader.parseHeader(headerText.text);if (header.format === "ascii") {- const bodyText = new TextDecoder("utf-8").decode(bytes.subarray(headerText.endOffset));+ const bodyText = new TextDecoder("utf-8").decode(+ bytes.subarray(headerText.endOffset),+ );return PlyLoader.parseAsciiBody(header, bodyText, options);}@@ -49,7 +63,10 @@ export class PlyLoader {);}- private static extractHeader(bytes: Uint8Array): { text: string; endOffset: number } {+ private static extractHeader(bytes: Uint8Array): {+ text: string;+ endOffset: number;+ } {const searchLen = Math.min(bytes.length, 10000);const slice = bytes.subarray(0, searchLen);const str = new TextDecoder("utf-8").decode(slice);@@ -76,7 +93,8 @@ export class PlyLoader {elements: PlyElement[];} {const lines = headerStr.split(/\r?\n/);- let format: "ascii" | "binary_little_endian" | "binary_big_endian" = "ascii";+ let format: "ascii" | "binary_little_endian" | "binary_big_endian" =+ "ascii";let version = "1.0";const elements: PlyElement[] = [];let currentElement: PlyElement | null = null;@@ -148,8 +166,12 @@ export class PlyLoader {const nxIdx = vProps.findIndex((p) => p.name === "nx");const nyIdx = vProps.findIndex((p) => p.name === "ny");const nzIdx = vProps.findIndex((p) => p.name === "nz");- const uIdx = vProps.findIndex((p) => p.name === "u" || p.name === "s" || p.name === "texture_u");- const vCoordIdx = vProps.findIndex((p) => p.name === "v" || p.name === "t" || p.name === "texture_v");+ const uIdx = vProps.findIndex(+ (p) => p.name === "u" || p.name === "s" || p.name === "texture_u",+ );+ const vCoordIdx = vProps.findIndex(+ (p) => p.name === "v" || p.name === "t" || p.name === "texture_v",+ );const rIdx = vProps.findIndex((p) => p.name === "red" || p.name === "r");const gIdx = vProps.findIndex((p) => p.name === "green" || p.name === "g");const bIdx = vProps.findIndex((p) => p.name === "blue" || p.name === "b");@@ -194,7 +216,10 @@ export class PlyLoader {const r = (Number.parseFloat(parts[rIdx] ?? "255") || 0) / 255;const g = (Number.parseFloat(parts[gIdx] ?? "255") || 0) / 255;const b = (Number.parseFloat(parts[bIdx] ?? "255") || 0) / 255;- const a = aIdx !== -1 ? (Number.parseFloat(parts[aIdx] ?? "255") || 0) / 255 : 1.0;+ const a =+ aIdx !== -1+ ? (Number.parseFloat(parts[aIdx] ?? "255") || 0) / 255+ : 1.0;rawColors.push(r, g, b, a);}}@@ -233,7 +258,11 @@ export class PlyLoader {isLittleEndian: boolean,options: LoadModelOptions,): Model3DData {- const view = new DataView(bodyBytes.buffer, bodyBytes.byteOffset, bodyBytes.byteLength);+ const view = new DataView(+ bodyBytes.buffer,+ bodyBytes.byteOffset,+ bodyBytes.byteLength,+ );let offset = 0;const vertexElement = header.elements.find((e) => e.name === "vertex");@@ -304,10 +333,18 @@ export class PlyLoader {};for (let v = 0; v < vertexCount; v++) {- let x = 0, y = 0, z = 0;- let nx = 0, ny = 0, nz = 0;- let u = 0, vCoord = 0;- let r = 1, g = 1, b = 1, a = 1;+ let x = 0,+ y = 0,+ z = 0;+ let nx = 0,+ ny = 0,+ nz = 0;+ let u = 0,+ vCoord = 0;+ let r = 1,+ g = 1,+ b = 1,+ a = 1;for (const prop of vProps) {const val = readType(prop.type);@@ -321,10 +358,14 @@ export class PlyLoader {else if (pName === "nz") nz = val;else if (pName === "u" || pName === "s") u = val;else if (pName === "v" || pName === "t") vCoord = val;- else if (pName === "red" || pName === "r") r = prop.type.includes("int") ? val / 255 : val;- else if (pName === "green" || pName === "g") g = prop.type.includes("int") ? val / 255 : val;- else if (pName === "blue" || pName === "b") b = prop.type.includes("int") ? val / 255 : val;- else if (pName === "alpha" || pName === "a") a = prop.type.includes("int") ? val / 255 : val;+ else if (pName === "red" || pName === "r")+ r = prop.type.includes("int") ? val / 255 : val;+ else if (pName === "green" || pName === "g")+ g = prop.type.includes("int") ? val / 255 : val;+ else if (pName === "blue" || pName === "b")+ b = prop.type.includes("int") ? val / 255 : val;+ else if (pName === "alpha" || pName === "a")+ a = prop.type.includes("int") ? val / 255 : val;}rawPositions.push(x, y, z);@@ -387,7 +428,8 @@ export class PlyLoader {const defaultMaterial: Material3D = {name: "ply_default",- diffuseColor: [0.8, 0.8, 0.85, 1.0],+ // Vertex colors carry the color; the neutral gray would only dim them.+ diffuseColor: rawColors ? [1, 1, 1, 1] : [0.8, 0.8, 0.85, 1.0],shininess: 32.0,specularIntensity: 0.5,...options.defaultMaterial,@@ -406,7 +448,6 @@ export class PlyLoader {},];-return {name: "PLY_Model",meshes,diff --git a/packages/webgpu-renderers/src/loaders/types.ts b/packages/webgpu-renderers/src/loaders/types.tsindex 23a86953..793c21e2 100644--- a/packages/webgpu-renderers/src/loaders/types.ts+++ b/packages/webgpu-renderers/src/loaders/types.ts@@ -7,7 +7,7 @@ export interface Mesh3DData {normals: Float32Array; // 3 floats per vertex (nx, ny, nz)uvs: Float32Array; // 2 floats per vertex (u, v)indices: Uint16Array | Uint32Array;- colors?: Float32Array; // 4 floats per vertex (r, g, b, a)+ colors?: Float32Array; // 4 floats per vertex (r, g, b, a), display (sRGB) 0..1; multiplies the base colortangents?: Float32Array; // 4 floats per vertex (x, y, z, w)// Skinned Mesh / Skeletal Animation attributesjointIndices?: Float32Array; // 4 bone indices per vertexdiff --git a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.tsindex f9a83739..cabeb82c 100644--- a/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts+++ b/packages/webgpu-renderers/src/renderer3d/mesh3d-pipeline.ts@@ -77,6 +77,8 @@ export interface GPUMeshBuffers {indexBuffer: GPUBuffer;indexCount: number;indexFormat: GPUIndexFormat;+ /** Per-vertex RGBA (display / sRGB), 4 floats each; absent = white. */+ colorBuffer?: GPUBuffer;}export class Mesh3DPipeline {@@ -96,6 +98,11 @@ export class Mesh3DPipeline {/** (0.5, 0.5, 1): an unperturbed tangent-space normal. */private flatNormalTexture: GPUTexture;private defaultSkinningBuffer: GPUBuffer;+ /** White vertex colors for meshes without any, grown to the largest mesh. */+ private whiteColorBuffer: GPUBuffer | null = null;+ private whiteColorCapacity = 0;+ /** Outgrown white buffers: earlier draws this frame may still read them. */+ private retiredColorBuffers: GPUBuffer[] = [];// 16 (model) + 16 (normal) + 4 (tint) + 4 (params) + 4 (materialParams) + 4 (shade) + 8 (pbr, emissive) + 4 (maps) = 60 floats (240 bytes)private modelData = new Float32Array(52 + SURFACE_PARAMS_FLOATS);@@ -199,6 +206,10 @@ export class Mesh3DPipeline {{ shaderLocation: 4, offset: 48, format: "float32x4" }, // weights],},+ {+ arrayStride: 16, // RGBA vertex color+ attributes: [{ shaderLocation: 5, offset: 0, format: "float32x4" }],+ },],};@@ -403,6 +414,7 @@ export class Mesh3DPipeline {pass.setBindGroup(2, textureBindGroup);pass.setBindGroup(3, lightsBindGroup);pass.setVertexBuffer(0, meshBuffers.vertexBuffer);+ pass.setVertexBuffer(1, this.colorsFor(meshBuffers));pass.setIndexBuffer(meshBuffers.indexBuffer, meshBuffers.indexFormat);pass.drawIndexed(meshBuffers.indexCount);}@@ -418,11 +430,35 @@ export class Mesh3DPipeline {pass.setBindGroup(0, lightCameraBindGroup);pass.setBindGroup(1, this.packModel(data));pass.setVertexBuffer(0, meshBuffers.vertexBuffer);+ pass.setVertexBuffer(1, this.colorsFor(meshBuffers));pass.setIndexBuffer(meshBuffers.indexBuffer, meshBuffers.indexFormat);pass.drawIndexed(meshBuffers.indexCount);}+ private colorsFor(meshBuffers: GPUMeshBuffers): GPUBuffer {+ if (meshBuffers.colorBuffer) return meshBuffers.colorBuffer;+ // 64 bytes per vertex in the main buffer.+ const needed = Math.max(1, Math.ceil(meshBuffers.vertexBuffer.size / 64));+ if (!this.whiteColorBuffer || this.whiteColorCapacity < needed) {+ if (this.whiteColorBuffer)+ this.retiredColorBuffers.push(this.whiteColorBuffer);+ const capacity = 2 ** Math.ceil(Math.log2(needed));+ this.whiteColorBuffer = this.device.createBuffer({+ label: "Mesh3DWhiteVertexColors",+ size: capacity * 16,+ usage: GPUBufferUsage.VERTEX,+ mappedAtCreation: true,+ });+ new Float32Array(this.whiteColorBuffer.getMappedRange()).fill(1);+ this.whiteColorBuffer.unmap();+ this.whiteColorCapacity = capacity;+ }+ return this.whiteColorBuffer;+ }+destroy(): void {+ this.whiteColorBuffer?.destroy();+ for (const buffer of this.retiredColorBuffers) buffer.destroy();this.modelPool.destroy();this.skinningPool.destroy();this.defaultWhiteTexture.destroy();diff --git a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts b/packages/webgpu-renderers/src/renderer3d/renderer3d.tsindex 00d1355a..094f75f7 100644--- a/packages/webgpu-renderers/src/renderer3d/renderer3d.ts+++ b/packages/webgpu-renderers/src/renderer3d/renderer3d.ts@@ -1169,11 +1169,26 @@ export class Renderer3D {}indexBuffer.unmap();+ let colorBuffer: GPUBuffer | undefined;+ if (mesh.colors && mesh.colors.length >= numVerts * 4) {+ colorBuffer = this.device.createBuffer({+ label: "Mesh3DVertexColors",+ size: Math.max(16, numVerts * 16),+ usage: GPUBufferUsage.VERTEX,+ mappedAtCreation: true,+ });+ new Float32Array(colorBuffer.getMappedRange()).set(+ mesh.colors.subarray(0, numVerts * 4),+ );+ colorBuffer.unmap();+ }+cached = {vertexBuffer,indexBuffer,indexCount: mesh.indices.length,indexFormat: is32Bit ? "uint32" : "uint16",+ colorBuffer,};this.meshBuffersCache.set(mesh, cached);return cached;@@ -1207,6 +1222,7 @@ export class Renderer3D {indexBuffer: meshBuffers.indexBuffer,indexCount: meshBuffers.indexCount,indexFormat: meshBuffers.indexFormat,+ colorBuffer: meshBuffers.colorBuffer,};}diff --git a/packages/webgpu-renderers/src/shaders/mesh3d.ts b/packages/webgpu-renderers/src/shaders/mesh3d.tsindex 1bf89c65..b8dc3a95 100644--- a/packages/webgpu-renderers/src/shaders/mesh3d.ts+++ b/packages/webgpu-renderers/src/shaders/mesh3d.ts@@ -47,6 +47,7 @@ struct VertexInput {@location(2) normal : vec3<f32>,@location(3) joints : vec4<f32>,@location(4) weights : vec4<f32>,+ @location(5) color : vec4<f32>, // vertex color (display / sRGB), white when absent};struct VertexOutput {@@ -55,6 +56,7 @@ struct VertexOutput {@location(1) worldPos : vec3<f32>,@location(2) worldNormal : vec3<f32>,@location(3) linearDepth : f32,+ @location(4) color : vec4<f32>,};@vertex@@ -99,6 +101,7 @@ fn vs_main(in: VertexInput) -> VertexOutput {out.clipPos = cam.viewProjMatrix * worldPos4;out.uv = in.uv;+ out.color = in.color;out.linearDepth = dot(worldPos4.xyz - cam.cameraPos.xyz, cam.cameraDir.xyz);return out;@@ -243,12 +246,12 @@ fn shadeMesh(in: VertexOutput, isFront: bool) -> MeshFragment {if (texColor.a < model.shade.a) {discard;}- let alpha = texColor.a * model.params.x * model.colorTint.a * matteCoverage(in.clipPos.xy);+ let alpha = texColor.a * in.color.a * model.params.x * model.colorTint.a * matteCoverage(in.clipPos.xy);if (alpha <= 0.001) {discard;}- let baseRgb = texColor.rgb * model.colorTint.rgb;+ let baseRgb = texColor.rgb * in.color.rgb * model.colorTint.rgb;var out: MeshFragment;if (model.materialParams.w > 1.5) {var toon = baseRgb;