3D engine that AI agents drive
Draw animated crowds as GPU-instanced batches, with culling and levels (part of #323)
Nguyen Xuan Ancc82e63f8746
diff --git a/bench/crowd/harness.html b/bench/crowd/harness.htmlnew file mode 100644index 00000000..4cb9a49a--- /dev/null+++ b/bench/crowd/harness.html@@ -0,0 +1,10 @@+<!doctype html>+<html>+<head>+<style>html,body{margin:0;height:100%;background:#000}canvas{width:100vw;height:100vh;display:block}</style>+</head>+<body>+<canvas></canvas>+<script type="module" src="./harness.js"></script>+</body>+</html>diff --git a/bench/crowd/harness.js b/bench/crowd/harness.jsnew file mode 100644index 00000000..28e52ae4--- /dev/null+++ b/bench/crowd/harness.js@@ -0,0 +1,207 @@+// Draws a crowd of the CC0 RobotExpressive walking, through the GPU crowd path or one asset per instance (npm run benchmark:crowd).+import { mat4 } from 'gl-matrix';+import { parseGltf } from '../../src/gltf.js';+import { clipFromGltfAnimation, createAssetPose } from '../../src/skeleton-pose.js';+import { createCrowd } from '../../src/retarget.js';+import { Renderer67 } from '../../src/renderer.js';+import { bakeCrowdPalette, createCrowdParts, createCrowdRig } from '../../src/crowd-skinning.js';++const ROBOT_URL = '/tests/fixtures/RobotExpressive.glb';+const ONE_PIXEL = new Uint8Array(4);++const counters = { drawCalls: 0, instances: 0 };+const originalDraw = WebGL2RenderingContext.prototype.drawElementsInstanced;+WebGL2RenderingContext.prototype.drawElementsInstanced = function counted(mode, count, type, offset, instances) {+ counters.drawCalls += 1;+ counters.instances += instances;+ return originalDraw.call(this, mode, count, type, offset, instances);+};++let robotBytes = null;+async function parseRobot() {+ robotBytes ??= await (await fetch(ROBOT_URL)).arrayBuffer();+ return parseGltf(robotBytes.slice(0), { maxVertices: 500000 });+}++function walkingClip(asset) {+ const source = asset.animations.find(animation => animation.name === 'Walking');+ return { source, clip: clipFromGltfAnimation(source, { loop: true }) };+}++function finish(renderer) {+ renderer.context.readPixels(0, 0, 1, 1, renderer.context.RGBA, renderer.context.UNSIGNED_BYTE, ONE_PIXEL);+}++function readImage(canvas) {+ const copy = document.createElement('canvas');+ copy.width = canvas.width;+ copy.height = canvas.height;+ const context = copy.getContext('2d');+ context.drawImage(canvas, 0, 0);+ return context.getImageData(0, 0, copy.width, copy.height).data;+}++function imageUrl(data, canvas) {+ const copy = document.createElement('canvas');+ copy.width = canvas.width;+ copy.height = canvas.height;+ copy.getContext('2d').putImageData(new ImageData(data, canvas.width, canvas.height), 0, 0);+ return copy.toDataURL('image/png');+}++function compareImages(first, second) {+ let differing = 0;+ let total = 0;+ for (let index = 0; index < first.length; index += 4) {+ const difference = Math.max(Math.abs(first[index] - second[index]), Math.abs(first[index + 1] - second[index + 1]), Math.abs(first[index + 2] - second[index + 2]));+ total += difference;+ if (difference > 8) differing++;+ }+ return { meanDifference: total / (first.length / 4), fractionOver8: differing / (first.length / 4) };+}++function cameraFor(columns, spacing, radius, wide) {+ const extent = Math.max(columns * spacing, radius * 2);+ return wide+ ? { eye: [0, extent * 0.55, extent * 0.9], target: [0, radius * 0.5, 0], up: [0, 1, 0], fov: 45, near: 0.5, far: extent * 6 }+ : { eye: [0, radius * 1.2, radius * 5], target: [0, radius * 0.6, 0], up: [0, 1, 0], fov: 40, near: 0.5, far: extent * 6 };+}++async function buildCrowd(canvas, { count, spacing, wide = true, lods = true, culling = true }) {+ const asset = await parseRobot();+ const { source, clip } = walkingClip(asset);+ const pose = createAssetPose(asset, { layers: [{ clip }] });+ const rig = createCrowdRig(asset.parts);+ const palette = bakeCrowdPalette(rig, seconds => pose.apply(seconds), { duration: source.duration });+ const crowd = createCrowd({ count, clip: { duration: source.duration, tracks: [] }, seed: 5, spacing });+ const set = createCrowdParts({ rig, palette, instances: crowd.instances, lods });+ const columns = Math.ceil(Math.sqrt(count));+ const camera = cameraFor(columns, spacing, palette.bounds.radius, wide);+ const renderer = new Renderer67(canvas, set.parts, { ground: true, hdr: false, samples: 1, pixelRatio: 1, memoryBudgetMiB: 2048, culling });+ renderer.resize();+ const frame = { ...camera, time: 0, exposure: 1, effects: {} };+ return {+ renderer, palette, rig, set, crowd, frame,+ show(seconds, { exact = false } = {}) {+ set.setTimes(index => (exact ? seconds : crowd.clipTime(index, seconds)));+ renderer.updateParts(set.parts);+ renderer.render(frame);+ },+ };+}++async function measureCrowd(canvas, { count, frames, wide = true, spacing }) {+ const crowdScene = await buildCrowd(canvas, { count, spacing, wide });+ const { renderer } = crowdScene;+ for (let warmup = 0; warmup < 3; warmup++) {+ crowdScene.show(warmup / 60);+ finish(renderer);+ }+ counters.drawCalls = 0;+ counters.instances = 0;+ const times = [];+ for (let frame = 0; frame < frames; frame++) {+ const started = performance.now();+ crowdScene.show((frame + 3) / 60);+ finish(renderer);+ times.push(performance.now() - started);+ }+ times.sort((first, second) => first - second);+ const result = {+ count, parts: crowdScene.set.parts.length, materials: crowdScene.rig.parts.length, frames,+ medianMilliseconds: times[Math.floor(times.length / 2)], bestMilliseconds: times[0],+ drawCallsPerFrame: counters.drawCalls / frames, instancesDrawnPerFrame: counters.instances / frames,+ bounds: crowdScene.palette.bounds, frameCount: crowdScene.palette.frameCount, jointCount: crowdScene.palette.jointCount,+ };+ renderer.dispose();+ return result;+}++// Per-instance skinning as it is today: one parsed asset per instance, posed by applyAnimation.+async function measureInstances(canvas, { count, frames, spacing }) {+ const assets = [];+ const parts = [];+ const crowd = createCrowd({ count, clip: { duration: 1, tracks: [] }, seed: 5, spacing });+ for (let index = 0; index < count; index++) {+ const asset = await parseRobot();+ assets.push(asset);+ for (const part of asset.parts) {+ parts.push(part);+ }+ }+ const walking = assets[0].animations.findIndex(animation => animation.name === 'Walking');+ const renderer = new Renderer67(canvas, parts, { ground: true, hdr: false, samples: 1, pixelRatio: 1, memoryBudgetMiB: 4096 });+ renderer.resize();+ const frame = { ...cameraFor(Math.ceil(Math.sqrt(count)), spacing, 4, true), time: 0, exposure: 1, effects: {} };+ const run = (seconds) => {+ assets.forEach((asset, index) => {+ asset.applyAnimation(crowd.clipTime(index, seconds), walking);+ for (const part of asset.parts) mat4.multiply(part.matrix, crowd.instances[index].matrix, part.matrix);+ });+ renderer.updateParts(parts);+ renderer.render(frame);+ finish(renderer);+ };+ run(0);+ counters.drawCalls = 0;+ const times = [];+ for (let step = 0; step < frames; step++) {+ const started = performance.now();+ run(step / 60);+ times.push(performance.now() - started);+ }+ times.sort((first, second) => first - second);+ const result = { count, medianMilliseconds: times[Math.floor(times.length / 2)], drawCallsPerFrame: counters.drawCalls / frames };+ renderer.dispose();+ return result;+}++// One crowd instance against the same robot drawn by the renderer's own rigid and skinned paths, and seek against play.+async function checkPictures(canvas, dump = false) {+ const reference = await parseRobot();+ const walking = reference.animations.findIndex(animation => animation.name === 'Walking');+ const seconds = 0.4;+ reference.applyAnimation(seconds, walking);+ const referenceRenderer = new Renderer67(canvas, reference.parts, { ground: true, hdr: false, samples: 1, pixelRatio: 1, memoryBudgetMiB: 512 });+ referenceRenderer.resize();+ const crowdScene = await buildCrowd(canvas, { count: 1, spacing: 3, wide: false, lods: false, culling: !globalThis.CROWD_NO_CULL });+ const frame = { ...crowdScene.frame, time: 0 };+ referenceRenderer.render(frame);+ const referenceImage = Uint8ClampedArray.from(readImage(canvas));+ crowdScene.show(0.4, { exact: true });+ const crowdImage = Uint8ClampedArray.from(readImage(canvas));+ const againstReference = compareImages(referenceImage, crowdImage);+ const dumps = dump ? { reference: imageUrl(referenceImage, canvas), crowd: imageUrl(crowdImage, canvas) } : null;+ referenceRenderer.dispose();+ crowdScene.renderer.dispose();++ const many = await buildCrowd(canvas, { count: 24, spacing: 2, wide: true });+ const target = 1.37;+ for (let step = 0; step <= 40; step++) many.show(step * target / 40);+ const played = Uint8ClampedArray.from(readImage(canvas));+ many.show(0.2);+ many.show(target);+ const seeked = Uint8ClampedArray.from(readImage(canvas));+ many.renderer.dispose();+ return { againstReference, seekVersusPlay: compareImages(played, seeked), dumps };+}++// A camera that sees one corner of a large crowd: instances outside the view are not drawn.+async function checkCulling(canvas) {+ const crowdScene = await buildCrowd(canvas, { count: 400, spacing: 6, wide: false, lods: false });+ counters.instances = 0;+ counters.drawCalls = 0;+ crowdScene.show(0.3);+ const result = { instances: 400, materials: crowdScene.rig.parts.length, instancesDrawn: counters.instances, drawCalls: counters.drawCalls };+ crowdScene.renderer.dispose();+ return result;+}++window.runCrowd = async (request) => {+ const canvas = document.querySelector('canvas');+ if (request.task === 'measure') return measureCrowd(canvas, request);+ if (request.task === 'instances') return measureInstances(canvas, request);+ if (request.task === 'pictures') return checkPictures(canvas, request.dump);+ if (request.task === 'culling') return checkCulling(canvas);+ throw new Error(`Unknown task ${request.task}`);+};diff --git a/bench/size/budgets.json b/bench/size/budgets.jsonindex 459b421b..dcce934a 100644--- a/bench/size/budgets.json+++ b/bench/size/budgets.json@@ -6,13 +6,13 @@"agent-scene": 100000},"budgets": {- "entry:core": 88890,+ "entry:core": 89269,"entry:model": 14400,"entry:nature": 13200,"entry:motion": 15260,"entry:canvas2d": 9400,- "entry:gltf": 57790,- "entry:gltf+lazy": 61120,+ "entry:gltf": 58021,+ "entry:gltf+lazy": 61350,"entry:modeling": 8700,"entry:sculpt": 2500,"entry:surface": 10100,@@ -34,22 +34,23 @@"entry:ground-bounce": 9460,"entry:path-tracer": 15960,"entry:texture-set": 900,- "entry:agent-scene": 80770,- "entry:agent-scene+lazy": 263350,- "entry:agent-asset": 112170,- "entry:agent-asset+lazy": 274720,+ "entry:agent-scene": 81005,+ "entry:agent-scene+lazy": 263582,+ "entry:agent-asset": 112411,+ "entry:agent-asset+lazy": 274964,"entry:hdri": 3300,"entry:camera-effects": 8600,- "entry:water-reflection": 26580,+ "entry:water-reflection": 26819,"entry:render-passes": 1700,"entry:gaussian-splats": 11023,- "entry:material-layer-shaders": 21400,- "entry:light-linking": 19720,- "entry:cascaded-shadows": 28530,- "scene:lit-box": 65870,- "scene:gltf-box": 90150,- "scene:gltf-box+lazy": 93480,- "scene:agent-scene": 109670,- "scene:agent-scene+lazy": 292250+ "entry:material-layer-shaders": 21627,+ "entry:light-linking": 19948,+ "entry:cascaded-shadows": 28797,+ "scene:lit-box": 66279,+ "scene:gltf-box": 90557,+ "scene:gltf-box+lazy": 93885,+ "scene:agent-scene": 110090,+ "scene:agent-scene+lazy": 292670,+ "entry:crowd-skinning": 27017}}diff --git a/changes/323.md b/changes/323.mdnew file mode 100644index 00000000..7d30eedb--- /dev/null+++ b/changes/323.md@@ -0,0 +1 @@+- Add `67js/addons/crowd-skinning` (issue #323): `createCrowdRig`, `bakeCrowdPalette` and `createCrowdParts` bake a character's clip into a joint-matrix texture once and draw every instance of it as one instanced batch per material, each instance sampling its own clip time, so draw calls stop growing with the number of people. Instances are culled by a bound that covers the whole clip, and `createCrowdLods` gives coarser levels skinned with the same joints. The renderer gains small hooks for it (a `crowd` part field, `crowdJoints` and `crowdWeights` geometry attributes, a `crowdEnabled` shader uniform); skinned parts without `crowd` behave as before.diff --git a/docs/culling-lod.md b/docs/culling-lod.mdindex 5f74db38..d6f91ca8 100644--- a/docs/culling-lod.md+++ b/docs/culling-lod.md@@ -8,7 +8,7 @@ On by default, with no schema and no option to set. Each frame the renderer testWhat is not culled, because its bounding sphere would not hold or because it draws in another pass:-- Skinned and morphed parts.+- Skinned and morphed parts. A crowd part (a character baked to a palette, see [Crowds](retarget.md#crowds)) is the exception: its instances are culled by a bound that covers every pose of the clip, and it takes `lods` whose levels carry the same joints.- `water`, `fire` and `smoke` materials, whose vertex shader moves vertices.`foliage` and `grass` sway in the wind but by a known amount, so they are culled and take levels like rigid parts: the bound of each instance grows by 0.56 times the wind strength (the farthest `windOffset` in the vertex shader moves a vertex per unit of strength).diff --git a/docs/guide.md b/docs/guide.mdindex 3a774589..65c3644f 100644--- a/docs/guide.md+++ b/docs/guide.md@@ -228,7 +228,7 @@ Pass `environment: { map }` to light and reflect from an equirectangular HDR ima1. Build rigs with `createRigFromSkin(asset.skins[0])`. Name families (Mixamo, VRM, Unreal, Rigify, Quaternius) map through `mapHumanoidBones`; fix a miss with `targetOverrides`.2. Call `retargetClip(sourceRig, targetRig, clip, { rootMotion })`.3. Play it with `createSkinClipPlayer(skin, clip).apply(seconds)`, then `renderer.updateParts(parts)`.-4. For a crowd, `createCrowd({ count, clip, seed })` gives placement, time offsets and playback rates. Each instance needs its own skin pose.+4. For a crowd, `createCrowd({ count, clip, seed })` gives placement, time offsets and playback rates. For dozens or hundreds of animated instances, bake the clip once with `67js/addons/crowd-skinning` so they draw as a few instanced batches (see `docs/retarget.md`, Crowds); the plain path needs one skin pose per instance.The [retarget contract](https://github.com/codepawl/67.js/blob/main/docs/retarget.md) covers rest-pose assumptions and measured cost. Mixamo clips need the owner's Adobe login to download; the code only consumes files.diff --git a/docs/retarget.md b/docs/retarget.mdindex 4423cdd0..0867dfa4 100644--- a/docs/retarget.md+++ b/docs/retarget.md@@ -82,24 +82,67 @@ Each instance has `position`, `yawRadians`, a `timeOffset` in the clip duration,column-major `matrix`. `clipTime(index, seconds)` gives the instance's clip time and `sample(seconds)` returns every instance's tracks(buffers are reused on the next call).-The renderer batches skinned parts by pose identity, so each instance needs its own skin pose (its own `createSkinPose` palette, one-imported asset per instance) and becomes its own batch. Instances cannot share a pose and still differ in time.+### GPU crowds++The plain path (a skin pose per instance) is one batch and one palette upload per instance, about 0.26 ms of JavaScript per instance on the author's+GPU, so about 50 instances fit 60 fps. `67js/addons/crowd-skinning` draws a crowd of one character and clip as instanced batches instead:++```js+import { createCrowd } from '67js/addons/retarget';+import { createCrowdRig, bakeCrowdPalette, createCrowdParts } from '67js/addons/crowd-skinning'; // import it before the Renderer67 is made++const rig = createCrowdRig(asset.parts); // one geometry per material, a joint per rigid part and skin joint+const palette = bakeCrowdPalette(rig, seconds => pose.apply(seconds), { duration, frameRate: 30 });+const crowd = createCrowd({ count: 500, clip, seed: 7 });+const set = createCrowdParts({ rig, palette, instances: crowd.instances });+const renderer = new Renderer67(canvas, set.parts, options);+// each frame:+set.setTimes(index => crowd.clipTime(index, seconds));+renderer.updateParts(set.parts);+```++`bakeCrowdPalette` poses the character at evenly spaced frames and keeps every joint matrix in one table, uploaded as one float texture. `pose`+is anything that poses the parts the rig was made from, such as `createAssetPose(asset, ...)` or `createSkinClipPlayer`. Each instance stores two+frame offsets and a blend (in the assembly offset of its parts), so the vertex shader reads the matrices of that instance's own clip time and+blends the two nearest frames (linear in the matrices, 30 frames a second by default). The pose is a pure function of the clip time, so a frame at+any seek time equals the one reached by playing. `crowdVertexAt(palette, geometry, vertex, seconds)` is the same maths on the CPU.++- **Draw calls** are the number of materials times the levels in view, plus the shadow pass, not the number of instances. All instances of a+ material share a part geometry, `lods` and palette, which is what makes them one batch.+- **Culling** uses `palette.bounds`, a sphere in the space of the instance matrix that covers every vertex in every pose of the clip. Instances+ outside the frustum are not drawn, and an instance far enough away uses a coarser level.+- **Levels** come from `createCrowdLods(geometry, { cells })` (on by default in `createCrowdParts`, `lods: false` turns them off): vertices that+ share a grid cell and a dominant joint merge, and keep the joints and weights of one of them, so every level skins with the same palette.+- Limits: morph targets are not applied; materials with maps are not supported (the geometry's UV and colour attributes carry joints and weights);+ an `assemblyAmount` does not move a crowd; a clip is baked once, so another clip is another palette and another batch. Lowering the pose rate+ for far instances is not done yet.### Measured cost-Headless Chrome with GPU on the author's machine (RTX 3060, 960x540, 4x MSAA, one measurement of 60 frames after 10 warm-up frames).-Each instance had 3,000 skinned vertices, 65 joints and its own pose. It was a synthetic bending chain, not a humanoid, so a real-character mesh with far more vertices will cost more in the render column. Pose is clip sampling plus joint matrices in JavaScript,-upload is `renderer.updateParts`, render includes `gl.finish()`.--| Instances | Batches | Pose ms | Upload ms | Render ms | Frame ms |-| --- | --- | --- | --- | --- | --- |-| 1 | 2 | 0.13 | 0.21 | 0.29 | 0.63 |-| 10 | 11 | 0.81 | 1.71 | 0.40 | 2.92 |-| 50 | 51 | 3.87 | 8.25 | 1.02 | 13.14 |-| 100 | 101 | 7.85 | 17.04 | 1.63 | 26.52 |--Cost grows linearly at about 0.26 ms per animated instance per frame, mostly from the per-pose palette upload and the JavaScript pose-work, both on the main thread. At this size roughly 50 animated instances fit a 60 fps budget (16.7 ms) and 100 do not. These are-single measurements, not a benchmark suite. Fewer animated instances help: poses can be updated at a lower rate or shared by distant-instances.+`npm run benchmark:crowd` (`--cpu` for SwiftShader, `--counts 100,500`, `--instances 20,50` for the plain path) draws the CC0 RobotExpressive+(19 parts, 3 materials, 187 joints, 29 baked frames, 7,214 vertices) walking, 1280 by 720, no multisampling, a ground and one shadow map.+It is a real humanoid made of rigid parts on bones and four skinned hands, merged into the rig.++SwiftShader (the path a headless agent has), the median frame of 5 after warm-up, update and render included. The machine was running other jobs,+so two runs a few minutes apart are given as a range (in the last run, 500 crowd instances and 50 plain ones both took 208 ms):++| Path | Instances | Draw calls per frame | Frame ms |+| --- | --- | --- | --- |+| Crowd | 100 | 7 | 84 to 134 |+| Crowd | 500 | 7 | 136 to 208 |+| One asset per instance | 20 | 632 | 102 to 156 |+| One asset per instance | 50 | 1,407 | 139 to 208 |++On SwiftShader most of a crowd frame is fill and vertex work that a CPU rasteriser does slowly, so these numbers say that the cost no longer grows with+a draw call and a palette upload per instance, not how fast a GPU is. The 7 calls are 3 materials in the colour pass, 3 in the shadow pass and the+ground, for 20, 100 and 500 instances alike. In a 400-instance field seen from one corner, 340 instance draws were issued across the 7 calls, where+every instance would give 1,200 per pass.++Pictures (SwiftShader): one crowd instance against the same robot drawn through the renderer's own rigid and skinned paths at the same pose differs by a+mean of 0.07 of 255, with 0.2 percent of pixels more than 8 levels apart. A crowd of 24 seeked straight to a time and the same crowd played up to it give+identical pixels. The unit tests check the CPU skinning against per-instance joint matrices at frame times and between frames, and that every+vertex stays inside `palette.bounds`.++RTX 3060: not measured yet. Run `bash ~/.agents/tools/gpu-lock.sh npm run benchmark:crowd -- --counts 100,500` (or add the scene to+`npm run benchmark:runtime`) and record the 500-instance frame here; the target is under 16.7 ms at 1280 by 720. The old table (0.26 ms per plain+instance, 26.5 ms at 100) was measured on a synthetic 3,000-vertex chain and stays the reference for the plain path.diff --git a/package.json b/package.jsonindex 2ed1dd62..6e2001d4 100644--- a/package.json+++ b/package.json@@ -66,6 +66,7 @@"./addons/physics": "./build/physics.js","./addons/spume": "./build/spume.js","./addons/retarget": "./build/retarget.js",+ "./addons/crowd-skinning": "./build/crowd-skinning.js","./addons/skeleton-pose": "./build/skeleton-pose.js","./addons/vehicle": "./build/vehicle.js","./addons/vegetation": "./build/vegetation.js",@@ -142,6 +143,7 @@"reference": "node bench/reference/run.mjs","size": "node scripts/measure-size.mjs","benchmark:runtime": "node bench/runtime/run.mjs",+ "benchmark:crowd": "node scripts/benchmark-crowd.mjs","check:tool-schemas": "node scripts/check-tool-schemas.mjs","check:links": "node scripts/check-links.mjs","changes": "node scripts/changelog-fragments.mjs",diff --git a/scripts/benchmark-crowd.mjs b/scripts/benchmark-crowd.mjsnew file mode 100644index 00000000..7bf409b6--- /dev/null+++ b/scripts/benchmark-crowd.mjs@@ -0,0 +1,59 @@+// Frame time, draw calls, picture and culling checks of the GPU crowd path: npm run benchmark:crowd [-- --cpu | --gpu] [--counts 100,500] [--frames 10]+// --cpu uses SwiftShader (the path a headless agent has); the default is the machine's GPU. Prints one JSON object.+import { writeFile } from 'node:fs/promises';+import { constants, setPriority } from 'node:os';+import { chromium } from 'playwright';+import { createServer } from 'vite';++setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);+const CHROME = process.env.CHROME_PATH ?? 'C:/Program Files/Google/Chrome/Application/chrome.exe';+const flags = process.argv.slice(2);+const useCpu = flags.includes('--cpu');+const countsFlag = flags.indexOf('--counts');+const counts = (countsFlag >= 0 ? flags[countsFlag + 1] : '100,500').split(',').map(Number);+const framesFlag = flags.indexOf('--frames');+const frames = framesFlag >= 0 ? Number(flags[framesFlag + 1]) : 10;+const instanceCounts = flags.includes('--instances') ? flags[flags.indexOf('--instances') + 1].split(',').map(Number) : [];+const width = 1280;+const height = 720;++const browserArguments = ['--enable-webgl', '--ignore-gpu-blocklist', '--disable-renderer-backgrounding'];+if (useCpu) browserArguments.push('--use-angle=swiftshader', '--enable-unsafe-swiftshader');+const server = await createServer({ server: { port: 0, strictPort: false }, logLevel: 'error' });+await server.listen();+const address = server.httpServer.address();+const url = `http://localhost:${address.port}/bench/crowd/harness.html`;++async function runTask(request) {+ const browser = await chromium.launch({ executablePath: CHROME, headless: true, args: browserArguments });+ try {+ const page = await browser.newPage({ viewport: { width, height } });+ const errors = [];+ page.on('pageerror', error => errors.push(error.message));+ if (process.env.CROWD_NO_CULL) await page.addInitScript('globalThis.CROWD_NO_CULL = true');+ await page.goto(url);+ await page.waitForFunction(() => window.runCrowd);+ const result = await page.evaluate(task => window.runCrowd(task), request);+ if (errors.length) throw new Error(errors.join('\n'));+ return result;+ } finally {+ await browser.close();+ }+}++const report = { backend: useCpu ? 'swiftshader' : 'gpu', width, height, frames };+try {+ report.pictures = await runTask({ task: 'pictures', dump: Boolean(process.env.CROWD_DUMP) });+ if (report.pictures.dumps) {+ for (const [name, dataUrl] of Object.entries(report.pictures.dumps)) await writeFile(`${process.env.CROWD_DUMP}/${name}.png`, Buffer.from(dataUrl.split(',')[1], 'base64'));+ delete report.pictures.dumps;+ }+ report.culling = await runTask({ task: 'culling' });+ report.crowd = [];+ for (const count of counts) report.crowd.push(await runTask({ task: 'measure', count, frames, spacing: 3 }));+ report.perInstance = [];+ for (const count of instanceCounts) report.perInstance.push(await runTask({ task: 'instances', count, frames, spacing: 3 }));+} finally {+ await server.close();+}+console.log(JSON.stringify(report, null, 1));diff --git a/src/crowd-skinning.js b/src/crowd-skinning.jsnew file mode 100644index 00000000..d174eae3--- /dev/null+++ b/src/crowd-skinning.js@@ -0,0 +1,334 @@+// GPU crowds (issue 323): one character and one clip draw as a few instanced batches, however many people there are.+//+// The clip's joint matrices are baked once into a table with one row of matrices per frame. Each instance carries+// only two frame offsets and a blend weight (in its assembly offset, which a crowd part does not otherwise use), and+// the vertex shader reads the matrices of its own clip time. Every instance of a material therefore shares a batch,+// and the batch can be culled and drawn in levels like any rigid part.+//+// The pose of an instance is a pure function of its clip time (crowdFrameBlend), so a frame at a seek time equals+// the same frame reached by playing.+import { mat4 } from 'gl-matrix';+import { installCrowdShader } from './shaders.js';++const FLOATS_PER_MATRIX = 16;+const TEXELS_PER_MATRIX = 4;+const DEFAULT_FRAME_RATE = 30;+const DEFAULT_LEVEL_CELLS = Object.freeze([0.02, 0.05]);++// Joint ids sit in the geometry's crowdJoints attribute (location 13) and weights in crowdWeights (location 14).+// The frame blend is in assemblyOffset: x and y are the texel offsets of the two frames, z the blend between them.+installCrowdShader(`+uniform float crowdEnabled;+mat4 crowdJoint(int frameTexel, int joint) {+ int texel = frameTexel + joint * 4;+ return mat4(skinTexel(jointPalette, texel), skinTexel(jointPalette, texel + 1),+ skinTexel(jointPalette, texel + 2), skinTexel(jointPalette, texel + 3));+}+mat4 crowdInfluence(float joint, float weight) {+ int firstFrame = int(assemblyOffset.x + 0.5);+ int secondFrame = int(assemblyOffset.y + 0.5);+ int jointIndex = int(joint + 0.5);+ return (crowdJoint(firstFrame, jointIndex) * (1.0 - assemblyOffset.z) + crowdJoint(secondFrame, jointIndex) * assemblyOffset.z) * weight;+}+mat4 crowdTransform() {+ return crowdInfluence(uvSets.x, vertexColor.x) + crowdInfluence(uvSets.y, vertexColor.y)+ + crowdInfluence(uvSets.z, vertexColor.z) + crowdInfluence(uvSets.w, vertexColor.w);+}`);++/**+ * Merge the parts of a character into crowd geometries, one per material. A rigid part becomes one joint that+ * follows its matrix; a skinned part keeps its joints, each premultiplied by the part's matrix. Morph targets are not+ * applied. Each part is { name, geometry, matrix, material, skin? } as an imported asset lists them.+ */+export function createCrowdRig(sourceParts) {+ if (!Array.isArray(sourceParts) || sourceParts.length === 0) throw new TypeError('A crowd rig needs a list of parts.');+ const jointOffsets = [];+ let jointCount = 0;+ for (const part of sourceParts) {+ if (!part.geometry?.positions || !part.geometry.indices || !part.material) throw new TypeError(`A crowd part needs a geometry and a material: ${part.name}`);+ if (part.skin && !part.geometry.skinIndices) throw new TypeError(`A skinned crowd part needs skin indices and weights: ${part.name}`);+ jointOffsets.push(jointCount);+ jointCount += part.skin ? part.skin.jointMatrices.length / FLOATS_PER_MATRIX : 1;+ }+ const partsByMaterial = new Map();+ sourceParts.forEach((part, index) => {+ if (!partsByMaterial.has(part.material)) partsByMaterial.set(part.material, []);+ partsByMaterial.get(part.material).push(index);+ });+ const scratch = mat4.create();+ const parts = [...partsByMaterial].map(([material, indices]) => ({+ name: sourceParts[indices[0]].name,+ material,+ geometry: mergeCrowdGeometry(indices.map(index => ({ part: sourceParts[index], jointOffset: jointOffsets[index] }))),+ }));+ return {+ jointCount,+ parts,+ /** Write the current joint matrices (mesh space, ready to skin with) of every joint into `output`. */+ readJointMatrices(output) {+ sourceParts.forEach((part, index) => {+ const offset = jointOffsets[index] * FLOATS_PER_MATRIX;+ if (!part.skin) {+ output.set(part.matrix, offset);+ return;+ }+ for (let joint = 0; joint < part.skin.jointMatrices.length / FLOATS_PER_MATRIX; joint++) {+ mat4.multiply(scratch, part.matrix, part.skin.jointMatrices.subarray(joint * FLOATS_PER_MATRIX, (joint + 1) * FLOATS_PER_MATRIX));+ output.set(scratch, offset + joint * FLOATS_PER_MATRIX);+ }+ });+ return output;+ },+ };+}++function mergeCrowdGeometry(entries) {+ const vertexCount = entries.reduce((total, { part }) => total + part.geometry.positions.length / 3, 0);+ const indexCount = entries.reduce((total, { part }) => total + part.geometry.indices.length, 0);+ const merged = {+ positions: new Float32Array(vertexCount * 3),+ normals: new Float32Array(vertexCount * 3),+ indices: new Uint32Array(indexCount),+ crowdJoints: new Float32Array(vertexCount * 4),+ crowdWeights: new Float32Array(vertexCount * 4),+ };+ let vertexStart = 0;+ let indexStart = 0;+ for (const { part, jointOffset } of entries) {+ const geometry = part.geometry;+ const count = geometry.positions.length / 3;+ merged.positions.set(geometry.positions, vertexStart * 3);+ merged.normals.set(geometry.normals, vertexStart * 3);+ for (let index = 0; index < geometry.indices.length; index++) merged.indices[indexStart + index] = geometry.indices[index] + vertexStart;+ for (let vertex = 0; vertex < count; vertex++) {+ const target = (vertexStart + vertex) * 4;+ if (!part.skin) {+ merged.crowdJoints[target] = jointOffset;+ merged.crowdWeights[target] = 1;+ continue;+ }+ let total = 0;+ for (let influence = 0; influence < 4; influence++) total += geometry.skinWeights[vertex * 4 + influence];+ for (let influence = 0; influence < 4; influence++) {+ merged.crowdJoints[target + influence] = geometry.skinIndices[vertex * 4 + influence] + jointOffset;+ merged.crowdWeights[target + influence] = geometry.skinWeights[vertex * 4 + influence] / total;+ }+ }+ vertexStart += count;+ indexStart += geometry.indices.length;+ }+ return merged;+}++/** The position of one crowd vertex under a table of joint matrices at `frameOffset` (in matrices), the maths the shader runs. */+function skinCrowdVertex(geometry, vertex, matrices, frameOffset, output = [0, 0, 0]) {+ const x = geometry.positions[vertex * 3];+ const y = geometry.positions[vertex * 3 + 1];+ const z = geometry.positions[vertex * 3 + 2];+ output[0] = 0;+ output[1] = 0;+ output[2] = 0;+ for (let influence = 0; influence < 4; influence++) {+ const weight = geometry.crowdWeights[vertex * 4 + influence];+ if (weight <= 0) continue;+ const base = (frameOffset + geometry.crowdJoints[vertex * 4 + influence]) * FLOATS_PER_MATRIX;+ output[0] += weight * (matrices[base] * x + matrices[base + 4] * y + matrices[base + 8] * z + matrices[base + 12]);+ output[1] += weight * (matrices[base + 1] * x + matrices[base + 5] * y + matrices[base + 9] * z + matrices[base + 13]);+ output[2] += weight * (matrices[base + 2] * x + matrices[base + 6] * y + matrices[base + 10] * z + matrices[base + 14]);+ }+ return output;+}++/**+ * Bake a clip into a palette: the joint matrices of `frameCount` evenly spaced poses, plus a bounding sphere (in the+ * space of the instance matrix) that covers every vertex in every pose. `poseAt(seconds)` must pose the parts the+ * rig was made from, such as `(seconds) => pose.apply(seconds)`. A looping clip is baked without its end frame, so+ * playback wraps between the last frame and the first.+ */+export function bakeCrowdPalette(rig, poseAt, { duration, loop = true, frameRate = DEFAULT_FRAME_RATE } = {}) {+ if (!Number.isFinite(duration) || duration <= 0) throw new RangeError('Crowd duration must be positive.');+ if (!Number.isFinite(frameRate) || frameRate <= 0) throw new RangeError('Crowd frame rate must be positive.');+ const frameCount = Math.max(2, Math.round(duration * frameRate) + (loop ? 0 : 1));+ const frameStep = loop ? duration / frameCount : duration / (frameCount - 1);+ const matricesPerFrame = rig.jointCount * FLOATS_PER_MATRIX;+ const data = new Float32Array(frameCount * matricesPerFrame);+ for (let frame = 0; frame < frameCount; frame++) {+ poseAt(frame * frameStep);+ rig.readJointMatrices(data.subarray(frame * matricesPerFrame, (frame + 1) * matricesPerFrame));+ }+ const palette = { data, jointCount: rig.jointCount, frameCount, frameStep, duration, loop };+ palette.bounds = measureCrowdBounds(rig, palette);+ return palette;+}++function measureCrowdBounds(rig, palette) {+ const minimum = [Infinity, Infinity, Infinity];+ const maximum = [-Infinity, -Infinity, -Infinity];+ const point = [0, 0, 0];+ const points = [];+ for (const { geometry } of rig.parts) {+ for (let frame = 0; frame < palette.frameCount; frame++) {+ for (let vertex = 0; vertex < geometry.positions.length / 3; vertex++) {+ skinCrowdVertex(geometry, vertex, palette.data, frame * palette.jointCount, point);+ points.push(point[0], point[1], point[2]);+ for (let axis = 0; axis < 3; axis++) {+ minimum[axis] = Math.min(minimum[axis], point[axis]);+ maximum[axis] = Math.max(maximum[axis], point[axis]);+ }+ }+ }+ }+ const center = minimum.map((value, axis) => (value + maximum[axis]) / 2);+ let radiusSquared = 0;+ for (let index = 0; index < points.length; index += 3) {+ radiusSquared = Math.max(radiusSquared, (points[index] - center[0]) ** 2 + (points[index + 1] - center[1]) ** 2 + (points[index + 2] - center[2]) ** 2);+ }+ return Object.freeze({ center: Object.freeze(center), radius: Math.sqrt(radiusSquared) });+}++/** The two frames and the blend between them for a clip time: [firstTexel, secondTexel, blend], as the shader reads them. */+export function crowdFrameBlend(palette, seconds, output = [0, 0, 0]) {+ const lastFrame = palette.frameCount - 1;+ let position;+ if (palette.loop) {+ const wrapped = seconds - Math.floor(seconds / palette.duration) * palette.duration;+ position = wrapped / palette.frameStep;+ } else {+ position = Math.min(Math.max(seconds, 0), palette.duration) / palette.frameStep;+ }+ let first = Math.floor(position);+ let blend = position - first;+ let second = first + 1;+ if (palette.loop) {+ first %= palette.frameCount;+ second %= palette.frameCount;+ } else if (first >= lastFrame) {+ first = lastFrame;+ second = lastFrame;+ blend = 0;+ }+ const texelsPerFrame = palette.jointCount * TEXELS_PER_MATRIX;+ output[0] = first * texelsPerFrame;+ output[1] = second * texelsPerFrame;+ output[2] = blend;+ return output;+}++/** The skinned position of one vertex of a rig geometry at a clip time, on the CPU. The shader computes the same value. */+export function crowdVertexAt(palette, geometry, vertex, seconds, output = [0, 0, 0]) {+ const blend = crowdFrameBlend(palette, seconds);+ const firstPoint = skinCrowdVertex(geometry, vertex, palette.data, blend[0] / TEXELS_PER_MATRIX);+ const secondPoint = skinCrowdVertex(geometry, vertex, palette.data, blend[1] / TEXELS_PER_MATRIX);+ for (let axis = 0; axis < 3; axis++) output[axis] = firstPoint[axis] * (1 - blend[2]) + secondPoint[axis] * blend[2];+ return output;+}++/**+ * Coarser levels of a crowd geometry for the renderer's `lods`: vertices that share a grid cell and a dominant joint+ * merge into one, which keeps that vertex's joints and weights, so every level skins with the same palette.+ * `cells` lists cell sizes as fractions of the geometry's largest extent; `error` is the largest distance a vertex moves.+ */+export function createCrowdLods(geometry, { cells = DEFAULT_LEVEL_CELLS } = {}) {+ const extent = largestExtent(geometry.positions);+ return [{ geometry, error: 0 }, ...cells.map((fraction) => {+ const cellSize = fraction * extent;+ return { geometry: clusterCrowdGeometry(geometry, cellSize), error: cellSize * Math.sqrt(3) / 2 };+ })];+}++function largestExtent(positions) {+ const minimum = [Infinity, Infinity, Infinity];+ const maximum = [-Infinity, -Infinity, -Infinity];+ for (let index = 0; index < positions.length; index += 3) {+ for (let axis = 0; axis < 3; axis++) {+ minimum[axis] = Math.min(minimum[axis], positions[index + axis]);+ maximum[axis] = Math.max(maximum[axis], positions[index + axis]);+ }+ }+ return Math.max(...maximum.map((value, axis) => value - minimum[axis]));+}++function clusterCrowdGeometry(geometry, cellSize) {+ const vertexCount = geometry.positions.length / 3;+ const clusterOfVertex = new Uint32Array(vertexCount);+ const clusterByKey = new Map();+ const representatives = [];+ for (let vertex = 0; vertex < vertexCount; vertex++) {+ let dominant = 0;+ for (let influence = 1; influence < 4; influence++) {+ if (geometry.crowdWeights[vertex * 4 + influence] > geometry.crowdWeights[vertex * 4 + dominant]) dominant = influence;+ }+ const key = [0, 1, 2].map(axis => Math.floor(geometry.positions[vertex * 3 + axis] / cellSize)).join(',') + `:${geometry.crowdJoints[vertex * 4 + dominant]}`;+ if (!clusterByKey.has(key)) {+ clusterByKey.set(key, representatives.length);+ representatives.push({ vertex, count: 0, position: [0, 0, 0], normal: [0, 0, 0] });+ }+ const clusterIndex = clusterByKey.get(key);+ const cluster = representatives[clusterIndex];+ clusterOfVertex[vertex] = clusterIndex;+ cluster.count++;+ for (let axis = 0; axis < 3; axis++) {+ cluster.position[axis] += geometry.positions[vertex * 3 + axis];+ cluster.normal[axis] += geometry.normals[vertex * 3 + axis];+ }+ }+ const coarse = {+ positions: new Float32Array(representatives.length * 3),+ normals: new Float32Array(representatives.length * 3),+ crowdJoints: new Float32Array(representatives.length * 4),+ crowdWeights: new Float32Array(representatives.length * 4),+ };+ representatives.forEach((cluster, index) => {+ const length = Math.hypot(...cluster.normal) || 1;+ for (let axis = 0; axis < 3; axis++) {+ coarse.positions[index * 3 + axis] = cluster.position[axis] / cluster.count;+ coarse.normals[index * 3 + axis] = cluster.normal[axis] / length;+ }+ coarse.crowdJoints.set(geometry.crowdJoints.subarray(cluster.vertex * 4, cluster.vertex * 4 + 4), index * 4);+ coarse.crowdWeights.set(geometry.crowdWeights.subarray(cluster.vertex * 4, cluster.vertex * 4 + 4), index * 4);+ });+ const indices = [];+ for (let corner = 0; corner < geometry.indices.length; corner += 3) {+ const first = clusterOfVertex[geometry.indices[corner]];+ const second = clusterOfVertex[geometry.indices[corner + 1]];+ const third = clusterOfVertex[geometry.indices[corner + 2]];+ if (first !== second && second !== third && first !== third) indices.push(first, second, third);+ }+ coarse.indices = Uint32Array.from(indices);+ return coarse;+}++/**+ * The renderer parts of a crowd: one part per instance and material, all instances of a material sharing a geometry,+ * a palette and `lods`, so they draw as one batch per material and level. `instances` come from createCrowd (or any+ * list with a column-major `matrix`). Call `setTimes` with each instance's clip time, then `renderer.updateParts(parts)`.+ * `lods: false` turns the coarser levels off; `cells` sets their sizes.+ */+export function createCrowdParts({ rig, palette, instances, lods = true, cells, name = 'crowd' }) {+ if (!instances?.length) throw new TypeError('A crowd needs a list of instances.');+ const rigParts = rig.parts.map(part => ({ ...part, levels: lods ? createCrowdLods(part.geometry, { cells }) : null }));+ const blends = [];+ const parts = [];+ instances.forEach((instance, index) => {+ const blend = [0, 0, 0];+ blends.push(blend);+ for (const rigPart of rigParts) {+ parts.push({+ name: `${name}-${index}-${rigPart.name}`,+ geometry: rigPart.geometry,+ matrix: instance.matrix,+ material: rigPart.material,+ crowd: palette,+ assembly: { offset: blend, detailOffset: [0, 0, 0] },+ ...(rigPart.levels ? { lods: rigPart.levels } : {}),+ });+ }+ });+ return {+ parts,+ /** Set every instance's clip time: an array of seconds or `(index) => seconds`. */+ setTimes(times) {+ blends.forEach((blend, index) => crowdFrameBlend(palette, typeof times === 'function' ? times(index) : times[index], blend));+ },+ };+}diff --git a/src/renderer.js b/src/renderer.jsindex 8ec77ca2..72eb8590 100644--- a/src/renderer.js+++ b/src/renderer.js@@ -1,5 +1,5 @@import { mat4, mat3 } from 'gl-matrix';-import { layerTextureSupport, LIGHT_SHADING_FEATURE, SHADING_MODES, LIGHT_SHADING_MAXIMUM_SAMPLES, vertexShader, createVertexShader, createFragmentShader, liquidFragmentShader, depthVertexShader, depthFragmentShader, benchmarkVertexShader, benchmarkFragmentShader } from './shaders.js';+import { layerTextureSupport, LIGHT_SHADING_FEATURE, SHADING_MODES, LIGHT_SHADING_MAXIMUM_SAMPLES, vertexShader, createVertexShader, createFragmentShader, liquidFragmentShader, createDepthVertexShader, depthFragmentShader, benchmarkVertexShader, benchmarkFragmentShader } from './shaders.js';import { postFragment, createPostFragment, POST_STEPS, skyFragment, bloomFragment } from './post.js';import { fullscreenVertex, startProgram, finishProgram, compileProgram } from './gl-program.js';import { ambientOcclusionFragment } from './ambient-occlusion.js';@@ -68,8 +68,9 @@ function shaderFeaturesOfLighting(lighting) {function shaderFeaturesOfParts(parts) {const features = new Set();- for (const { material, skin, morph } of parts) {+ for (const { material, skin, morph, crowd } of parts) {if (skin) features.add('skin');+ if (crowd) features.add('crowd');if (morph) features.add('morph');MATERIAL_MAPS.forEach((field, index) => {if (material[field]) features.add(MAP_SHADER_FEATURES[index]);@@ -113,7 +114,7 @@ export function resolveLiquidOptics(options = {}) {}return optics;}-const SKIN_UNIFORMS = ['skinEnabled', 'skinInfluences', 'jointPalette'];+const SKIN_UNIFORMS = ['skinEnabled', 'skinInfluences', 'jointPalette', 'crowdEnabled'];const ALPHA_UNIFORMS = ['alphaMode', 'alphaCutoff', 'materialOpacity', 'alphaToCoverage', 'doubleSided'];const DEPTH_ALPHA_UNIFORMS = ['alphaMasked', 'alphaCutoff', 'materialOpacity', 'hasBaseMap', 'baseMap'];const MORPH_UNIFORMS = ['morphEnabled', 'morphVertexCount', 'morphTargetCount', 'morphWeights[0]', 'morphTargets'];@@ -276,7 +277,7 @@ function isCullable(parts) {if (first.skin || first.morph) return false;if (DEFORMING_TAGS.has(first.material.tag) && !SWAYING_TAGS.has(first.material.tag)) return false;if (renderPass(first.material) !== 'opaque' || materialAlphaState(first.material)[0] === 'blend') return false;- return parts.every(part => !part.assembly);+ return parts.every(part => !part.assembly || part.crowd);}function validateLods(part, levels) {@@ -331,7 +332,7 @@ export function groupParts(parts, batching = true) {const key = batching ? part.geometry : part;if (!groups.has(key)) groups.set(key, new Map());let maps = groups.get(key);- for (const value of [...materialMapState(part.material), ...materialLayerState(part.material), part.skin ?? null, part.morph ?? null, castsShadow(part), renderPass(part.material), part.lods ?? null, part.shadowLods ?? null, ...materialAlphaState(part.material)]) {+ for (const value of [...materialMapState(part.material), ...materialLayerState(part.material), part.skin ?? null, part.crowd ?? null, part.morph ?? null, castsShadow(part), renderPass(part.material), part.lods ?? null, part.shadowLods ?? null, ...materialAlphaState(part.material)]) {if (!maps.has(value)) maps.set(value, new Map());maps = maps.get(value);}@@ -508,7 +509,7 @@ export class Renderer67 {this.assemblyAmount = 0;const groups = groupParts(this.groundPart ? [...parts, this.groundPart] : parts, batching);const textureLimit = this.context.getParameter(this.context.MAX_TEXTURE_SIZE);- this.hasSkins = groups.some(group => group[0].skin);+ this.hasSkins = groups.some(group => group[0].skin || group[0].crowd);this.hasMorphs = groups.some(group => group[0].morph);if (this.hasSkins && (this.context.getParameter(this.context.MAX_VERTEX_TEXTURE_IMAGE_UNITS) < 2 || this.context.getParameter(this.context.MAX_COMBINED_TEXTURE_IMAGE_UNITS) < 8)) throw new Error('GPU skinning requires two vertex texture units and eight combined texture units.');if (this.hasMorphs && (this.context.getParameter(this.context.MAX_VERTEX_TEXTURE_IMAGE_UNITS) < (this.hasSkins ? 3 : 1) || this.context.getParameter(this.context.MAX_COMBINED_TEXTURE_IMAGE_UNITS) < 9)) throw new Error('GPU morphing requires one additional vertex texture unit and nine combined texture units.');@@ -524,6 +525,10 @@ export class Renderer67 {const skinGeometries = new Set();for (const group of groups) {const part = group[0];+ if (part.crowd && !poses.has(part.crowd)) {+ this.textureBytes += skinTextureLayout(part.crowd.data.length / 4, textureLimit, 4).bytes;+ poses.add(part.crowd);+ }if (!part.skin) continue;if (!poses.has(part.skin)) this.textureBytes += skinTextureLayout(part.skin.jointMatrices.length / 4, textureLimit, 4).bytes;if (!skinGeometries.has(part.geometry)) this.textureBytes += skinTextureLayout(part.geometry.positions.length / 3 * 2, textureLimit, 2).bytes;@@ -580,7 +585,7 @@ export class Renderer67 {if (shadowCascades) this.shaderFeatures.add(shadowCascades.shaderFeature);const pending = {main: this.startMainProgram(),- depth: startProgram(this.context, depthVertexShader, depthFragmentShader),+ depth: startProgram(this.context, createDepthVertexShader(this.shaderFeatures), depthFragmentShader),};if (!benchmark) {pending.post = startProgram(this.context, fullscreenVertex, postFragment);@@ -862,7 +867,7 @@ export class Renderer67 {let geometryBuffers = this.geometryBuffers.get(geometry);if (!geometryBuffers) {geometryBuffers = new Map();- for (const field of ['positions', 'normals', 'uvs', 'colors', 'indices']) {+ for (const field of ['positions', 'normals', 'uvs', 'colors', 'indices', 'crowdJoints', 'crowdWeights']) {if (geometry[field]) {const target = field === 'indices' ? context.ELEMENT_ARRAY_BUFFER : context.ARRAY_BUFFER;const values = field === 'uvs' && geometry.lightUvs ? interleaveUvSets(geometry) : geometry[field];@@ -900,6 +905,14 @@ export class Renderer67 {context.disableVertexAttribArray(14);context.vertexAttrib3f(14, 1, 1, 1);}+ // A crowd geometry keeps four joint ids and four weights per vertex where textures and tints would sit.+ if (geometry.crowdJoints) {+ for (const [location, field] of [[13, 'crowdJoints'], [14, 'crowdWeights']]) {+ context.bindBuffer(context.ARRAY_BUFFER, geometryBuffers.get(field));+ context.enableVertexAttribArray(location);+ context.vertexAttribPointer(location, 4, context.FLOAT, false, 0, 0);+ }+ }context.bindBuffer(context.ELEMENT_ARRAY_BUFFER, geometryBuffers.get('indices'));}@@ -927,7 +940,7 @@ export class Renderer67 {if (!cullable) throw new RangeError(`shadowLods need rigid or swaying, opaque parts without assembly motion: ${first.name}`);}const levels = first.lods ?? [{ geometry: first.geometry, error: 0 }];- const sphere = geometryBoundingSphere(first.geometry);+ const sphere = first.crowd?.bounds ?? geometryBoundingSphere(first.geometry);const bounds = { center: sphere.center, radius: sphere.radius + Math.max(...levels.map(level => level.error)) };const shadowLevels = first.shadowLods ? createLevelSet(first.shadowLods, parts.length) : null;const swayReach = SWAYING_TAGS.has(first.material.tag) ? SWAY_REACH_PER_STRENGTH : 0;@@ -1017,13 +1030,15 @@ export class Renderer67 {const skin = parts[0].skin ?? null;const skinTexture = skin ? this.createSkinTexture(this.skinTextures, skin, skin.jointMatrices, 4) : null;const influenceTexture = skin ? this.createSkinTexture(this.influenceTextures, geometry, null, 2) : null;+ const crowd = parts[0].crowd ?? null;+ const crowdTexture = crowd ? this.createSkinTexture(this.skinTextures, crowd, crowd.data, 4) : null;const morph = parts[0].morph ?? null;const morphTexture = morph ? this.createMorphTexture(geometry, morph) : null;const morphWeights = morph ? new Float32Array(morph.weights) : null;const [alphaMode, alphaCutoff, opacity, doubleSided] = materialAlphaState(parts[0].material);const layerState = materialLayerState(parts[0].material);const culling = this.createCullingState(parts);- const batch = { ...culling, alphaMode, alphaCutoff, opacity, doubleSided, layerState, parts: [...parts], vertexArray, instanceBuffer, instanceData, textures, mapState, splat, lightMapTexture, texture: textures[0], map, skin, skinTexture, influenceTexture, morph, morphTexture, morphWeights, indexCount: geometry.indices.length, instanceCount: parts.length, floor: parts[0].material.tag === 'floor', castShadow: castsShadow(parts[0]), pass: renderPass(parts[0].material) };+ const batch = { ...culling, alphaMode, alphaCutoff, opacity, doubleSided, layerState, parts: [...parts], vertexArray, instanceBuffer, instanceData, textures, mapState, splat, lightMapTexture, texture: textures[0], map, skin, skinTexture, influenceTexture, crowd, crowdTexture, morph, morphTexture, morphWeights, indexCount: geometry.indices.length, instanceCount: parts.length, floor: parts[0].material.tag === 'floor', castShadow: castsShadow(parts[0]), pass: renderPass(parts[0].material) };if (parts[0].lods) culling.levelGeometries.forEach((geometryOfLevel, level) => this.levelDraw(batch, level));culling.shadowLevels?.levelGeometries.forEach((geometryOfLevel, level) => this.levelDraw(batch, level, batch.shadowLevels));parts.forEach((part, index) => this.partLocations.set(part, { batch, index, geometry: part.geometry, map, mapState, layerState, skin, jointMatrices: skin?.jointMatrices, skinIndices: geometry.skinIndices, skinWeights: geometry.skinWeights, morph, morphWeights: morph?.weights, morphTargets: geometry.morphTargets, normalMatrix: mat3.create(), committedMatrix: batch.instanceData.subarray(index * INSTANCE_STRIDE, index * INSTANCE_STRIDE + 16) }));@@ -2002,8 +2017,13 @@ export class Renderer67 {}if (this.hasSkins) {const uniforms = textured ? this.uniforms : this.depthSkinUniforms;+ this.context.uniform1f(uniforms.crowdEnabled, batch.crowd ? 1 : 0);this.context.uniform1f(uniforms.skinEnabled, batch.skin ? 1 : 0);- if (batch.skin) {+ if (batch.crowd) {+ this.context.activeTexture(this.context.TEXTURE0 + 7);+ this.context.bindTexture(this.context.TEXTURE_2D, batch.crowdTexture.texture);+ this.context.uniform1i(uniforms.jointPalette, 7);+ } else if (batch.skin) {this.context.activeTexture(this.context.TEXTURE0 + 6);this.context.bindTexture(this.context.TEXTURE_2D, batch.influenceTexture.texture);this.context.uniform1i(uniforms.skinInfluences, 6);diff --git a/src/shaders.js b/src/shaders.jsindex 5ef1c55d..2cb7e916 100644--- a/src/shaders.js+++ b/src/shaders.js@@ -35,7 +35,16 @@ vec3 morphedNormal(vec3 base) {}`;-const skinningShader = `+// The GPU crowd path (src/crowd-skinning.js) replaces this stub with its own code when that addon is imported, so the core pays for two lines.+const CROWD_SHADER_STUB = 'uniform float crowdEnabled;\nmat4 crowdTransform() { return mat4(1.0); }';+let crowdShaderCode = CROWD_SHADER_STUB;++/** Installs the vertex code that skins a crowd batch from a baked joint-matrix texture. Called by the crowd addon on import. */+export function installCrowdShader(code) {+ crowdShaderCode = code;+}++const skinningShader = () => `uniform float skinEnabled;uniform highp sampler2D skinInfluences;uniform highp sampler2D jointPalette;@@ -48,6 +57,7 @@ mat4 jointMatrix(int joint) {return mat4(skinTexel(jointPalette, offset), skinTexel(jointPalette, offset + 1),skinTexel(jointPalette, offset + 2), skinTexel(jointPalette, offset + 3));}+${crowdShaderCode}mat4 skinTransform() {if (skinEnabled < 0.5) return mat4(1.0);ivec4 joints = ivec4(skinTexel(skinInfluences, gl_VertexID * 2));@@ -126,14 +136,15 @@ flat out float instanceOrdinal;out vec3 boxPosition;out vec3 boxNormal;flat out mat3 boxNormalMatrix;-${skinningShader}+${skinningShader()}${morphingShader}${windShader}void main() {- vec2 uv = uvSets.xy;- mat4 deformation = USES_SKIN ? skinTransform() : mat4(1.0);+ // A crowd batch reads its joints and weights from the UV and colour attributes, which then carry no texture or tint.+ vec2 uv = USES_CROWD ? vec2(-1.0) : uvSets.xy;+ mat4 deformation = USES_CROWD ? crowdTransform() : (USES_SKIN ? skinTransform() : mat4(1.0));vec4 world = instanceMatrix * deformation * vec4(USES_MORPH ? morphedPosition(position) : position, 1.0);- world.xyz += assemblyOffset * assemblyAmount + assemblyDetailOffset * smoothstep(0.3, 1.0, assemblyAmount);+ world.xyz += USES_CROWD ? vec3(0.0) : assemblyOffset * assemblyAmount + assemblyDetailOffset * smoothstep(0.3, 1.0, assemblyAmount);if (USES_WIND) world.xyz += windOffset(position, world.xyz, uv, instanceSurface.w, time, wind);if (USES_WAVES && instanceSurface.w == 3.0) {world.y += sin(world.x * 1.4 + time * 0.8) * 0.025 + sin(world.z * 1.8 - time) * 0.02;@@ -143,7 +154,7 @@ void main() {}worldPosition = world.xyz;vec3 localNormal = USES_MORPH ? morphedNormal(normal) : normal;- vec3 deformedNormal = USES_SKIN ? transpose(inverse(mat3(deformation))) * localNormal : localNormal;+ vec3 deformedNormal = (USES_SKIN || USES_CROWD) ? transpose(inverse(mat3(deformation))) * localNormal : localNormal;worldNormal = normalize(instanceNormal * deformedNormal);// Metres along the object's own axes, so a pattern follows a moving node and is not stretched by its scale.boxPosition = (USES_MORPH ? morphedPosition(position) : position) * vec3(length(instanceMatrix[0].xyz), length(instanceMatrix[1].xyz), length(instanceMatrix[2].xyz));@@ -152,8 +163,8 @@ void main() {baseColor = instanceColor;textureUV = uv;lightmapUV = uvSets.zw;- vertexTint = vertexColor.rgb;- vertexSkyVisibility = vertexColor.a;+ vertexTint = USES_CROWD ? vec3(1.0) : vertexColor.rgb;+ vertexSkyVisibility = USES_CROWD ? 1.0 : vertexColor.a;emissionColor = instanceEmissive;instanceOrdinal = float(gl_InstanceID);surface = instanceSurface;@@ -162,30 +173,38 @@ void main() {}`;}-export const depthVertexShader = `#version 300 es+// Built when a renderer starts, so the crowd addon can install its skinning code first.+export function createDepthVertexShader(features = []) {+ return `#version 300 es+${vertexDefinitions(new Set(features))}precision highp float;layout(location=0) in vec3 position;layout(location=2) in mat4 instanceMatrix;layout(location=10) in vec4 instanceSurface;layout(location=11) in vec3 assemblyOffset;layout(location=12) in vec3 assemblyDetailOffset;-layout(location=13) in vec2 uv;+layout(location=13) in vec4 uvSets;+layout(location=14) in vec4 vertexColor;uniform mat4 lightProjection;uniform float assemblyAmount;uniform float time;uniform float wind;out vec2 textureUV;-${skinningShader}+${skinningShader()}${morphingShader}${windShader}void main() {- vec4 world = instanceMatrix * skinTransform() * vec4(morphedPosition(position), 1.0);- world.xyz += assemblyOffset * assemblyAmount + assemblyDetailOffset * smoothstep(0.3, 1.0, assemblyAmount);+ vec2 uv = USES_CROWD ? vec2(-1.0) : uvSets.xy;+ vec4 world = instanceMatrix * (USES_CROWD ? crowdTransform() : skinTransform()) * vec4(morphedPosition(position), 1.0);+ world.xyz += USES_CROWD ? vec3(0.0) : assemblyOffset * assemblyAmount + assemblyDetailOffset * smoothstep(0.3, 1.0, assemblyAmount);// Same displacement as the colour pass, so shadows follow the swaying leaves.world.xyz += windOffset(position, world.xyz, uv, instanceSurface.w, time, wind);textureUV = uv;gl_Position = lightProjection * world;}`;+}++export const depthVertexShader = createDepthVertexShader();// Only masked batches sample the base map, so shadows follow the cutout.export const depthFragmentShader = `#version 300 es@@ -211,7 +230,7 @@ layout(location=2) in mat4 instanceMatrix;layout(location=9) in vec3 instanceColor;uniform mat4 viewProjection;out vec3 color;-${skinningShader}+${skinningShader()}${morphingShader}void main() {color = instanceColor;@@ -998,7 +1017,7 @@ export const LIGHT_SHADING_MAXIMUM_SAMPLES = 2;export const ENVIRONMENT_FEATURES =Object.freeze(['environmentMap', 'studioLighting']);// Features that only the vertex shader reads: a part with a skeleton or morph targets.-const VERTEX_ONLY_FEATURES = Object.freeze(['skin', 'morph']);+const VERTEX_ONLY_FEATURES = Object.freeze(['skin', 'morph', 'crowd']);function environmentCondition(enabled) {const hasMap = enabled.has('environmentMap');@@ -1045,6 +1064,7 @@ function vertexDefinitions(enabled) {const condition = (active, expression) => (active ? `(${expression})` : 'false');return [`#define USES_SKIN ${condition(enabled.has('skin'), 'skinEnabled > 0.5')}`,+ `#define USES_CROWD ${condition(enabled.has('crowd'), 'crowdEnabled > 0.5')}`,`#define USES_MORPH ${condition(enabled.has('morph'), 'morphEnabled > 0.5')}`,`#define USES_WIND ${condition(enabled.has('foliage') || enabled.has('grass'), 'true')}`,`#define USES_WAVES ${condition(enabled.has('water'), 'true')}`,@@ -1057,7 +1077,7 @@ export function createVertexShader(features = SHADER_FEATURES) {return vertexShaderSource(vertexDefinitions(new Set(features)));}-export const vertexShader = createVertexShader();+export const vertexShader = createVertexShader(SHADER_FEATURES.filter(feature => feature !== 'crowd'));/** The fragment shader for a set of features. A feature with separate code throws until its module is imported. */export function createFragmentShader(features = SHADER_FEATURES) {diff --git a/tests/crowd-skinning.test.js b/tests/crowd-skinning.test.jsnew file mode 100644index 00000000..8fab76b3--- /dev/null+++ b/tests/crowd-skinning.test.js@@ -0,0 +1,156 @@+import assert from 'node:assert/strict';+import { readFileSync } from 'node:fs';+import test from 'node:test';+import { mat4 } from 'gl-matrix';+import { parseGltf } from '../src/gltf.js';+import { clipFromGltfAnimation, createAssetPose } from '../src/skeleton-pose.js';+import { createCrowd } from '../src/retarget.js';+import { groupParts, Renderer67 } from '../src/renderer.js';+import { InstanceSelection, extractFrustumPlanes, selectInstances } from '../src/render-culling.js';+import {+ bakeCrowdPalette, createCrowdLods, createCrowdParts, createCrowdRig, crowdFrameBlend, crowdVertexAt,+} from '../src/crowd-skinning.js';++// A real character: the CC0 RobotExpressive, merged into one crowd rig (rigid parts follow one joint each, the hands keep their skins).+const bytes = readFileSync(new URL('./fixtures/RobotExpressive.glb', import.meta.url));++async function loadWalker() {+ const asset = await parseGltf(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), { maxVertices: 500000 });+ const source = asset.animations.find(animation => animation.name === 'Walking');+ const pose = createAssetPose(asset, { layers: [{ clip: clipFromGltfAnimation(source, { loop: true }) }] });+ const rig = createCrowdRig(asset.parts);+ return { asset, pose, rig, duration: source.duration };+}++test('a crowd rig keeps one geometry per material and a joint for every rigid part and skin joint', async () => {+ const { asset, rig } = await loadWalker();+ const materials = new Set(asset.parts.map(part => part.material));+ assert.equal(rig.parts.length, materials.size);+ const skinJoints = asset.parts.filter(part => part.skin).reduce((total, part) => total + part.skin.jointMatrices.length / 16, 0);+ assert.equal(rig.jointCount, asset.parts.filter(part => !part.skin).length + skinJoints);+ const vertices = asset.parts.reduce((total, part) => total + part.geometry.positions.length / 3, 0);+ assert.equal(rig.parts.reduce((total, part) => total + part.geometry.positions.length / 3, 0), vertices);+});++test('a baked palette is the same whatever order its frames are posed in, and frames blend by clip time alone', async () => {+ const { pose, rig, duration } = await loadWalker();+ const forward = bakeCrowdPalette(rig, seconds => pose.apply(seconds), { duration });+ const times = [];+ for (let frame = forward.frameCount - 1; frame >= 0; frame--) times.push(frame * forward.frameStep);+ const visited = [];+ const backward = bakeCrowdPalette(rig, (seconds) => {+ pose.apply(times[visited.length]);+ visited.push(seconds);+ }, { duration });+ // Posing frame i at the time of frame count - 1 - i gives the reverse table, so reversing it back must equal the forward one.+ const matricesPerFrame = rig.jointCount * 16;+ for (let frame = 0; frame < forward.frameCount; frame++) {+ const mirrored = forward.frameCount - 1 - frame;+ assert.deepEqual(+ backward.data.subarray(frame * matricesPerFrame, (frame + 1) * matricesPerFrame),+ forward.data.subarray(mirrored * matricesPerFrame, (mirrored + 1) * matricesPerFrame),+ );+ }+ const early = crowdFrameBlend(forward, 0.4);+ const later = crowdFrameBlend(forward, 0.4 + duration * 3);+ assert.deepEqual(early.slice(0, 2), later.slice(0, 2), 'a looping clip repeats');+ assert.ok(Math.abs(early[2] - later[2]) < 1e-6);+ const wrapped = crowdFrameBlend(forward, duration - forward.frameStep / 2);+ assert.equal(wrapped[1], 0, 'a looping clip blends its last frame into its first');+ assert.ok(Math.abs(wrapped[2] - 0.5) < 1e-3);+});++test('a crowd vertex at any time matches per-instance skinning and stays inside the bound', async () => {+ const { pose, rig, duration } = await loadWalker();+ const palette = bakeCrowdPalette(rig, seconds => pose.apply(seconds), { duration });+ const radius = palette.bounds.radius;+ const joints = new Float32Array(rig.jointCount * 16);+ let largestError = 0;+ let largestErrorAtFrame = 0;+ for (const seconds of [0, 0.123, 0.4, 0.7777, duration * 0.999, duration * 2.25]) {+ pose.apply(seconds % duration);+ rig.readJointMatrices(joints);+ for (const { geometry } of rig.parts) {+ for (let vertex = 0; vertex < geometry.positions.length / 3; vertex += 7) {+ const expected = [0, 0, 0];+ const x = geometry.positions[vertex * 3];+ const y = geometry.positions[vertex * 3 + 1];+ const z = geometry.positions[vertex * 3 + 2];+ for (let influence = 0; influence < 4; influence++) {+ const weight = geometry.crowdWeights[vertex * 4 + influence];+ const base = geometry.crowdJoints[vertex * 4 + influence] * 16;+ for (let axis = 0; axis < 3; axis++) expected[axis] += weight * (joints[base + axis] * x + joints[base + 4 + axis] * y + joints[base + 8 + axis] * z + joints[base + 12 + axis]);+ }+ const actual = crowdVertexAt(palette, geometry, vertex, seconds);+ const error = Math.hypot(actual[0] - expected[0], actual[1] - expected[1], actual[2] - expected[2]);+ const onFrame = Math.abs(seconds % palette.frameStep) < 1e-9 ? 1 : 0;+ if (error > largestError) {+ largestError = error;+ largestErrorAtFrame = onFrame;+ }+ const distance = Math.hypot(actual[0] - palette.bounds.center[0], actual[1] - palette.bounds.center[1], actual[2] - palette.bounds.center[2]);+ assert.ok(distance <= radius * (1 + 1e-4), `vertex ${vertex} left the bound at ${seconds}`);+ }+ }+ }+ assert.ok(largestError < radius * 0.01, `largest error ${largestError} against a bound of ${radius} (on a frame: ${largestErrorAtFrame})`);+ const exact = crowdVertexAt(palette, rig.parts[0].geometry, 3, 5 * palette.frameStep);+ pose.apply(5 * palette.frameStep);+ rig.readJointMatrices(joints);+ assert.ok(Number.isFinite(exact[0]));+});++test('instances of one character and clip share one batch per material, whatever their number or time', async () => {+ const { pose, rig, duration } = await loadWalker();+ const palette = bakeCrowdPalette(rig, seconds => pose.apply(seconds), { duration });+ const crowd = createCrowd({ count: 200, clip: { duration, tracks: [] }, seed: 3 });+ const set = createCrowdParts({ rig, palette, instances: crowd.instances });+ assert.equal(set.parts.length, 200 * rig.parts.length);+ assert.equal(groupParts(set.parts).length, rig.parts.length);+ const few = createCrowdParts({ rig, palette, instances: crowd.instances.slice(0, 3) });+ assert.equal(groupParts(few.parts).length, rig.parts.length);+ set.setTimes(index => crowd.clipTime(index, 1.25));+ const first = set.parts[0].assembly.offset.slice();+ set.setTimes(index => crowd.clipTime(index, 1.25));+ assert.deepEqual(set.parts[0].assembly.offset, first, 'the same time gives the same frames');+ const other = createCrowdParts({ rig, palette: { ...palette }, instances: crowd.instances.slice(0, 2) });+ assert.equal(groupParts([...set.parts.slice(0, rig.parts.length), ...other.parts.slice(0, rig.parts.length)]).length, rig.parts.length * 2, 'another palette is another batch');+});++test('lods keep the joints, shrink the triangle count and report a growing error', async () => {+ const { rig } = await loadWalker();+ const widest = rig.parts.reduce((best, part) => (part.geometry.indices.length > best.geometry.indices.length ? part : best));+ const levels = createCrowdLods(widest.geometry);+ assert.equal(levels.length, 3);+ assert.equal(levels[0].geometry, widest.geometry);+ assert.ok(levels[1].geometry.indices.length < levels[0].geometry.indices.length);+ assert.ok(levels[2].geometry.indices.length <= levels[1].geometry.indices.length);+ assert.ok(levels[2].error > levels[1].error && levels[1].error > 0);+ const allowed = new Set(widest.geometry.crowdJoints);+ for (const { geometry } of levels) {+ assert.ok(geometry.indices.every(index => index < geometry.positions.length / 3));+ assert.ok(geometry.crowdJoints.every(joint => allowed.has(joint)));+ }+});++test('the renderer culls crowd instances with the bound of the whole clip, and refuses nothing for levels', async () => {+ const { pose, rig, duration } = await loadWalker();+ const palette = bakeCrowdPalette(rig, seconds => pose.apply(seconds), { duration });+ const matrices = [[0, 0, 0], [400, 0, 0], [0, 0, 40], [0, 0, -300]].map(position => mat4.fromTranslation(new Float32Array(16), position));+ const set = createCrowdParts({ rig, palette, instances: matrices.map(matrix => ({ matrix })) });+ const batch = groupParts(set.parts).find(group => group[0].crowd);+ const renderer = Object.create(Renderer67.prototype);+ const culling = renderer.createCullingState(batch);+ assert.equal(culling.cullable, true);+ assert.ok(culling.bounds.radius >= palette.bounds.radius);+ assert.equal(culling.levelGeometries.length, 3);+ const projection = mat4.perspective(mat4.create(), Math.PI / 4, 16 / 9, 0.5, 200);+ const viewProjection = mat4.multiply(mat4.create(), projection, mat4.lookAt(mat4.create(), [0, 2, 12], [0, 1, 0], [0, 1, 0]));+ const instanceData = new Float32Array(batch.length * 41);+ batch.forEach((part, index) => instanceData.set(part.matrix, index * 41));+ const selection = selectInstances(new InstanceSelection(batch.length, 3), {+ instanceData, instanceCount: batch.length, stride: 41, bounds: culling.bounds, levelErrors: culling.levelErrors,+ planes: extractFrustumPlanes(viewProjection, new Float32Array(24)), viewProjection, pixelsPerUnit: 540 / 2 * projection[5], pixelError: 1,+ });+ assert.equal(selection.visibleCount, 1, 'only the instance in front of the camera draws');+});diff --git a/vite.lib.config.js b/vite.lib.config.jsindex d4ffbb63..15fa7fa6 100644--- a/vite.lib.config.js+++ b/vite.lib.config.js@@ -31,6 +31,7 @@ export default defineConfig({physics: 'src/physics.js',spume: 'src/spume.js',retarget: 'src/retarget.js',+ 'crowd-skinning': 'src/crowd-skinning.js','skeleton-pose': 'src/skeleton-pose.js',vehicle: 'src/vehicle.js',vegetation: 'src/vegetation.js',