3D engine that AI agents drive
Let agents set shadowOnly and keep it out of Cook camera rays (#958)
Nguyen Xuan An40a377630f95
diff --git a/bench/agent-modeling/results/bench-958-haiku.json b/bench/agent-modeling/results/bench-958-haiku.jsonnew file mode 100644index 00000000..ac22ad18--- /dev/null+++ b/bench/agent-modeling/results/bench-958-haiku.json@@ -0,0 +1,223 @@+{+ "run": "bench-958-haiku",+ "model": "claude-haiku-5-5",+ "date": "2026-11-12T08:02:24.256Z",+ "skill": false,+ "branches": "on",+ "relations": "on",+ "runsPerTask": 3,+ "passed": 3,+ "tasks": 3,+ "toolCalls": 24,+ "costUsd": 0.04,+ "byModel": {+ "claude-haiku-5-5": {+ "runs": 3,+ "passed": 3,+ "successRate": 1,+ "toolCalls": 24,+ "toolErrors": 0,+ "inputTokens": 1174553,+ "outputTokens": 12176,+ "costUsd": 0.04,+ "tasks": {+ "crate": {+ "runs": 1,+ "passed": 1,+ "successRate": 1,+ "toolCalls": 11,+ "toolErrors": 0,+ "inputTokens": 538306,+ "outputTokens": 5954,+ "costUsd": 0.02+ },+ "bolt": {+ "runs": 1,+ "passed": 1,+ "successRate": 1,+ "toolCalls": 7,+ "toolErrors": 0,+ "inputTokens": 345622,+ "outputTokens": 3463,+ "costUsd": 0.01+ },+ "table": {+ "runs": 1,+ "passed": 1,+ "successRate": 1,+ "toolCalls": 6,+ "toolErrors": 0,+ "inputTokens": 290625,+ "outputTokens": 2759,+ "costUsd": 0.01+ }+ }+ }+ },+ "records": [+ {+ "task": "crate",+ "attempt": 1,+ "pass": true,+ "passed": 8,+ "total": 8,+ "failed": [],+ "model": "claude-haiku-5-5",+ "toolCalls": 11,+ "toolErrors": 0,+ "toolsByName": {+ "ToolSearch": 1,+ "scene_capabilities": 6,+ "scene_create": 1,+ "mesh_edit": 1,+ "scene_look": 1,+ "scene_export_recipe": 1,+ "scene_dispose": 1+ },+ "branchCalls": 0,+ "relationCalls": 0,+ "pathQueries": 0,+ "features": {+ "geometryTypes": [+ "mesh"+ ],+ "meshOperations": [+ "set_position",+ "inset_individual",+ "extrude_individual"+ ],+ "motionIk": [],+ "motionTracks": 0+ },+ "look": {+ "silhouette": {+ "views": {+ "front": 1,+ "side": 1,+ "three-quarter": 1+ },+ "mean": 1+ },+ "images": [+ "artifacts/agent-modeling/bench-958-haiku/look/1/crate-front.png",+ "artifacts/agent-modeling/bench-958-haiku/look/1/crate-side.png",+ "artifacts/agent-modeling/bench-958-haiku/look/1/crate-three-quarter.png"+ ]+ },+ "turns": 13,+ "costUsd": 0.01654611,+ "inputTokens": 538306,+ "outputTokens": 5954,+ "wallSeconds": 64,+ "exitCode": 0,+ "stopped": "success"+ },+ {+ "task": "bolt",+ "attempt": 1,+ "pass": true,+ "passed": 7,+ "total": 7,+ "failed": [],+ "model": "claude-haiku-5-5",+ "toolCalls": 7,+ "toolErrors": 0,+ "toolsByName": {+ "ToolSearch": 1,+ "scene_capabilities": 1,+ "scene_create": 1,+ "mesh_inspect": 1,+ "scene_apply": 1,+ "scene_look": 1,+ "scene_export_recipe": 1,+ "scene_dispose": 1+ },+ "branchCalls": 0,+ "relationCalls": 0,+ "pathQueries": 0,+ "features": {+ "geometryTypes": [+ "cylinder"+ ],+ "meshOperations": [],+ "motionIk": [],+ "motionTracks": 0+ },+ "look": {+ "silhouette": {+ "views": {+ "front": 1,+ "side": 1,+ "three-quarter": 0.9997+ },+ "mean": 0.9999+ },+ "images": [+ "artifacts/agent-modeling/bench-958-haiku/look/1/bolt-front.png",+ "artifacts/agent-modeling/bench-958-haiku/look/1/bolt-side.png",+ "artifacts/agent-modeling/bench-958-haiku/look/1/bolt-three-quarter.png"+ ]+ },+ "turns": 9,+ "costUsd": 0.01215609,+ "inputTokens": 345622,+ "outputTokens": 3463,+ "wallSeconds": 54,+ "exitCode": 0,+ "stopped": "success"+ },+ {+ "task": "table",+ "attempt": 1,+ "pass": true,+ "passed": 6,+ "total": 6,+ "failed": [],+ "model": "claude-haiku-5-5",+ "toolCalls": 6,+ "toolErrors": 0,+ "toolsByName": {+ "ToolSearch": 1,+ "scene_create": 1,+ "mesh_inspect": 1,+ "scene_inspect": 1,+ "scene_look": 1,+ "scene_export_recipe": 1,+ "scene_dispose": 1+ },+ "branchCalls": 0,+ "relationCalls": 0,+ "pathQueries": 0,+ "features": {+ "geometryTypes": [+ "box"+ ],+ "meshOperations": [],+ "motionIk": [],+ "motionTracks": 0+ },+ "look": {+ "silhouette": {+ "views": {+ "front": 1,+ "side": 0.1684,+ "three-quarter": 0.6073+ },+ "mean": 0.5919+ },+ "images": [+ "artifacts/agent-modeling/bench-958-haiku/look/1/table-front.png",+ "artifacts/agent-modeling/bench-958-haiku/look/1/table-side.png",+ "artifacts/agent-modeling/bench-958-haiku/look/1/table-three-quarter.png"+ ]+ },+ "turns": 8,+ "costUsd": 0.01106565,+ "inputTokens": 290625,+ "outputTokens": 2759,+ "wallSeconds": 43,+ "exitCode": 0,+ "stopped": "success"+ }+ ]+}diff --git a/changes/958-shadow-only-cook.md b/changes/958-shadow-only-cook.mdnew file mode 100644index 00000000..8e18b0ec--- /dev/null+++ b/changes/958-shadow-only-cook.md@@ -0,0 +1 @@+- `scene_apply` nodes take `shadowOnly: true`, and the Cook cpu backend leaves a shadow-only node out of camera rays while it still blocks shadow rays and shows in bounces (#958)diff --git a/docs/agent-scene.md b/docs/agent-scene.mdindex 472e5b6a..23853b08 100644--- a/docs/agent-scene.md+++ b/docs/agent-scene.md@@ -345,7 +345,7 @@ Five node fields change how a node renders; `add-node` and `set-node` take them,- `castShadow: false` and `receiveShadow: false` (mesh nodes) stop a mesh casting or receiving shadows. Both default to true; glass casts none unless `castShadow` is set.- `holdout: true` writes zero colour and zero alpha where the mesh covers, and still hides what is behind it and still casts its shadow. `shadowCatcher: true` draws only the shadow it receives, black with the shadow as premultiplied alpha, and casts none. Both are for compositing over footage and need a transparent canvas: `new Renderer67(canvas, parts, { transparent: true })`. A node is never both. The shadow catcher shows the sun shadow only.-- `shadowOnly: true` (mesh nodes) keeps the node in the shadow passes and leaves it out of the picture, the same as Blender's Ray Visibility > Camera off. Use it for a gobo, such as a window frame outside the shot that throws bars of sun onto a wall. It is also left out of depth, object ID and wireframe passes. It defaults to false, so no scene changes. A node cannot set `shadowOnly` together with `castShadow: false`, `holdout` or `shadowCatcher`. The raster renderer honours it; the Cook path tracer does not yet, and traces the node as visible geometry.+- `shadowOnly: true` (mesh nodes) keeps the node in the shadow passes and leaves it out of the picture, the same as Blender's Ray Visibility > Camera off. Use it for a gobo, such as a window frame outside the shot that throws bars of sun onto a wall. It is also left out of depth, object ID and wireframe passes. It defaults to false, so no scene changes. A node cannot set `shadowOnly` together with `castShadow: false`, `holdout` or `shadowCatcher`. The raster renderer and the Cook cpu backend honour it: Cook leaves the node out of camera rays and keeps it for shadow and bounce rays, so a mirror or glass can still show it. The GPU path tracers still draw it, and say so in their ignored-feature notes.Parts with different `castShadow`, `receiveShadow`, `holdout`, `shadowCatcher` or `shadowOnly` values batch apart; `visible` never splits a batch. `receiveShadow`, `holdout`, `shadowCatcher` and `shadowOnly` changes need `replaceParts`, not `updateParts`.diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex 8d3fc2f3..5ff1ab21 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -62,6 +62,7 @@ const nodeFields = {receiveShadow: z.boolean().optional(),holdout: z.boolean().optional(),shadowCatcher: z.boolean().optional(),+ shadowOnly: z.boolean().optional(),blend: z.strictObject({ radius: z.number().min(0.005).max(2) }).optional().describe('Soft seam into another mesh, radius in m; docs/mesh-blend.md.'),label: z.string().max(256).optional().describe('Human-readable label. Identity remains the node ID.'),parent: identifier.nullable().optional().describe('Parent node ID, or null for a root. Local transforms stay unchanged.'),diff --git a/src/path-tracer-scene.js b/src/path-tracer-scene.jsindex 66430114..efd57651 100644--- a/src/path-tracer-scene.js+++ b/src/path-tracer-scene.js@@ -336,6 +336,7 @@ export function buildTraceScene(allParts, environment = {}) {...geometry,bvh,materials: parts.map(part => resolveMaterial(part.material ?? {})),+ shadowOnlyParts: parts.some(part => part.shadowOnly === true) ? Uint8Array.from(parts, part => (part.shadowOnly === true ? 1 : 0)) : null,lights: resolveLights(environment),environment: resolveEnvironment(environment.map),volume: resolveTraceVolume(environment.volumeFog),diff --git a/src/path-tracer.js b/src/path-tracer.jsindex 50396905..fc4a5d71 100644--- a/src/path-tracer.js+++ b/src/path-tracer.js@@ -392,8 +392,11 @@ function fillHit(scene, triangleId, originX, originY, originZ, directionX, direcreturn alpha;}-/** The nearest solid hit along a ray, stepping through the transparent texels of cutout materials. */-function intersect(scene, originX, originY, originZ, directionX, directionY, directionZ, hit) {+/**+ * The nearest solid hit along a ray, stepping through the transparent texels of cutout materials. A camera ray also+ * steps through a shadow-only part, which stays in the scene for shadow and bounce rays (Blender's camera visibility off).+ */+function intersect(scene, originX, originY, originZ, directionX, directionY, directionZ, hit, isCameraRay = false) {let startX = originX;let startY = originY;let startZ = originZ;@@ -401,8 +404,9 @@ function intersect(scene, originX, originY, originZ, directionX, directionY, dirfor (let attempt = 0; attempt < ALPHA_STEP_LIMIT; attempt++) {const found = scene.tracer.trace(startX, startY, startZ, directionX, directionY, directionZ, scene.farDistance, false);if (found.nearestTriangle < 0) return false;- const alpha = fillHit(scene, found.nearestTriangle, startX, startY, startZ, directionX, directionY, directionZ, found.nearestDistance, hit);- if (hit.material.alphaCutoff === 0 || alpha >= hit.material.alphaCutoff) {+ const seenByCamera = !(isCameraRay && scene.shadowOnlyParts?.[scene.triangleMaterial[found.nearestTriangle]] === 1);+ const alpha = seenByCamera ? fillHit(scene, found.nearestTriangle, startX, startY, startZ, directionX, directionY, directionZ, found.nearestDistance, hit) : 0;+ if (seenByCamera && (hit.material.alphaCutoff === 0 || alpha >= hit.material.alphaCutoff)) {hit.distance += travelled;return true;}@@ -866,7 +870,7 @@ function createPathTracer(scene, options) {let previousPdf = 0;let afterMirror = true;for (let depth = 0; depth <= options.bounces; depth++) {- const found = intersect(scene, positionX, positionY, positionZ, direction[0], direction[1], direction[2], hit);+ const found = intersect(scene, positionX, positionY, positionZ, direction[0], direction[1], direction[2], hit, depth === 0);const scattersInFog = scene.volume && scatterInFog(positionX, positionY, positionZ, found ? hit.distance : scene.farDistance);if (crossLocalVolumes) {const segmentLength = scattersInFog ? Math.hypot(hit.point[0] - positionX, hit.point[1] - positionY, hit.point[2] - positionZ) : (found ? hit.distance : scene.farDistance);@@ -978,6 +982,7 @@ export function describeIgnoredFeatures(parts, environment, backend = 'cpu') {if (!environment?.map) notes.add('There is no environment map, so the sky is black and only the lights shine.');for (const part of parts) {const material = part.material ?? {};+ if (part.shadowOnly === true && backend !== 'cpu') notes.add('shadowOnly is left out of camera rays by the cpu backend only; the GPU path tracers draw the part.');for (const field of IGNORED_MAP_FIELDS) if (material[field]) notes.add(`${field} is ignored.`);if (material.tag && material.tag !== 'surface') notes.add(`${material.tag} materials are traced as plain surfaces.`);if (material.alphaMode === 'blend') notes.add(hasGraphOpacity(material) ? 'A graph opacity output is traced as a cutout at 0.5.' : 'Blended transparency is traced as opaque.');diff --git a/tests/mcp-shadow-only-schema.test.js b/tests/mcp-shadow-only-schema.test.jsnew file mode 100644index 00000000..e247e5fc--- /dev/null+++ b/tests/mcp-shadow-only-schema.test.js@@ -0,0 +1,27 @@+import './support/load-agent-scene-modules.mjs';+import test from 'node:test';+import assert from 'node:assert/strict';+import { Client } from '@modelcontextprotocol/client';+import { InMemoryTransport } from '@modelcontextprotocol/server';+import { createAgentMcpServer } from '../scripts/mcp-server.mjs';++async function listedTools() {+ const server = createAgentMcpServer({ async close() {}, async capabilities() { return { scene: {}, phase: 'idle' }; } });+ const client = new Client({ name: '67js-shadow-only-schema-test', version: '1' });+ const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();+ await server.connect(serverTransport);+ await client.connect(clientTransport);+ try {+ return (await client.listTools()).tools;+ } finally {+ await client.close();+ await server.close();+ }+}++test('scene_apply lists shadowOnly as a boolean node field next to the other render flags', async () => {+ const tools = await listedTools();+ const text = JSON.stringify(tools.find(tool => tool.name === 'scene_apply').inputSchema);+ assert.match(text, /"shadowOnly":\{"type":"boolean"\}/);+ assert.match(text, /"shadowCatcher":\{"type":"boolean"\}/);+});diff --git a/tests/path-tracer-shadow-only.test.js b/tests/path-tracer-shadow-only.test.jsnew file mode 100644index 00000000..837b547f--- /dev/null+++ b/tests/path-tracer-shadow-only.test.js@@ -0,0 +1,51 @@+import assert from 'node:assert/strict';+import test from 'node:test';++import { box } from '../src/geometry.js';+import { describeIgnoredFeatures, renderPathTracedImage } from '../src/path-tracer.js';++function placed(translation, scale) {+ return new Float32Array([scale[0], 0, 0, 0, 0, scale[1], 0, 0, 0, 0, scale[2], 0, translation[0], translation[1], translation[2], 1]);+}++function surface(color) {+ return { tag: 'surface', metallic: 0, roughness: 1, emission: 0, color };+}++const environment = { keyColor: [3, 3, 3], lightDirection: [0, 1, 0] };+const camera = { eye: [0, 6, 0], target: [0, 0, 0], up: [0, 0, -1], fov: 30 };+const OPTIONS = { width: 16, height: 16, samples: 16, bounces: 2, denoise: false, threads: 1, backend: 'cpu' };++function floor() {+ return { name: 'floor', geometry: box(0, 1), matrix: placed([0, -0.05, 0], [20, 0.1, 20]), material: surface([0.5, 0.5, 0.5]) };+}++function caster(extra) {+ return { name: 'caster', geometry: box(0, 1), matrix: placed([0, 1.5, 0], [3, 0.2, 3]), material: surface([0.9, 0.2, 0.1]), ...extra };+}++async function centreBrightness(parts) {+ const { linear } = await renderPathTracedImage({ parts, environment, camera }, OPTIONS);+ const centre = (8 * 16 + 8) * 3;+ return linear[centre] + linear[centre + 1] + linear[centre + 2];+}++test('Cook draws the shadow of a shadow-only caster and not the caster', async () => {+ const lit = await centreBrightness([floor()]);+ const visibleCaster = await centreBrightness([floor(), caster()]);+ const shadowOnly = await centreBrightness([floor(), caster({ shadowOnly: true })]);+ assert.ok(lit > 0.5, 'the bare floor is lit by the sun');+ assert.ok(visibleCaster > lit * 0.5, 'a visible caster is drawn in the middle of the picture');+ assert.ok(shadowOnly < lit * 0.15, `the floor under a shadow-only caster is in its shadow: ${shadowOnly} against ${lit}`);+ assert.ok(visibleCaster > shadowOnly * 5, 'the shadow-only caster is not drawn');+});++test('a part without the flag leaves the trace scene and the notes unchanged', () => {+ assert.deepEqual(describeIgnoredFeatures([floor(), caster()], {}, 'cpu').filter(note => note.startsWith('shadowOnly')), []);+});++test('the GPU path tracers say they still draw a shadow-only part', () => {+ const notes = describeIgnoredFeatures([floor(), caster({ shadowOnly: true })], {}, 'gpu');+ assert.ok(notes.some(note => note.startsWith('shadowOnly')));+ assert.deepEqual(describeIgnoredFeatures([floor(), caster({ shadowOnly: true })], {}, 'cpu').filter(note => note.startsWith('shadowOnly')), []);+});