Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Let agents set shadowOnly and keep it out of Cook camera rays (#958)

Nguyen Xuan An40a377630f95
diff --git a/bench/agent-modeling/results/bench-958-haiku.json b/bench/agent-modeling/results/bench-958-haiku.json
new file mode 100644
index 00000000..ac22ad18
--- /dev/null
+++ b/bench/agent-modeling/results/bench-958-haiku.json
@@ -0,0 +1,223 @@
+{
+ "run": "bench-958-haiku",
+ "model": "claude-haiku-5-5",
+ "date": "2026-11-12T08:02:24.256Z",
+ "skill": false,
+ "branches": "on",
+ "relations": "on",
+ "runsPerTask": 3,
+ "passed": 3,
+ "tasks": 3,
+ "toolCalls": 24,
+ "costUsd": 0.04,
+ "byModel": {
+ "claude-haiku-5-5": {
+ "runs": 3,
+ "passed": 3,
+ "successRate": 1,
+ "toolCalls": 24,
+ "toolErrors": 0,
+ "inputTokens": 1174553,
+ "outputTokens": 12176,
+ "costUsd": 0.04,
+ "tasks": {
+ "crate": {
+ "runs": 1,
+ "passed": 1,
+ "successRate": 1,
+ "toolCalls": 11,
+ "toolErrors": 0,
+ "inputTokens": 538306,
+ "outputTokens": 5954,
+ "costUsd": 0.02
+ },
+ "bolt": {
+ "runs": 1,
+ "passed": 1,
+ "successRate": 1,
+ "toolCalls": 7,
+ "toolErrors": 0,
+ "inputTokens": 345622,
+ "outputTokens": 3463,
+ "costUsd": 0.01
+ },
+ "table": {
+ "runs": 1,
+ "passed": 1,
+ "successRate": 1,
+ "toolCalls": 6,
+ "toolErrors": 0,
+ "inputTokens": 290625,
+ "outputTokens": 2759,
+ "costUsd": 0.01
+ }
+ }
+ }
+ },
+ "records": [
+ {
+ "task": "crate",
+ "attempt": 1,
+ "pass": true,
+ "passed": 8,
+ "total": 8,
+ "failed": [],
+ "model": "claude-haiku-5-5",
+ "toolCalls": 11,
+ "toolErrors": 0,
+ "toolsByName": {
+ "ToolSearch": 1,
+ "scene_capabilities": 6,
+ "scene_create": 1,
+ "mesh_edit": 1,
+ "scene_look": 1,
+ "scene_export_recipe": 1,
+ "scene_dispose": 1
+ },
+ "branchCalls": 0,
+ "relationCalls": 0,
+ "pathQueries": 0,
+ "features": {
+ "geometryTypes": [
+ "mesh"
+ ],
+ "meshOperations": [
+ "set_position",
+ "inset_individual",
+ "extrude_individual"
+ ],
+ "motionIk": [],
+ "motionTracks": 0
+ },
+ "look": {
+ "silhouette": {
+ "views": {
+ "front": 1,
+ "side": 1,
+ "three-quarter": 1
+ },
+ "mean": 1
+ },
+ "images": [
+ "artifacts/agent-modeling/bench-958-haiku/look/1/crate-front.png",
+ "artifacts/agent-modeling/bench-958-haiku/look/1/crate-side.png",
+ "artifacts/agent-modeling/bench-958-haiku/look/1/crate-three-quarter.png"
+ ]
+ },
+ "turns": 13,
+ "costUsd": 0.01654611,
+ "inputTokens": 538306,
+ "outputTokens": 5954,
+ "wallSeconds": 64,
+ "exitCode": 0,
+ "stopped": "success"
+ },
+ {
+ "task": "bolt",
+ "attempt": 1,
+ "pass": true,
+ "passed": 7,
+ "total": 7,
+ "failed": [],
+ "model": "claude-haiku-5-5",
+ "toolCalls": 7,
+ "toolErrors": 0,
+ "toolsByName": {
+ "ToolSearch": 1,
+ "scene_capabilities": 1,
+ "scene_create": 1,
+ "mesh_inspect": 1,
+ "scene_apply": 1,
+ "scene_look": 1,
+ "scene_export_recipe": 1,
+ "scene_dispose": 1
+ },
+ "branchCalls": 0,
+ "relationCalls": 0,
+ "pathQueries": 0,
+ "features": {
+ "geometryTypes": [
+ "cylinder"
+ ],
+ "meshOperations": [],
+ "motionIk": [],
+ "motionTracks": 0
+ },
+ "look": {
+ "silhouette": {
+ "views": {
+ "front": 1,
+ "side": 1,
+ "three-quarter": 0.9997
+ },
+ "mean": 0.9999
+ },
+ "images": [
+ "artifacts/agent-modeling/bench-958-haiku/look/1/bolt-front.png",
+ "artifacts/agent-modeling/bench-958-haiku/look/1/bolt-side.png",
+ "artifacts/agent-modeling/bench-958-haiku/look/1/bolt-three-quarter.png"
+ ]
+ },
+ "turns": 9,
+ "costUsd": 0.01215609,
+ "inputTokens": 345622,
+ "outputTokens": 3463,
+ "wallSeconds": 54,
+ "exitCode": 0,
+ "stopped": "success"
+ },
+ {
+ "task": "table",
+ "attempt": 1,
+ "pass": true,
+ "passed": 6,
+ "total": 6,
+ "failed": [],
+ "model": "claude-haiku-5-5",
+ "toolCalls": 6,
+ "toolErrors": 0,
+ "toolsByName": {
+ "ToolSearch": 1,
+ "scene_create": 1,
+ "mesh_inspect": 1,
+ "scene_inspect": 1,
+ "scene_look": 1,
+ "scene_export_recipe": 1,
+ "scene_dispose": 1
+ },
+ "branchCalls": 0,
+ "relationCalls": 0,
+ "pathQueries": 0,
+ "features": {
+ "geometryTypes": [
+ "box"
+ ],
+ "meshOperations": [],
+ "motionIk": [],
+ "motionTracks": 0
+ },
+ "look": {
+ "silhouette": {
+ "views": {
+ "front": 1,
+ "side": 0.1684,
+ "three-quarter": 0.6073
+ },
+ "mean": 0.5919
+ },
+ "images": [
+ "artifacts/agent-modeling/bench-958-haiku/look/1/table-front.png",
+ "artifacts/agent-modeling/bench-958-haiku/look/1/table-side.png",
+ "artifacts/agent-modeling/bench-958-haiku/look/1/table-three-quarter.png"
+ ]
+ },
+ "turns": 8,
+ "costUsd": 0.01106565,
+ "inputTokens": 290625,
+ "outputTokens": 2759,
+ "wallSeconds": 43,
+ "exitCode": 0,
+ "stopped": "success"
+ }
+ ]
+}
diff --git a/changes/958-shadow-only-cook.md b/changes/958-shadow-only-cook.md
new file mode 100644
index 00000000..8e18b0ec
--- /dev/null
+++ b/changes/958-shadow-only-cook.md
@@ -0,0 +1 @@
+- `scene_apply` nodes take `shadowOnly: true`, and the Cook cpu backend leaves a shadow-only node out of camera rays while it still blocks shadow rays and shows in bounces (#958)
diff --git a/docs/agent-scene.md b/docs/agent-scene.md
index 472e5b6a..23853b08 100644
--- a/docs/agent-scene.md
+++ b/docs/agent-scene.md
@@ -345,7 +345,7 @@ Five node fields change how a node renders; `add-node` and `set-node` take them,
- `castShadow: false` and `receiveShadow: false` (mesh nodes) stop a mesh casting or receiving shadows. Both default to true; glass casts none unless `castShadow` is set.
- `holdout: true` writes zero colour and zero alpha where the mesh covers, and still hides what is behind it and still casts its shadow. `shadowCatcher: true` draws only the shadow it receives, black with the shadow as premultiplied alpha, and casts none. Both are for compositing over footage and need a transparent canvas: `new Renderer67(canvas, parts, { transparent: true })`. A node is never both. The shadow catcher shows the sun shadow only.
-- `shadowOnly: true` (mesh nodes) keeps the node in the shadow passes and leaves it out of the picture, the same as Blender's Ray Visibility > Camera off. Use it for a gobo, such as a window frame outside the shot that throws bars of sun onto a wall. It is also left out of depth, object ID and wireframe passes. It defaults to false, so no scene changes. A node cannot set `shadowOnly` together with `castShadow: false`, `holdout` or `shadowCatcher`. The raster renderer honours it; the Cook path tracer does not yet, and traces the node as visible geometry.
+- `shadowOnly: true` (mesh nodes) keeps the node in the shadow passes and leaves it out of the picture, the same as Blender's Ray Visibility > Camera off. Use it for a gobo, such as a window frame outside the shot that throws bars of sun onto a wall. It is also left out of depth, object ID and wireframe passes. It defaults to false, so no scene changes. A node cannot set `shadowOnly` together with `castShadow: false`, `holdout` or `shadowCatcher`. The raster renderer and the Cook cpu backend honour it: Cook leaves the node out of camera rays and keeps it for shadow and bounce rays, so a mirror or glass can still show it. The GPU path tracers still draw it, and say so in their ignored-feature notes.
Parts with different `castShadow`, `receiveShadow`, `holdout`, `shadowCatcher` or `shadowOnly` values batch apart; `visible` never splits a batch. `receiveShadow`, `holdout`, `shadowCatcher` and `shadowOnly` changes need `replaceParts`, not `updateParts`.
diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjs
index 8d3fc2f3..5ff1ab21 100644
--- a/scripts/mcp-server.mjs
+++ b/scripts/mcp-server.mjs
@@ -62,6 +62,7 @@ const nodeFields = {
receiveShadow: z.boolean().optional(),
holdout: z.boolean().optional(),
shadowCatcher: z.boolean().optional(),
+ shadowOnly: z.boolean().optional(),
blend: z.strictObject({ radius: z.number().min(0.005).max(2) }).optional().describe('Soft seam into another mesh, radius in m; docs/mesh-blend.md.'),
label: z.string().max(256).optional().describe('Human-readable label. Identity remains the node ID.'),
parent: identifier.nullable().optional().describe('Parent node ID, or null for a root. Local transforms stay unchanged.'),
diff --git a/src/path-tracer-scene.js b/src/path-tracer-scene.js
index 66430114..efd57651 100644
--- a/src/path-tracer-scene.js
+++ b/src/path-tracer-scene.js
@@ -336,6 +336,7 @@ export function buildTraceScene(allParts, environment = {}) {
...geometry,
bvh,
materials: parts.map(part => resolveMaterial(part.material ?? {})),
+ shadowOnlyParts: parts.some(part => part.shadowOnly === true) ? Uint8Array.from(parts, part => (part.shadowOnly === true ? 1 : 0)) : null,
lights: resolveLights(environment),
environment: resolveEnvironment(environment.map),
volume: resolveTraceVolume(environment.volumeFog),
diff --git a/src/path-tracer.js b/src/path-tracer.js
index 50396905..fc4a5d71 100644
--- a/src/path-tracer.js
+++ b/src/path-tracer.js
@@ -392,8 +392,11 @@ function fillHit(scene, triangleId, originX, originY, originZ, directionX, direc
return alpha;
}
-/** The nearest solid hit along a ray, stepping through the transparent texels of cutout materials. */
-function intersect(scene, originX, originY, originZ, directionX, directionY, directionZ, hit) {
+/**
+ * The nearest solid hit along a ray, stepping through the transparent texels of cutout materials. A camera ray also
+ * steps through a shadow-only part, which stays in the scene for shadow and bounce rays (Blender's camera visibility off).
+ */
+function intersect(scene, originX, originY, originZ, directionX, directionY, directionZ, hit, isCameraRay = false) {
let startX = originX;
let startY = originY;
let startZ = originZ;
@@ -401,8 +404,9 @@ function intersect(scene, originX, originY, originZ, directionX, directionY, dir
for (let attempt = 0; attempt < ALPHA_STEP_LIMIT; attempt++) {
const found = scene.tracer.trace(startX, startY, startZ, directionX, directionY, directionZ, scene.farDistance, false);
if (found.nearestTriangle < 0) return false;
- const alpha = fillHit(scene, found.nearestTriangle, startX, startY, startZ, directionX, directionY, directionZ, found.nearestDistance, hit);
- if (hit.material.alphaCutoff === 0 || alpha >= hit.material.alphaCutoff) {
+ const seenByCamera = !(isCameraRay && scene.shadowOnlyParts?.[scene.triangleMaterial[found.nearestTriangle]] === 1);
+ const alpha = seenByCamera ? fillHit(scene, found.nearestTriangle, startX, startY, startZ, directionX, directionY, directionZ, found.nearestDistance, hit) : 0;
+ if (seenByCamera && (hit.material.alphaCutoff === 0 || alpha >= hit.material.alphaCutoff)) {
hit.distance += travelled;
return true;
}
@@ -866,7 +870,7 @@ function createPathTracer(scene, options) {
let previousPdf = 0;
let afterMirror = true;
for (let depth = 0; depth <= options.bounces; depth++) {
- const found = intersect(scene, positionX, positionY, positionZ, direction[0], direction[1], direction[2], hit);
+ const found = intersect(scene, positionX, positionY, positionZ, direction[0], direction[1], direction[2], hit, depth === 0);
const scattersInFog = scene.volume && scatterInFog(positionX, positionY, positionZ, found ? hit.distance : scene.farDistance);
if (crossLocalVolumes) {
const segmentLength = scattersInFog ? Math.hypot(hit.point[0] - positionX, hit.point[1] - positionY, hit.point[2] - positionZ) : (found ? hit.distance : scene.farDistance);
@@ -978,6 +982,7 @@ export function describeIgnoredFeatures(parts, environment, backend = 'cpu') {
if (!environment?.map) notes.add('There is no environment map, so the sky is black and only the lights shine.');
for (const part of parts) {
const material = part.material ?? {};
+ if (part.shadowOnly === true && backend !== 'cpu') notes.add('shadowOnly is left out of camera rays by the cpu backend only; the GPU path tracers draw the part.');
for (const field of IGNORED_MAP_FIELDS) if (material[field]) notes.add(`${field} is ignored.`);
if (material.tag && material.tag !== 'surface') notes.add(`${material.tag} materials are traced as plain surfaces.`);
if (material.alphaMode === 'blend') notes.add(hasGraphOpacity(material) ? 'A graph opacity output is traced as a cutout at 0.5.' : 'Blended transparency is traced as opaque.');
diff --git a/tests/mcp-shadow-only-schema.test.js b/tests/mcp-shadow-only-schema.test.js
new file mode 100644
index 00000000..e247e5fc
--- /dev/null
+++ b/tests/mcp-shadow-only-schema.test.js
@@ -0,0 +1,27 @@
+import './support/load-agent-scene-modules.mjs';
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { Client } from '@modelcontextprotocol/client';
+import { InMemoryTransport } from '@modelcontextprotocol/server';
+import { createAgentMcpServer } from '../scripts/mcp-server.mjs';
+
+async function listedTools() {
+ const server = createAgentMcpServer({ async close() {}, async capabilities() { return { scene: {}, phase: 'idle' }; } });
+ const client = new Client({ name: '67js-shadow-only-schema-test', version: '1' });
+ const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+ await server.connect(serverTransport);
+ await client.connect(clientTransport);
+ try {
+ return (await client.listTools()).tools;
+ } finally {
+ await client.close();
+ await server.close();
+ }
+}
+
+test('scene_apply lists shadowOnly as a boolean node field next to the other render flags', async () => {
+ const tools = await listedTools();
+ const text = JSON.stringify(tools.find(tool => tool.name === 'scene_apply').inputSchema);
+ assert.match(text, /"shadowOnly":\{"type":"boolean"\}/);
+ assert.match(text, /"shadowCatcher":\{"type":"boolean"\}/);
+});
diff --git a/tests/path-tracer-shadow-only.test.js b/tests/path-tracer-shadow-only.test.js
new file mode 100644
index 00000000..837b547f
--- /dev/null
+++ b/tests/path-tracer-shadow-only.test.js
@@ -0,0 +1,51 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { box } from '../src/geometry.js';
+import { describeIgnoredFeatures, renderPathTracedImage } from '../src/path-tracer.js';
+
+function placed(translation, scale) {
+ return new Float32Array([scale[0], 0, 0, 0, 0, scale[1], 0, 0, 0, 0, scale[2], 0, translation[0], translation[1], translation[2], 1]);
+}
+
+function surface(color) {
+ return { tag: 'surface', metallic: 0, roughness: 1, emission: 0, color };
+}
+
+const environment = { keyColor: [3, 3, 3], lightDirection: [0, 1, 0] };
+const camera = { eye: [0, 6, 0], target: [0, 0, 0], up: [0, 0, -1], fov: 30 };
+const OPTIONS = { width: 16, height: 16, samples: 16, bounces: 2, denoise: false, threads: 1, backend: 'cpu' };
+
+function floor() {
+ return { name: 'floor', geometry: box(0, 1), matrix: placed([0, -0.05, 0], [20, 0.1, 20]), material: surface([0.5, 0.5, 0.5]) };
+}
+
+function caster(extra) {
+ return { name: 'caster', geometry: box(0, 1), matrix: placed([0, 1.5, 0], [3, 0.2, 3]), material: surface([0.9, 0.2, 0.1]), ...extra };
+}
+
+async function centreBrightness(parts) {
+ const { linear } = await renderPathTracedImage({ parts, environment, camera }, OPTIONS);
+ const centre = (8 * 16 + 8) * 3;
+ return linear[centre] + linear[centre + 1] + linear[centre + 2];
+}
+
+test('Cook draws the shadow of a shadow-only caster and not the caster', async () => {
+ const lit = await centreBrightness([floor()]);
+ const visibleCaster = await centreBrightness([floor(), caster()]);
+ const shadowOnly = await centreBrightness([floor(), caster({ shadowOnly: true })]);
+ assert.ok(lit > 0.5, 'the bare floor is lit by the sun');
+ assert.ok(visibleCaster > lit * 0.5, 'a visible caster is drawn in the middle of the picture');
+ assert.ok(shadowOnly < lit * 0.15, `the floor under a shadow-only caster is in its shadow: ${shadowOnly} against ${lit}`);
+ assert.ok(visibleCaster > shadowOnly * 5, 'the shadow-only caster is not drawn');
+});
+
+test('a part without the flag leaves the trace scene and the notes unchanged', () => {
+ assert.deepEqual(describeIgnoredFeatures([floor(), caster()], {}, 'cpu').filter(note => note.startsWith('shadowOnly')), []);
+});
+
+test('the GPU path tracers say they still draw a shadow-only part', () => {
+ const notes = describeIgnoredFeatures([floor(), caster({ shadowOnly: true })], {}, 'gpu');
+ assert.ok(notes.some(note => note.startsWith('shadowOnly')));
+ assert.deepEqual(describeIgnoredFeatures([floor(), caster({ shadowOnly: true })], {}, 'cpu').filter(note => note.startsWith('shadowOnly')), []);
+});