3D engine that AI agents drive
Return structured look findings an agent can act on without reading the image (#905)
Nguyen Xuan Ana7bf48cb1a0f
diff --git a/changes/905.md b/changes/905.mdnew file mode 100644index 00000000..5b325f89--- /dev/null+++ b/changes/905.md@@ -0,0 +1 @@+- `scene_look` lists floating parts, intersecting parts, parts outside the camera and the share of a blown-out or black frame as findings with part names and numbers, so an agent can catch them without reading the picture (#905)diff --git a/docs/mcp.md b/docs/mcp.mdindex 8e5eb978..a16bc2c0 100644--- a/docs/mcp.md+++ b/docs/mcp.md@@ -166,6 +166,17 @@ Every tool schema is read on every turn, so `node scripts/measure-tool-cost.mjs``mesh_inspect` lint adds warnings for `intersecting-parts`, `self-intersection`, `thin-wall`, `floating-part` and `asymmetric`. They never set `ok` to false, run within face budgets, and list any check they skipped in `skippedChecks`. `scene_inspect` returns a `diagnostics` list only when there is something to say: ignored material fields, blended transmission, unused materials or geometries, and for imported assets the decoder's not-drawn features, rigs with no joints or no skin, and assets with no clips. Each item has `kind`, `severity`, `target`, `message` and `fix`.+`scene_look` returns the same kind of `diagnostics` list next to the picture, so an agent can check a frame without reading it. The findings carry the part names and the numbers in fields, and each list is capped (8 parts per finding, 8 findings per kind, with `nodeCount` or `droppedCount` saying how many were left out):++| kind | fields | meaning |+| --- | --- | --- |+| `floating-node` | `nodes`, `gapMeters`, `nearest` | The part is more than 2 cm from every other part and from the floor level (the lowest point of the scene). `nearest` is the closest part or `floor`. |+| `intersecting-nodes` | `nodes`, `penetrationMeters` | The bounds of two parts overlap by 2 cm or more on every axis. Bounds are axis aligned, so a rotated part can report an overlap its surface does not have; a part sunk into the ground is reported too. |+| `part-outside-frame` | `nodes`, `outsideCount`, `partCount` | Parts whose bounding sphere is outside the camera frustum or behind the camera. When more than half are outside, `subject-out-of-frame` is returned with the same fields instead. |+| `exposure-clipped`, `exposure-black` | `whitePercent`, `blackPercent` | Share of a 64 by 64 copy of the frame that is clipped to white (35 percent or more) or black (98 percent or more) after tone mapping. |++The checks run on part bounds and on that small copy of the frame, so they cost almost nothing. Scenes of over 256 parts get no floating or intersection findings. `src/frame-checks.js` holds them.+The recipe uses `format: '67js.scene'` and `schemaVersion: 1`.Supported geometry types are `box`, `cylinder`, `sphere`, `torus`, `lathe`, `extrude`, `tube`, `curve`, `sdf`, `text`, `grid`, `terrain`, and `mesh`. The fields of `curve` and `text` are listed by `scene_capabilities` (`curveText.details`) and not repeated in every tool schema. Text uses the bundled font only; the MCP accepts no font file or URL.A `mesh` geometry starts from one of the others as `base` and replays an edit log; see [mesh.md](mesh.md).diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.jsindex 2fd120fb..019c535e 100644--- a/scripts/agent-preview-browser.js+++ b/scripts/agent-preview-browser.js@@ -31,7 +31,7 @@ import { clipFromGltfAnimation } from '../src/skeleton-pose.js';import { sessionProblem, SCENE_STORE_LIMITS } from '../src/agent-errors.js';import { AGENT_SCENE_DEFAULT_LIMITS } from '../src/agent-scene-capabilities.js';import { describeApplication } from '../src/agent-apply-report.js';-import { framingProblems, exposureProblems, lumaShares } from '../src/frame-checks.js';+import { framingProblems, placementProblems, exposureProblems, lumaShares } from '../src/frame-checks.js';import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../src/agent-branches.js';import { agentAssetDecodeOptions, withCatalogLimitAdvice } from '../src/agent-asset-limits.js';import { describeMarks, validateMarks } from '../src/agent-annotate.js';@@ -188,7 +188,10 @@ function sampleFrameShares() {function viewProblems(record, camera, exposure) {const problems = [];const boxes = record.scene.worldBounds?.();- if (boxes) problems.push(...framingProblems({ camera, aspect: canvas.width / canvas.height, boxes }));+ if (boxes) {+ problems.push(...framingProblems({ camera, aspect: canvas.width / canvas.height, boxes }));+ problems.push(...placementProblems({ boxes }));+ }if (record.scene.bakeStaleness) problems.push(...record.scene.bakeStaleness());if (exposure !== undefined) problems.push(...exposureProblems({ shares: sampleFrameShares(), exposure }));return problems;diff --git a/src/frame-checks.js b/src/frame-checks.jsindex cfe0ec20..195ba519 100644--- a/src/frame-checks.js+++ b/src/frame-checks.js@@ -10,8 +10,17 @@ const WHITE_LUMA = 250 / 255;const BLACK_FRAME_SHARE = 0.98;const CLIPPED_FRAME_SHARE = 0.35;-function warning(kind, target, message, fix) {- return { kind, severity: 'warning', target, message, fix };+const MAXIMUM_FINDING_NODES = 8;++/** A finding names the parts it concerns in `nodes` and carries its numbers as extra fields. */+function warning(kind, target, message, fix, details = {}) {+ return { kind, severity: 'warning', target, message, fix, ...details };+}++/** At most 8 part names for a finding, with `nodeCount` when more were left out. */+export function namedNodes(names) {+ const nodes = names.slice(0, MAXIMUM_FINDING_NODES);+ return names.length > nodes.length ? { nodes, nodeCount: names.length } : { nodes };}function subtract(first, second) {@@ -86,26 +95,40 @@ export function framingProblems({ camera, aspect, boxes }) {const upward = cross(right, forward);const sideScale = Math.sqrt(1 + tangent * tangent);const wideScale = Math.sqrt(1 + tangentWide * tangentWide);+ const outside = [];const visibleBoxes = boxes.filter(box => {const { center, radius } = sphereOf(box);const offset = subtract(center, camera.eye);const depth = dot(offset, forward);- if (depth + radius < camera.near || depth - radius > camera.far) return false;+ if (depth + radius < camera.near || depth - radius > camera.far) {+ outside.push(box.name);+ return false;+ }const sideways = (Math.abs(dot(offset, right)) - tangentWide * depth) / wideScale;const vertical = (Math.abs(dot(offset, upward)) - tangent * depth) / sideScale;- return sideways <= radius && vertical <= radius;+ const inView = sideways <= radius && vertical <= radius;+ if (!inView) outside.push(box.name);+ return inView;});+ const outsideDetails = { ...namedNodes(outside), outsideCount: outside.length, partCount: boxes.length };if (visibleBoxes.length * 2 < boxes.length) {return [warning('subject-out-of-frame', 'camera',`${boxes.length - visibleBoxes.length} of ${boxes.length} parts are outside the camera frustum.`,- `${framingSuggestion(camera, boxes, tangent)} to put every part in view.`)];+ `${framingSuggestion(camera, boxes, tangent)} to put every part in view.`, outsideDetails)];}const { center, radius } = sphereOf(unionOf(visibleBoxes));const depth = dot(subtract(center, camera.eye), forward);if (depth > 0 && radius / (depth * tangent) < TINY_SUBJECT_RATIO) {return [warning('subject-too-small', 'camera',`The visible parts fill under ${Math.round(TINY_SUBJECT_RATIO * 100)} percent of the frame height.`,- `${framingSuggestion(camera, visibleBoxes, tangent)} to fill the frame, or lower camera.fov.`)];+ `${framingSuggestion(camera, visibleBoxes, tangent)} to fill the frame, or lower camera.fov.`,+ { heightPercent: Math.round(radius / (depth * tangent) * 100) })];+ }+ if (outside.length > 0) {+ return [warning('part-outside-frame', 'camera',+ `${outside.length} of ${boxes.length} parts are outside the camera frustum, so the picture does not show them.`,+ `Move them into view with scene_apply set-node, or ${framingSuggestion(camera, boxes, tangent)} to put every part in view.`,+ outsideDetails)];}return [];}@@ -123,21 +146,89 @@ export function lumaShares(rgba) {return { black: black / pixels, white: white / pixels };}+function percents(shares) {+ return { blackPercent: Math.round(shares.black * 100), whitePercent: Math.round(shares.white * 100) };+}+/** Warnings for a frame that is nearly all black or has a large clipped area; `exposure` is the current effects.exposure. */export function exposureProblems({ shares, exposure }) {if (shares.black >= BLACK_FRAME_SHARE) {return [warning('exposure-black', 'effects.exposure',`${Math.round(shares.black * 100)} percent of the frame is black after tone mapping.`,- `Raise effects.exposure from ${exposure} to ${round(Math.max(exposure, 0.25) * 4)}, raise environment.keyIntensity, or aim a light at the subject.`)];+ `Raise effects.exposure from ${exposure} to ${round(Math.max(exposure, 0.25) * 4)}, raise environment.keyIntensity, or aim a light at the subject.`, percents(shares))];}if (shares.white >= CLIPPED_FRAME_SHARE) {return [warning('exposure-clipped', 'effects.exposure',`${Math.round(shares.white * 100)} percent of the frame is clipped to white after tone mapping.`,- `Lower effects.exposure from ${exposure} to ${round(exposure / 2)}, or lower environment.keyIntensity.`)];+ `Lower effects.exposure from ${exposure} to ${round(exposure / 2)}, or lower environment.keyIntensity.`, percents(shares))];}return [];}+const FLOATING_GAP = 0.02;+const MINIMUM_PENETRATION = 0.02;+const MAXIMUM_PAIR_PARTS = 256;+const MAXIMUM_PLACEMENT_FINDINGS = 8;++/** Distance between two boxes along each axis; 0 where they touch or overlap on that axis. */+function axisGaps(first, second) {+ return [0, 1, 2].map(axis => Math.max(first.min[axis] - second.max[axis], second.min[axis] - first.max[axis], 0));+}++/** How far two boxes overlap along each axis, or null when they are apart on any axis. */+function axisPenetrations(first, second) {+ const depths = [0, 1, 2].map(axis => Math.min(first.max[axis], second.max[axis]) - Math.max(first.min[axis], second.min[axis]));+ return depths.every(depth => depth > 0) ? depths : null;+}++function droppedCount(total) {+ return total > MAXIMUM_PLACEMENT_FINDINGS ? { droppedCount: total - MAXIMUM_PLACEMENT_FINDINGS } : {};+}++/**+ * Findings about where parts sit, from world bounds ({ name, min, max }, y up): a part with nothing within 2 cm of it+ * that is above the lowest point of the scene (`floating-node`, `gapMeters` to the nearest part or floor), and two+ * parts whose bounds overlap by 2 cm or more on every axis (`intersecting-nodes`, `penetrationMeters`). Bounds are+ * axis aligned, so a rotated part may report an overlap its surface does not have. Each kind lists at most 8 and says+ * how many more there were in `droppedCount`. Scenes of over 256 parts are not checked.+ */+export function placementProblems({ boxes }) {+ if (boxes.length < 1 || boxes.length > MAXIMUM_PAIR_PARTS) return [];+ const floorHeight = Math.min(...boxes.map(box => box.min[1]));+ const floating = [];+ const intersecting = [];+ boxes.forEach((box, index) => {+ let nearest = { gap: box.min[1] - floorHeight, name: 'floor' };+ for (let otherIndex = 0; otherIndex < boxes.length; otherIndex++) {+ if (otherIndex === index) continue;+ const other = boxes[otherIndex];+ const gap = Math.hypot(...axisGaps(box, other));+ if (gap < nearest.gap) nearest = { gap, name: other.name };+ const penetrations = otherIndex > index ? axisPenetrations(box, other) : null;+ if (penetrations && Math.min(...penetrations) >= MINIMUM_PENETRATION) {+ intersecting.push({ nodes: [box.name, other.name], penetrationMeters: round(Math.min(...penetrations)) });+ }+ }+ if (nearest.gap > FLOATING_GAP) floating.push({ node: box.name, gapMeters: round(nearest.gap), nearest: nearest.name });+ });+ const problems = [];+ const floatingWorstFirst = floating.sort((first, second) => second.gapMeters - first.gapMeters);+ for (const entry of floatingWorstFirst.slice(0, MAXIMUM_PLACEMENT_FINDINGS)) {+ problems.push(warning('floating-node', entry.node,+ `${entry.node} touches nothing: it is ${entry.gapMeters} m from ${entry.nearest === 'floor' ? 'the floor level' : entry.nearest}.`,+ 'Lower it with scene_apply set-node, or give it a relation that rests it on a part, unless it is meant to fly.',+ { nodes: [entry.node], gapMeters: entry.gapMeters, nearest: entry.nearest, ...droppedCount(floating.length) }));+ }+ const intersectingDeepestFirst = intersecting.sort((first, second) => second.penetrationMeters - first.penetrationMeters);+ for (const entry of intersectingDeepestFirst.slice(0, MAXIMUM_PLACEMENT_FINDINGS)) {+ problems.push(warning('intersecting-nodes', entry.nodes[0],+ `${entry.nodes[0]} and ${entry.nodes[1]} overlap by ${entry.penetrationMeters} m inside their bounds.`,+ 'Move one of them apart with scene_apply set-node, or ignore it where the overlap is meant, such as a part sunk into the ground.',+ { nodes: entry.nodes, penetrationMeters: entry.penetrationMeters, ...droppedCount(intersecting.length) }));+ }+ return problems;+}+/** Adds a frame's diagnostics to `metrics.diagnostics`, keeping one entry per kind, target and message across a job's frames. */export function mergeFrameDiagnostics(metrics, frameDiagnostics) {if (!frameDiagnostics?.length) return;diff --git a/tests/frame-checks.test.js b/tests/frame-checks.test.jsindex 6200cbc4..8dd354e9 100644--- a/tests/frame-checks.test.js+++ b/tests/frame-checks.test.js@@ -2,7 +2,7 @@ import './support/load-agent-scene-modules.mjs';import assert from 'node:assert/strict';import test from 'node:test';import { createAgentScene } from '../src/agent-scene.js';-import { exposureProblems, framingProblems, lumaShares, mergeFrameDiagnostics } from '../src/frame-checks.js';+import { exposureProblems, framingProblems, lumaShares, mergeFrameDiagnostics, placementProblems } from '../src/frame-checks.js';import { closedRoom, closedRoomWithoutBake } from './support/agent-bake-rooms.mjs';function sceneWithCamera(camera) {@@ -111,3 +111,75 @@ test('job metrics keep one entry per warning across frames', () => {mergeFrameDiagnostics(metrics, undefined);assert.equal(metrics.diagnostics.length, 1);});++function boxScene(nodes) {+ return {+ format: '67js.scene', schemaVersion: 1,+ geometries: [{ id: 'box', type: 'box' }],+ materials: [{ id: 'red', color: [0.8, 0.2, 0.2] }],+ nodes: nodes.map(node => ({ geometry: 'box', material: 'red', ...node })),+ camera: { eye: [0, 3, 8], target: [0, 0.5, 0], fov: 45 },+ };+}++function placementOf(nodes) {+ return placementProblems({ boxes: createAgentScene(boxScene(nodes)).worldBounds() });+}++test('a box in the air is a floating part with its gap in metres and the nearest part', () => {+ const problems = placementOf([+ { id: 'floor', scale: [10, 0.2, 10], translation: [0, 0, 0] },+ { id: 'hover', translation: [0, 1.1, 0] },+ ]);+ assert.deepEqual(kinds(problems), ['floating-node']);+ assert.deepEqual(problems[0].nodes, ['hover']);+ assert.equal(problems[0].nearest, 'floor');+ assert.equal(problems[0].gapMeters, 0.5);+});++test('a box resting on the floor or on another box is not floating', () => {+ assert.deepEqual(placementOf([+ { id: 'floor', scale: [10, 0.2, 10], translation: [0, 0, 0] },+ { id: 'rest', translation: [0, 0.6, 0] },+ { id: 'stacked', scale: [0.5, 0.5, 0.5], translation: [0, 1.35, 0] },+ ]), []);+});++test('two boxes that overlap name both parts and how deep', () => {+ const problems = placementOf([+ { id: 'floor', scale: [10, 0.2, 10], translation: [0, 0, 0] },+ { id: 'first', translation: [-0.4, 0.6, 0] },+ { id: 'second', translation: [0.4, 0.6, 0] },+ ]);+ assert.deepEqual(kinds(problems), ['intersecting-nodes']);+ assert.deepEqual(problems[0].nodes, ['first', 'second']);+ assert.equal(problems[0].penetrationMeters, 0.2);+});++test('a box behind the camera is named as outside the frame while the rest stays in view', () => {+ const scene = createAgentScene(boxScene([+ { id: 'seen', translation: [0, 0, 0] },+ { id: 'also-seen', translation: [1, 0, 0] },+ { id: 'behind', translation: [0, 0, 20] },+ ]));+ const problems = framingProblems({ camera: scene.camera, aspect: 1.5, boxes: scene.worldBounds() });+ assert.deepEqual(kinds(problems), ['part-outside-frame']);+ assert.deepEqual(problems[0].nodes, ['behind']);+ assert.equal(problems[0].outsideCount, 1);+ assert.equal(problems[0].partCount, 3);+});++test('placement findings are capped and say how many were left out', () => {+ const nodes = Array.from({ length: 12 }, (_, index) => ({ id: `hover-${index}`, translation: [index * 3, 1 + index, 0] }));+ const problems = placementOf([{ id: 'floor', scale: [60, 0.2, 10], translation: [15, 0, 0] }, ...nodes]);+ assert.equal(problems.length, 8);+ assert.equal(problems[0].droppedCount, 4);+});++test('a blown-out frame reports how much is clipped', () => {+ const [finding] = exposureProblems({ shares: lumaShares(frameOf(255)), exposure: 1 });+ assert.equal(finding.kind, 'exposure-clipped');+ assert.equal(finding.whitePercent, 100);+ const [black] = exposureProblems({ shares: lumaShares(frameOf(0)), exposure: 1 });+ assert.equal(black.blackPercent, 100);+});