Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Return structured look findings an agent can act on without reading the image (#905)

Nguyen Xuan Ana7bf48cb1a0f
diff --git a/changes/905.md b/changes/905.md
new file mode 100644
index 00000000..5b325f89
--- /dev/null
+++ b/changes/905.md
@@ -0,0 +1 @@
+- `scene_look` lists floating parts, intersecting parts, parts outside the camera and the share of a blown-out or black frame as findings with part names and numbers, so an agent can catch them without reading the picture (#905)
diff --git a/docs/mcp.md b/docs/mcp.md
index 8e5eb978..a16bc2c0 100644
--- a/docs/mcp.md
+++ b/docs/mcp.md
@@ -166,6 +166,17 @@ Every tool schema is read on every turn, so `node scripts/measure-tool-cost.mjs`
`mesh_inspect` lint adds warnings for `intersecting-parts`, `self-intersection`, `thin-wall`, `floating-part` and `asymmetric`. They never set `ok` to false, run within face budgets, and list any check they skipped in `skippedChecks`. `scene_inspect` returns a `diagnostics` list only when there is something to say: ignored material fields, blended transmission, unused materials or geometries, and for imported assets the decoder's not-drawn features, rigs with no joints or no skin, and assets with no clips. Each item has `kind`, `severity`, `target`, `message` and `fix`.
+`scene_look` returns the same kind of `diagnostics` list next to the picture, so an agent can check a frame without reading it. The findings carry the part names and the numbers in fields, and each list is capped (8 parts per finding, 8 findings per kind, with `nodeCount` or `droppedCount` saying how many were left out):
+
+| kind | fields | meaning |
+| --- | --- | --- |
+| `floating-node` | `nodes`, `gapMeters`, `nearest` | The part is more than 2 cm from every other part and from the floor level (the lowest point of the scene). `nearest` is the closest part or `floor`. |
+| `intersecting-nodes` | `nodes`, `penetrationMeters` | The bounds of two parts overlap by 2 cm or more on every axis. Bounds are axis aligned, so a rotated part can report an overlap its surface does not have; a part sunk into the ground is reported too. |
+| `part-outside-frame` | `nodes`, `outsideCount`, `partCount` | Parts whose bounding sphere is outside the camera frustum or behind the camera. When more than half are outside, `subject-out-of-frame` is returned with the same fields instead. |
+| `exposure-clipped`, `exposure-black` | `whitePercent`, `blackPercent` | Share of a 64 by 64 copy of the frame that is clipped to white (35 percent or more) or black (98 percent or more) after tone mapping. |
+
+The checks run on part bounds and on that small copy of the frame, so they cost almost nothing. Scenes of over 256 parts get no floating or intersection findings. `src/frame-checks.js` holds them.
+
The recipe uses `format: '67js.scene'` and `schemaVersion: 1`.
Supported geometry types are `box`, `cylinder`, `sphere`, `torus`, `lathe`, `extrude`, `tube`, `curve`, `sdf`, `text`, `grid`, `terrain`, and `mesh`. The fields of `curve` and `text` are listed by `scene_capabilities` (`curveText.details`) and not repeated in every tool schema. Text uses the bundled font only; the MCP accepts no font file or URL.
A `mesh` geometry starts from one of the others as `base` and replays an edit log; see [mesh.md](mesh.md).
diff --git a/scripts/agent-preview-browser.js b/scripts/agent-preview-browser.js
index 2fd120fb..019c535e 100644
--- a/scripts/agent-preview-browser.js
+++ b/scripts/agent-preview-browser.js
@@ -31,7 +31,7 @@ import { clipFromGltfAnimation } from '../src/skeleton-pose.js';
import { sessionProblem, SCENE_STORE_LIMITS } from '../src/agent-errors.js';
import { AGENT_SCENE_DEFAULT_LIMITS } from '../src/agent-scene-capabilities.js';
import { describeApplication } from '../src/agent-apply-report.js';
-import { framingProblems, exposureProblems, lumaShares } from '../src/frame-checks.js';
+import { framingProblems, placementProblems, exposureProblems, lumaShares } from '../src/frame-checks.js';
import { BRANCH_LIMITS, createBranchStore, printableLabel } from '../src/agent-branches.js';
import { agentAssetDecodeOptions, withCatalogLimitAdvice } from '../src/agent-asset-limits.js';
import { describeMarks, validateMarks } from '../src/agent-annotate.js';
@@ -188,7 +188,10 @@ function sampleFrameShares() {
function viewProblems(record, camera, exposure) {
const problems = [];
const boxes = record.scene.worldBounds?.();
- if (boxes) problems.push(...framingProblems({ camera, aspect: canvas.width / canvas.height, boxes }));
+ if (boxes) {
+ problems.push(...framingProblems({ camera, aspect: canvas.width / canvas.height, boxes }));
+ problems.push(...placementProblems({ boxes }));
+ }
if (record.scene.bakeStaleness) problems.push(...record.scene.bakeStaleness());
if (exposure !== undefined) problems.push(...exposureProblems({ shares: sampleFrameShares(), exposure }));
return problems;
diff --git a/src/frame-checks.js b/src/frame-checks.js
index cfe0ec20..195ba519 100644
--- a/src/frame-checks.js
+++ b/src/frame-checks.js
@@ -10,8 +10,17 @@ const WHITE_LUMA = 250 / 255;
const BLACK_FRAME_SHARE = 0.98;
const CLIPPED_FRAME_SHARE = 0.35;
-function warning(kind, target, message, fix) {
- return { kind, severity: 'warning', target, message, fix };
+const MAXIMUM_FINDING_NODES = 8;
+
+/** A finding names the parts it concerns in `nodes` and carries its numbers as extra fields. */
+function warning(kind, target, message, fix, details = {}) {
+ return { kind, severity: 'warning', target, message, fix, ...details };
+}
+
+/** At most 8 part names for a finding, with `nodeCount` when more were left out. */
+export function namedNodes(names) {
+ const nodes = names.slice(0, MAXIMUM_FINDING_NODES);
+ return names.length > nodes.length ? { nodes, nodeCount: names.length } : { nodes };
}
function subtract(first, second) {
@@ -86,26 +95,40 @@ export function framingProblems({ camera, aspect, boxes }) {
const upward = cross(right, forward);
const sideScale = Math.sqrt(1 + tangent * tangent);
const wideScale = Math.sqrt(1 + tangentWide * tangentWide);
+ const outside = [];
const visibleBoxes = boxes.filter(box => {
const { center, radius } = sphereOf(box);
const offset = subtract(center, camera.eye);
const depth = dot(offset, forward);
- if (depth + radius < camera.near || depth - radius > camera.far) return false;
+ if (depth + radius < camera.near || depth - radius > camera.far) {
+ outside.push(box.name);
+ return false;
+ }
const sideways = (Math.abs(dot(offset, right)) - tangentWide * depth) / wideScale;
const vertical = (Math.abs(dot(offset, upward)) - tangent * depth) / sideScale;
- return sideways <= radius && vertical <= radius;
+ const inView = sideways <= radius && vertical <= radius;
+ if (!inView) outside.push(box.name);
+ return inView;
});
+ const outsideDetails = { ...namedNodes(outside), outsideCount: outside.length, partCount: boxes.length };
if (visibleBoxes.length * 2 < boxes.length) {
return [warning('subject-out-of-frame', 'camera',
`${boxes.length - visibleBoxes.length} of ${boxes.length} parts are outside the camera frustum.`,
- `${framingSuggestion(camera, boxes, tangent)} to put every part in view.`)];
+ `${framingSuggestion(camera, boxes, tangent)} to put every part in view.`, outsideDetails)];
}
const { center, radius } = sphereOf(unionOf(visibleBoxes));
const depth = dot(subtract(center, camera.eye), forward);
if (depth > 0 && radius / (depth * tangent) < TINY_SUBJECT_RATIO) {
return [warning('subject-too-small', 'camera',
`The visible parts fill under ${Math.round(TINY_SUBJECT_RATIO * 100)} percent of the frame height.`,
- `${framingSuggestion(camera, visibleBoxes, tangent)} to fill the frame, or lower camera.fov.`)];
+ `${framingSuggestion(camera, visibleBoxes, tangent)} to fill the frame, or lower camera.fov.`,
+ { heightPercent: Math.round(radius / (depth * tangent) * 100) })];
+ }
+ if (outside.length > 0) {
+ return [warning('part-outside-frame', 'camera',
+ `${outside.length} of ${boxes.length} parts are outside the camera frustum, so the picture does not show them.`,
+ `Move them into view with scene_apply set-node, or ${framingSuggestion(camera, boxes, tangent)} to put every part in view.`,
+ outsideDetails)];
}
return [];
}
@@ -123,21 +146,89 @@ export function lumaShares(rgba) {
return { black: black / pixels, white: white / pixels };
}
+function percents(shares) {
+ return { blackPercent: Math.round(shares.black * 100), whitePercent: Math.round(shares.white * 100) };
+}
+
/** Warnings for a frame that is nearly all black or has a large clipped area; `exposure` is the current effects.exposure. */
export function exposureProblems({ shares, exposure }) {
if (shares.black >= BLACK_FRAME_SHARE) {
return [warning('exposure-black', 'effects.exposure',
`${Math.round(shares.black * 100)} percent of the frame is black after tone mapping.`,
- `Raise effects.exposure from ${exposure} to ${round(Math.max(exposure, 0.25) * 4)}, raise environment.keyIntensity, or aim a light at the subject.`)];
+ `Raise effects.exposure from ${exposure} to ${round(Math.max(exposure, 0.25) * 4)}, raise environment.keyIntensity, or aim a light at the subject.`, percents(shares))];
}
if (shares.white >= CLIPPED_FRAME_SHARE) {
return [warning('exposure-clipped', 'effects.exposure',
`${Math.round(shares.white * 100)} percent of the frame is clipped to white after tone mapping.`,
- `Lower effects.exposure from ${exposure} to ${round(exposure / 2)}, or lower environment.keyIntensity.`)];
+ `Lower effects.exposure from ${exposure} to ${round(exposure / 2)}, or lower environment.keyIntensity.`, percents(shares))];
}
return [];
}
+const FLOATING_GAP = 0.02;
+const MINIMUM_PENETRATION = 0.02;
+const MAXIMUM_PAIR_PARTS = 256;
+const MAXIMUM_PLACEMENT_FINDINGS = 8;
+
+/** Distance between two boxes along each axis; 0 where they touch or overlap on that axis. */
+function axisGaps(first, second) {
+ return [0, 1, 2].map(axis => Math.max(first.min[axis] - second.max[axis], second.min[axis] - first.max[axis], 0));
+}
+
+/** How far two boxes overlap along each axis, or null when they are apart on any axis. */
+function axisPenetrations(first, second) {
+ const depths = [0, 1, 2].map(axis => Math.min(first.max[axis], second.max[axis]) - Math.max(first.min[axis], second.min[axis]));
+ return depths.every(depth => depth > 0) ? depths : null;
+}
+
+function droppedCount(total) {
+ return total > MAXIMUM_PLACEMENT_FINDINGS ? { droppedCount: total - MAXIMUM_PLACEMENT_FINDINGS } : {};
+}
+
+/**
+ * Findings about where parts sit, from world bounds ({ name, min, max }, y up): a part with nothing within 2 cm of it
+ * that is above the lowest point of the scene (`floating-node`, `gapMeters` to the nearest part or floor), and two
+ * parts whose bounds overlap by 2 cm or more on every axis (`intersecting-nodes`, `penetrationMeters`). Bounds are
+ * axis aligned, so a rotated part may report an overlap its surface does not have. Each kind lists at most 8 and says
+ * how many more there were in `droppedCount`. Scenes of over 256 parts are not checked.
+ */
+export function placementProblems({ boxes }) {
+ if (boxes.length < 1 || boxes.length > MAXIMUM_PAIR_PARTS) return [];
+ const floorHeight = Math.min(...boxes.map(box => box.min[1]));
+ const floating = [];
+ const intersecting = [];
+ boxes.forEach((box, index) => {
+ let nearest = { gap: box.min[1] - floorHeight, name: 'floor' };
+ for (let otherIndex = 0; otherIndex < boxes.length; otherIndex++) {
+ if (otherIndex === index) continue;
+ const other = boxes[otherIndex];
+ const gap = Math.hypot(...axisGaps(box, other));
+ if (gap < nearest.gap) nearest = { gap, name: other.name };
+ const penetrations = otherIndex > index ? axisPenetrations(box, other) : null;
+ if (penetrations && Math.min(...penetrations) >= MINIMUM_PENETRATION) {
+ intersecting.push({ nodes: [box.name, other.name], penetrationMeters: round(Math.min(...penetrations)) });
+ }
+ }
+ if (nearest.gap > FLOATING_GAP) floating.push({ node: box.name, gapMeters: round(nearest.gap), nearest: nearest.name });
+ });
+ const problems = [];
+ const floatingWorstFirst = floating.sort((first, second) => second.gapMeters - first.gapMeters);
+ for (const entry of floatingWorstFirst.slice(0, MAXIMUM_PLACEMENT_FINDINGS)) {
+ problems.push(warning('floating-node', entry.node,
+ `${entry.node} touches nothing: it is ${entry.gapMeters} m from ${entry.nearest === 'floor' ? 'the floor level' : entry.nearest}.`,
+ 'Lower it with scene_apply set-node, or give it a relation that rests it on a part, unless it is meant to fly.',
+ { nodes: [entry.node], gapMeters: entry.gapMeters, nearest: entry.nearest, ...droppedCount(floating.length) }));
+ }
+ const intersectingDeepestFirst = intersecting.sort((first, second) => second.penetrationMeters - first.penetrationMeters);
+ for (const entry of intersectingDeepestFirst.slice(0, MAXIMUM_PLACEMENT_FINDINGS)) {
+ problems.push(warning('intersecting-nodes', entry.nodes[0],
+ `${entry.nodes[0]} and ${entry.nodes[1]} overlap by ${entry.penetrationMeters} m inside their bounds.`,
+ 'Move one of them apart with scene_apply set-node, or ignore it where the overlap is meant, such as a part sunk into the ground.',
+ { nodes: entry.nodes, penetrationMeters: entry.penetrationMeters, ...droppedCount(intersecting.length) }));
+ }
+ return problems;
+}
+
/** Adds a frame's diagnostics to `metrics.diagnostics`, keeping one entry per kind, target and message across a job's frames. */
export function mergeFrameDiagnostics(metrics, frameDiagnostics) {
if (!frameDiagnostics?.length) return;
diff --git a/tests/frame-checks.test.js b/tests/frame-checks.test.js
index 6200cbc4..8dd354e9 100644
--- a/tests/frame-checks.test.js
+++ b/tests/frame-checks.test.js
@@ -2,7 +2,7 @@ import './support/load-agent-scene-modules.mjs';
import assert from 'node:assert/strict';
import test from 'node:test';
import { createAgentScene } from '../src/agent-scene.js';
-import { exposureProblems, framingProblems, lumaShares, mergeFrameDiagnostics } from '../src/frame-checks.js';
+import { exposureProblems, framingProblems, lumaShares, mergeFrameDiagnostics, placementProblems } from '../src/frame-checks.js';
import { closedRoom, closedRoomWithoutBake } from './support/agent-bake-rooms.mjs';
function sceneWithCamera(camera) {
@@ -111,3 +111,75 @@ test('job metrics keep one entry per warning across frames', () => {
mergeFrameDiagnostics(metrics, undefined);
assert.equal(metrics.diagnostics.length, 1);
});
+
+function boxScene(nodes) {
+ return {
+ format: '67js.scene', schemaVersion: 1,
+ geometries: [{ id: 'box', type: 'box' }],
+ materials: [{ id: 'red', color: [0.8, 0.2, 0.2] }],
+ nodes: nodes.map(node => ({ geometry: 'box', material: 'red', ...node })),
+ camera: { eye: [0, 3, 8], target: [0, 0.5, 0], fov: 45 },
+ };
+}
+
+function placementOf(nodes) {
+ return placementProblems({ boxes: createAgentScene(boxScene(nodes)).worldBounds() });
+}
+
+test('a box in the air is a floating part with its gap in metres and the nearest part', () => {
+ const problems = placementOf([
+ { id: 'floor', scale: [10, 0.2, 10], translation: [0, 0, 0] },
+ { id: 'hover', translation: [0, 1.1, 0] },
+ ]);
+ assert.deepEqual(kinds(problems), ['floating-node']);
+ assert.deepEqual(problems[0].nodes, ['hover']);
+ assert.equal(problems[0].nearest, 'floor');
+ assert.equal(problems[0].gapMeters, 0.5);
+});
+
+test('a box resting on the floor or on another box is not floating', () => {
+ assert.deepEqual(placementOf([
+ { id: 'floor', scale: [10, 0.2, 10], translation: [0, 0, 0] },
+ { id: 'rest', translation: [0, 0.6, 0] },
+ { id: 'stacked', scale: [0.5, 0.5, 0.5], translation: [0, 1.35, 0] },
+ ]), []);
+});
+
+test('two boxes that overlap name both parts and how deep', () => {
+ const problems = placementOf([
+ { id: 'floor', scale: [10, 0.2, 10], translation: [0, 0, 0] },
+ { id: 'first', translation: [-0.4, 0.6, 0] },
+ { id: 'second', translation: [0.4, 0.6, 0] },
+ ]);
+ assert.deepEqual(kinds(problems), ['intersecting-nodes']);
+ assert.deepEqual(problems[0].nodes, ['first', 'second']);
+ assert.equal(problems[0].penetrationMeters, 0.2);
+});
+
+test('a box behind the camera is named as outside the frame while the rest stays in view', () => {
+ const scene = createAgentScene(boxScene([
+ { id: 'seen', translation: [0, 0, 0] },
+ { id: 'also-seen', translation: [1, 0, 0] },
+ { id: 'behind', translation: [0, 0, 20] },
+ ]));
+ const problems = framingProblems({ camera: scene.camera, aspect: 1.5, boxes: scene.worldBounds() });
+ assert.deepEqual(kinds(problems), ['part-outside-frame']);
+ assert.deepEqual(problems[0].nodes, ['behind']);
+ assert.equal(problems[0].outsideCount, 1);
+ assert.equal(problems[0].partCount, 3);
+});
+
+test('placement findings are capped and say how many were left out', () => {
+ const nodes = Array.from({ length: 12 }, (_, index) => ({ id: `hover-${index}`, translation: [index * 3, 1 + index, 0] }));
+ const problems = placementOf([{ id: 'floor', scale: [60, 0.2, 10], translation: [15, 0, 0] }, ...nodes]);
+ assert.equal(problems.length, 8);
+ assert.equal(problems[0].droppedCount, 4);
+});
+
+test('a blown-out frame reports how much is clipped', () => {
+ const [finding] = exposureProblems({ shares: lumaShares(frameOf(255)), exposure: 1 });
+ assert.equal(finding.kind, 'exposure-clipped');
+ assert.equal(finding.whitePercent, 100);
+ const [black] = exposureProblems({ shares: lumaShares(frameOf(0)), exposure: 1 });
+ assert.equal(black.blackPercent, 100);
+});