3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Add a lasso mark to scene_ask_user (#922)
Nguyen Xuan An9f6e91ecdca9
diff --git a/bench/agent-modeling/README.md b/bench/agent-modeling/README.mdindex de7c2d3c..65c71b0c 100644--- a/bench/agent-modeling/README.md+++ b/bench/agent-modeling/README.md@@ -149,6 +149,9 @@ absolute `.json` path under 64 KB: `{"marks":[{"pixel":[x,y],"note":"..."}]}` wiat most 280 characters (an optional `motionTime` is checked and ignored). The marks go through the same pick as a browser answer, so the agentgets the part, point and normal under the pixel. `ask-user-pin` pins the statue of the stage scene with the note "make this red"; its checkspass when the statue's material is red and the plinth is not recoloured. A test pins the fixture pixel to the statue at the stage camera.+A mark may carry `"lasso":[[x,y],...]` (3 to 64 points inside the frame) instead of `pixel`; the answer then lists the parts under the outline.+`ask-user-lasso` (`ask-user-lasso.json`) outlines the statue and the plinth with the note "make these blue"; its checks pass when both are blue+and the floor, which shares the plinth's stone material, is not.### Several runs and several modelsdiff --git a/bench/agent-modeling/ask-user-lasso.json b/bench/agent-modeling/ask-user-lasso.jsonnew file mode 100644index 00000000..c93a27bc--- /dev/null+++ b/bench/agent-modeling/ask-user-lasso.json@@ -0,0 +1,5 @@+{+ "marks": [+ { "lasso": [[372, 222], [396, 222], [396, 270], [416, 274], [416, 314], [356, 314], [356, 274], [372, 270]], "note": "make these blue" }+ ]+}diff --git a/bench/agent-modeling/scene-checks.mjs b/bench/agent-modeling/scene-checks.mjsindex c1958c4c..008bb50d 100644--- a/bench/agent-modeling/scene-checks.mjs+++ b/bench/agent-modeling/scene-checks.mjs@@ -131,6 +131,11 @@ function isRed([red, green, blue]) {return red >= 0.6 && red >= 2 * green && red >= 2 * blue;}+// Blue: a strong blue channel, at least twice the red and the green.+function isBlue([red, green, blue]) {+ return blue >= 0.6 && blue >= 2 * red && blue >= 2 * green;+}+function materialOf(check, recipe) {const node = recipe.nodes.find(candidate => candidate.id === check.node);const material = recipe.materials.find(candidate => candidate.id === node?.material);@@ -141,6 +146,7 @@ function materialOf(check, recipe) {if (check.roughness && !inRange(roughness, check.roughness)) problems.push(`roughness ${roughness}`);if (check.warm && !(color[0] > color[2])) problems.push('colour is not warm');if (check.red !== undefined && isRed(color) !== check.red) problems.push(check.red ? `colour ${color.map(round)} is not red` : `colour ${color.map(round)} is red`);+ if (check.blue !== undefined && isBlue(color) !== check.blue) problems.push(check.blue ? `colour ${color.map(round)} is not blue` : `colour ${color.map(round)} is blue`);return { pass: problems.length === 0, detail: problems.join('; ') || `metallic ${metallic}, roughness ${roughness}` };}diff --git a/bench/agent-modeling/scenes.mjs b/bench/agent-modeling/scenes.mjsindex 43e631c8..96b853eb 100644--- a/bench/agent-modeling/scenes.mjs+++ b/bench/agent-modeling/scenes.mjs@@ -93,6 +93,15 @@ const ASK_USER_PIN_CHECKS = [{ kind: 'materialOf', node: 'plinth', red: false, label: 'the plinth is not recoloured' },];+// The plinth and the floor share the stone material, so colouring the plinth alone needs a material of its own.+const ASK_USER_LASSO_CHECKS = [+ { kind: 'lint' },+ { kind: 'parts', min: 3, max: 3 },+ { kind: 'materialOf', node: 'statue', blue: true, label: 'the lassoed statue is blue' },+ { kind: 'materialOf', node: 'plinth', blue: true, label: 'the lassoed plinth is blue' },+ { kind: 'materialOf', node: 'floor', blue: false, label: 'the floor outside the lasso is not recoloured' },+];+export const SCENE_TASKS = [{id: 'camera-closeup',@@ -171,6 +180,16 @@ colour so it turns warmer (red up, blue down) between time 0 and time 4. ${FINISRead their pin with scene_ask_user (it returns the part under the pin and their note), then make that change to that part and nothing else. ${FINISH}`,checks: ASK_USER_PIN_CHECKS,},+ {+ // The answer comes from bench/agent-modeling/ask-user-lasso.json, drawn around the statue and the plinth.+ id: 'ask-user-lasso',+ askUserFixture: 'ask-user-lasso.json',+ startRecipe: stageRecipe(),+ ownPreamble: true,+ prompt: `${PLAIN_PREAMBLE} ${STAGE} A person has drawn a lasso around a region of the picture of this scene and left a note about a change to make there.+Read their lasso with scene_ask_user (it returns the parts under the outline and their note), then make that change to those parts and nothing else. ${FINISH}`,+ checks: ASK_USER_LASSO_CHECKS,+ },{id: 'film-reveal',startRecipe: stageRecipe(),@@ -239,6 +258,13 @@ export const SCENE_REFERENCES = {recipe.materials = recipe.materials.map(material => (material.id === 'bronze' ? { id: 'bronze', color: [0.85, 0.12, 0.1] } : material));return recipe;})(),+ 'ask-user-lasso': (() => {+ const recipe = stageRecipe();+ recipe.materials = recipe.materials.map(material => (material.id === 'bronze' ? { id: 'bronze', color: [0.1, 0.2, 0.85] } : material));+ recipe.materials.push({ id: 'plinth-stone', color: [0.1, 0.2, 0.85] });+ recipe.nodes = recipe.nodes.map(node => (node.id === 'plinth' ? { ...node, material: 'plinth-stone' } : node));+ return recipe;+ })(),'film-reveal': stageRecipe({motion: {duration: 6,diff --git a/bench/agent-modeling/suite.mjs b/bench/agent-modeling/suite.mjsindex d268a30c..1fc5ae59 100644--- a/bench/agent-modeling/suite.mjs+++ b/bench/agent-modeling/suite.mjs@@ -28,6 +28,7 @@ export const BENCH_TASKS = [{ id: 'look-metal', kind: 'look' },{ id: 'look-fog', kind: 'look' },{ id: 'ask-user-pin', kind: 'look', topics: ['askUser'] },+ { id: 'ask-user-lasso', kind: 'look', topics: ['askUser'] },{ id: 'render-pick', kind: 'light' },{ id: 'light-pick-hard', kind: 'light' },{ id: 'light-key-fill', kind: 'light' },diff --git a/changes/922-lasso.md b/changes/922-lasso.mdnew file mode 100644index 00000000..65a00fcf--- /dev/null+++ b/changes/922-lasso.md@@ -0,0 +1 @@+- `scene_ask_user`: the annotate page has a lasso tool; a lasso mark comes back with `kind` `"lasso"`, its `points` and the `parts` under the outline (most covered first, with `coverage`), and the test-only answer fixture accepts `lasso` marks (#922).diff --git a/docs/mcp.md b/docs/mcp.mdindex 9c445eb9..ad4b84a3 100644--- a/docs/mcp.md+++ b/docs/mcp.md@@ -221,7 +221,9 @@ Set operations use `set-node`, `set-material`, or `set-camera`, with fields direAn explicit `null` restores the authored base pose. Release `0.8.0` also supports imported source-node motion.Imported clip sampling remains available through `set-animation`.-`scene_ask_user` is for a change the person cannot put in words. It renders the current view, opens it in their default browser on a loopback address under a random path, and waits up to `timeoutSeconds` (default 300, at most 900) for them to pin up to 16 points with a note of at most 280 characters each and press Send. Each mark comes back with its pixel, the `partId` and world `point` and `normal` the pin lands on (all `null` on empty sky), the `distance` from the camera and the part's `material`, plus the `view` camera and a PNG with the numbered pins. A cancelled or timed-out page returns `outcome` `cancelled` or `timeout` with no marks. Nothing leaves the machine and the tool takes no code or remote URL. Not yet marks: lasso regions, dragged transforms, colours, saved views and film times (issue #922 lists them).+`scene_ask_user` is for a change the person cannot put in words. It renders the current view, opens it in their default browser on a loopback address under a random path, and waits up to `timeoutSeconds` (default 300, at most 900) for them to pin up to 16 points with a note of at most 280 characters each and press Send. Each mark comes back with its pixel, the `partId` and world `point` and `normal` the pin lands on (all `null` on empty sky), the `distance` from the camera and the part's `material`, plus the `view` camera and a PNG with the numbered pins. A cancelled or timed-out page returns `outcome` `cancelled` or `timeout` with no marks. Nothing leaves the machine and the tool takes no code or remote URL.++The page also has a lasso tool: the person draws a closed outline around a region (3 to 64 points, thinned from the pen stroke) and adds a note, so one mark can mean "these parts". Such a mark counts toward the 16 and comes back with the pin fields plus `kind` `"lasso"`, `points` and `parts`. `parts` lists the parts under the outline, most covered first, each as `{ partId, samples, coverage, material }`: the server picks a fixed 24 by 24 grid of pixels over the outline's bounding box, keeps the ones inside it (even-odd rule) and runs the same pick as a pin on each, so `coverage` is the share of those pixels a part holds (pixels on empty sky count toward no part, so the shares may add up to less than 1). `pixel`, `partId`, `point`, `normal`, `distance` and `material` are those of the most covered part, taken at the picked pixel nearest the outline's centre; `parts` is empty and `partId` `null` when the outline holds only sky. A sliver of a part behind the region can show up with a small `coverage`: read it before acting on every part listed. The inline chat view and the review page take pins only. Not yet marks: dragged transforms, colours, saved views and film times (issue #922 lists them).#### Inline in chat apps (MCP Apps)diff --git a/scripts/annotate-session.mjs b/scripts/annotate-session.mjsindex 78cedc3f..c825ae62 100644--- a/scripts/annotate-session.mjs+++ b/scripts/annotate-session.mjs@@ -16,6 +16,12 @@ h1{margin:0;font-size:18px;font-weight:600}main{display:flex;gap:20px;align-items:flex-start;flex-wrap:wrap}.stage{position:relative;line-height:0;flex:0 1 ${width}px;max-width:100%;cursor:crosshair;user-select:none}.stage img{width:100%;height:auto;border-radius:8px;display:block}+.stage.lasso{cursor:cell;touch-action:none}+.outline{position:absolute;inset:0;width:100%;height:100%;pointer-events:none;overflow:visible}+.outline polygon,.outline polyline{fill:rgba(255,90,79,.22);stroke:#ff5a4f;stroke-width:2;stroke-linejoin:round;vector-effect:non-scaling-stroke}+.outline polyline{fill:none;stroke:#fff}+.tools{display:flex;gap:4px}+button[aria-pressed=true]{background:#26343b;box-shadow:inset 0 0 0 2px #6fb7ff}.pin{position:absolute;width:26px;height:26px;margin:-13px 0 0 -13px;border-radius:50%;background:#ff5a4f;color:#fff;font:600 13px/26px system-ui,sans-serif;text-align:center;box-shadow:0 0 0 2px #fff;pointer-events:none}aside{flex:1 1 280px;max-width:380px;display:flex;flex-direction:column;gap:12px;user-select:none}.row{display:flex;gap:10px;align-items:center}@@ -44,13 +50,17 @@ function pageHtml({ prompt, width, height, maxMarks, maxNoteLength }) {<main><div class="stage" id="stage"><img id="frame" src="frame.png" width="${width}" height="${height}" alt="Rendered scene"></div><aside id="side">-<div class="hint" id="hint">Click the picture to drop a pin, then say what to change there.</div>+<div class="tools" role="group" aria-label="Mark type">+<button class="icon" id="tool-pin" aria-pressed="true" aria-label="Pin" title="Pin: click a spot"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 21s-6-5.5-6-11a6 6 0 0 1 12 0c0 5.500-6 11-6 11z"/><circle cx="12" cy="10" r="2"/></svg></button>+<button class="icon" id="tool-lasso" aria-pressed="false" aria-label="Lasso" title="Lasso: draw around a region"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 4C7 4 3 6.500 3 10s4 6 9 6 9-2.500 9-6-4-6-9-6z"/><path d="M9 16c-1 2-1 3.500 1 4"/></svg></button>+</div>+<div class="hint" id="hint">Click the picture to drop a pin, or draw around a region with the lasso, then say what to change.</div><div id="rows" style="display:flex;flex-direction:column;gap:8px"></div><div class="count" id="count"></div><div class="actions"><button class="primary" id="send">Send</button><button id="cancel">Cancel</button></div></aside></main>-<script type="application/json" id="limits">${JSON.stringify({ width, height, maxMarks, maxNoteLength })}</script>+<script type="application/json" id="limits">${JSON.stringify({ width, height, maxMarks, maxNoteLength, minLassoPoints: ANNOTATE_LIMITS.minLassoPoints, maxLassoPoints: ANNOTATE_LIMITS.maxLassoPoints })}</script><script src="app.js"></script></body></html>`;}@@ -65,15 +75,51 @@ const rows = document.getElementById('rows');const count = document.getElementById('count');const sendButton = document.getElementById('send');const trash = '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 6h18M8 6V4h8v2M6 6l1 14h10l1-14"/></svg>';+const svgNamespace = 'http://www.w3.org/2000/svg';+const outline = document.createElementNS(svgNamespace, 'svg');+outline.setAttribute('class', 'outline');+outline.setAttribute('viewBox', '0 0 ' + limits.width + ' ' + limits.height);+outline.setAttribute('preserveAspectRatio', 'none');+stage.appendChild(outline);+let tool = 'pin';+let drawing = null;+const toolButtons = { pin: document.getElementById('tool-pin'), lasso: document.getElementById('tool-lasso') };+function chooseTool(name) {+ tool = name;+ stage.classList.toggle('lasso', name === 'lasso');+ Object.entries(toolButtons).forEach(([key, button]) => button.setAttribute('aria-pressed', String(key === name)));+}+toolButtons.pin.addEventListener('click', () => chooseTool('pin'));+toolButtons.lasso.addEventListener('click', () => chooseTool('lasso'));+function pointsText(points) {+ return points.map(point => point[0].toFixed(1) + ',' + point[1].toFixed(1)).join(' ');+}+function markCentre(mark) {+ if (mark.points === undefined) return [mark.x, mark.y];+ const total = mark.points.reduce((sum, point) => [sum[0] + point[0], sum[1] + point[1]], [0, 0]);+ return [total[0] / mark.points.length, total[1] / mark.points.length];+}+// A pen stroke has hundreds of points; the answer allows maxLassoPoints, so keep evenly spaced ones.+function thinned(points) {+ if (points.length <= limits.maxLassoPoints) return points;+ return Array.from({ length: limits.maxLassoPoints }, (unused, index) => points[Math.floor(index * points.length / limits.maxLassoPoints)]);+}function render() {stage.querySelectorAll('.pin').forEach(pin => pin.remove());+ outline.textContent = '';rows.textContent = '';marks.forEach((mark, index) => {+ const centre = markCentre(mark);+ if (mark.points !== undefined) {+ const shape = document.createElementNS(svgNamespace, 'polygon');+ shape.setAttribute('points', pointsText(mark.points));+ outline.appendChild(shape);+ }const pin = document.createElement('div');pin.className = 'pin';pin.textContent = index + 1;- pin.style.left = (mark.x / limits.width * 100) + '%';- pin.style.top = (mark.y / limits.height * 100) + '%';+ pin.style.left = (centre[0] / limits.width * 100) + '%';+ pin.style.top = (centre[1] / limits.height * 100) + '%';stage.appendChild(pin);const row = document.createElement('div');row.className = 'row';@@ -87,26 +133,64 @@ function render() {input.addEventListener('input', () => { mark.note = input.value; });const remove = document.createElement('button');remove.className = 'icon';- remove.title = 'Remove pin';- remove.setAttribute('aria-label', 'Remove pin ' + (index + 1));+ remove.title = 'Remove mark';+ remove.setAttribute('aria-label', 'Remove mark ' + (index + 1));remove.innerHTML = trash;remove.addEventListener('click', () => { marks.splice(index, 1); render(); });row.append(badge, input, remove);rows.appendChild(row);mark.input = input;});- count.textContent = marks.length + ' of ' + limits.maxMarks + ' pins';+ count.textContent = marks.length + ' of ' + limits.maxMarks + ' marks';sendButton.disabled = marks.length === 0;}-stage.addEventListener('click', event => {- if (marks.length >= limits.maxMarks) return;+function pixelOf(event) {const box = frame.getBoundingClientRect();- const x = (event.clientX - box.left) / box.width * limits.width;- const y = (event.clientY - box.top) / box.height * limits.height;+ const x = Math.min(limits.width, Math.max(0, (event.clientX - box.left) / box.width * limits.width));+ const y = Math.min(limits.height, Math.max(0, (event.clientY - box.top) / box.height * limits.height));+ return [x, y];+}+function polygonArea(points) {+ let twice = 0;+ points.forEach((point, index) => {+ const next = points[(index + 1) % points.length];+ twice += point[0] * next[1] - next[0] * point[1];+ });+ return Math.abs(twice) / 2;+}+stage.addEventListener('click', event => {+ if (tool !== 'pin' || marks.length >= limits.maxMarks) return;+ const [x, y] = pixelOf(event);marks.push({ x, y, note: '' });render();marks[marks.length - 1].input.focus();});+stage.addEventListener('pointerdown', event => {+ if (tool !== 'lasso' || marks.length >= limits.maxMarks || event.button !== 0) return;+ event.preventDefault();+ stage.setPointerCapture(event.pointerId);+ const trail = document.createElementNS(svgNamespace, 'polyline');+ outline.appendChild(trail);+ drawing = { points: [pixelOf(event)], trail };+});+stage.addEventListener('pointermove', event => {+ if (!drawing) return;+ const point = pixelOf(event);+ const last = drawing.points[drawing.points.length - 1];+ if (Math.hypot(point[0] - last[0], point[1] - last[1]) < 3) return;+ drawing.points.push(point);+ drawing.trail.setAttribute('points', pointsText(drawing.points));+});+function endLasso() {+ if (!drawing) return;+ const points = thinned(drawing.points);+ drawing = null;+ if (points.length >= limits.minLassoPoints && polygonArea(points) >= 1) marks.push({ kind: 'lasso', points, note: '' });+ render();+ if (marks.length && marks[marks.length - 1].points === points) marks[marks.length - 1].input.focus();+}+stage.addEventListener('pointerup', endLasso);+stage.addEventListener('pointercancel', endLasso);function markedImage() {const canvas = document.createElement('canvas');canvas.width = limits.width;@@ -116,16 +200,28 @@ function markedImage() {context.font = '600 15px system-ui, sans-serif';context.textAlign = 'center';context.textBaseline = 'middle';+ marks.forEach(mark => {+ if (mark.points === undefined) return;+ context.beginPath();+ mark.points.forEach((point, index) => (index === 0 ? context.moveTo(point[0], point[1]) : context.lineTo(point[0], point[1])));+ context.closePath();+ context.fillStyle = 'rgba(255,90,79,0.22)';+ context.fill();+ context.lineWidth = 2;+ context.strokeStyle = '#ff5a4f';+ context.stroke();+ });marks.forEach((mark, index) => {+ const [centreX, centreY] = markCentre(mark);context.beginPath();- context.arc(mark.x, mark.y, 14, 0, Math.PI * 2);+ context.arc(centreX, centreY, 14, 0, Math.PI * 2);context.fillStyle = '#ff5a4f';context.fill();context.lineWidth = 2;context.strokeStyle = '#fff';context.stroke();context.fillStyle = '#fff';- context.fillText(String(index + 1), mark.x, mark.y + 1);+ context.fillText(String(index + 1), centreX, centreY + 1);});return canvas.toDataURL('image/png').split(',')[1];}@@ -142,7 +238,7 @@ async function post(path, body) {sendButton.addEventListener('click', async () => {sendButton.disabled = true;try {- await post('send', { marks: marks.map(({ x, y, note }) => ({ x, y, note })), image: markedImage() });+ await post('send', { marks: marks.map(mark => (mark.points === undefined ? { x: mark.x, y: mark.y, note: mark.note } : { kind: 'lasso', points: mark.points, note: mark.note })), image: markedImage() });finish('ok', 'Sent to the agent. You can close this tab.');} catch (error) {sendButton.disabled = false;diff --git a/scripts/ask-user-fixture.mjs b/scripts/ask-user-fixture.mjsindex e3357a66..28217c63 100644--- a/scripts/ask-user-fixture.mjs+++ b/scripts/ask-user-fixture.mjs@@ -39,21 +39,40 @@ function readFixtureText(filePath) {}}+function isPixelPair(value) {+ return Array.isArray(value) && value.length === 2+ && value.every(coordinate => Number.isFinite(coordinate) && coordinate >= 0 && coordinate <= ASK_USER_FIXTURE_LIMITS.maxPixel);+}++function validatePixel(pixel, where) {+ if (!isPixelPair(pixel)) throw fixtureError(`${where} pixel must be [x, y], two numbers from 0 to ${ASK_USER_FIXTURE_LIMITS.maxPixel}. Or give lasso [[x, y], ...] instead.`);+}++function validateLassoPoints(lasso, where) {+ const { minLassoPoints, maxLassoPoints } = ANNOTATE_LIMITS;+ if (!Array.isArray(lasso) || lasso.length < minLassoPoints || lasso.length > maxLassoPoints) {+ throw fixtureError(`${where} lasso must be an array of ${minLassoPoints} to ${maxLassoPoints} [x, y] points.`);+ }+ const badIndex = lasso.findIndex(point => !isPixelPair(point));+ if (badIndex >= 0) throw fixtureError(`${where} lasso point ${badIndex + 1} must be [x, y], two numbers from 0 to ${ASK_USER_FIXTURE_LIMITS.maxPixel}.`);+}+function validateFixtureMark(mark, index, name) {const where = `${name} mark ${index + 1}`;if (!mark || typeof mark !== 'object' || Array.isArray(mark)) throw fixtureError(`${where} must be an object with pixel and note.`);- const { pixel, note = '', motionTime } = mark;- const validPixel = Array.isArray(pixel) && pixel.length === 2- && pixel.every(value => Number.isFinite(value) && value >= 0 && value <= ASK_USER_FIXTURE_LIMITS.maxPixel);- if (!validPixel) throw fixtureError(`${where} pixel must be [x, y], two numbers from 0 to ${ASK_USER_FIXTURE_LIMITS.maxPixel}.`);+ const { pixel, lasso, note = '', motionTime } = mark;+ if (pixel !== undefined && lasso !== undefined) throw fixtureError(`${where} has both pixel and lasso. Keep one of them.`);+ if (lasso !== undefined) validateLassoPoints(lasso, where);+ else validatePixel(pixel, where);if (typeof note !== 'string' || note.length > ANNOTATE_LIMITS.maxNoteLength) {throw fixtureError(`${where} note must be text of at most ${ANNOTATE_LIMITS.maxNoteLength} characters.`);}if (motionTime !== undefined && !(Number.isFinite(motionTime) && motionTime >= 0 && motionTime <= ASK_USER_FIXTURE_LIMITS.maxMotionTime)) {throw fixtureError(`${where} motionTime must be a number of seconds from 0 to ${ASK_USER_FIXTURE_LIMITS.maxMotionTime}.`);}- // Marks of the browser flow are { x, y, note }; motionTime is checked and then ignored, the pick is of the still frame.- return { x: pixel[0], y: pixel[1], note };+ // Marks of the browser flow are { x, y, note } or { kind: 'lasso', points, note }; motionTime is checked and then ignored, the+ // pick is of the still frame.+ return lasso === undefined ? { x: pixel[0], y: pixel[1], note } : { kind: 'lasso', points: lasso, note };}/**diff --git a/src/agent-annotate.js b/src/agent-annotate.jsindex fb431e12..694c444f 100644--- a/src/agent-annotate.js+++ b/src/agent-annotate.js@@ -1,6 +1,7 @@// Marks a person leaves on a rendered view for scene_ask_user (issue #922): pure helpers shared by the preview page and the-// local annotate server. A mark is a pixel and a short note; the page turns each pixel into a ray and the scene query turns-// the ray into a part, a world point and a surface normal.+// local annotate server. A pin is a pixel and a short note; the page turns each pixel into a ray and the scene query turns+// the ray into a part, a world point and a surface normal. A lasso is a closed outline and a note; the pick runs on a grid of+// pixels inside it and the answer lists the parts those pixels land on.import { mat4, vec4 } from 'gl-matrix';import { applyProjection } from './camera.js';@@ -11,6 +12,9 @@ export const ANNOTATE_LIMITS = Object.freeze({defaultTimeoutSeconds: 300,maxTimeoutSeconds: 900,maxImageBytes: 1024 * 1024,+ minLassoPoints: 3,+ maxLassoPoints: 64,+ lassoSamplesPerAxis: 24,});const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47];@@ -19,25 +23,81 @@ function invalidMarks(detail) {return new TypeError(`The annotate marks are invalid: ${detail}`);}+function roundedPixel(value) {+ return Math.round(value * 10) / 10;+}++function validNote(note, index) {+ if (typeof note !== 'string' || note.length > ANNOTATE_LIMITS.maxNoteLength) {+ throw invalidMarks(`mark ${index + 1} note must be text of at most ${ANNOTATE_LIMITS.maxNoteLength} characters.`);+ }+ return note.trim();+}++function validatePinMark(mark, index, { width, height }) {+ const { x, y, note = '' } = mark;+ if (!Number.isFinite(x) || x < 0 || x > width) throw invalidMarks(`mark ${index + 1} x must be a pixel from 0 to ${width}.`);+ if (!Number.isFinite(y) || y < 0 || y > height) throw invalidMarks(`mark ${index + 1} y must be a pixel from 0 to ${height}.`);+ return { x: roundedPixel(x), y: roundedPixel(y), note: validNote(note, index) };+}++function validateLassoMark(mark, index, { width, height }) {+ const { points, note = '' } = mark;+ const { minLassoPoints, maxLassoPoints } = ANNOTATE_LIMITS;+ if (!Array.isArray(points) || points.length < minLassoPoints || points.length > maxLassoPoints) {+ const got = Array.isArray(points) ? `${points.length} points` : typeof points;+ throw invalidMarks(`mark ${index + 1} points must be an array of ${minLassoPoints} to ${maxLassoPoints} [x, y] pixels, got ${got}.`);+ }+ const clean = points.map((point, pointIndex) => {+ const valid = Array.isArray(point) && point.length === 2+ && Number.isFinite(point[0]) && point[0] >= 0 && point[0] <= width+ && Number.isFinite(point[1]) && point[1] >= 0 && point[1] <= height;+ if (!valid) throw invalidMarks(`mark ${index + 1} points[${pointIndex}] must be [x, y] with x from 0 to ${width} and y from 0 to ${height}.`);+ return [roundedPixel(point[0]), roundedPixel(point[1])];+ });+ if (polygonArea(clean) < 1) throw invalidMarks(`mark ${index + 1} points must outline an area of at least 1 square pixel, not a line or a dot.`);+ return { kind: 'lasso', points: clean, note: validNote(note, index) };+}+/**- * Checks marks sent by the annotate page. Returns clean { x, y, note } records with pixel positions inside the view.- * Throws a TypeError that says which mark failed, so a broken page cannot slip unbounded data to the agent.+ * Checks marks sent by the annotate page. Returns clean records with pixel positions inside the view: { x, y, note } for a pin+ * and { kind: 'lasso', points, note } for a lasso. Throws a TypeError that says which mark failed, so a broken page cannot slip+ * unbounded data to the agent.*/export function validateMarks(marks, { width, height, maxMarks = ANNOTATE_LIMITS.maxMarks }) {if (!Array.isArray(marks)) throw invalidMarks('marks must be an array.');if (marks.length > maxMarks) throw invalidMarks(`at most ${maxMarks} marks, got ${marks.length}.`);return marks.map((mark, index) => {if (!mark || typeof mark !== 'object' || Array.isArray(mark)) throw invalidMarks(`mark ${index + 1} must be an object.`);- const { x, y, note = '' } = mark;- if (!Number.isFinite(x) || x < 0 || x > width) throw invalidMarks(`mark ${index + 1} x must be a pixel from 0 to ${width}.`);- if (!Number.isFinite(y) || y < 0 || y > height) throw invalidMarks(`mark ${index + 1} y must be a pixel from 0 to ${height}.`);- if (typeof note !== 'string' || note.length > ANNOTATE_LIMITS.maxNoteLength) {- throw invalidMarks(`mark ${index + 1} note must be text of at most ${ANNOTATE_LIMITS.maxNoteLength} characters.`);- }- return { x: Math.round(x * 10) / 10, y: Math.round(y * 10) / 10, note: note.trim() };+ if (mark.kind === 'lasso') return validateLassoMark(mark, index, { width, height });+ if (mark.kind !== undefined && mark.kind !== 'pin') throw invalidMarks(`mark ${index + 1} kind must be "pin" or "lasso", got ${JSON.stringify(mark.kind)}.`);+ return validatePinMark(mark, index, { width, height });});}+/** Area of a polygon of [x, y] points, by the shoelace formula. */+export function polygonArea(points) {+ let twiceArea = 0;+ points.forEach(([x, y], index) => {+ const [nextX, nextY] = points[(index + 1) % points.length];+ twiceArea += x * nextY - nextX * y;+ });+ return Math.abs(twiceArea) / 2;+}++/** True when the pixel lies inside the polygon (even-odd rule: a loop the outline crosses itself into counts as outside). */+export function pointInPolygon(points, x, y) {+ let inside = false;+ for (let index = 0, previous = points.length - 1; index < points.length; previous = index++) {+ const [currentX, currentY] = points[index];+ const [previousX, previousY] = points[previous];+ const crosses = (currentY > y) !== (previousY > y)+ && x < (previousX - currentX) * (y - currentY) / (previousY - currentY) + currentX;+ if (crosses) inside = !inside;+ }+ return inside;+}+/** True when the base64 text is a PNG within the image budget. */export function isBoundedPng(base64) {if (typeof base64 !== 'string' || !/^[A-Za-z0-9+/]+={0,2}$/.test(base64)) return false;@@ -68,19 +128,71 @@ function roundedVector(vector) {return Array.from(vector, value => Math.round(value * 1000) / 1000);}-/** Numbers the marks and attaches what each one points at: part ID, world point, normal and distance, or null on empty sky. */+function hitFields(hit) {+ return {+ partId: hit ? hit.id : null,+ point: hit ? roundedVector(hit.point) : null,+ normal: hit ? roundedVector(hit.normal) : null,+ distance: hit ? Math.round(hit.distance * 1000) / 1000 : null,+ material: hit?.material ?? null,+ };+}++function polygonCentre(points) {+ const sum = points.reduce((total, [x, y]) => [total[0] + x, total[1] + y], [0, 0]);+ return [roundedPixel(sum[0] / points.length), roundedPixel(sum[1] / points.length)];+}++/**+ * The pixels a lasso is picked at: the centres of a grid over its bounding box that fall inside the outline, plus the outline's+ * centre when that is inside too. The grid is fixed, so the same outline always picks the same pixels.+ */+export function lassoSamplePixels(points) {+ const [left, right] = [Math.min(...points.map(point => point[0])), Math.max(...points.map(point => point[0]))];+ const [top, bottom] = [Math.min(...points.map(point => point[1])), Math.max(...points.map(point => point[1]))];+ const perAxis = ANNOTATE_LIMITS.lassoSamplesPerAxis;+ const samples = [];+ for (let row = 0; row < perAxis; row++) {+ for (let column = 0; column < perAxis; column++) {+ const pixel = [left + (column + 0.5) / perAxis * (right - left), top + (row + 0.5) / perAxis * (bottom - top)];+ if (pointInPolygon(points, pixel[0], pixel[1])) samples.push(pixel);+ }+ }+ const centre = polygonCentre(points);+ if (pointInPolygon(points, centre[0], centre[1])) samples.push(centre);+ return samples;+}++// The parts under a lasso, most covered first. Each carries its share of the picked pixels; the lasso's own hit fields are the+// most covered part's hit nearest the outline's centre.+function describeLasso(mark, index, camera, { width, height, raycast }) {+ const centre = polygonCentre(mark.points);+ const samples = lassoSamplePixels(mark.points);+ const groups = new Map();+ for (const [pixelX, pixelY] of samples) {+ const hit = raycast(pixelRay(camera, pixelX, pixelY, width, height));+ if (!hit) continue;+ const offset = Math.hypot(pixelX - centre[0], pixelY - centre[1]);+ const group = groups.get(hit.id) ?? { count: 0, hit, offset };+ group.count += 1;+ if (offset < group.offset) Object.assign(group, { hit, offset });+ groups.set(hit.id, group);+ }+ const ranked = [...groups.entries()].sort(([firstId, first], [secondId, second]) => second.count - first.count || (firstId < secondId ? -1 : 1));+ const parts = ranked.map(([partId, group]) => ({+ partId, samples: group.count, coverage: Math.round(group.count / samples.length * 100) / 100, material: group.hit.material ?? null,+ }));+ return { number: index + 1, kind: 'lasso', pixel: centre, points: mark.points, note: mark.note, ...hitFields(ranked[0]?.[1].hit), parts };+}++/**+ * Numbers the marks and attaches what each one points at: part ID, world point, normal and distance, or null on empty sky.+ * A lasso also returns kind, points and parts (the parts under its outline); its partId and point are the most covered part's.+ */export function describeMarks(marks, camera, { width, height, raycast }) {return marks.map((mark, index) => {+ if (mark.kind === 'lasso') return describeLasso(mark, index, camera, { width, height, raycast });const hit = raycast(pixelRay(camera, mark.x, mark.y, width, height));- return {- number: index + 1,- pixel: [mark.x, mark.y],- note: mark.note,- partId: hit ? hit.id : null,- point: hit ? roundedVector(hit.point) : null,- normal: hit ? roundedVector(hit.normal) : null,- distance: hit ? Math.round(hit.distance * 1000) / 1000 : null,- material: hit?.material ?? null,- };+ return { number: index + 1, pixel: [mark.x, mark.y], note: mark.note, ...hitFields(hit) };});}diff --git a/src/agent-scene-capabilities.js b/src/agent-scene-capabilities.jsindex e0a343d0..83fe4bdd 100644--- a/src/agent-scene-capabilities.js+++ b/src/agent-scene-capabilities.js@@ -236,7 +236,7 @@ export const AGENT_SCENE_CAPABILITIES = Object.freeze({terrain: 'One mesh, x and z from -size/2 to size/2, hills about y = 0. size 40 m (1-2000), resolution 64 cells (2-256), seed 1, amplitude 1 m (0-200), holes up to 16 of {center:[x,z],radius}, no surface inside; radius a few cells or more.',vehicleDamage: 'Geometry dented {source, impacts:[{point, direction, impulse N s}] 1-8, stiffness 2000, maxDepth 0.25 m, radius 0.4}: the source (not a fragment or dent) pushed in around each point by min(maxDepth, impulse/stiffness); a part that comes off is a body on a fixed joint with breakImpulse. docs/vehicle.md.',placedAssets: 'Node field asset:{id or name,materials?,flatten?,clip?,drive?} turns a group node into a catalog GLB (id from asset_catalog, or name: its exact file name, stored as the id; give one). Its transform places it. scene_apply add-node with an asset field places the first or any later instance, so a scene need not start with its assets. materials:[{name,id?,color,metallic,roughness,opacity,emission,emissiveColor}] override source materials by name; an id (such as "stool-heat") gives that material a handle that a set-motion material track can drive: emission and emissiveColor fade a heat glow, one id per placement. emissionMask:{axis:x|y|z,from,to} (metres in the own space of the part, smooth ramp, from above to flips it) multiplies emission so the glow sits only at one end while texture and shading stay; rasterizer only, path tracers ignore it. flatten:{color?,unlit?} flattens every material of the placement in one call: unlit true draws them without lighting, color [r,g,b] paints them one colour (textures dropped), both give a flat silhouette; a named override still wins. clip:{index,start=0,speed=1,loop=true} plays that clip from scene time start; set-motion tracks move the node and scene time seeks the clip. drive:{path | legs, cruiseSpeed | speed, wheels, lights} drives a car along a path in the placement frame (docs/vehicle.md). Max 512 placements, 16 assets, 16 animated.',- askUser: 'scene_ask_user opens the current view in their browser; they pin up to maxMarks (16) points, each with a note, and press Send. Ask when a change is hard to say in words ("warmer here", "this bit"), not for facts you can inspect. Each mark returns partId, point, normal, note, distance and material (null on sky), plus view.camera and a PNG with numbered pins. Apply the change the note names to that partId, then to similar parts (same material or name prefix) when the note says so. outcome cancelled or timeout returns no marks: ask in text. Edits made while they mark change revision; the result warns.',+ askUser: 'scene_ask_user opens the current view in their browser; they pin up to maxMarks (16) points, each with a note, and press Send. Ask when a change is hard to say in words ("warmer here", "this bit"), not for facts you can inspect. Each mark returns partId, point, normal, note, distance and material (null on sky), plus view.camera and a PNG with numbered pins. Apply the change the note names to that partId, then to similar parts (same material or name prefix) when the note says so. A mark with kind "lasso" is an outline around a region: apply the note to every part in its parts list (partId, coverage, material, most covered first), and skip a part with a tiny coverage that the note does not fit. Parts that share a material need their own material before one of them is recoloured. outcome cancelled or timeout returns no marks: ask in text. Edits made while they mark change revision; the result warns.',navigation: 'scene_inspect path:{ground,from:[x,z],to:[x,z]} returns a route over the ground node around its holes, steep slopes and the nodes in avoid:[ids] (box footprints), as keyframes [{time,value:[x,y,z]}] for a translation track with interpolation LINEAR. radius 0.3 m; maxSlope 40 degrees; cellSize defaults to radius; seconds or speed (1.4 m/s); lift adds y.',meshEdits: Object.freeze({ operations: Object.freeze(Object.keys(MESH_EDIT_OPERATIONS)), maxEdits: MAX_MESH_EDITS }),meshStack: Object.freeze({ operations: STACK_OPERATIONS, maxNodes: MAX_STACK_NODES }),diff --git a/tests/agent-annotate-lasso.test.js b/tests/agent-annotate-lasso.test.jsnew file mode 100644index 00000000..c7ea5282--- /dev/null+++ b/tests/agent-annotate-lasso.test.js@@ -0,0 +1,108 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import { ANNOTATE_LIMITS, validateMarks, describeMarks, pointInPolygon, polygonArea, lassoSamplePixels } from '../src/agent-annotate.js';+import { runAnnotateSession } from '../scripts/annotate-session.mjs';++// The lasso mark of scene_ask_user (#922): an outline the person draws, resolved to the parts under it.+const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=';+const view = { width: 400, height: 200 };+const camera = { eye: [0, 0, 10], target: [0, 0, 0], up: [0, 1, 0], fov: 60, near: 0.1, far: 100 };+const square = [[100, 50], [200, 50], [200, 150], [100, 150]];++test('pointInPolygon and polygonArea follow the even-odd rule for a square, a notch and a self-crossing bow tie', () => {+ assert.equal(polygonArea(square), 10000);+ assert.equal(pointInPolygon(square, 150, 100), true);+ assert.equal(pointInPolygon(square, 99, 100), false);+ assert.equal(pointInPolygon(square, 150, 151), false);+ const notch = [[0, 0], [100, 0], [100, 100], [60, 100], [60, 40], [40, 40], [40, 100], [0, 100]];+ assert.equal(pointInPolygon(notch, 50, 70), false, 'the notch cut into the bottom is outside');+ assert.equal(pointInPolygon(notch, 20, 70), true);+ assert.equal(pointInPolygon(notch, 50, 20), true);+ const bowTie = [[0, 0], [100, 100], [100, 0], [0, 100]];+ assert.equal(pointInPolygon(bowTie, 10, 50), true, 'the left lobe of the bow tie');+ assert.equal(pointInPolygon(bowTie, 90, 50), true, 'the right lobe');+ assert.equal(pointInPolygon(bowTie, 50, 10), false, 'the open top between the lobes');+ assert.equal(pointInPolygon(bowTie, 50, 90), false, 'the open bottom between the lobes');+});++test('validateMarks keeps a lasso as rounded points and names the field of a bad one', () => {+ const [lasso] = validateMarks([{ kind: 'lasso', points: [[10.04, 20], [100, 20], [100, 80.96]], note: ' these ' }], view);+ assert.deepEqual(lasso, { kind: 'lasso', points: [[10, 20], [100, 20], [100, 81]], note: 'these' });+ const lassoWith = fields => [{ kind: 'lasso', points: square, ...fields }];+ assert.throws(() => validateMarks(lassoWith({ points: [[0, 0], [10, 10]] }), view), /mark 1 points must be an array of 3 to 64 \[x, y\] pixels, got 2 points/);+ assert.throws(() => validateMarks(lassoWith({ points: undefined }), view), /mark 1 points must be an array of 3 to 64 .*got undefined/);+ const manyPoints = Array.from({ length: ANNOTATE_LIMITS.maxLassoPoints + 1 }, (unused, index) => [200 + Math.round(90 * Math.cos(index / 65 * 2 * Math.PI)), 100 + Math.round(90 * Math.sin(index / 65 * 2 * Math.PI))]);+ assert.throws(() => validateMarks(lassoWith({ points: manyPoints }), view), /got 65 points/);+ assert.equal(validateMarks(lassoWith({ points: manyPoints.slice(0, 64) }), view).length, 1);+ assert.throws(() => validateMarks(lassoWith({ points: [[0, 0], [401, 0], [0, 10]] }), view), /points\[1\] must be \[x, y\] with x from 0 to 400 and y from 0 to 200/);+ assert.throws(() => validateMarks(lassoWith({ points: [[0, 0], [10, -1], [0, 10]] }), view), /points\[1\]/);+ assert.throws(() => validateMarks(lassoWith({ points: [[0, 0], [10, 'a'], [0, 10]] }), view), /points\[1\]/);+ assert.throws(() => validateMarks(lassoWith({ points: [[0, 0], [10, 10, 5], [0, 10]] }), view), /points\[1\]/);+ assert.throws(() => validateMarks(lassoWith({ points: [[0, 0], [10, 10], [20, 20]] }), view), /outline an area/);+ assert.throws(() => validateMarks(lassoWith({ note: 'x'.repeat(281) }), view), /mark 1 note must be text of at most 280 characters/);+ assert.throws(() => validateMarks([{ kind: 'box', x: 1, y: 1 }], view), /mark 1 kind must be "pin" or "lasso", got "box"/);+ assert.deepEqual(validateMarks([{ kind: 'pin', x: 1, y: 2, note: 'a' }], view), [{ x: 1, y: 2, note: 'a' }]);+ assert.equal(validateMarks(Array(ANNOTATE_LIMITS.maxMarks).fill({ kind: 'lasso', points: square }), view).length, ANNOTATE_LIMITS.maxMarks);+});++test('lassoSamplePixels is a fixed grid inside the outline, so the same outline always picks the same pixels', () => {+ const samples = lassoSamplePixels(square);+ assert.equal(samples.length, ANNOTATE_LIMITS.lassoSamplesPerAxis ** 2 + 1);+ assert.ok(samples.every(([x, y]) => x > 100 && x < 200 && y > 50 && y < 150));+ assert.deepEqual(lassoSamplePixels(square), samples);+ const triangle = lassoSamplePixels([[0, 0], [100, 0], [0, 100]]);+ assert.ok(triangle.length > 250 && triangle.length < 330, `about half of 576, got ${triangle.length}`);+ assert.ok(triangle.every(([x, y]) => x + y <= 100));+});++// The camera looks down -z; a ray leaving left of the axis hits the wall, a ray near the axis the lamp, the rest is sky.+function wallLampSky(ray) {+ if (ray.direction[0] < -0.0001) return { id: 'wall', point: [-1, 0, 0], normal: [0, 0, 1], distance: 10, material: 'plaster' };+ if (ray.direction[0] < 0.1) return { id: 'lamp', point: [0.5, 0, 0], normal: [0, 0, 1], distance: 9, material: 'brass' };+ return null;+}++test('describeMarks resolves a lasso to the parts under it, most covered first, and keeps the pin shape for pins', () => {+ const [region, sky, pin] = describeMarks([+ { kind: 'lasso', points: [[100, 50], [300, 50], [300, 150], [100, 150]], note: 'these' },+ { kind: 'lasso', points: [[390, 5], [399, 5], [399, 20]], note: 'sky' },+ { x: 100, y: 100, note: 'pin' },+ ], camera, { ...view, raycast: wallLampSky });+ assert.equal(region.number, 1);+ assert.equal(region.kind, 'lasso');+ assert.deepEqual(region.points, [[100, 50], [300, 50], [300, 150], [100, 150]]);+ assert.deepEqual(region.pixel, [200, 100]);+ assert.deepEqual(region.parts.map(part => part.partId), ['wall', 'lamp']);+ assert.ok(region.parts[0].samples > region.parts[1].samples);+ assert.ok(region.parts.every(part => part.coverage > 0 && part.coverage < 1));+ assert.equal(region.parts[0].material, 'plaster');+ assert.equal(region.partId, 'wall');+ assert.equal(region.material, 'plaster');+ assert.deepEqual(region.normal, [0, 0, 1]);+ assert.equal(sky.partId, null);+ assert.deepEqual(sky.parts, []);+ assert.equal('kind' in pin, false);+ assert.equal(pin.partId, 'wall');+});++test('describeMarks breaks a tie between equally covered parts by part ID', () => {+ const raycast = ray => ({ id: ray.direction[0] < 0 ? 'zebra' : 'apple', point: [0, 0, 0], normal: [0, 1, 0], distance: 1, material: null });+ const lasso = { kind: 'lasso', points: [[100, 50], [300, 50], [300, 150], [100, 150]], note: '' };+ const [mark] = describeMarks([lasso], camera, { ...view, raycast });+ assert.deepEqual(mark.parts.map(part => part.partId).sort(), ['apple', 'zebra']);+ assert.equal(mark.parts[0].samples >= mark.parts[1].samples, true);+});++test('the annotate page offers the lasso tool and returns a validated lasso mark on Send', async () => {+ const opened = Promise.withResolvers();+ const session = runAnnotateSession({ png: PNG, width: view.width, height: view.height, waitMs: 5000, open: async url => opened.resolve(url) });+ const url = await opened.promise;+ assert.match(await (await fetch(`${url}app.js`)).text(), /pointerdown/);+ const page = await (await fetch(url)).text();+ assert.match(page, /id="tool-lasso"/);+ assert.match(page, /"maxLassoPoints":64/);+ const send = marks => fetch(`${url}send`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ marks, image: PNG }) });+ assert.equal((await send([{ kind: 'lasso', points: [[0, 0], [10, 10]], note: '' }])).status, 400);+ assert.equal((await send([{ kind: 'lasso', points: square, note: 'these' }])).status, 200);+ assert.deepEqual((await session).marks, [{ kind: 'lasso', points: square, note: 'these' }]);+});diff --git a/tests/agent-bench-subsets.test.js b/tests/agent-bench-subsets.test.jsindex abd12699..12908561 100644--- a/tests/agent-bench-subsets.test.js+++ b/tests/agent-bench-subsets.test.js@@ -79,19 +79,19 @@ test('bench.mjs --dry-run lists the selected tasks and rejects unknown values wiassert.throws(() => execFileSync(process.execPath, [benchPath, '--topics', 'nope', '--dry-run'], { encoding: 'utf8', stdio: 'pipe' }), /Unknown topic nope/);});-test('--tasks accepts a scene task that the default run leaves out, and the default run stays on the 31 bench tasks', () => {+test('--tasks accepts a scene task that the default run leaves out, and the default run stays on the 32 bench tasks', () => {assert.deepEqual(OPTIONAL_TASK_IDS, ['findings-fix']);assert.deepEqual(selectTasks({ tasks: 'findings-fix' }, BENCH_TASKS, OPTIONAL_TASK_IDS), ['findings-fix']);assert.deepEqual(selectTasks({ tasks: 'findings-fix,bolt' }, BENCH_TASKS, OPTIONAL_TASK_IDS), ['bolt', 'findings-fix']);- assert.equal(selectTasks({}, BENCH_TASKS, OPTIONAL_TASK_IDS).length, 31);+ assert.equal(selectTasks({}, BENCH_TASKS, OPTIONAL_TASK_IDS).length, 32);assert.equal(selectTasks({ kinds: 'light' }, BENCH_TASKS, OPTIONAL_TASK_IDS).includes('findings-fix'), false);assert.throws(() => selectTasks({ tasks: 'nope' }, BENCH_TASKS, OPTIONAL_TASK_IDS), /Unknown task nope\. Valid tasks: crate,.*findings-fix\./);});-test('bench.mjs --tasks findings-fix --dry-run selects it, an unknown id lists the valid ids, the default dry run selects 31 tasks', () => {+test('bench.mjs --tasks findings-fix --dry-run selects it, an unknown id lists the valid ids, the default dry run selects 32 tasks', () => {const run = argumentsList => execFileSync(process.execPath, [benchPath, ...argumentsList, '--dry-run'], { encoding: 'utf8', stdio: 'pipe' });- assert.match(run(['--tasks', 'findings-fix']), /^1 of 31 tasks: findings-fix/);- assert.match(run([]), /^31 of 31 tasks: crate,/);+ assert.match(run(['--tasks', 'findings-fix']), /^1 of 32 tasks: findings-fix/);+ assert.match(run([]), /^32 of 32 tasks: crate,/);assert.throws(() => run(['--tasks', 'nope']), /Unknown task nope\. Valid tasks: crate,.*findings-fix\./s);});diff --git a/tests/agent-bench.test.js b/tests/agent-bench.test.jsindex 354820f4..c0b19d27 100644--- a/tests/agent-bench.test.js+++ b/tests/agent-bench.test.js@@ -38,6 +38,13 @@ test('the camera, light, look and film tasks reject plausible wrong answers', asassert.deepEqual(failedKinds(await wrong('sunset-timelapse', recipe => { recipe.motion.tracks[1].keyframes[1].value = [1, 1, 1]; })), ['sun colour gets warmer']);assert.deepEqual(failedKinds(await wrong('film-reveal', recipe => { recipe.motion.tracks[0].keyframes[1].value = [0, 3, 9]; })), ['cameraDolly']);assert.deepEqual(failedKinds(await wrong('film-reveal', recipe => { recipe.motion.tracks.pop(); })), ['exposure fades up']);+ assert.deepEqual(failedKinds(await wrong('ask-user-lasso', recipe => { recipe.materials.find(material => material.id === 'bronze').color = [0.45, 0.3, 0.2]; })), ['the lassoed statue is blue']);+ assert.deepEqual(failedKinds(await wrong('ask-user-lasso', recipe => { recipe.nodes.find(node => node.id === 'plinth').material = 'stone'; })), ['the lassoed plinth is blue']);+ // The shared stone material: colouring it blue colours the floor too, which the lasso did not cover.+ assert.deepEqual(failedKinds(await wrong('ask-user-lasso', recipe => {+ recipe.materials.find(material => material.id === 'stone').color = [0.1, 0.2, 0.85];+ recipe.nodes.find(node => node.id === 'plinth').material = 'stone';+ })), ['the floor outside the lasso is not recoloured']);});const record = (task, pass, extra = {}) => ({ task, pass, toolCalls: 10, toolErrors: 1, inputTokens: 1000, outputTokens: 100, wallSeconds: 30, ...extra });diff --git a/tests/agent-runtime.test.js b/tests/agent-runtime.test.jsindex 2c9647be..8079309f 100644--- a/tests/agent-runtime.test.js+++ b/tests/agent-runtime.test.js@@ -697,6 +697,29 @@ runTest('a scene_ask_user fixture answers without a browser, in the same shape aawait assert.rejects(fixture.runtime.askUser({ sessionId: fixture.sessionId, maxMarks: 1 }), /at most 1 marks/);});+runTest('a lasso mark resolves to the parts under its outline, through the same pick as a pin, and a lasso over sky has none (#922)', async () => {+ const wholeView = [[0, 0], [768, 0], [768, 512], [0, 512]];+ const skyCorner = [[10, 10], [60, 10], [60, 100], [10, 100]];+ const fixture = await reviewRuntimeHarness({+ runtimeOptions: { askUserFixture: { name: 'lasso.json', marks: [+ { kind: 'lasso', points: wholeView, note: 'make these blue' }, { kind: 'lasso', points: skyCorner, note: 'nothing here' }, { x: 384, y: 256, note: 'a pin' }] } } });+ const answer = JSON.parse(JSON.stringify((await fixture.runtime.askUser({ sessionId: fixture.sessionId })).content));+ const [whole, corner, pin] = answer.marks;+ assert.equal(whole.kind, 'lasso');+ assert.deepEqual(whole.points, wholeView);+ assert.equal(whole.note, 'make these blue');+ assert.equal(whole.partId, 'lamp');+ assert.deepEqual(whole.parts.map(part => part.partId), ['lamp']);+ assert.ok(whole.parts[0].coverage > 0.5 && whole.parts[0].coverage < 1, 'the left of the view is sky, so the lamp covers only part of the outline');+ assert.equal(whole.parts[0].material, 'brass');+ assert.equal(corner.partId, null);+ assert.deepEqual(corner.parts, []);+ assert.equal('kind' in pin, false, 'a pin keeps its old shape');+ assert.equal(pin.partId, 'lamp');+ const pickCall = fixture.sourceCalls.find(call => call.method === 'pickMarks');+ assert.equal(pickCall.input.marks[0].kind, 'lasso', 'the lasso reaches the page as an outline, not as a pixel');+});+runTest('an inline scene_ask_user hands its nonce to the view only and resolves marks once (#923)', async () => {let browserSessions = 0;const { runtime, sessionId } = await reviewRuntimeHarness({ contextOverrides: { runAnnotateSession: async () => { browserSessions++; return { outcome: 'timeout' }; } } });diff --git a/tests/ask-user-fixture.test.js b/tests/ask-user-fixture.test.jsindex 5a5552e2..6bba2d48 100644--- a/tests/ask-user-fixture.test.js+++ b/tests/ask-user-fixture.test.js@@ -70,6 +70,48 @@ test('the loader checks the marks against the limits of real marks', async conte}});+test('a fixture mark may carry a lasso instead of a pixel, and the loader checks its points and count (#922)', async context => {+ const outline = [[10, 10], [100, 10], [100, 80]];+ const filePath = await fixtureFile(context, 'lasso.json', { marks: [{ lasso: outline, note: 'make these blue' }, { pixel: [1, 2] }] });+ assert.deepEqual(loadFrom(filePath), { name: 'lasso.json', marks: [{ kind: 'lasso', points: outline, note: 'make these blue' }, { x: 1, y: 2, note: '' }] });+ const tooMany = Array.from({ length: 65 }, (unused, index) => [index, 5]);+ const cases = [+ [{ marks: [{ lasso: [[1, 1], [2, 2]] }] }, /mark 1 lasso must be an array of 3 to 64 \[x, y\] points/],+ [{ marks: [{ lasso: tooMany }] }, /mark 1 lasso must be an array of 3 to 64 \[x, y\] points/],+ [{ marks: [{ lasso: 'around the statue' }] }, /mark 1 lasso must be an array of 3 to 64/],+ [{ marks: [{ lasso: [[1, 1], [2, 2], [3]] }] }, /mark 1 lasso point 3 must be \[x, y\], two numbers from 0 to 1024/],+ [{ marks: [{ lasso: [[1, 1], [2, 2], [3, 1025]] }] }, /mark 1 lasso point 3 must be \[x, y\]/],+ [{ marks: [{ lasso: [[1, 1], [2, 2], [-3, 1]] }] }, /mark 1 lasso point 3 must be \[x, y\]/],+ [{ marks: [{ lasso: outline, pixel: [1, 1] }] }, /mark 1 has both pixel and lasso/],+ [{ marks: [{ lasso: outline, note: 'x'.repeat(281) }] }, /mark 1 note must be text of at most 280 characters/],+ ];+ for (const [content, message] of cases) {+ const caseFile = await fixtureFile(context, 'case.json', content);+ assert.throws(() => loadFrom(caseFile), message, JSON.stringify(content).slice(0, 80));+ }+ const atLimit = await fixtureFile(context, 'limit.json', { marks: [{ lasso: tooMany.slice(0, 64).map(([x]) => [x * 10, 500 + x]) }] });+ assert.equal(loadFrom(atLimit).marks[0].points.length, 64);+});++test('the bench lasso outlines the statue and the plinth, not the floor, at the stage camera (#922)', async () => {+ const task = taskById('ask-user-lasso');+ assert.ok(BENCH_TASKS.find(candidate => candidate.id === 'ask-user-lasso').topics.includes('askUser'));+ const lassoFixture = join(import.meta.dirname, '..', 'bench', 'agent-modeling', 'ask-user-lasso.json');+ assert.deepEqual(askUserEnvironment(task), { [ASK_USER_FIXTURE_VARIABLE]: resolve(lassoFixture) });++ const fixture = loadFrom(lassoFixture);+ const { preview } = previewHarness();+ preview.create({ recipe: structuredClone(task.startRecipe), sessionId: 'lasso', backend: 'cpu', memoryBudgetMiB: 64 });+ const frame = preview.annotateFrame({ sessionId: 'lasso' });+ const picked = await preview.pickMarks({ sessionId: 'lasso', camera: frame.camera, width: frame.width, height: frame.height, marks: fixture.marks });+ const [mark] = picked.marks;+ assert.equal(mark.kind, 'lasso');+ assert.equal(mark.note, 'make these blue');+ const covered = Object.fromEntries(mark.parts.map(part => [part.partId, part.coverage]));+ assert.ok(covered.statue > 0.25 && covered.plinth > 0.25, JSON.stringify(covered));+ assert.ok((covered.floor ?? 0) < 0.15, `the floor stays a sliver: ${JSON.stringify(covered)}`);+});+function startServer(environmentOverrides) {const child = spawn(process.execPath, [serverScript], { env: { ...process.env, ...environmentOverrides }, stdio: ['pipe', 'ignore', 'pipe'], windowsHide: true });let errors = '';