ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(pixel): explain a failed click and a rendered hidden overlay to the agent (#8267)
Mike Bradley40a51f170d64
diff --git a/ods/extensions/services/pixel-agent/plugin/workspace-preview-inspect.mjs b/ods/extensions/services/pixel-agent/plugin/workspace-preview-inspect.mjsindex a897001cd..b9edc5652 100644--- a/ods/extensions/services/pixel-agent/plugin/workspace-preview-inspect.mjs+++ b/ods/extensions/services/pixel-agent/plugin/workspace-preview-inspect.mjs@@ -143,6 +143,18 @@ function unchangedPaintNote(result) {}return '';}+// The capsule counts elements that carry the hidden attribute yet are visibly+// rendered, because an author style such as display: flex overrides it. A+// generated memory game shipped its win overlay that way: it covered the board+// from load, every click landed on it, and the agent blamed the inspector+// (Workstation06, 2026-10-27).+function renderedHiddenNote(result) {+ const count = result.diagnostics?.renderedHiddenAttributeCount ?? 0;+ if (count < 1) return '';+ return ` Note: ${count === 1 ? 'one element carries the hidden attribute but is' : `${count} elements carry the hidden attribute but are`} rendered anyway, ` ++ 'because an author style such as display: flex overrides hidden. An overlay, dialog or panel meant to stay hidden then shows, and can cover the page and take its clicks. ' ++ 'Keep the display value for its shown state only, or add [hidden] { display: none !important; }.';+}// A passed paint step whose box is almost all one color: a cleared canvas with a// few specks (dust, a cursor) shows two colors and passes, yet looks blank to a// player. Informational, never a failure: menus and dark scenes can be mostly@@ -188,6 +200,14 @@ function transitionCoverageFeedback(request, result) {if (paintFailure?.errorCode === 'uniform_paint') return `Step ${paintFailure.index+1} (assert-painted) captured one color only (${paintFailure.paint.dominant.hex}) in the element's on-screen box after the listed actions, throughout the observation window: a blank, black or single-color render. Read the render path in the source (canvas size, drawing calls, the animation loop, asset paths and script errors) and the actions before this step. Correct a mistaken plan on the same published snapshot, or fix the rendering, republish and inspect the new snapshot. If the element is meant to be one solid color, use assert-visible for it instead. This inspector's Chromium has no GPU and may not render WebGL: when the element draws with WebGL, keep the page unchanged for that limitation and report rendering as unverified. Do not add placeholder paint only to pass this check. Rendering remains unverified.`;// not_in_viewport means no sample captured anything, for any of three reasons.if (paintFailure) return `Step ${paintFailure.index+1} (assert-painted) captured nothing. Either less than 8x8 CSS pixels of the element's content box, inside any rounded corners, lay within the ${request.viewport.width}x${request.viewport.height} viewport; or the element is too thin for its length to keep a captured pixel inside it once its longer side is downscaled to 256 pixels, such as a long narrow bar; or it stopped being one visible match while the paint was sampled, for example a splash canvas that hides or replaces itself. Target the rendered drawing element itself, such as the game canvas, that stays visible after the listed actions, with a viewport large enough to show it as loaded (the inspector does not scroll), and retry the same published snapshot. Rendering remains unverified.`;+ // click_failed: the target was found, but Playwright could not click it within+ // 2 seconds. Without a reason the agent called the inspector unable to click.+ const clickFailure = result.steps?.find(step => step.action === 'click' && step.errorCode === 'click_failed');+ if (clickFailure) return 'Requested behavior remains unverified; a failed inspection does not establish a visibility transition. ' ++ `Step ${clickFailure.index+1} (click) found its target but could not click it within 2 seconds, so later steps did not run. ` ++ 'The inspector clicks the way a person does, at the center of a visible, enabled element that has stopped moving. ' ++ "Most often another element covers that point, such as an overlay, modal or backdrop, and it takes a person's click as well: that is a page defect to fix, not an inspector limit. " ++ 'Otherwise the target is disabled, or animates without pause. Read the source for which it is, fix the page, republish, and inspect the new snapshot.';if (result.status !== 'passed') return 'Requested behavior remains unverified; a failed inspection does not establish a visibility transition.';if (downloadPlan(request)) return 'Chromium captured the declared snapshot file bytes inside the isolated capsule after the listed click. No file was exported to the user computer. This does not validate PDF quality, ZIP members, or future downloads beyond the bounded observation window.';const paintPass = paintPlan(request) ? ' ' + PAINT_PASS : '';@@ -531,7 +551,7 @@ export function createWorkspacePreviewInspectTool({request,transport='unix',tranconst summary=pageErrors? `Preview inspection ${result.status==='passed'?'steps passed, but':'failed, and'} ${pageErrorFeedback(pageErrors)}`: requirement ? transitionIncompleteFeedback(normalized, requirement)- : `Preview inspection ${result.status}. ${transitionCoverageFeedback(normalized, result)}${unchangedPaintNote(result)}${nearlyOneColorNote(result)}`;+ : `Preview inspection ${result.status}. ${transitionCoverageFeedback(normalized, result)}${unchangedPaintNote(result)}${nearlyOneColorNote(result)}${renderedHiddenNote(result)}`;// The palette is stated once, as its fixed line; the evidence copy omits// it and the load-time control names (used only for locator feedback).const {renderedColors,controls,...rest}=result;diff --git a/ods/extensions/services/pixel-agent/tests/inspection_click_feedback.test.mjs b/ods/extensions/services/pixel-agent/tests/inspection_click_feedback.test.mjsnew file mode 100644index 000000000..4976b6cbc--- /dev/null+++ b/ods/extensions/services/pixel-agent/tests/inspection_click_feedback.test.mjs@@ -0,0 +1,61 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import {createWorkspacePreviewInspectTool, inspectionPlanHash, INSPECTION_KIND, INSPECTION_SCOPE,+ normalizeWorkspacePreviewInspectionParams as normalize}+ from '../plugin/workspace-preview-inspect.mjs';++// Workstation06 (2026-10-27): a generated memory game styled its win overlay with+// display: flex, overriding the overlay's hidden attribute, so the overlay covered+// the board from load. Every card click failed as click_failed with no reason,+// and the agent told the owner the inspector could not click while the game worked.+const digest = 'b'.repeat(64);+const request = () => ({siteId: `site-${digest.slice(0,24)}`, sha256:digest, viewport:{width:800,height:600},+ steps:[{action:'click',locator:{selector:'.card:nth-child(1)'}},+ {action:'assert-text',locator:{selector:'#moves'},expectedText:'1'}]});+const state = text => ({count:1,visible:true,display:'block',visibility:'visible',opacity:'1',hidden:false,+ hiddenUntilFound:false,rectCount:1,...(text === undefined ? {} : {text:{actual:text,truncated:false}})});+function receipt(params, {clickFails = false, renderedHidden = 0} = {}) {+ const steps = clickFails+ ? [{index:0,...structuredClone(params.steps[0]),before:state(),stable:true,status:'failed',errorCode:'click_failed'}]+ : [{index:0,...structuredClone(params.steps[0]),before:state(),after:state(),stable:true,status:'passed'},+ {index:1,...structuredClone(params.steps[1]),before:state('1'),stable:true,status:'passed'}];+ return {schemaVersion:1,kind:INSPECTION_KIND,status:clickFails ? 'failed' : 'passed',+ siteId:params.siteId,sha256:params.sha256,planSha256:inspectionPlanHash(normalize(params)),viewport:params.viewport,+ scope:INSPECTION_SCOPE,diagnostics:{renderedHiddenAttributeCount:renderedHidden,hiddenUntilFoundCount:0},+ blockedRequests:[],steps};+}+const summary = result => result.content[0].text.split(' Evidence: ')[0];+const inspect = (options, id = 'click') => {+ const params = request();+ return createWorkspacePreviewInspectTool({request:async () => receipt(params, options)}).execute(id, params);+};++test('a failed click is a page problem to find, not an inspector limit', async () => {+ const result = await inspect({clickFails:true});+ assert.equal(result.isError, true);+ const text = summary(result);+ assert.match(text, /Step 1 \(click\) found its target but could not click it/);+ assert.match(text, /covers that point.*person's click as well/);+ assert.match(text, /not an inspector limit/);+ assert.match(text, /disabled, or animates without pause/);+ assert.match(text, /fix the page, republish, and inspect the new snapshot/);+ assert.match(text, /Requested behavior remains unverified/);+ assert.doesNotMatch(text, /hidden attribute/);+});++test('a rendered hidden overlay is named beside the failed click', async () => {+ const text = summary(await inspect({clickFails:true, renderedHidden:1}));+ assert.match(text, /could not click it/);+ assert.match(text, /one element carries the hidden attribute but is rendered anyway/);+ assert.match(text, /display: flex overrides hidden/);+ assert.match(text, /\[hidden\] \{ display: none !important; \}/);+});++test('a passed inspection still notes hidden elements that are rendered', async () => {+ const noted = summary(await inspect({renderedHidden:2}, 'passed-noted'));+ assert.match(noted, /^Preview inspection passed\./);+ assert.match(noted, /2 elements carry the hidden attribute but are rendered anyway/);+ const clean = summary(await inspect({}, 'passed-clean'));+ assert.match(clean, /^Preview inspection passed\./);+ assert.doesNotMatch(clean, /hidden attribute|could not click/);+});