ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(pixel): name the step that keeps a passing preview check from counting (#8181)
Mike Bradleyf6606907f257
diff --git a/ods/extensions/services/pixel-agent/COMPLETION-RELIABILITY.md b/ods/extensions/services/pixel-agent/COMPLETION-RELIABILITY.mdindex e9587e304..5af187ed1 100644--- a/ods/extensions/services/pixel-agent/COMPLETION-RELIABILITY.md+++ b/ods/extensions/services/pixel-agent/COMPLETION-RELIABILITY.md@@ -890,6 +890,37 @@ through the real guard, inspection receipt and ingress, plus the regressioncases (success claims in many forms, empty, silent and narration-only replies,imitated notes, passing and pending checks, local links, code fences).+## Passing checks that do not count++On the RTX Spark (stopwatch), the first interaction check failed at step 2.+The agent edited the page, republished, and inspected the new snapshot. The+tool answered "Preview inspection passed. These steps tested opposite+visibility states of the same element around a click." Delivery still said+"The attempted preview interaction checks have not passed browser inspection."++A passing check of the latest snapshot does satisfy a check attempted on an+earlier one; a pass of an older snapshot does not. But the tool reports the+capsule's result, and the run guard counts a plan as interaction evidence only+when each click has an assertion of its result before the next action and the+last action has one too (`boundPreviewBehavior`). A plan can pass a transition+around one click and then click another control with nothing asserted. Nothing+told the model: the guard's next step for that snapshot had already gone out+with the publication, so the coaching interval kept it off the passing result,+and the pinned harness drops a `before_agent_finalize` revision after any+plugin tool call.++That result now names the step first, for example "The latest inspection of+this snapshot passed, but step 5 (click) is followed directly by step 6+(click) with no assertion of the click's own result, so this plan does not+count as interaction evidence." `unassertedBehaviorStep` finds the step, and+`boundPreviewBehavior` uses the same function, so the two cannot disagree. The+step is named only for a passing, error-free receipt bound to the current+snapshot, in a run with an attempted interaction check. The evidence policy+and the delivery text are unchanged.++`tests/preview_interaction_assurance.test.mjs` replays the stopwatch sequence+(failed check, edit, republish, passing check), direct and through Tool Search.+## Focused verificationFor an explicitly published repair naming one workspace file, recognizeddiff --git a/ods/extensions/services/pixel-agent/plugin/preview-interaction-assurance.mjs b/ods/extensions/services/pixel-agent/plugin/preview-interaction-assurance.mjsindex a350547fa..203c9662c 100644--- a/ods/extensions/services/pixel-agent/plugin/preview-interaction-assurance.mjs+++ b/ods/extensions/services/pixel-agent/plugin/preview-interaction-assurance.mjs@@ -185,29 +185,46 @@ export function attemptedPreviewBehavior(params, preview) {? Object.freeze({requiresInteraction: true}) : undefined;}-export function boundPreviewBehavior(params, result, preview, attempted) {- if (!attempted) return undefined;- return boundInspection(params, result, preview, request => {- if (request.steps.some(step => step.action === 'download')) return false;- if (!request.steps.some(step => BEHAVIOR_ACTIONS.includes(step.action))) return false;- // Fill/select can prepare a click, but each click needs a postcondition- // before another action. A value change without a click needs one too.- // Key presses may chain (Space to start, then ArrowRight); the last- // action before the end still needs a postcondition.- let pending;- for (const step of request.steps) {- if (BEHAVIOR_ASSERTIONS.includes(step.action)) pending = undefined;- else if (BEHAVIOR_ACTIONS.includes(step.action)) {- if (pending === 'click') return false;- pending = step.action;- }+// The step that keeps an action plan from counting as behavior evidence: the+// first click followed directly by another action (that action is `next`),+// else a last action with no assertion after it. Fill/select can prepare a+// click, but each click needs a postcondition before another action. A value+// change without a click needs one too. Key presses may chain (Space to start,+// then ArrowRight); the last action before the end still needs a postcondition.+export function unassertedBehaviorStep(steps) {+ let pending;+ for (const [index, step] of steps.entries()) {+ if (BEHAVIOR_ASSERTIONS.includes(step.action)) pending = undefined;+ else if (BEHAVIOR_ACTIONS.includes(step.action)) {+ if (pending?.action === 'click') return Object.freeze({...pending, next: Object.freeze({index, action: step.action})});+ pending = {index, action: step.action};}- return pending === undefined;- });+ }+ return pending ? Object.freeze(pending) : undefined;}-export function previewBehaviorInstruction(preview, pageErrors) {- return pageErrorRepairInstruction(preview, pageErrors) ??+export function boundPreviewBehavior(params, result, preview, attempted) {+ if (!attempted) return undefined;+ return boundInspection(params, result, preview, request =>+ !request.steps.some(step => step.action === 'download') &&+ request.steps.some(step => BEHAVIOR_ACTIONS.includes(step.action)) &&+ !unassertedBehaviorStep(request.steps));+}++// `gap` is the unassertedBehaviorStep of this snapshot's latest passing+// inspection. That inspection's own result says "passed", and the generic+// instruction below usually went out already with the publication, so the+// step to correct is named first. Never evidence.+function unassertedStepLine(preview, gap) {+ if (!visibilityInspectionMatches(gap, preview)) return '';+ const lead = 'The latest inspection of this snapshot passed, but ';+ return gap.next+ ? `${lead}step ${gap.index + 1} (click) is followed directly by step ${gap.next.index + 1} (${gap.next.action}) with no assertion of the click's own result, so this plan does not count as interaction evidence. Insert an assertion of that click's visible result before step ${gap.next.index + 1} and retry the same published snapshot. `+ : `${lead}no assertion follows its last action, step ${gap.index + 1} (${gap.action}), so this plan does not count as interaction evidence. Add an assertion of that action's visible result after it and retry the same published snapshot. `;+}++export function previewBehaviorInstruction(preview, pageErrors, gap) {+ return pageErrorRepairInstruction(preview, pageErrors) ?? unassertedStepLine(preview, gap) +'The preview interaction check remains unverified. Test the actual controls and visible results needed for the owner\'s request. ' +`Read the actual controls and affected elements from the source; do not replace the behavior checks with a heading-only assertion. ` +`Call ${PREVIEW_INSPECTION_TOOL} with siteId ${JSON.stringify(preview.siteId)}, sha256 ${JSON.stringify(preview.sha256)}, viewport {width,height}, and valid steps. ` +diff --git a/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs b/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjsindex 553812c2a..3dda7e542 100644--- a/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs+++ b/ods/extensions/services/pixel-agent/plugin/tool-loop-guard.mjs@@ -48,7 +48,7 @@ import { PREVIEW_INSPECTION_TOOL, requestsVisibilityInteraction, requestsBehavioboundInspectionPageErrors, boundInspectionControls, boundRenderedPreviewInspection, renderedInspectionInstruction,pageErrorRepairInstruction, visibilityInspectionMatches,visibilityInspectionInstruction, requestedVisibilityTransition, inheritedVisibilityTransition,- attemptedPreviewBehavior, boundPreviewBehavior, previewBehaviorInstruction } from './preview-interaction-assurance.mjs';+ attemptedPreviewBehavior, boundPreviewBehavior, previewBehaviorInstruction, unassertedBehaviorStep } from './preview-interaction-assurance.mjs';import { workspaceRevalidationCandidate, workspaceReadOnlyCall, settledRevalidationReceipt, boundedPreviewVerification } from "./preview-revalidation.mjs";import { boundedPreviewDelivery } from './preview-delivery-recovery.mjs';import { explainedPreviewDelivery, inspectionFailureLine } from './preview-explanation.mjs';@@ -7832,6 +7832,14 @@ export function createToolLoopGuard({? attempt : undefined;}+ // The step that kept the current snapshot's latest passing inspection from+ // counting as behavior evidence, for the next-step instruction. Never evidence.+ function workspaceBehaviorGap(state) {+ const gap = state.workspaceBehaviorGap;+ return visibilityInspectionMatches(gap, state.workspacePreview) &&+ gap.sessionId === state.currentSessionId && gap.sessionKey === state.currentSessionKey ? gap : undefined;+ }+// The failed step of the current snapshot's latest inspection, for the note// beside a kept explanation (preview-explanation.mjs). Never evidence.function workspaceInspectionFailureLine(state) {@@ -8171,6 +8179,7 @@ export function createToolLoopGuard({state.workspaceBehaviorInspection = undefined;state.workspaceRenderedInspection = undefined;state.workspaceInspectionFailure = undefined;+ state.workspaceBehaviorGap = undefined;state.workspaceInspectionGeneration = (state.workspaceInspectionGeneration ?? 0) + 1;}// First-attempt obligation: record the duty to check an interaction@@ -11325,6 +11334,13 @@ export function createToolLoopGuard({? boundRenderedPreviewInspection(inspected.params, inspected.result, state.workspacePreview) : undefined;state.workspaceRenderedInspection = renderedProof ? Object.freeze({...renderedProof,sessionId: state.currentSessionId, sessionKey: state.currentSessionKey}) : undefined;+ // A passing plan that the behavior policy above rejects only for an+ // unasserted action: name that step in the next instruction.+ const behaviorGap = behaviorBound && renderedProof && !state.workspaceBehaviorInspection &&+ !inspected.params.steps.some(step => step.action === 'download')+ ? unassertedBehaviorStep(inspected.params.steps) : undefined;+ state.workspaceBehaviorGap = behaviorGap ? Object.freeze({...behaviorGap, siteId: renderedProof.siteId,+ sha256: renderedProof.sha256, sessionId: state.currentSessionId, sessionKey: state.currentSessionKey}) : undefined;// Names the first failed step to the owner beside a kept explanation.const failureLine = inspectionFailureLine(inspected.result?.details, state.workspacePreview);state.workspaceInspectionFailure = failureLine ? Object.freeze({siteId: state.workspacePreview.siteId,@@ -12761,7 +12777,7 @@ export function createToolLoopGuard({!state.workspaceVisibilityInspectionUnavailable) return {stage: 'workspace-preview-behavior',instruction: previewBehaviorInstruction(state.workspacePreview,- state.workspaceInspectionPageErrors),+ state.workspaceInspectionPageErrors, workspaceBehaviorGap(state)),};if (controlNamesUninspected(state) && !state.workspaceVisibilityInspectionUnavailable) return {stage: 'workspace-preview-control-name-inspection',@@ -13143,7 +13159,7 @@ export function createToolLoopGuard({return '[ODS Pixel next step] ' + (state.workspaceVisibilityInspectionUnavailable? 'Keep the published preview, but report the attempted interaction checks as unverified because inspection is unavailable. Do not claim they work.': previewBehaviorInstruction(state.workspacePreview,- state.workspaceInspectionPageErrors));+ state.workspaceInspectionPageErrors, workspaceBehaviorGap(state)));}// Any inspection of this snapshot reports its load-time names.if (controlNamesUninspected(state) && !state.workspaceVisibilityInspectionUnavailable) {@@ -14401,6 +14417,7 @@ export function createToolLoopGuard({state.workspaceBehaviorInspection = undefined;state.workspaceRenderedInspection = undefined;state.workspaceInspectionFailure = undefined;+ state.workspaceBehaviorGap = undefined;state.previewVerificationGeneration = (state.previewVerificationGeneration ?? 0) + 1;sessionPreviews.delete(state.currentSessionId);sessionPreviewVisibilityObligations.delete(state.currentSessionId);diff --git a/ods/extensions/services/pixel-agent/tests/preview_interaction_assurance.test.mjs b/ods/extensions/services/pixel-agent/tests/preview_interaction_assurance.test.mjsindex fca85a343..1847bb3c9 100644--- a/ods/extensions/services/pixel-agent/tests/preview_interaction_assurance.test.mjs+++ b/ods/extensions/services/pixel-agent/tests/preview_interaction_assurance.test.mjs@@ -823,7 +823,7 @@ test('a different run and session cannot inherit the behavior obligation',()=>{// --- Pure helper tests for attemptedPreviewBehavior/boundPreviewBehavior ---import {FILL_INSPECTION_SCOPE} from '../plugin/workspace-preview-inspect.mjs';-import {attemptedPreviewBehavior, boundPreviewBehavior} from '../plugin/preview-interaction-assurance.mjs';+import {attemptedPreviewBehavior, boundPreviewBehavior, unassertedBehaviorStep} from '../plugin/preview-interaction-assurance.mjs';function purePreview(sha='a'.repeat(64)) { return {siteId:'site-'+sha.slice(0,24),sha256:sha}; }@@ -953,6 +953,160 @@ test('several fields may prepare one click, but two clicks cannot share a finalassert.equal(boundPreviewBehavior(noPostcondition,{details:pureReceipt(noPostcondition)},preview,attempted),undefined);});+test('unassertedBehaviorStep names exactly the step boundPreviewBehavior rejects',()=>{+ const preview=purePreview(),at=selector=>({selector});+ const step=(action,selector)=>action==='assert-text' ? {action,locator:at(selector),expectedText:'ok'}+ : action==='fill' ? {action,locator:at(selector),value:'x'} : action==='press' ? {action,locator:at(selector),key:'Space'} : {action,locator:at(selector)};+ for (const [actions,gap] of [+ [['fill','fill','click','assert-text'],undefined],+ [['click','assert-visible','click','assert-hidden'],undefined],+ [['press','press','assert-painted'],undefined],+ [['assert-hidden','click','click','assert-visible'],{index:1,action:'click',next:{index:2,action:'click'}}],+ [['click','press','assert-painted'],{index:0,action:'click',next:{index:1,action:'press'}}],+ [['click','assert-visible','fill'],{index:2,action:'fill'}],+ [['fill','click'],{index:1,action:'click'}],+ ]) {+ const params=pureParams(preview,actions.map((action,index)=>step(action,'#s'+index)));+ assert.deepEqual(unassertedBehaviorStep(params.steps),gap,actions.join(','));+ // pureReceipt builds click, fill and assertion evidence, not press or paint.+ if (actions.some(action=>['press','assert-painted'].includes(action))) continue;+ assert.equal(Boolean(boundPreviewBehavior(params,{details:pureReceipt(params)},preview,{requiresInteraction:true})),gap===undefined,actions.join(','));+ }+});++// RTX Spark stopwatch run (2026-10-25): the first check failed at step 2, the+// model edited the page and republished, and its check of the new snapshot+// passed ("tested opposite visibility states of the same element around a+// click"). Delivery still said the attempted checks had not passed.+const stopwatchPrompt='Make me a simple stopwatch web page with start, stop and lap buttons';+const stopwatchButton=name=>({role:'button',name,exact:true});+function stopwatchPlan(preview,steps=[+ {action:'assert-visible',locator:stopwatchButton('Start')},+ {action:'assert-hidden',locator:{selector:'#stop'}},+ {action:'click',locator:stopwatchButton('Start')},+ {action:'assert-hidden',locator:{selector:'#start'}},+ {action:'assert-visible',locator:{selector:'#stop'}},+ {action:'click',locator:stopwatchButton('Stop')},+ {action:'assert-visible',locator:{selector:'#start'}},+]) { return {siteId:preview.siteId,sha256:preview.sha256,viewport:{width:480,height:800},steps}; }+// Lap is clicked and Stop follows at once: the Start transition passes, but+// the Lap click's own result is never asserted.+function lapThenStopPlan(preview,lapResult) { return stopwatchPlan(preview,[+ {action:'assert-visible',locator:stopwatchButton('Start')},+ {action:'assert-hidden',locator:{selector:'#stop'}},+ {action:'click',locator:stopwatchButton('Start')},+ {action:'assert-visible',locator:{selector:'#stop'}},+ {action:'click',locator:stopwatchButton('Lap')},+ ...(lapResult ? [{action:'assert-visible',locator:{selector:'#laps li'}}] : []),+ {action:'click',locator:stopwatchButton('Stop')},+ {action:'assert-visible',locator:{selector:'#start'}},+ {action:'assert-hidden',locator:{selector:'#stop'}},+]); }+function persistObserved(guard,{event,ctx},result) {+ return guard.toolResultPersist({toolName:event.toolName,toolCallId:event.toolCallId,+ message:{role:'toolResult',toolName:event.toolName,toolCallId:event.toolCallId,content:[{type:'text',text:'result'}],...result}},ctx);+}+const nextSteps=persisted=>(persisted?.message?.content ?? []).filter(block=>/^\[ODS Pixel next step\]/.test(block?.text ?? '')).map(block=>block.text);+// The first check fails at its step 2 (assert-hidden on a disabled, still+// visible button); the model edits the page and publishes a new snapshot.+function failThenRepublish(guard,preview,wrapped) {+ const first=behaviorInspection(guard,stopwatchPlan(preview),{wrapped,id:'first-check'});+ const failed=structuredClone(first.result), inner=wrapped?failed.details.result:failed;+ inner.isError=true;+ inner.details.status='failed';+ inner.details.steps=inner.details.steps.slice(0,2);+ inner.details.steps[1]={...inner.details.steps[1],status:'failed',errorCode:'visibility_mismatch',+ before:{count:1,visible:true,display:'inline-block',visibility:'visible',opacity:'0.4',hidden:false,hiddenUntilFound:false,rectCount:1}};+ guard.afterToolCall({...first.event,result:failed},first.ctx);+ persistObserved(guard,first,failed);+ assert.equal(guard.verificationForRun('run').status,'failed');+ assert.match(guard.verificationForRun('run').text,/attempted preview interaction checks have not passed/);+ const content='<!doctype html><button>Show details</button><p hidden>Details</p>';+ const edit=call(guard,'edit',{path:preview.relativeDirectory+'/index.html',edits:[{oldText:'<p hidden>',newText:'<p hidden class="details">'}]},+ 'repair',{content:[{type:'text',text:'Successfully replaced 1 block(s).'}]});+ persistObserved(guard,edit,{content:[{type:'text',text:'Successfully replaced 1 block(s).'}]});+ const data=Buffer.from(content.replace('<p hidden>','<p hidden class="details">')),name=Buffer.from('index.html'),a=Buffer.alloc(4),b=Buffer.alloc(8);+ a.writeUInt32BE(name.length);b.writeBigUInt64BE(BigInt(data.length));+ const sha256=createHash('sha256').update(a).update(name).update(b).update(data).digest('hex'),siteId='site-'+sha256.slice(0,24);+ const next={...preview,sha256,siteId,entrySha256:createHash('sha256').update(data).digest('hex'),bytes:data.length,+ url:`http://${siteId}.localhost:9437/${siteId}/`};+ const published=call(guard,'pixel_ods_workspace_preview',{relativeDirectory:preview.relativeDirectory},'republish',{details:next});+ // Its result already carries the generic behavior step for this snapshot.+ assert.equal(nextSteps(persistObserved(guard,published,{details:next})).length,1);+ return next;+}++for (const wrapped of [false,true]) test(`a passing check of the republished snapshot satisfies the check that failed on the earlier one (${wrapped?'deferred':'direct'})`,()=>{+ const {guard,preview}=setup({prompt:stopwatchPrompt});+ const next=failThenRepublish(guard,preview,wrapped);+ assert.notEqual(next.sha256,preview.sha256);+ assert.equal(guard.verificationForRun('run').status,'failed','the earlier attempt still needs a check of the new snapshot');+ // A pass of the older snapshot cannot certify the new one.+ const stale=behaviorInspection(guard,stopwatchPlan(preview),{wrapped,id:'stale-check'});+ guard.afterToolCall({...stale.event,result:stale.result},stale.ctx);+ assert.equal(guard.verificationForRun('run').status,'failed');+ const latest=behaviorInspection(guard,stopwatchPlan(next),{wrapped,id:'latest-check'});+ guard.afterToolCall({...latest.event,result:latest.result},latest.ctx);+ const delivered=guard.verificationForRun('run');+ assert.equal(delivered.status,'passed');+ assert.equal(delivered.preview.sha256,next.sha256);+ assert.match(delivered.text,/Browser inspection passed for the submitted interaction checks only/);+ assert.doesNotMatch(delivered.text,/attempted preview interaction checks have not passed/);+ assert.equal(guard.beforeAgentFinalize({},context),undefined);+});++for (const wrapped of [false,true]) test(`a passing plan that leaves a click unasserted says which step, in its own result (${wrapped?'deferred':'direct'})`,()=>{+ const {guard,preview}=setup({prompt:stopwatchPrompt});+ const next=failThenRepublish(guard,preview,wrapped);+ const gap=behaviorInspection(guard,lapThenStopPlan(next),{wrapped,id:'lap-then-stop'});+ guard.afterToolCall({...gap.event,result:gap.result},gap.ctx);+ // The evidence policy is unchanged: this plan proves no behavior.+ assert.equal(guard.verificationForRun('run').status,'failed');+ assert.match(guard.verificationForRun('run').text,/attempted preview interaction checks have not passed/);+ // The pinned harness drops a finalize revision after a plugin tool call, so+ // this result is the last point that can tell the model what to correct.+ const [step]=nextSteps(persistObserved(guard,gap,gap.result));+ assert.ok(step,'the passing result carries its own next step');+ assert.match(step,/The latest inspection of this snapshot passed, but step 5 \(click\) is followed directly by step 6 \(click\) with no assertion of the click's own result/);+ assert.ok(step.includes(next.sha256));+ assert.match(guard.beforeAgentFinalize({},context)?.retry?.instruction ?? '',/step 5 \(click\) is followed directly by step 6 \(click\)/);+ const corrected=behaviorInspection(guard,lapThenStopPlan(next,true),{wrapped,id:'lap-asserted'});+ guard.afterToolCall({...corrected.event,result:corrected.result},corrected.ctx);+ assert.equal(guard.verificationForRun('run').status,'passed');+ assert.deepEqual(nextSteps(persistObserved(guard,corrected,corrected.result)).filter(text=>/latest inspection of this snapshot passed, but/.test(text)),[]);+ assert.equal(guard.beforeAgentFinalize({},context),undefined);+});++test('a Tool Search child that binds the passing plan names the unasserted step on its parent result',()=>{+ const {guard,preview}=setup({prompt:stopwatchPrompt});+ const next=failThenRepublish(guard,preview,true);+ const id='nested-gap',params=lapThenStopPlan(next);+ const outer=behaviorInspection(guard,params,{wrapped:true,id});+ const child=call(guard,PREVIEW_INSPECTION_TOOL,params,`tool_search_code:${id}:${PREVIEW_INSPECTION_TOOL}:1`);+ guard.afterToolCall({...child.event,result:outer.result.details.result},child.ctx);+ guard.afterToolCall({...outer.event,result:outer.result},outer.ctx);+ assert.equal(guard.verificationForRun('run').status,'failed');+ assert.match(nextSteps(persistObserved(guard,outer,outer.result)).join('\n'),/step 5 \(click\) is followed directly by step 6 \(click\)/);+});++for (const fault of ['failed','trailing','older-snapshot','static']) test(`only a passing current check names an unasserted step: ${fault}`,()=>{+ const {guard,preview}=setup({prompt:stopwatchPrompt});+ const next=failThenRepublish(guard,preview,false);+ const params=fault==='older-snapshot' ? lapThenStopPlan(preview)+ : fault==='static' ? stopwatchPlan(next,[{action:'assert-visible',locator:stopwatchButton('Start')}])+ : fault==='trailing' ? stopwatchPlan(next,[...stopwatchPlan(next).steps,{action:'click',locator:stopwatchButton('Lap')}])+ : lapThenStopPlan(next);+ const observed=behaviorInspection(guard,params,{id:'check-'+fault});+ const result=structuredClone(observed.result);+ if (fault==='failed') { result.isError=true; result.details.status='failed'; result.details.steps=result.details.steps.slice(0,1);+ result.details.steps[0]={...result.details.steps[0],status:'failed',errorCode:'no_match',before:{count:0}}; }+ guard.afterToolCall({...observed.event,result},observed.ctx);+ assert.equal(guard.verificationForRun('run').status,'failed');+ const steps=nextSteps(persistObserved(guard,observed,result)).join('\n');+ if (fault==='trailing') assert.match(steps,/The latest inspection of this snapshot passed, but no assertion follows its last action, step 8 \(click\)/);+ else assert.doesNotMatch(steps,/latest inspection of this snapshot passed, but/);+});+// A live laptop page was published with unguarded localStorage and called// functional despite a sandbox SecurityError. HTTP readback alone must not// certify model-authored pages when browser inspection is installed.