Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(agent): preserve completion fallback and evidence freshness
Emre Sokullu9e3d903020dc
diff --git a/package.json b/package.jsonindex 0d8281485..1a3fa32ff 100644--- a/package.json+++ b/package.json@@ -58,7 +58,7 @@"test:rich-text-dom": "node test/ui-content-lifecycle.mjs --dom","test:firefox-bidi": "node --test test/firefox-bidi.mjs test/firefox-full-page-screenshot.mjs","test:firefox-bidi:e2e": "node test/firefox-bidi-e2e.mjs",- "test:systemone": "node --test test/systemone.mjs test/completion-verifier.mjs",+ "test:systemone": "node --test test/systemone.mjs test/completion-verifier.mjs test/completion-document.mjs","test:systemone:ui": "node test/systemone-ui.mjs","test:systemone:fast": "node --test test/systemone-fast.mjs && node test/systemone-fast-dom.mjs && node --test test/jev/fixtures.test.mjs","benchmark:jev": "node test/jev/benchmark.mjs",diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 45e372b95..8c0047551 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -1,6 +1,7 @@import { verifyBrowserCompletion } from './completion-runtime.js';import { DECISION_SETTINGS_KEYS, resolveDecisionConfig } from './decision-config.js';import { completionStopError } from './completion-verifier.js';+import { COMPLETION_DOCUMENT_STAMP_SCRIPT } from './completion-document.js';import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText } from './systemone-evidence.js';import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';@@ -7190,7 +7191,7 @@ export class Agent extends LoopDetector {}async _completionDocumentStamp(tabId) {- const result = await cdpClient.evaluate(tabId, "(() => { const text = (document.body?.innerText || '').slice(0,20000) + Array.from(document.querySelectorAll('input,textarea,select')).slice(0,100).map(e=>e.value).join('|'); let hash=2166136261; for(const c of text) hash=Math.imul(hash ^ c.charCodeAt(0),16777619); return [location.href,performance.timeOrigin,text.length,hash].join('|'); })()");+ const result = await cdpClient.evaluate(tabId, COMPLETION_DOCUMENT_STAMP_SCRIPT);return result?.result?.value || '';}@@ -15150,10 +15151,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* Whether another automatic screenshot is allowed under maxScreenshotsPerTurn* (issue #311). 0 means unlimited. Does not mutate the counter.*/- _canTakeAutoScreenshot(tabId) {+ _canTakeAutoScreenshot(tabId, slots = 1) {const cap = Number(this.maxScreenshotsPerTurn) || 0;if (cap <= 0) return true;- return (this.autoScreenshotCount.get(tabId) || 0) < cap;+ return (this.autoScreenshotCount.get(tabId) || 0) + slots <= cap;}/**diff --git a/src/chrome/src/agent/completion-document.js b/src/chrome/src/agent/completion-document.jsnew file mode 100644index 000000000..f86167e66--- /dev/null+++ b/src/chrome/src/agent/completion-document.js@@ -0,0 +1,20 @@+// Executed in the document in both browsers. Only the fingerprint leaves it.+// Visible text and values alone omit checked, disabled and ARIA state.+export function completionDocumentStamp() {+ const states = [];+ for (const element of document.querySelectorAll('*')) {+ const attributes = Array.from(element.attributes).filter(attribute =>+ attribute.name.startsWith('aria-') || ['role', 'hidden', 'inert', 'disabled', 'readonly', 'required'].includes(attribute.name));+ if (!attributes.length && !/^(INPUT|TEXTAREA|SELECT|OPTION|BUTTON)$/.test(element.tagName)) continue;+ states.push([element.tagName, element.id, element.value, element.checked, element.indeterminate,+ element.disabled, element.readOnly, element.selected, element.selectedIndex,+ attributes.map(attribute => [attribute.name, attribute.value]).sort()]);+ if (states.length >= 1000) break;+ }+ const text = (document.body?.innerText || '').slice(0, 20000) + JSON.stringify(states);+ let hash = 2166136261;+ for (const character of text) hash = Math.imul(hash ^ character.charCodeAt(0), 16777619);+ return [location.href, performance.timeOrigin, text.length, hash].join('|');+}++export const COMPLETION_DOCUMENT_STAMP_SCRIPT = `(${completionDocumentStamp.toString()})()`;diff --git a/src/chrome/src/agent/completion-runtime.js b/src/chrome/src/agent/completion-runtime.jsindex cf52bf1d3..7a3662c35 100644--- a/src/chrome/src/agent/completion-runtime.js+++ b/src/chrome/src/agent/completion-runtime.js@@ -37,10 +37,11 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrlconst run = agent.completionInvariants.get(tabId);const scope = JSON.stringify([run?.runToken, run?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)]);agent._completionVerdicts ??= new Map();- const currentIdentity = async evidence => !!evidence.identity && context.isCurrent() && !signal?.aborted && agent._activeProvider(tabId) === provider+ const currentRun = () => context.isCurrent() && !signal?.aborted && agent._activeProvider(tabId) === provider+ && scope === JSON.stringify([agent.completionInvariants.get(tabId)?.runToken, agent.completionInvariants.get(tabId)?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)]);+ const currentIdentity = async evidence => !!evidence.identity && currentRun()&& JSON.stringify(resolveDecisionConfig(await chrome.storage.local.get(DECISION_SETTINGS_KEYS), compass)) === JSON.stringify(config)- && scope === JSON.stringify([agent.completionInvariants.get(tabId)?.runToken, agent.completionInvariants.get(tabId)?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)])- && evidence.identity === await agent._completionDocumentStamp(tabId);+ && evidence.identity === await agent._completionDocumentStamp(tabId) && currentRun();const current = async evidence => {if (!await currentIdentity(evidence)) return false;if (evidence.modality !== 'vision') return true;@@ -61,7 +62,11 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrlconst capture = async (modality, refresh = false) => {const before = await agent._completionDocumentStamp(tabId);const cached = captures.get(modality);- if (!refresh && cached?.identity === before && context.isCurrent()) return cached;+ const reusable = cached?.identity === before && context.isCurrent();+ // A new vision judgment needs both its evidence and its post-request+ // freshness capture. With only one slot left, use AX before paying a judge.+ if (modality === 'vision' && agent._canTakeAutoScreenshot?.(tabId, refresh || reusable ? 1 : 2) === false) return null;+ if (!refresh && reusable) return cached;let observation;if (modality === 'vision') {let pixels = await agent._captureCompletionJudgeImage(tabId);diff --git a/src/chrome/src/agent/completion-verifier.js b/src/chrome/src/agent/completion-verifier.jsindex 2800b4ecc..53556b030 100644--- a/src/chrome/src/agent/completion-verifier.js+++ b/src/chrome/src/agent/completion-verifier.js@@ -48,9 +48,10 @@ export async function verifyCompletion({ decision, llm, capture, isCurrent, signif (verdict.outcome !== 'uncertain') return { ...verdict, engine: engine.name, modality, identity: evidence.identity, evidenceKey: evidence.key };} catch (error) {if (completionStopError(error, signal)) throw error;- onAttempt({ engine: engine.name, modality, outcome: 'uncertain', reason: error?.code || (error?.status ? `http_${error.status}` : 'unavailable') });+ const status = error?.status ?? error?.httpStatus;+ onAttempt({ engine: engine.name, modality, outcome: 'uncertain', reason: error?.code || (status ? `http_${status}` : 'unavailable') });// Only an image-contract error can retry a failed request as AX.- if (!(modality === 'vision' && [400, 413, 415, 422].includes(error?.status))) break;+ if (!(modality === 'vision' && [400, 413, 415, 422].includes(status))) break;}}}diff --git a/src/chrome/src/agent/decision-vision-probe.js b/src/chrome/src/agent/decision-vision-probe.jsindex 29c6441cd..50905560c 100644--- a/src/chrome/src/agent/decision-vision-probe.js+++ b/src/chrome/src/agent/decision-vision-probe.js@@ -1,13 +1,16 @@// Test pixels, not HTTP acceptance. Two independently randomized solid colors// avoid treating a model that always guesses one label as vision-capable.-export async function probeDecisionVision(config, evaluate, { random = () => crypto.getRandomValues(new Uint32Array(1))[0] } = {}) {+export async function probeDecisionVision(config, evaluate, { random = Math.random } = {}) {const colors = ['red', 'blue', 'green', 'yellow'];- const first = random() % colors.length;- const second = (first + 1 + random() % 3) % colors.length;- for (const index of [first, second]) {- if (index === second) await new Promise(resolve => setTimeout(resolve, 1100));+ // These test labels need variety, not cryptographic secrecy. Sample without+ // replacement rather than applying modulo arithmetic to a secure integer.+ const first = colors[Math.floor(random() * colors.length)];+ const remaining = colors.filter(color => color !== first);+ const second = remaining[Math.floor(random() * remaining.length)];+ for (const color of [first, second]) {+ if (color === second) await new Promise(resolve => setTimeout(resolve, 1100));const canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(64, 64) : Object.assign(document.createElement('canvas'), { width: 64, height: 64 });- const context = canvas.getContext('2d'); context.fillStyle = colors[index]; context.fillRect(0, 0, 64, 64);+ const context = canvas.getContext('2d'); context.fillStyle = color; context.fillRect(0, 0, 64, 64);const blob = canvas.convertToBlob ? await canvas.convertToBlob({ type: 'image/png' }) : await new Promise(resolve => canvas.toBlob(resolve, 'image/png'));const bytes = new Uint8Array(await blob.arrayBuffer());const url = 'data:image/png;base64,' + btoa(String.fromCharCode(...bytes));@@ -15,7 +18,7 @@ export async function probeDecisionVision(config, evaluate, { random = () => crystate: [{ type: 'image_url', image_url: { url } }],questions: { image_color: { type: 'choice', instructions: 'Which color fills the attached image? Answer from the pixels only.', criteria: Object.fromEntries(colors.map(c => [c, c + ' pixels'])) } } });const answer = result.answers.image_color;- if (answer.choice !== colors[index] || answer.probabilities[colors[index]] < .9) return false;+ if (answer.choice !== color || answer.probabilities[color] < .9) return false;}return true;}diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex c7c6d46ef..f1ae234f6 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -1,6 +1,7 @@import { verifyBrowserCompletion } from './completion-runtime.js';import { DECISION_SETTINGS_KEYS, resolveDecisionConfig } from './decision-config.js';import { completionStopError } from './completion-verifier.js';+import { COMPLETION_DOCUMENT_STAMP_SCRIPT } from './completion-document.js';import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText } from './systemone-evidence.js';import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';@@ -7041,7 +7042,7 @@ export class Agent extends LoopDetector {}async _completionDocumentStamp(tabId) {- const result = await browser.tabs.executeScript(tabId, { code: "(() => { const text = (document.body?.innerText || '').slice(0,20000) + Array.from(document.querySelectorAll('input,textarea,select')).slice(0,100).map(e=>e.value).join('|'); let hash=2166136261; for(const c of text) hash=Math.imul(hash ^ c.charCodeAt(0),16777619); return [location.href,performance.timeOrigin,text.length,hash].join('|'); })()" });+ const result = await browser.tabs.executeScript(tabId, { code: COMPLETION_DOCUMENT_STAMP_SCRIPT });return result?.[0] || '';}@@ -14463,10 +14464,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* Whether another automatic screenshot is allowed under maxScreenshotsPerTurn* (issue #311). 0 means unlimited. Does not mutate the counter.*/- _canTakeAutoScreenshot(tabId) {+ _canTakeAutoScreenshot(tabId, slots = 1) {const cap = Number(this.maxScreenshotsPerTurn) || 0;if (cap <= 0) return true;- return (this.autoScreenshotCount.get(tabId) || 0) < cap;+ return (this.autoScreenshotCount.get(tabId) || 0) + slots <= cap;}/**diff --git a/src/firefox/src/agent/completion-document.js b/src/firefox/src/agent/completion-document.jsnew file mode 100644index 000000000..f86167e66--- /dev/null+++ b/src/firefox/src/agent/completion-document.js@@ -0,0 +1,20 @@+// Executed in the document in both browsers. Only the fingerprint leaves it.+// Visible text and values alone omit checked, disabled and ARIA state.+export function completionDocumentStamp() {+ const states = [];+ for (const element of document.querySelectorAll('*')) {+ const attributes = Array.from(element.attributes).filter(attribute =>+ attribute.name.startsWith('aria-') || ['role', 'hidden', 'inert', 'disabled', 'readonly', 'required'].includes(attribute.name));+ if (!attributes.length && !/^(INPUT|TEXTAREA|SELECT|OPTION|BUTTON)$/.test(element.tagName)) continue;+ states.push([element.tagName, element.id, element.value, element.checked, element.indeterminate,+ element.disabled, element.readOnly, element.selected, element.selectedIndex,+ attributes.map(attribute => [attribute.name, attribute.value]).sort()]);+ if (states.length >= 1000) break;+ }+ const text = (document.body?.innerText || '').slice(0, 20000) + JSON.stringify(states);+ let hash = 2166136261;+ for (const character of text) hash = Math.imul(hash ^ character.charCodeAt(0), 16777619);+ return [location.href, performance.timeOrigin, text.length, hash].join('|');+}++export const COMPLETION_DOCUMENT_STAMP_SCRIPT = `(${completionDocumentStamp.toString()})()`;diff --git a/src/firefox/src/agent/completion-runtime.js b/src/firefox/src/agent/completion-runtime.jsindex 4db5344d7..5b384d9da 100644--- a/src/firefox/src/agent/completion-runtime.js+++ b/src/firefox/src/agent/completion-runtime.js@@ -37,10 +37,11 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrlconst run = agent.completionInvariants.get(tabId);const scope = JSON.stringify([run?.runToken, run?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)]);agent._completionVerdicts ??= new Map();- const currentIdentity = async evidence => !!evidence.identity && context.isCurrent() && !signal?.aborted && agent._activeProvider(tabId) === provider+ const currentRun = () => context.isCurrent() && !signal?.aborted && agent._activeProvider(tabId) === provider+ && scope === JSON.stringify([agent.completionInvariants.get(tabId)?.runToken, agent.completionInvariants.get(tabId)?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)]);+ const currentIdentity = async evidence => !!evidence.identity && currentRun()&& JSON.stringify(resolveDecisionConfig(await browser.storage.local.get(DECISION_SETTINGS_KEYS), compass)) === JSON.stringify(config)- && scope === JSON.stringify([agent.completionInvariants.get(tabId)?.runToken, agent.completionInvariants.get(tabId)?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)])- && evidence.identity === await agent._completionDocumentStamp(tabId);+ && evidence.identity === await agent._completionDocumentStamp(tabId) && currentRun();const current = async evidence => {if (!await currentIdentity(evidence)) return false;if (evidence.modality !== 'vision') return true;@@ -61,7 +62,11 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrlconst capture = async (modality, refresh = false) => {const before = await agent._completionDocumentStamp(tabId);const cached = captures.get(modality);- if (!refresh && cached?.identity === before && context.isCurrent()) return cached;+ const reusable = cached?.identity === before && context.isCurrent();+ // A new vision judgment needs both its evidence and its post-request+ // freshness capture. With only one slot left, use AX before paying a judge.+ if (modality === 'vision' && agent._canTakeAutoScreenshot?.(tabId, refresh || reusable ? 1 : 2) === false) return null;+ if (!refresh && reusable) return cached;let observation;if (modality === 'vision') {let pixels = await agent._captureCompletionJudgeImage(tabId);diff --git a/src/firefox/src/agent/completion-verifier.js b/src/firefox/src/agent/completion-verifier.jsindex 2800b4ecc..53556b030 100644--- a/src/firefox/src/agent/completion-verifier.js+++ b/src/firefox/src/agent/completion-verifier.js@@ -48,9 +48,10 @@ export async function verifyCompletion({ decision, llm, capture, isCurrent, signif (verdict.outcome !== 'uncertain') return { ...verdict, engine: engine.name, modality, identity: evidence.identity, evidenceKey: evidence.key };} catch (error) {if (completionStopError(error, signal)) throw error;- onAttempt({ engine: engine.name, modality, outcome: 'uncertain', reason: error?.code || (error?.status ? `http_${error.status}` : 'unavailable') });+ const status = error?.status ?? error?.httpStatus;+ onAttempt({ engine: engine.name, modality, outcome: 'uncertain', reason: error?.code || (status ? `http_${status}` : 'unavailable') });// Only an image-contract error can retry a failed request as AX.- if (!(modality === 'vision' && [400, 413, 415, 422].includes(error?.status))) break;+ if (!(modality === 'vision' && [400, 413, 415, 422].includes(status))) break;}}}diff --git a/src/firefox/src/agent/decision-vision-probe.js b/src/firefox/src/agent/decision-vision-probe.jsindex 29c6441cd..50905560c 100644--- a/src/firefox/src/agent/decision-vision-probe.js+++ b/src/firefox/src/agent/decision-vision-probe.js@@ -1,13 +1,16 @@// Test pixels, not HTTP acceptance. Two independently randomized solid colors// avoid treating a model that always guesses one label as vision-capable.-export async function probeDecisionVision(config, evaluate, { random = () => crypto.getRandomValues(new Uint32Array(1))[0] } = {}) {+export async function probeDecisionVision(config, evaluate, { random = Math.random } = {}) {const colors = ['red', 'blue', 'green', 'yellow'];- const first = random() % colors.length;- const second = (first + 1 + random() % 3) % colors.length;- for (const index of [first, second]) {- if (index === second) await new Promise(resolve => setTimeout(resolve, 1100));+ // These test labels need variety, not cryptographic secrecy. Sample without+ // replacement rather than applying modulo arithmetic to a secure integer.+ const first = colors[Math.floor(random() * colors.length)];+ const remaining = colors.filter(color => color !== first);+ const second = remaining[Math.floor(random() * remaining.length)];+ for (const color of [first, second]) {+ if (color === second) await new Promise(resolve => setTimeout(resolve, 1100));const canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(64, 64) : Object.assign(document.createElement('canvas'), { width: 64, height: 64 });- const context = canvas.getContext('2d'); context.fillStyle = colors[index]; context.fillRect(0, 0, 64, 64);+ const context = canvas.getContext('2d'); context.fillStyle = color; context.fillRect(0, 0, 64, 64);const blob = canvas.convertToBlob ? await canvas.convertToBlob({ type: 'image/png' }) : await new Promise(resolve => canvas.toBlob(resolve, 'image/png'));const bytes = new Uint8Array(await blob.arrayBuffer());const url = 'data:image/png;base64,' + btoa(String.fromCharCode(...bytes));@@ -15,7 +18,7 @@ export async function probeDecisionVision(config, evaluate, { random = () => crystate: [{ type: 'image_url', image_url: { url } }],questions: { image_color: { type: 'choice', instructions: 'Which color fills the attached image? Answer from the pixels only.', criteria: Object.fromEntries(colors.map(c => [c, c + ' pixels'])) } } });const answer = result.answers.image_color;- if (answer.choice !== colors[index] || answer.probabilities[colors[index]] < .9) return false;+ if (answer.choice !== color || answer.probabilities[color] < .9) return false;}return true;}diff --git a/test/chat-steering-ui.mjs b/test/chat-steering-ui.mjsindex 3187a78fa..483e35206 100644--- a/test/chat-steering-ui.mjs+++ b/test/chat-steering-ui.mjs@@ -12,6 +12,7 @@ function extract(source, name) {}for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {+ const { DECISION_SETTINGS_KEYS } = await import(`../src/${build}/src/agent/decision-config.js`);test(`${build}: native composer honors delivery settings, explicitly steers, and preserves drafts across races`, async () => {const browser = await engine.launch({ headless: true });try {@@ -73,6 +74,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} };var browser = chrome;var AUTO_GROUP_TABS_KEY = 'autoGroupTabs';+ var DECISION_SETTINGS_KEYS = ${JSON.stringify(DECISION_SETTINGS_KEYS)};var DOWNLOAD_DIRECTORY_STORAGE_KEY = 'downloadDirectory';var CLOUD_BRIDGE_ENABLED_KEY = 'cloudBridgeEnabled', CLOUD_BRIDGE_URL_KEY = 'cloudBridgeUrl';var composerDeliveryModeSelect = document.getElementById('select-composer-delivery-mode');diff --git a/test/completion-document.mjs b/test/completion-document.mjsnew file mode 100644index 000000000..0d4fe6fff--- /dev/null+++ b/test/completion-document.mjs@@ -0,0 +1,77 @@+import assert from 'node:assert/strict';+import { test } from 'node:test';+import { chromium, firefox } from 'playwright';++for (const [build, browserType] of [['chrome', chromium], ['firefox', firefox]]) {+ const { COMPLETION_DOCUMENT_STAMP_SCRIPT } = await import(`../src/${build}/src/agent/completion-document.js`);+ const { verifyBrowserCompletion } = await import(`../src/${build}/src/agent/completion-runtime.js`);+ const { probeDecisionVision } = await import(`../src/${build}/src/agent/decision-vision-probe.js`);+ test(`${build}: completion stamp detects accessibility-only changes and rejects reverted checkbox success`, async () => {+ const browser = await browserType.launch();+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;+ try {+ const page = await browser.newPage();+ await page.setContent('<input id="check" type="checkbox" checked><input id="text" value="unchanged"><button id="button">Save</button><div id="state" role="status">Published</div>');+ const stamp = () => page.evaluate(COMPLETION_DOCUMENT_STAMP_SCRIPT);+ const initial = await stamp();+ for (const [selector, property] of [['#check', 'checked'], ['#check', 'indeterminate'], ['#text', 'disabled'], ['#text', 'readOnly']]) {+ await page.locator(selector).evaluate((element, property) => { element[property] = !element[property]; }, property);+ assert.notEqual(await stamp(), initial, property);+ await page.locator(selector).evaluate((element, property) => { element[property] = !element[property]; }, property);+ assert.equal(await stamp(), initial);+ }+ for (const attribute of ['aria-checked', 'aria-expanded', 'aria-busy', 'aria-disabled', 'aria-hidden', 'aria-selected', 'aria-pressed', 'aria-label', 'aria-valuenow']) {+ await page.locator('#state').evaluate((element, name) => element.setAttribute(name, 'true'), attribute);+ assert.notEqual(await stamp(), initial, attribute);+ await page.locator('#state').evaluate((element, name) => element.removeAttribute(name), attribute);+ assert.equal(await stamp(), initial);+ }+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({ decisionProvider: 'local', systemOneEnabled: true }) } } };+ const provider = { model: 'active', supportsVision: false, config: { category: 'local' } };+ const agent = {+ _activeProvider: () => provider, _runAbortSignal: () => null, systemOneContext: () => ({ isCurrent: () => true }),+ _latestTaskText: () => 'Check this checkbox', _originalTaskText: () => '', _progressTaskKeyHash: () => 'check',+ _planExecutionGuards: new Map(), completionInvariants: new Map([[1, { runToken: 'run' }]]),+ _completionSubmitStates: new Map(), conversationIds: new Map(), conversations: new Map(), _completionDocumentStamp: stamp,+ executeTool: async () => ({ success: true, pageContent: 'Checkbox checked' }), recordSystemOneVerdict() {},+ evaluateSystemOne: async () => {+ await page.locator('#check').evaluate(element => { element.checked = false; });+ return { model: 'kev-latest', answers: { task_outcome: { choice: 'succeeded', probabilities: { succeeded: .99 } } } };+ },+ };+ const verdict = await verifyBrowserCompletion(agent, 1);+ assert.equal(verdict.outcome, 'pending');+ assert.equal(verdict.engine, 'freshness');+ assert.equal(await page.locator('#check').inputValue(), 'on', 'the value did not change with checked state');+ } finally {+ globalThis.chrome = previousChrome; globalThis.browser = previousBrowser;+ await browser.close();+ }+ });+ test(`${build}: vision capability probe uses two distinct pixel facts and rejects constant answers`, async () => {+ const browser = await browserType.launch();+ try {+ const page = await browser.newPage();+ const result = await page.evaluate(async source => {+ const probe = (0, eval)(`(${source})`);+ const colors = [];+ const consume = async request => {+ const image = new Image(); image.src = request.state[0].image_url.url;+ await image.decode();+ const canvas = document.createElement('canvas'); canvas.width = canvas.height = 64;+ const context = canvas.getContext('2d'); context.drawImage(image, 0, 0);+ const [r, g, b] = context.getImageData(32, 32, 1, 1).data;+ const choice = r && g ? 'yellow' : r ? 'red' : g ? 'green' : b ? 'blue' : 'unknown';+ colors.push(choice);+ return { answers: { image_color: { choice, probabilities: { [choice]: .99 } } } };+ };+ const accepted = await probe({}, consume, { random: () => .999 });+ const constantRejected = !await probe({}, async () => ({ answers: { image_color: { choice: 'red', probabilities: { red: .99 } } } }), { random: () => 0 });+ return { accepted, colors, constantRejected };+ }, probeDecisionVision.toString());+ assert.equal(result.accepted, true);+ assert.equal(new Set(result.colors).size, 2);+ assert.equal(result.constantRejected, true);+ } finally { await browser.close(); }+ });+}diff --git a/test/completion-verifier.mjs b/test/completion-verifier.mjsindex df080d59b..c24ca2583 100644--- a/test/completion-verifier.mjs+++ b/test/completion-verifier.mjs@@ -73,12 +73,13 @@ for (const build of ['chrome', 'firefox']) {assert.equal(result.modality, 'ax', 'capture failures should retry with AX');});test(`${build}: auth/transport skip engine; unsupported image retries AX`, async () => {- for (const status of [401, 422]) {+ for (const field of ['status', 'httpStatus']) for (const status of [401, 500, 400, 413, 415, 422]) for (const name of ['decision', 'llm']) {const calls = [];- const decision = { name: 'decision', supportsVision: true, evaluate: async (_e, modality) => { calls.push(modality); if (modality === 'vision') throw Object.assign(new Error('HTTP'), { status }); return { outcome: 'failed' }; } };- const result = await verifier.verifyCompletion({ decision, capture: async () => ({ identity: 'fresh' }), isCurrent: () => true });- assert.deepEqual(calls, status === 401 ? ['vision'] : ['vision', 'ax']);- assert.equal(result.outcome, status === 401 ? 'uncertain' : 'failed');+ const engine = { name, supportsVision: true, evaluate: async (_e, modality) => { calls.push(modality); if (modality === 'vision') throw Object.assign(new Error('HTTP'), { [field]: status }); return { outcome: 'failed' }; } };+ const result = await verifier.verifyCompletion({ [name]: engine, capture: async () => ({ identity: 'fresh' }), isCurrent: () => true });+ const unsupportedImage = [400, 413, 415, 422].includes(status);+ assert.deepEqual(calls, unsupportedImage ? ['vision', 'ax'] : ['vision'], `${name}/${field}/${status}`);+ assert.equal(result.outcome, unsupportedImage ? 'failed' : 'uncertain');}});test(`${build}: quota, cancellation, budget and stale evidence never fall through`, async () => {@@ -216,4 +217,59 @@ for (const build of ['chrome', 'firefox']) {if (navigatorDescriptor) Object.defineProperty(globalThis, 'navigator', navigatorDescriptor); else delete globalThis.navigator;}});+ test(`${build}: finite screenshot budgets preserve freshness capacity or select AX`, async () => {+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;+ const navigatorDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'navigator');+ Object.defineProperty(globalThis, 'navigator', { configurable: true, value: { onLine: true } });+ try {+ for (const engine of ['decision', 'llm']) for (const [cap, used, modality] of [[0, 10, 'vision'], [1, 0, 'ax'], [2, 0, 'vision'], [3, 2, 'ax'], [3, 1, 'vision'], [2, 2, 'ax']]) {+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({ decisionProvider: 'local', systemOneEnabled: engine === 'decision', decisionVisionMode: 'on' }) } } };+ let captures = 0, spent = used, requests = 0;+ const active = { model: 'local-model', supportsVision: true, config: { category: 'local' }, chat() {} };+ const agent = {+ _activeProvider: () => active, _runAbortSignal: () => null, systemOneContext: () => ({ isCurrent: () => true }),+ _latestTaskText: () => 'Verify publication', _originalTaskText: () => '', _progressTaskKeyHash: () => 'task',+ _planExecutionGuards: new Map(), completionInvariants: new Map([[1, { runToken: 'run' }]]),+ _completionSubmitStates: new Map(), conversationIds: new Map(), conversations: new Map(),+ _completionDocumentStamp: async () => 'stable',+ _canTakeAutoScreenshot: (_tabId, slots = 1) => cap === 0 || spent + slots <= cap,+ _captureCompletionJudgeImage: async () => { if (cap && spent >= cap) return null; spent++; captures++; return 'data:image/png;base64,stable'; },+ _budgetForCapture: () => ({ maxTargetPx: 1408, maxTargetTokens: 1400 }), _shrinkImageForBudget: async dataUrl => ({ dataUrl }),+ executeTool: async () => ({ success: true, pageContent: 'Published requested content' }),+ _newCostRunState: () => ({}), recordSystemOneVerdict() {},+ evaluateSystemOne: async () => { requests++; const result = response(); result.model = 'kev-latest'; return result; },+ _chatWithCostAllowance: async () => { requests++; return { content: '{"outcome":"succeeded","reason":"Published"}' }; },+ };+ const verdict = await runtime.verifyBrowserCompletion(agent, 1);+ assert.equal(verdict.outcome, 'succeeded', `${engine}: cap=${cap}, used=${used}`);+ assert.equal(verdict.modality, modality);+ assert.equal(captures, modality === 'vision' ? 2 : 0);+ assert.equal(requests, 1);+ if (cap) assert.ok(spent <= cap, 'freshness must honor the user screenshot cap');+ }+ } finally {+ globalThis.chrome = previousChrome; globalThis.browser = previousBrowser;+ if (navigatorDescriptor) Object.defineProperty(globalThis, 'navigator', navigatorDescriptor); else delete globalThis.navigator;+ }+ });+ test(`${build}: a replaced run during the final document read cannot establish completion`, async () => {+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({ decisionProvider: 'local', systemOneEnabled: true }) } } };+ let judged = false, current = true;+ const provider = { model: 'local', config: { category: 'local' } };+ const agent = {+ _activeProvider: () => provider, _runAbortSignal: () => null, systemOneContext: () => ({ isCurrent: () => current }),+ _latestTaskText: () => 'Verify publication', _originalTaskText: () => '', _progressTaskKeyHash: () => 'task',+ _planExecutionGuards: new Map(), completionInvariants: new Map([[1, { runToken: 'run' }]]),+ _completionSubmitStates: new Map(), conversationIds: new Map(), conversations: new Map(),+ _completionDocumentStamp: async () => { if (judged) current = false; return 'stable'; },+ executeTool: async () => ({ success: true, pageContent: 'Published' }), recordSystemOneVerdict() {},+ evaluateSystemOne: async () => { judged = true; const result = response(); result.model = 'kev-latest'; return result; },+ };+ try {+ const verdict = await runtime.verifyBrowserCompletion(agent, 1);+ assert.equal(verdict.outcome, 'pending');+ assert.equal(verdict.engine, 'freshness');+ } finally { globalThis.chrome = previousChrome; globalThis.browser = previousBrowser; }+ });}diff --git a/test/run.js b/test/run.jsindex 8fad887dc..75435c418 100644--- a/test/run.js+++ b/test/run.js@@ -20665,6 +20665,7 @@ test('image budget helpers: auto-screenshot counter + failed capture does not buconst agent = new AgentClass({});const tabId = 42;agent.maxScreenshotsPerTurn = 1;+ assert.equal(agent._canTakeAutoScreenshot(tabId, 2), false, `${AgentClass.name}: vision verification must reserve its refresh slot`);assert.equal(agent._canTakeAutoScreenshot(tabId), true);agent._recordAutoScreenshot(tabId);