Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(agent): preserve completion fallback and evidence freshness

Emre Sokullu9e3d903020dc
diff --git a/package.json b/package.json
index 0d8281485..1a3fa32ff 100644
--- a/package.json
+++ b/package.json
@@ -58,7 +58,7 @@
"test:rich-text-dom": "node test/ui-content-lifecycle.mjs --dom",
"test:firefox-bidi": "node --test test/firefox-bidi.mjs test/firefox-full-page-screenshot.mjs",
"test:firefox-bidi:e2e": "node test/firefox-bidi-e2e.mjs",
- "test:systemone": "node --test test/systemone.mjs test/completion-verifier.mjs",
+ "test:systemone": "node --test test/systemone.mjs test/completion-verifier.mjs test/completion-document.mjs",
"test:systemone:ui": "node test/systemone-ui.mjs",
"test:systemone:fast": "node --test test/systemone-fast.mjs && node test/systemone-fast-dom.mjs && node --test test/jev/fixtures.test.mjs",
"benchmark:jev": "node test/jev/benchmark.mjs",
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 45e372b95..8c0047551 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -1,6 +1,7 @@
import { verifyBrowserCompletion } from './completion-runtime.js';
import { DECISION_SETTINGS_KEYS, resolveDecisionConfig } from './decision-config.js';
import { completionStopError } from './completion-verifier.js';
+import { COMPLETION_DOCUMENT_STAMP_SCRIPT } from './completion-document.js';
import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';
import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText } from './systemone-evidence.js';
import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
@@ -7190,7 +7191,7 @@ export class Agent extends LoopDetector {
}
async _completionDocumentStamp(tabId) {
- const result = await cdpClient.evaluate(tabId, "(() => { const text = (document.body?.innerText || '').slice(0,20000) + Array.from(document.querySelectorAll('input,textarea,select')).slice(0,100).map(e=>e.value).join('|'); let hash=2166136261; for(const c of text) hash=Math.imul(hash ^ c.charCodeAt(0),16777619); return [location.href,performance.timeOrigin,text.length,hash].join('|'); })()");
+ const result = await cdpClient.evaluate(tabId, COMPLETION_DOCUMENT_STAMP_SCRIPT);
return result?.result?.value || '';
}
@@ -15150,10 +15151,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* Whether another automatic screenshot is allowed under maxScreenshotsPerTurn
* (issue #311). 0 means unlimited. Does not mutate the counter.
*/
- _canTakeAutoScreenshot(tabId) {
+ _canTakeAutoScreenshot(tabId, slots = 1) {
const cap = Number(this.maxScreenshotsPerTurn) || 0;
if (cap <= 0) return true;
- return (this.autoScreenshotCount.get(tabId) || 0) < cap;
+ return (this.autoScreenshotCount.get(tabId) || 0) + slots <= cap;
}
/**
diff --git a/src/chrome/src/agent/completion-document.js b/src/chrome/src/agent/completion-document.js
new file mode 100644
index 000000000..f86167e66
--- /dev/null
+++ b/src/chrome/src/agent/completion-document.js
@@ -0,0 +1,20 @@
+// Executed in the document in both browsers. Only the fingerprint leaves it.
+// Visible text and values alone omit checked, disabled and ARIA state.
+export function completionDocumentStamp() {
+ const states = [];
+ for (const element of document.querySelectorAll('*')) {
+ const attributes = Array.from(element.attributes).filter(attribute =>
+ attribute.name.startsWith('aria-') || ['role', 'hidden', 'inert', 'disabled', 'readonly', 'required'].includes(attribute.name));
+ if (!attributes.length && !/^(INPUT|TEXTAREA|SELECT|OPTION|BUTTON)$/.test(element.tagName)) continue;
+ states.push([element.tagName, element.id, element.value, element.checked, element.indeterminate,
+ element.disabled, element.readOnly, element.selected, element.selectedIndex,
+ attributes.map(attribute => [attribute.name, attribute.value]).sort()]);
+ if (states.length >= 1000) break;
+ }
+ const text = (document.body?.innerText || '').slice(0, 20000) + JSON.stringify(states);
+ let hash = 2166136261;
+ for (const character of text) hash = Math.imul(hash ^ character.charCodeAt(0), 16777619);
+ return [location.href, performance.timeOrigin, text.length, hash].join('|');
+}
+
+export const COMPLETION_DOCUMENT_STAMP_SCRIPT = `(${completionDocumentStamp.toString()})()`;
diff --git a/src/chrome/src/agent/completion-runtime.js b/src/chrome/src/agent/completion-runtime.js
index cf52bf1d3..7a3662c35 100644
--- a/src/chrome/src/agent/completion-runtime.js
+++ b/src/chrome/src/agent/completion-runtime.js
@@ -37,10 +37,11 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrl
const run = agent.completionInvariants.get(tabId);
const scope = JSON.stringify([run?.runToken, run?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)]);
agent._completionVerdicts ??= new Map();
- const currentIdentity = async evidence => !!evidence.identity && context.isCurrent() && !signal?.aborted && agent._activeProvider(tabId) === provider
+ const currentRun = () => context.isCurrent() && !signal?.aborted && agent._activeProvider(tabId) === provider
+ && scope === JSON.stringify([agent.completionInvariants.get(tabId)?.runToken, agent.completionInvariants.get(tabId)?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)]);
+ const currentIdentity = async evidence => !!evidence.identity && currentRun()
&& JSON.stringify(resolveDecisionConfig(await chrome.storage.local.get(DECISION_SETTINGS_KEYS), compass)) === JSON.stringify(config)
- && scope === JSON.stringify([agent.completionInvariants.get(tabId)?.runToken, agent.completionInvariants.get(tabId)?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)])
- && evidence.identity === await agent._completionDocumentStamp(tabId);
+ && evidence.identity === await agent._completionDocumentStamp(tabId) && currentRun();
const current = async evidence => {
if (!await currentIdentity(evidence)) return false;
if (evidence.modality !== 'vision') return true;
@@ -61,7 +62,11 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrl
const capture = async (modality, refresh = false) => {
const before = await agent._completionDocumentStamp(tabId);
const cached = captures.get(modality);
- if (!refresh && cached?.identity === before && context.isCurrent()) return cached;
+ const reusable = cached?.identity === before && context.isCurrent();
+ // A new vision judgment needs both its evidence and its post-request
+ // freshness capture. With only one slot left, use AX before paying a judge.
+ if (modality === 'vision' && agent._canTakeAutoScreenshot?.(tabId, refresh || reusable ? 1 : 2) === false) return null;
+ if (!refresh && reusable) return cached;
let observation;
if (modality === 'vision') {
let pixels = await agent._captureCompletionJudgeImage(tabId);
diff --git a/src/chrome/src/agent/completion-verifier.js b/src/chrome/src/agent/completion-verifier.js
index 2800b4ecc..53556b030 100644
--- a/src/chrome/src/agent/completion-verifier.js
+++ b/src/chrome/src/agent/completion-verifier.js
@@ -48,9 +48,10 @@ export async function verifyCompletion({ decision, llm, capture, isCurrent, sign
if (verdict.outcome !== 'uncertain') return { ...verdict, engine: engine.name, modality, identity: evidence.identity, evidenceKey: evidence.key };
} catch (error) {
if (completionStopError(error, signal)) throw error;
- onAttempt({ engine: engine.name, modality, outcome: 'uncertain', reason: error?.code || (error?.status ? `http_${error.status}` : 'unavailable') });
+ const status = error?.status ?? error?.httpStatus;
+ onAttempt({ engine: engine.name, modality, outcome: 'uncertain', reason: error?.code || (status ? `http_${status}` : 'unavailable') });
// Only an image-contract error can retry a failed request as AX.
- if (!(modality === 'vision' && [400, 413, 415, 422].includes(error?.status))) break;
+ if (!(modality === 'vision' && [400, 413, 415, 422].includes(status))) break;
}
}
}
diff --git a/src/chrome/src/agent/decision-vision-probe.js b/src/chrome/src/agent/decision-vision-probe.js
index 29c6441cd..50905560c 100644
--- a/src/chrome/src/agent/decision-vision-probe.js
+++ b/src/chrome/src/agent/decision-vision-probe.js
@@ -1,13 +1,16 @@
// Test pixels, not HTTP acceptance. Two independently randomized solid colors
// avoid treating a model that always guesses one label as vision-capable.
-export async function probeDecisionVision(config, evaluate, { random = () => crypto.getRandomValues(new Uint32Array(1))[0] } = {}) {
+export async function probeDecisionVision(config, evaluate, { random = Math.random } = {}) {
const colors = ['red', 'blue', 'green', 'yellow'];
- const first = random() % colors.length;
- const second = (first + 1 + random() % 3) % colors.length;
- for (const index of [first, second]) {
- if (index === second) await new Promise(resolve => setTimeout(resolve, 1100));
+ // These test labels need variety, not cryptographic secrecy. Sample without
+ // replacement rather than applying modulo arithmetic to a secure integer.
+ const first = colors[Math.floor(random() * colors.length)];
+ const remaining = colors.filter(color => color !== first);
+ const second = remaining[Math.floor(random() * remaining.length)];
+ for (const color of [first, second]) {
+ if (color === second) await new Promise(resolve => setTimeout(resolve, 1100));
const canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(64, 64) : Object.assign(document.createElement('canvas'), { width: 64, height: 64 });
- const context = canvas.getContext('2d'); context.fillStyle = colors[index]; context.fillRect(0, 0, 64, 64);
+ const context = canvas.getContext('2d'); context.fillStyle = color; context.fillRect(0, 0, 64, 64);
const blob = canvas.convertToBlob ? await canvas.convertToBlob({ type: 'image/png' }) : await new Promise(resolve => canvas.toBlob(resolve, 'image/png'));
const bytes = new Uint8Array(await blob.arrayBuffer());
const url = 'data:image/png;base64,' + btoa(String.fromCharCode(...bytes));
@@ -15,7 +18,7 @@ export async function probeDecisionVision(config, evaluate, { random = () => cry
state: [{ type: 'image_url', image_url: { url } }],
questions: { image_color: { type: 'choice', instructions: 'Which color fills the attached image? Answer from the pixels only.', criteria: Object.fromEntries(colors.map(c => [c, c + ' pixels'])) } } });
const answer = result.answers.image_color;
- if (answer.choice !== colors[index] || answer.probabilities[colors[index]] < .9) return false;
+ if (answer.choice !== color || answer.probabilities[color] < .9) return false;
}
return true;
}
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index c7c6d46ef..f1ae234f6 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -1,6 +1,7 @@
import { verifyBrowserCompletion } from './completion-runtime.js';
import { DECISION_SETTINGS_KEYS, resolveDecisionConfig } from './decision-config.js';
import { completionStopError } from './completion-verifier.js';
+import { COMPLETION_DOCUMENT_STAMP_SCRIPT } from './completion-document.js';
import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';
import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText } from './systemone-evidence.js';
import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
@@ -7041,7 +7042,7 @@ export class Agent extends LoopDetector {
}
async _completionDocumentStamp(tabId) {
- const result = await browser.tabs.executeScript(tabId, { code: "(() => { const text = (document.body?.innerText || '').slice(0,20000) + Array.from(document.querySelectorAll('input,textarea,select')).slice(0,100).map(e=>e.value).join('|'); let hash=2166136261; for(const c of text) hash=Math.imul(hash ^ c.charCodeAt(0),16777619); return [location.href,performance.timeOrigin,text.length,hash].join('|'); })()" });
+ const result = await browser.tabs.executeScript(tabId, { code: COMPLETION_DOCUMENT_STAMP_SCRIPT });
return result?.[0] || '';
}
@@ -14463,10 +14464,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* Whether another automatic screenshot is allowed under maxScreenshotsPerTurn
* (issue #311). 0 means unlimited. Does not mutate the counter.
*/
- _canTakeAutoScreenshot(tabId) {
+ _canTakeAutoScreenshot(tabId, slots = 1) {
const cap = Number(this.maxScreenshotsPerTurn) || 0;
if (cap <= 0) return true;
- return (this.autoScreenshotCount.get(tabId) || 0) < cap;
+ return (this.autoScreenshotCount.get(tabId) || 0) + slots <= cap;
}
/**
diff --git a/src/firefox/src/agent/completion-document.js b/src/firefox/src/agent/completion-document.js
new file mode 100644
index 000000000..f86167e66
--- /dev/null
+++ b/src/firefox/src/agent/completion-document.js
@@ -0,0 +1,20 @@
+// Executed in the document in both browsers. Only the fingerprint leaves it.
+// Visible text and values alone omit checked, disabled and ARIA state.
+export function completionDocumentStamp() {
+ const states = [];
+ for (const element of document.querySelectorAll('*')) {
+ const attributes = Array.from(element.attributes).filter(attribute =>
+ attribute.name.startsWith('aria-') || ['role', 'hidden', 'inert', 'disabled', 'readonly', 'required'].includes(attribute.name));
+ if (!attributes.length && !/^(INPUT|TEXTAREA|SELECT|OPTION|BUTTON)$/.test(element.tagName)) continue;
+ states.push([element.tagName, element.id, element.value, element.checked, element.indeterminate,
+ element.disabled, element.readOnly, element.selected, element.selectedIndex,
+ attributes.map(attribute => [attribute.name, attribute.value]).sort()]);
+ if (states.length >= 1000) break;
+ }
+ const text = (document.body?.innerText || '').slice(0, 20000) + JSON.stringify(states);
+ let hash = 2166136261;
+ for (const character of text) hash = Math.imul(hash ^ character.charCodeAt(0), 16777619);
+ return [location.href, performance.timeOrigin, text.length, hash].join('|');
+}
+
+export const COMPLETION_DOCUMENT_STAMP_SCRIPT = `(${completionDocumentStamp.toString()})()`;
diff --git a/src/firefox/src/agent/completion-runtime.js b/src/firefox/src/agent/completion-runtime.js
index 4db5344d7..5b384d9da 100644
--- a/src/firefox/src/agent/completion-runtime.js
+++ b/src/firefox/src/agent/completion-runtime.js
@@ -37,10 +37,11 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrl
const run = agent.completionInvariants.get(tabId);
const scope = JSON.stringify([run?.runToken, run?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)]);
agent._completionVerdicts ??= new Map();
- const currentIdentity = async evidence => !!evidence.identity && context.isCurrent() && !signal?.aborted && agent._activeProvider(tabId) === provider
+ const currentRun = () => context.isCurrent() && !signal?.aborted && agent._activeProvider(tabId) === provider
+ && scope === JSON.stringify([agent.completionInvariants.get(tabId)?.runToken, agent.completionInvariants.get(tabId)?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)]);
+ const currentIdentity = async evidence => !!evidence.identity && currentRun()
&& JSON.stringify(resolveDecisionConfig(await browser.storage.local.get(DECISION_SETTINGS_KEYS), compass)) === JSON.stringify(config)
- && scope === JSON.stringify([agent.completionInvariants.get(tabId)?.runToken, agent.completionInvariants.get(tabId)?.lastAction?.sequence, agent._progressTaskKeyHash(tabId), config.provider, config.model, config.url, provider.model, provider.supportsVision, JSON.stringify(config)])
- && evidence.identity === await agent._completionDocumentStamp(tabId);
+ && evidence.identity === await agent._completionDocumentStamp(tabId) && currentRun();
const current = async evidence => {
if (!await currentIdentity(evidence)) return false;
if (evidence.modality !== 'vision') return true;
@@ -61,7 +62,11 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrl
const capture = async (modality, refresh = false) => {
const before = await agent._completionDocumentStamp(tabId);
const cached = captures.get(modality);
- if (!refresh && cached?.identity === before && context.isCurrent()) return cached;
+ const reusable = cached?.identity === before && context.isCurrent();
+ // A new vision judgment needs both its evidence and its post-request
+ // freshness capture. With only one slot left, use AX before paying a judge.
+ if (modality === 'vision' && agent._canTakeAutoScreenshot?.(tabId, refresh || reusable ? 1 : 2) === false) return null;
+ if (!refresh && reusable) return cached;
let observation;
if (modality === 'vision') {
let pixels = await agent._captureCompletionJudgeImage(tabId);
diff --git a/src/firefox/src/agent/completion-verifier.js b/src/firefox/src/agent/completion-verifier.js
index 2800b4ecc..53556b030 100644
--- a/src/firefox/src/agent/completion-verifier.js
+++ b/src/firefox/src/agent/completion-verifier.js
@@ -48,9 +48,10 @@ export async function verifyCompletion({ decision, llm, capture, isCurrent, sign
if (verdict.outcome !== 'uncertain') return { ...verdict, engine: engine.name, modality, identity: evidence.identity, evidenceKey: evidence.key };
} catch (error) {
if (completionStopError(error, signal)) throw error;
- onAttempt({ engine: engine.name, modality, outcome: 'uncertain', reason: error?.code || (error?.status ? `http_${error.status}` : 'unavailable') });
+ const status = error?.status ?? error?.httpStatus;
+ onAttempt({ engine: engine.name, modality, outcome: 'uncertain', reason: error?.code || (status ? `http_${status}` : 'unavailable') });
// Only an image-contract error can retry a failed request as AX.
- if (!(modality === 'vision' && [400, 413, 415, 422].includes(error?.status))) break;
+ if (!(modality === 'vision' && [400, 413, 415, 422].includes(status))) break;
}
}
}
diff --git a/src/firefox/src/agent/decision-vision-probe.js b/src/firefox/src/agent/decision-vision-probe.js
index 29c6441cd..50905560c 100644
--- a/src/firefox/src/agent/decision-vision-probe.js
+++ b/src/firefox/src/agent/decision-vision-probe.js
@@ -1,13 +1,16 @@
// Test pixels, not HTTP acceptance. Two independently randomized solid colors
// avoid treating a model that always guesses one label as vision-capable.
-export async function probeDecisionVision(config, evaluate, { random = () => crypto.getRandomValues(new Uint32Array(1))[0] } = {}) {
+export async function probeDecisionVision(config, evaluate, { random = Math.random } = {}) {
const colors = ['red', 'blue', 'green', 'yellow'];
- const first = random() % colors.length;
- const second = (first + 1 + random() % 3) % colors.length;
- for (const index of [first, second]) {
- if (index === second) await new Promise(resolve => setTimeout(resolve, 1100));
+ // These test labels need variety, not cryptographic secrecy. Sample without
+ // replacement rather than applying modulo arithmetic to a secure integer.
+ const first = colors[Math.floor(random() * colors.length)];
+ const remaining = colors.filter(color => color !== first);
+ const second = remaining[Math.floor(random() * remaining.length)];
+ for (const color of [first, second]) {
+ if (color === second) await new Promise(resolve => setTimeout(resolve, 1100));
const canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(64, 64) : Object.assign(document.createElement('canvas'), { width: 64, height: 64 });
- const context = canvas.getContext('2d'); context.fillStyle = colors[index]; context.fillRect(0, 0, 64, 64);
+ const context = canvas.getContext('2d'); context.fillStyle = color; context.fillRect(0, 0, 64, 64);
const blob = canvas.convertToBlob ? await canvas.convertToBlob({ type: 'image/png' }) : await new Promise(resolve => canvas.toBlob(resolve, 'image/png'));
const bytes = new Uint8Array(await blob.arrayBuffer());
const url = 'data:image/png;base64,' + btoa(String.fromCharCode(...bytes));
@@ -15,7 +18,7 @@ export async function probeDecisionVision(config, evaluate, { random = () => cry
state: [{ type: 'image_url', image_url: { url } }],
questions: { image_color: { type: 'choice', instructions: 'Which color fills the attached image? Answer from the pixels only.', criteria: Object.fromEntries(colors.map(c => [c, c + ' pixels'])) } } });
const answer = result.answers.image_color;
- if (answer.choice !== colors[index] || answer.probabilities[colors[index]] < .9) return false;
+ if (answer.choice !== color || answer.probabilities[color] < .9) return false;
}
return true;
}
diff --git a/test/chat-steering-ui.mjs b/test/chat-steering-ui.mjs
index 3187a78fa..483e35206 100644
--- a/test/chat-steering-ui.mjs
+++ b/test/chat-steering-ui.mjs
@@ -12,6 +12,7 @@ function extract(source, name) {
}
for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
+ const { DECISION_SETTINGS_KEYS } = await import(`../src/${build}/src/agent/decision-config.js`);
test(`${build}: native composer honors delivery settings, explicitly steers, and preserves drafts across races`, async () => {
const browser = await engine.launch({ headless: true });
try {
@@ -73,6 +74,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} };
var browser = chrome;
var AUTO_GROUP_TABS_KEY = 'autoGroupTabs';
+ var DECISION_SETTINGS_KEYS = ${JSON.stringify(DECISION_SETTINGS_KEYS)};
var DOWNLOAD_DIRECTORY_STORAGE_KEY = 'downloadDirectory';
var CLOUD_BRIDGE_ENABLED_KEY = 'cloudBridgeEnabled', CLOUD_BRIDGE_URL_KEY = 'cloudBridgeUrl';
var composerDeliveryModeSelect = document.getElementById('select-composer-delivery-mode');
diff --git a/test/completion-document.mjs b/test/completion-document.mjs
new file mode 100644
index 000000000..0d4fe6fff
--- /dev/null
+++ b/test/completion-document.mjs
@@ -0,0 +1,77 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { chromium, firefox } from 'playwright';
+
+for (const [build, browserType] of [['chrome', chromium], ['firefox', firefox]]) {
+ const { COMPLETION_DOCUMENT_STAMP_SCRIPT } = await import(`../src/${build}/src/agent/completion-document.js`);
+ const { verifyBrowserCompletion } = await import(`../src/${build}/src/agent/completion-runtime.js`);
+ const { probeDecisionVision } = await import(`../src/${build}/src/agent/decision-vision-probe.js`);
+ test(`${build}: completion stamp detects accessibility-only changes and rejects reverted checkbox success`, async () => {
+ const browser = await browserType.launch();
+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;
+ try {
+ const page = await browser.newPage();
+ await page.setContent('<input id="check" type="checkbox" checked><input id="text" value="unchanged"><button id="button">Save</button><div id="state" role="status">Published</div>');
+ const stamp = () => page.evaluate(COMPLETION_DOCUMENT_STAMP_SCRIPT);
+ const initial = await stamp();
+ for (const [selector, property] of [['#check', 'checked'], ['#check', 'indeterminate'], ['#text', 'disabled'], ['#text', 'readOnly']]) {
+ await page.locator(selector).evaluate((element, property) => { element[property] = !element[property]; }, property);
+ assert.notEqual(await stamp(), initial, property);
+ await page.locator(selector).evaluate((element, property) => { element[property] = !element[property]; }, property);
+ assert.equal(await stamp(), initial);
+ }
+ for (const attribute of ['aria-checked', 'aria-expanded', 'aria-busy', 'aria-disabled', 'aria-hidden', 'aria-selected', 'aria-pressed', 'aria-label', 'aria-valuenow']) {
+ await page.locator('#state').evaluate((element, name) => element.setAttribute(name, 'true'), attribute);
+ assert.notEqual(await stamp(), initial, attribute);
+ await page.locator('#state').evaluate((element, name) => element.removeAttribute(name), attribute);
+ assert.equal(await stamp(), initial);
+ }
+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({ decisionProvider: 'local', systemOneEnabled: true }) } } };
+ const provider = { model: 'active', supportsVision: false, config: { category: 'local' } };
+ const agent = {
+ _activeProvider: () => provider, _runAbortSignal: () => null, systemOneContext: () => ({ isCurrent: () => true }),
+ _latestTaskText: () => 'Check this checkbox', _originalTaskText: () => '', _progressTaskKeyHash: () => 'check',
+ _planExecutionGuards: new Map(), completionInvariants: new Map([[1, { runToken: 'run' }]]),
+ _completionSubmitStates: new Map(), conversationIds: new Map(), conversations: new Map(), _completionDocumentStamp: stamp,
+ executeTool: async () => ({ success: true, pageContent: 'Checkbox checked' }), recordSystemOneVerdict() {},
+ evaluateSystemOne: async () => {
+ await page.locator('#check').evaluate(element => { element.checked = false; });
+ return { model: 'kev-latest', answers: { task_outcome: { choice: 'succeeded', probabilities: { succeeded: .99 } } } };
+ },
+ };
+ const verdict = await verifyBrowserCompletion(agent, 1);
+ assert.equal(verdict.outcome, 'pending');
+ assert.equal(verdict.engine, 'freshness');
+ assert.equal(await page.locator('#check').inputValue(), 'on', 'the value did not change with checked state');
+ } finally {
+ globalThis.chrome = previousChrome; globalThis.browser = previousBrowser;
+ await browser.close();
+ }
+ });
+ test(`${build}: vision capability probe uses two distinct pixel facts and rejects constant answers`, async () => {
+ const browser = await browserType.launch();
+ try {
+ const page = await browser.newPage();
+ const result = await page.evaluate(async source => {
+ const probe = (0, eval)(`(${source})`);
+ const colors = [];
+ const consume = async request => {
+ const image = new Image(); image.src = request.state[0].image_url.url;
+ await image.decode();
+ const canvas = document.createElement('canvas'); canvas.width = canvas.height = 64;
+ const context = canvas.getContext('2d'); context.drawImage(image, 0, 0);
+ const [r, g, b] = context.getImageData(32, 32, 1, 1).data;
+ const choice = r && g ? 'yellow' : r ? 'red' : g ? 'green' : b ? 'blue' : 'unknown';
+ colors.push(choice);
+ return { answers: { image_color: { choice, probabilities: { [choice]: .99 } } } };
+ };
+ const accepted = await probe({}, consume, { random: () => .999 });
+ const constantRejected = !await probe({}, async () => ({ answers: { image_color: { choice: 'red', probabilities: { red: .99 } } } }), { random: () => 0 });
+ return { accepted, colors, constantRejected };
+ }, probeDecisionVision.toString());
+ assert.equal(result.accepted, true);
+ assert.equal(new Set(result.colors).size, 2);
+ assert.equal(result.constantRejected, true);
+ } finally { await browser.close(); }
+ });
+}
diff --git a/test/completion-verifier.mjs b/test/completion-verifier.mjs
index df080d59b..c24ca2583 100644
--- a/test/completion-verifier.mjs
+++ b/test/completion-verifier.mjs
@@ -73,12 +73,13 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(result.modality, 'ax', 'capture failures should retry with AX');
});
test(`${build}: auth/transport skip engine; unsupported image retries AX`, async () => {
- for (const status of [401, 422]) {
+ for (const field of ['status', 'httpStatus']) for (const status of [401, 500, 400, 413, 415, 422]) for (const name of ['decision', 'llm']) {
const calls = [];
- const decision = { name: 'decision', supportsVision: true, evaluate: async (_e, modality) => { calls.push(modality); if (modality === 'vision') throw Object.assign(new Error('HTTP'), { status }); return { outcome: 'failed' }; } };
- const result = await verifier.verifyCompletion({ decision, capture: async () => ({ identity: 'fresh' }), isCurrent: () => true });
- assert.deepEqual(calls, status === 401 ? ['vision'] : ['vision', 'ax']);
- assert.equal(result.outcome, status === 401 ? 'uncertain' : 'failed');
+ const engine = { name, supportsVision: true, evaluate: async (_e, modality) => { calls.push(modality); if (modality === 'vision') throw Object.assign(new Error('HTTP'), { [field]: status }); return { outcome: 'failed' }; } };
+ const result = await verifier.verifyCompletion({ [name]: engine, capture: async () => ({ identity: 'fresh' }), isCurrent: () => true });
+ const unsupportedImage = [400, 413, 415, 422].includes(status);
+ assert.deepEqual(calls, unsupportedImage ? ['vision', 'ax'] : ['vision'], `${name}/${field}/${status}`);
+ assert.equal(result.outcome, unsupportedImage ? 'failed' : 'uncertain');
}
});
test(`${build}: quota, cancellation, budget and stale evidence never fall through`, async () => {
@@ -216,4 +217,59 @@ for (const build of ['chrome', 'firefox']) {
if (navigatorDescriptor) Object.defineProperty(globalThis, 'navigator', navigatorDescriptor); else delete globalThis.navigator;
}
});
+ test(`${build}: finite screenshot budgets preserve freshness capacity or select AX`, async () => {
+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;
+ const navigatorDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'navigator');
+ Object.defineProperty(globalThis, 'navigator', { configurable: true, value: { onLine: true } });
+ try {
+ for (const engine of ['decision', 'llm']) for (const [cap, used, modality] of [[0, 10, 'vision'], [1, 0, 'ax'], [2, 0, 'vision'], [3, 2, 'ax'], [3, 1, 'vision'], [2, 2, 'ax']]) {
+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({ decisionProvider: 'local', systemOneEnabled: engine === 'decision', decisionVisionMode: 'on' }) } } };
+ let captures = 0, spent = used, requests = 0;
+ const active = { model: 'local-model', supportsVision: true, config: { category: 'local' }, chat() {} };
+ const agent = {
+ _activeProvider: () => active, _runAbortSignal: () => null, systemOneContext: () => ({ isCurrent: () => true }),
+ _latestTaskText: () => 'Verify publication', _originalTaskText: () => '', _progressTaskKeyHash: () => 'task',
+ _planExecutionGuards: new Map(), completionInvariants: new Map([[1, { runToken: 'run' }]]),
+ _completionSubmitStates: new Map(), conversationIds: new Map(), conversations: new Map(),
+ _completionDocumentStamp: async () => 'stable',
+ _canTakeAutoScreenshot: (_tabId, slots = 1) => cap === 0 || spent + slots <= cap,
+ _captureCompletionJudgeImage: async () => { if (cap && spent >= cap) return null; spent++; captures++; return 'data:image/png;base64,stable'; },
+ _budgetForCapture: () => ({ maxTargetPx: 1408, maxTargetTokens: 1400 }), _shrinkImageForBudget: async dataUrl => ({ dataUrl }),
+ executeTool: async () => ({ success: true, pageContent: 'Published requested content' }),
+ _newCostRunState: () => ({}), recordSystemOneVerdict() {},
+ evaluateSystemOne: async () => { requests++; const result = response(); result.model = 'kev-latest'; return result; },
+ _chatWithCostAllowance: async () => { requests++; return { content: '{"outcome":"succeeded","reason":"Published"}' }; },
+ };
+ const verdict = await runtime.verifyBrowserCompletion(agent, 1);
+ assert.equal(verdict.outcome, 'succeeded', `${engine}: cap=${cap}, used=${used}`);
+ assert.equal(verdict.modality, modality);
+ assert.equal(captures, modality === 'vision' ? 2 : 0);
+ assert.equal(requests, 1);
+ if (cap) assert.ok(spent <= cap, 'freshness must honor the user screenshot cap');
+ }
+ } finally {
+ globalThis.chrome = previousChrome; globalThis.browser = previousBrowser;
+ if (navigatorDescriptor) Object.defineProperty(globalThis, 'navigator', navigatorDescriptor); else delete globalThis.navigator;
+ }
+ });
+ test(`${build}: a replaced run during the final document read cannot establish completion`, async () => {
+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;
+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({ decisionProvider: 'local', systemOneEnabled: true }) } } };
+ let judged = false, current = true;
+ const provider = { model: 'local', config: { category: 'local' } };
+ const agent = {
+ _activeProvider: () => provider, _runAbortSignal: () => null, systemOneContext: () => ({ isCurrent: () => current }),
+ _latestTaskText: () => 'Verify publication', _originalTaskText: () => '', _progressTaskKeyHash: () => 'task',
+ _planExecutionGuards: new Map(), completionInvariants: new Map([[1, { runToken: 'run' }]]),
+ _completionSubmitStates: new Map(), conversationIds: new Map(), conversations: new Map(),
+ _completionDocumentStamp: async () => { if (judged) current = false; return 'stable'; },
+ executeTool: async () => ({ success: true, pageContent: 'Published' }), recordSystemOneVerdict() {},
+ evaluateSystemOne: async () => { judged = true; const result = response(); result.model = 'kev-latest'; return result; },
+ };
+ try {
+ const verdict = await runtime.verifyBrowserCompletion(agent, 1);
+ assert.equal(verdict.outcome, 'pending');
+ assert.equal(verdict.engine, 'freshness');
+ } finally { globalThis.chrome = previousChrome; globalThis.browser = previousBrowser; }
+ });
}
diff --git a/test/run.js b/test/run.js
index 8fad887dc..75435c418 100644
--- a/test/run.js
+++ b/test/run.js
@@ -20665,6 +20665,7 @@ test('image budget helpers: auto-screenshot counter + failed capture does not bu
const agent = new AgentClass({});
const tabId = 42;
agent.maxScreenshotsPerTurn = 1;
+ assert.equal(agent._canTakeAutoScreenshot(tabId, 2), false, `${AgentClass.name}: vision verification must reserve its refresh slot`);
assert.equal(agent._canTakeAutoScreenshot(tabId), true);
agent._recordAutoScreenshot(tabId);