Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(agent): isolate completion evidence to the active run
Emre Sokulluda4b92fea6c8
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex bc4651a7c..e00ff536c 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -1226,7 +1226,9 @@ export class Agent extends LoopDetector {_beginCompletionInvariant(tabId) {this._completionRunCounter += 1;const token = `completion_${tabId}_${Date.now()}_${this._completionRunCounter}`;- this.completionInvariants.set(tabId, createCompletionInvariantState(token));+ const historyToolCallIdsBeforeRun = new Set((this.conversations.get(tabId) || [])+ .flatMap(message => [message.tool_call_id, ...(message.tool_calls || []).map(call => call.id)]).filter(Boolean));+ this.completionInvariants.set(tabId, { ...createCompletionInvariantState(token), historyToolCallIdsBeforeRun });this._completionSubmitStates.delete(tabId);this._completionVerdicts?.delete(tabId);this._doneBlockCount.delete(tabId);diff --git a/src/chrome/src/agent/completion-runtime.js b/src/chrome/src/agent/completion-runtime.jsindex 1226d8a23..24cf10dd9 100644--- a/src/chrome/src/agent/completion-runtime.js+++ b/src/chrome/src/agent/completion-runtime.js@@ -62,13 +62,17 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrlthrow Object.assign(new Error('Visual evidence changed or could not be refreshed.'), { code: 'COMPLETION_VISUAL_CHANGED' });};const messages = agent.conversations.get(tabId) || [];- const toolNames = new Map(messages.flatMap(m => (m.tool_calls || []).map(call => [call.id, call.function?.name])));- const reads = messages.filter(m => m.role === 'tool' && /read_page|get_accessibility_tree|verify_form/.test(toolNames.get(m.tool_call_id) || m.name || ''));+ // Tool IDs survive conversation compaction. Unknown provenance or reused+ // pre-run IDs cannot certify that this task performed a required read.+ const calls = run?.historyToolCallIdsBeforeRun instanceof Set+ ? messages.flatMap(m => m.tool_calls || []).filter(call => call.id && !run.historyToolCallIdsBeforeRun.has(call.id)) : [];+ const toolNames = new Map(calls.map(call => [call.id, call.function?.name]));+ const reads = messages.filter(m => m.role === 'tool' && /^(read_page|get_accessibility_tree|verify_form)$/.test(toolNames.get(m.tool_call_id) || ''));const selectedReads = [...new Set([...reads.filter(m => /rules|guidelines|moderation/i.test(String(m.content))).slice(-2), ...reads.slice(-2)])];const history = selectedReads.map(m => ({ tool: toolNames.get(m.tool_call_id) || m.name, evidence: wrapSystemOneData(boundedText(redactSystemOneText(String(m.content)), 450)) }));// Entered values can be secrets without recognizable labels. Action names// supply history without sending arbitrary tool arguments to another judge.- const recordedActions = messages.flatMap(m => m.tool_calls || []).filter(c => /^(type|fill|click|navigate|set_)/.test(c.function?.name || '')).slice(-8).map(c => ({ tool: c.function.name }));+ const recordedActions = calls.filter(c => /^(type|fill|click|navigate|set_)/.test(c.function?.name || '')).slice(-8).map(c => ({ tool: c.function.name }));const captures = new Map();const capture = async (modality, refresh = false) => {const before = await agent._completionDocumentStamp(tabId, true);@@ -93,10 +97,13 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrlconst identity = await agent._completionDocumentStamp(tabId, true);if (!identity || identity !== before || !context.isCurrent()) return null;const submit = agent._completionSubmitStates.get(tabId);+ const binding = submit?.workflowBinding;+ const workflowBinding = binding ? { adapterName: binding.adapterName, revision: binding.revision,+ job: binding.job, verificationKind: binding.verificationKind, recipientBound: binding.recipientBound === true } : null;const info = {task: boundedText(redactSystemOneText(String(task || '')), 2500),requirements: { requiresSubmission: guard?.requiresSubmission, requiresStateChange: guard?.requiresStateChange, workflow: boundedText(guard?.siteWorkflow?.job?.id, 120) },- action: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify({ lastAction: run?.lastAction, submit: submit ? { dispatched: submit.dispatched, originatingUrl: submit.originatingUrl, resultingUrl: submit.resultingUrl, workflowBinding: submit.workflowBinding } : null })), 1000)),+ action: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify({ lastAction: run?.lastAction, submit: submit ? { dispatched: submit.dispatched, originatingUrl: submit.originatingUrl, resultingUrl: submit.resultingUrl, workflowBinding } : null })), 1000)),recorded_reads: history, recorded_actions: recordedActions, document: boundedText(redactSystemOneText(pageUrl), 1000),page_errors: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify(pageState?.validationMessages || pageState?.errorMessages || [])), 500)),candidate_summary_not_proof: wrapSystemOneData(boundedText(redactSystemOneText(String(summary)), 600)),diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex ceee65a0d..b9a20ef16 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -1017,7 +1017,9 @@ export class Agent extends LoopDetector {_beginCompletionInvariant(tabId) {this._completionRunCounter += 1;const token = `completion_${tabId}_${Date.now()}_${this._completionRunCounter}`;- this.completionInvariants.set(tabId, createCompletionInvariantState(token));+ const historyToolCallIdsBeforeRun = new Set((this.conversations.get(tabId) || [])+ .flatMap(message => [message.tool_call_id, ...(message.tool_calls || []).map(call => call.id)]).filter(Boolean));+ this.completionInvariants.set(tabId, { ...createCompletionInvariantState(token), historyToolCallIdsBeforeRun });this._completionSubmitStates.delete(tabId);this._completionVerdicts?.delete(tabId);this._doneBlockCount.delete(tabId);diff --git a/src/firefox/src/agent/completion-runtime.js b/src/firefox/src/agent/completion-runtime.jsindex a343a06b4..f8c602ef0 100644--- a/src/firefox/src/agent/completion-runtime.js+++ b/src/firefox/src/agent/completion-runtime.js@@ -62,13 +62,17 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrlthrow Object.assign(new Error('Visual evidence changed or could not be refreshed.'), { code: 'COMPLETION_VISUAL_CHANGED' });};const messages = agent.conversations.get(tabId) || [];- const toolNames = new Map(messages.flatMap(m => (m.tool_calls || []).map(call => [call.id, call.function?.name])));- const reads = messages.filter(m => m.role === 'tool' && /read_page|get_accessibility_tree|verify_form/.test(toolNames.get(m.tool_call_id) || m.name || ''));+ // Tool IDs survive conversation compaction. Unknown provenance or reused+ // pre-run IDs cannot certify that this task performed a required read.+ const calls = run?.historyToolCallIdsBeforeRun instanceof Set+ ? messages.flatMap(m => m.tool_calls || []).filter(call => call.id && !run.historyToolCallIdsBeforeRun.has(call.id)) : [];+ const toolNames = new Map(calls.map(call => [call.id, call.function?.name]));+ const reads = messages.filter(m => m.role === 'tool' && /^(read_page|get_accessibility_tree|verify_form)$/.test(toolNames.get(m.tool_call_id) || ''));const selectedReads = [...new Set([...reads.filter(m => /rules|guidelines|moderation/i.test(String(m.content))).slice(-2), ...reads.slice(-2)])];const history = selectedReads.map(m => ({ tool: toolNames.get(m.tool_call_id) || m.name, evidence: wrapSystemOneData(boundedText(redactSystemOneText(String(m.content)), 450)) }));// Entered values can be secrets without recognizable labels. Action names// supply history without sending arbitrary tool arguments to another judge.- const recordedActions = messages.flatMap(m => m.tool_calls || []).filter(c => /^(type|fill|click|navigate|set_)/.test(c.function?.name || '')).slice(-8).map(c => ({ tool: c.function.name }));+ const recordedActions = calls.filter(c => /^(type|fill|click|navigate|set_)/.test(c.function?.name || '')).slice(-8).map(c => ({ tool: c.function.name }));const captures = new Map();const capture = async (modality, refresh = false) => {const before = await agent._completionDocumentStamp(tabId, true);@@ -93,10 +97,13 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrlconst identity = await agent._completionDocumentStamp(tabId, true);if (!identity || identity !== before || !context.isCurrent()) return null;const submit = agent._completionSubmitStates.get(tabId);+ const binding = submit?.workflowBinding;+ const workflowBinding = binding ? { adapterName: binding.adapterName, revision: binding.revision,+ job: binding.job, verificationKind: binding.verificationKind, recipientBound: binding.recipientBound === true } : null;const info = {task: boundedText(redactSystemOneText(String(task || '')), 2500),requirements: { requiresSubmission: guard?.requiresSubmission, requiresStateChange: guard?.requiresStateChange, workflow: boundedText(guard?.siteWorkflow?.job?.id, 120) },- action: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify({ lastAction: run?.lastAction, submit: submit ? { dispatched: submit.dispatched, originatingUrl: submit.originatingUrl, resultingUrl: submit.resultingUrl, workflowBinding: submit.workflowBinding } : null })), 1000)),+ action: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify({ lastAction: run?.lastAction, submit: submit ? { dispatched: submit.dispatched, originatingUrl: submit.originatingUrl, resultingUrl: submit.resultingUrl, workflowBinding } : null })), 1000)),recorded_reads: history, recorded_actions: recordedActions, document: boundedText(redactSystemOneText(pageUrl), 1000),page_errors: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify(pageState?.validationMessages || pageState?.errorMessages || [])), 500)),candidate_summary_not_proof: wrapSystemOneData(boundedText(redactSystemOneText(String(summary)), 600)),diff --git a/test/completion-verifier.mjs b/test/completion-verifier.mjsindex 98b0daef5..5aec56adf 100644--- a/test/completion-verifier.mjs+++ b/test/completion-verifier.mjs@@ -130,8 +130,12 @@ for (const build of ['chrome', 'firefox']) {strictSecretMode: false, _activeProvider: () => active, _runAbortSignal: () => null,systemOneContext: () => ({ isCurrent: () => true }), _latestTaskText: () => 'Read the community rules and publish a WebBrain post', _originalTaskText: () => '',_progressTaskKeyHash: () => 'task-key', _planExecutionGuards: new Map([[1, guard]]),- completionInvariants: new Map([[1, { runToken: 'run', lastAction: { sequence: 2, name: 'click' } }]]),- _completionSubmitStates: new Map([[1, { dispatched: true, observedAfterSubmit: true, currentUrl: 'https://old.reddit.com/comments/new' }]]),+ completionInvariants: new Map([[1, { runToken: 'run', historyToolCallIdsBeforeRun: new Set(), lastAction: { sequence: 2, name: 'click' } }]]),+ _completionSubmitStates: new Map([[1, { dispatched: true, observedAfterSubmit: true, currentUrl: 'https://old.reddit.com/comments/new', workflowBinding: {+ adapterName: 'reddit', revision: '1', job: 'publish', verificationKind: 'published_resource', recipientBound: true,+ messageBody: 'unlabelled-entered-value', composerBody: 'nested-entered-value', messageSubject: 'another-entered-value',+ recipientTargets: [{ address: 'opaque-entered-value' }], metadataRequirements: [{ field: 'title', value: 'metadata-entered-value' }],+ } }]]),conversationIds: new Map([[1, 'session']]), conversations: new Map([[1, [{ role: 'assistant', tool_calls: [{ id: 'rules', function: { name: 'read_page', arguments: '{}' } }] },{ role: 'tool', tool_call_id: 'rules', content: '{"text":"Community rules: relevant tools are allowed"}' },@@ -150,6 +154,7 @@ for (const build of ['chrome', 'firefox']) {assert.match(JSON.stringify(lastRequest.state), /Community rules/);assert.doesNotMatch(JSON.stringify(lastRequest.state), /entered-value/);assert.deepEqual(lastRequest.state[0].recorded_actions, ['type_ax', 'type_text', 'set_field', 'fill_form', 'navigate'].map(tool => ({ tool })));+ assert.match(lastRequest.state[0].action, /published_resource/);const initialState = JSON.stringify(lastRequest.state);await runtime.verifyBrowserCompletion(agent, 1, { pageUrl: 'https://old.reddit.com/comments/new' });assert.equal(JSON.stringify(lastRequest.state), initialState);@@ -194,6 +199,40 @@ for (const build of ['chrome', 'firefox']) {if (navigatorDescriptor) Object.defineProperty(globalThis, 'navigator', navigatorDescriptor); else delete globalThis.navigator;}});+ test(`${build}: completion history excludes previous runs and keeps current reads through compaction`, async () => {+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({ decisionProvider: 'local', systemOneEnabled: true }) } } };+ const provider = { model: 'local', supportsVision: false, config: { category: 'local' } };+ const messages = [+ { role: 'assistant', tool_calls: [{ id: 'prior-rules', function: { name: 'read_page' } }, { id: 'prior-type', function: { name: 'type_ax' } }] },+ { role: 'tool', tool_call_id: 'prior-rules', content: 'STALE community rules from another task and document' },+ { role: 'tool', name: 'read_page', content: 'STALE unbound rules result' },+ { role: 'assistant', tool_calls: [{ id: 'current-rules', function: { name: 'read_page' } }, { id: 'current-click', function: { name: 'click_ax' } }] },+ { role: 'tool', tool_call_id: 'current-rules', content: 'CURRENT community rules inspected in this run' },+ ];+ let state;+ const run = { runToken: 'current', historyToolCallIdsBeforeRun: new Set(['prior-rules', 'prior-type']) };+ const agent = {+ _activeProvider: () => provider, _runAbortSignal: () => null, systemOneContext: () => ({ isCurrent: () => true }),+ _latestTaskText: () => 'Read the rules and publish a post', _originalTaskText: () => '', _progressTaskKeyHash: () => 'task',+ _planExecutionGuards: new Map(), completionInvariants: new Map([[1, run]]), _completionSubmitStates: new Map(),+ conversationIds: new Map(), conversations: new Map([[1, messages]]), _completionDocumentStamp: async () => 'document',+ executeTool: async () => ({ success: true, pageContent: 'Requested post published' }), recordSystemOneVerdict() {},+ evaluateSystemOne: async (_tab, _client, args) => { state = args.state; const result = response(JSON.stringify(state).includes('CURRENT') ? 'succeeded' : 'uncertain'); result.model = 'kev-latest'; return result; },+ };+ try {+ for (const compacted of [false, true]) {+ if (compacted) agent.conversations.set(1, structuredClone(messages.slice(1)));+ agent._completionVerdicts?.clear();+ assert.equal((await runtime.verifyBrowserCompletion(agent, 1)).outcome, 'succeeded');+ assert.match(JSON.stringify(state), /CURRENT/); assert.doesNotMatch(JSON.stringify(state), /STALE/);+ assert.deepEqual(state[0].recorded_actions, [{ tool: 'click_ax' }]);+ }+ delete run.historyToolCallIdsBeforeRun; agent._completionVerdicts.clear();+ assert.equal((await runtime.verifyBrowserCompletion(agent, 1)).outcome, 'uncertain', 'unknown read provenance cannot establish task completion');+ assert.deepEqual(state[0].recorded_reads, []); assert.deepEqual(state[0].recorded_actions, []);+ } finally { globalThis.chrome = previousChrome; globalThis.browser = previousBrowser; }+ });test(`${build}: stale image verdicts cannot succeed without fresh evidence`, async () => {const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;const navigatorDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'navigator');diff --git a/test/run.js b/test/run.jsindex 75435c418..7ea095350 100644--- a/test/run.js+++ b/test/run.js@@ -28717,8 +28717,13 @@ test('completion invariant run tokens isolate overlapping and cleared runs', ()const agent = new AgentClass({});const tabId = 6049;agent.conversationModes.set(tabId, 'act');+ agent.conversations.set(tabId, [+ { role: 'assistant', tool_calls: [{ id: 'old-rules-read', function: { name: 'read_page' } }] },+ { role: 'tool', tool_call_id: 'old-rules-read', content: 'Rules from the previous task' },+ ]);const oldToken = agent._beginCompletionInvariant(tabId);+ assert.deepEqual([...agent.completionInvariants.get(tabId).historyToolCallIdsBeforeRun], ['old-rules-read']);agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_1' }, { success: true });assert.equal(agent.completionInvariants.get(tabId)?.verificationDebt, true);