Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(agent): isolate completion evidence to the active run

Emre Sokulluda4b92fea6c8
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index bc4651a7c..e00ff536c 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -1226,7 +1226,9 @@ export class Agent extends LoopDetector {
_beginCompletionInvariant(tabId) {
this._completionRunCounter += 1;
const token = `completion_${tabId}_${Date.now()}_${this._completionRunCounter}`;
- this.completionInvariants.set(tabId, createCompletionInvariantState(token));
+ const historyToolCallIdsBeforeRun = new Set((this.conversations.get(tabId) || [])
+ .flatMap(message => [message.tool_call_id, ...(message.tool_calls || []).map(call => call.id)]).filter(Boolean));
+ this.completionInvariants.set(tabId, { ...createCompletionInvariantState(token), historyToolCallIdsBeforeRun });
this._completionSubmitStates.delete(tabId);
this._completionVerdicts?.delete(tabId);
this._doneBlockCount.delete(tabId);
diff --git a/src/chrome/src/agent/completion-runtime.js b/src/chrome/src/agent/completion-runtime.js
index 1226d8a23..24cf10dd9 100644
--- a/src/chrome/src/agent/completion-runtime.js
+++ b/src/chrome/src/agent/completion-runtime.js
@@ -62,13 +62,17 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrl
throw Object.assign(new Error('Visual evidence changed or could not be refreshed.'), { code: 'COMPLETION_VISUAL_CHANGED' });
};
const messages = agent.conversations.get(tabId) || [];
- const toolNames = new Map(messages.flatMap(m => (m.tool_calls || []).map(call => [call.id, call.function?.name])));
- const reads = messages.filter(m => m.role === 'tool' && /read_page|get_accessibility_tree|verify_form/.test(toolNames.get(m.tool_call_id) || m.name || ''));
+ // Tool IDs survive conversation compaction. Unknown provenance or reused
+ // pre-run IDs cannot certify that this task performed a required read.
+ const calls = run?.historyToolCallIdsBeforeRun instanceof Set
+ ? messages.flatMap(m => m.tool_calls || []).filter(call => call.id && !run.historyToolCallIdsBeforeRun.has(call.id)) : [];
+ const toolNames = new Map(calls.map(call => [call.id, call.function?.name]));
+ const reads = messages.filter(m => m.role === 'tool' && /^(read_page|get_accessibility_tree|verify_form)$/.test(toolNames.get(m.tool_call_id) || ''));
const selectedReads = [...new Set([...reads.filter(m => /rules|guidelines|moderation/i.test(String(m.content))).slice(-2), ...reads.slice(-2)])];
const history = selectedReads.map(m => ({ tool: toolNames.get(m.tool_call_id) || m.name, evidence: wrapSystemOneData(boundedText(redactSystemOneText(String(m.content)), 450)) }));
// Entered values can be secrets without recognizable labels. Action names
// supply history without sending arbitrary tool arguments to another judge.
- const recordedActions = messages.flatMap(m => m.tool_calls || []).filter(c => /^(type|fill|click|navigate|set_)/.test(c.function?.name || '')).slice(-8).map(c => ({ tool: c.function.name }));
+ const recordedActions = calls.filter(c => /^(type|fill|click|navigate|set_)/.test(c.function?.name || '')).slice(-8).map(c => ({ tool: c.function.name }));
const captures = new Map();
const capture = async (modality, refresh = false) => {
const before = await agent._completionDocumentStamp(tabId, true);
@@ -93,10 +97,13 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrl
const identity = await agent._completionDocumentStamp(tabId, true);
if (!identity || identity !== before || !context.isCurrent()) return null;
const submit = agent._completionSubmitStates.get(tabId);
+ const binding = submit?.workflowBinding;
+ const workflowBinding = binding ? { adapterName: binding.adapterName, revision: binding.revision,
+ job: binding.job, verificationKind: binding.verificationKind, recipientBound: binding.recipientBound === true } : null;
const info = {
task: boundedText(redactSystemOneText(String(task || '')), 2500),
requirements: { requiresSubmission: guard?.requiresSubmission, requiresStateChange: guard?.requiresStateChange, workflow: boundedText(guard?.siteWorkflow?.job?.id, 120) },
- action: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify({ lastAction: run?.lastAction, submit: submit ? { dispatched: submit.dispatched, originatingUrl: submit.originatingUrl, resultingUrl: submit.resultingUrl, workflowBinding: submit.workflowBinding } : null })), 1000)),
+ action: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify({ lastAction: run?.lastAction, submit: submit ? { dispatched: submit.dispatched, originatingUrl: submit.originatingUrl, resultingUrl: submit.resultingUrl, workflowBinding } : null })), 1000)),
recorded_reads: history, recorded_actions: recordedActions, document: boundedText(redactSystemOneText(pageUrl), 1000),
page_errors: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify(pageState?.validationMessages || pageState?.errorMessages || [])), 500)),
candidate_summary_not_proof: wrapSystemOneData(boundedText(redactSystemOneText(String(summary)), 600)),
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index ceee65a0d..b9a20ef16 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -1017,7 +1017,9 @@ export class Agent extends LoopDetector {
_beginCompletionInvariant(tabId) {
this._completionRunCounter += 1;
const token = `completion_${tabId}_${Date.now()}_${this._completionRunCounter}`;
- this.completionInvariants.set(tabId, createCompletionInvariantState(token));
+ const historyToolCallIdsBeforeRun = new Set((this.conversations.get(tabId) || [])
+ .flatMap(message => [message.tool_call_id, ...(message.tool_calls || []).map(call => call.id)]).filter(Boolean));
+ this.completionInvariants.set(tabId, { ...createCompletionInvariantState(token), historyToolCallIdsBeforeRun });
this._completionSubmitStates.delete(tabId);
this._completionVerdicts?.delete(tabId);
this._doneBlockCount.delete(tabId);
diff --git a/src/firefox/src/agent/completion-runtime.js b/src/firefox/src/agent/completion-runtime.js
index a343a06b4..f8c602ef0 100644
--- a/src/firefox/src/agent/completion-runtime.js
+++ b/src/firefox/src/agent/completion-runtime.js
@@ -62,13 +62,17 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrl
throw Object.assign(new Error('Visual evidence changed or could not be refreshed.'), { code: 'COMPLETION_VISUAL_CHANGED' });
};
const messages = agent.conversations.get(tabId) || [];
- const toolNames = new Map(messages.flatMap(m => (m.tool_calls || []).map(call => [call.id, call.function?.name])));
- const reads = messages.filter(m => m.role === 'tool' && /read_page|get_accessibility_tree|verify_form/.test(toolNames.get(m.tool_call_id) || m.name || ''));
+ // Tool IDs survive conversation compaction. Unknown provenance or reused
+ // pre-run IDs cannot certify that this task performed a required read.
+ const calls = run?.historyToolCallIdsBeforeRun instanceof Set
+ ? messages.flatMap(m => m.tool_calls || []).filter(call => call.id && !run.historyToolCallIdsBeforeRun.has(call.id)) : [];
+ const toolNames = new Map(calls.map(call => [call.id, call.function?.name]));
+ const reads = messages.filter(m => m.role === 'tool' && /^(read_page|get_accessibility_tree|verify_form)$/.test(toolNames.get(m.tool_call_id) || ''));
const selectedReads = [...new Set([...reads.filter(m => /rules|guidelines|moderation/i.test(String(m.content))).slice(-2), ...reads.slice(-2)])];
const history = selectedReads.map(m => ({ tool: toolNames.get(m.tool_call_id) || m.name, evidence: wrapSystemOneData(boundedText(redactSystemOneText(String(m.content)), 450)) }));
// Entered values can be secrets without recognizable labels. Action names
// supply history without sending arbitrary tool arguments to another judge.
- const recordedActions = messages.flatMap(m => m.tool_calls || []).filter(c => /^(type|fill|click|navigate|set_)/.test(c.function?.name || '')).slice(-8).map(c => ({ tool: c.function.name }));
+ const recordedActions = calls.filter(c => /^(type|fill|click|navigate|set_)/.test(c.function?.name || '')).slice(-8).map(c => ({ tool: c.function.name }));
const captures = new Map();
const capture = async (modality, refresh = false) => {
const before = await agent._completionDocumentStamp(tabId, true);
@@ -93,10 +97,13 @@ async function evaluateBrowserCompletion(agent, tabId, { pageState = {}, pageUrl
const identity = await agent._completionDocumentStamp(tabId, true);
if (!identity || identity !== before || !context.isCurrent()) return null;
const submit = agent._completionSubmitStates.get(tabId);
+ const binding = submit?.workflowBinding;
+ const workflowBinding = binding ? { adapterName: binding.adapterName, revision: binding.revision,
+ job: binding.job, verificationKind: binding.verificationKind, recipientBound: binding.recipientBound === true } : null;
const info = {
task: boundedText(redactSystemOneText(String(task || '')), 2500),
requirements: { requiresSubmission: guard?.requiresSubmission, requiresStateChange: guard?.requiresStateChange, workflow: boundedText(guard?.siteWorkflow?.job?.id, 120) },
- action: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify({ lastAction: run?.lastAction, submit: submit ? { dispatched: submit.dispatched, originatingUrl: submit.originatingUrl, resultingUrl: submit.resultingUrl, workflowBinding: submit.workflowBinding } : null })), 1000)),
+ action: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify({ lastAction: run?.lastAction, submit: submit ? { dispatched: submit.dispatched, originatingUrl: submit.originatingUrl, resultingUrl: submit.resultingUrl, workflowBinding } : null })), 1000)),
recorded_reads: history, recorded_actions: recordedActions, document: boundedText(redactSystemOneText(pageUrl), 1000),
page_errors: wrapSystemOneData(boundedText(redactSystemOneText(JSON.stringify(pageState?.validationMessages || pageState?.errorMessages || [])), 500)),
candidate_summary_not_proof: wrapSystemOneData(boundedText(redactSystemOneText(String(summary)), 600)),
diff --git a/test/completion-verifier.mjs b/test/completion-verifier.mjs
index 98b0daef5..5aec56adf 100644
--- a/test/completion-verifier.mjs
+++ b/test/completion-verifier.mjs
@@ -130,8 +130,12 @@ for (const build of ['chrome', 'firefox']) {
strictSecretMode: false, _activeProvider: () => active, _runAbortSignal: () => null,
systemOneContext: () => ({ isCurrent: () => true }), _latestTaskText: () => 'Read the community rules and publish a WebBrain post', _originalTaskText: () => '',
_progressTaskKeyHash: () => 'task-key', _planExecutionGuards: new Map([[1, guard]]),
- completionInvariants: new Map([[1, { runToken: 'run', lastAction: { sequence: 2, name: 'click' } }]]),
- _completionSubmitStates: new Map([[1, { dispatched: true, observedAfterSubmit: true, currentUrl: 'https://old.reddit.com/comments/new' }]]),
+ completionInvariants: new Map([[1, { runToken: 'run', historyToolCallIdsBeforeRun: new Set(), lastAction: { sequence: 2, name: 'click' } }]]),
+ _completionSubmitStates: new Map([[1, { dispatched: true, observedAfterSubmit: true, currentUrl: 'https://old.reddit.com/comments/new', workflowBinding: {
+ adapterName: 'reddit', revision: '1', job: 'publish', verificationKind: 'published_resource', recipientBound: true,
+ messageBody: 'unlabelled-entered-value', composerBody: 'nested-entered-value', messageSubject: 'another-entered-value',
+ recipientTargets: [{ address: 'opaque-entered-value' }], metadataRequirements: [{ field: 'title', value: 'metadata-entered-value' }],
+ } }]]),
conversationIds: new Map([[1, 'session']]), conversations: new Map([[1, [
{ role: 'assistant', tool_calls: [{ id: 'rules', function: { name: 'read_page', arguments: '{}' } }] },
{ role: 'tool', tool_call_id: 'rules', content: '{"text":"Community rules: relevant tools are allowed"}' },
@@ -150,6 +154,7 @@ for (const build of ['chrome', 'firefox']) {
assert.match(JSON.stringify(lastRequest.state), /Community rules/);
assert.doesNotMatch(JSON.stringify(lastRequest.state), /entered-value/);
assert.deepEqual(lastRequest.state[0].recorded_actions, ['type_ax', 'type_text', 'set_field', 'fill_form', 'navigate'].map(tool => ({ tool })));
+ assert.match(lastRequest.state[0].action, /published_resource/);
const initialState = JSON.stringify(lastRequest.state);
await runtime.verifyBrowserCompletion(agent, 1, { pageUrl: 'https://old.reddit.com/comments/new' });
assert.equal(JSON.stringify(lastRequest.state), initialState);
@@ -194,6 +199,40 @@ for (const build of ['chrome', 'firefox']) {
if (navigatorDescriptor) Object.defineProperty(globalThis, 'navigator', navigatorDescriptor); else delete globalThis.navigator;
}
});
+ test(`${build}: completion history excludes previous runs and keeps current reads through compaction`, async () => {
+ const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;
+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({ decisionProvider: 'local', systemOneEnabled: true }) } } };
+ const provider = { model: 'local', supportsVision: false, config: { category: 'local' } };
+ const messages = [
+ { role: 'assistant', tool_calls: [{ id: 'prior-rules', function: { name: 'read_page' } }, { id: 'prior-type', function: { name: 'type_ax' } }] },
+ { role: 'tool', tool_call_id: 'prior-rules', content: 'STALE community rules from another task and document' },
+ { role: 'tool', name: 'read_page', content: 'STALE unbound rules result' },
+ { role: 'assistant', tool_calls: [{ id: 'current-rules', function: { name: 'read_page' } }, { id: 'current-click', function: { name: 'click_ax' } }] },
+ { role: 'tool', tool_call_id: 'current-rules', content: 'CURRENT community rules inspected in this run' },
+ ];
+ let state;
+ const run = { runToken: 'current', historyToolCallIdsBeforeRun: new Set(['prior-rules', 'prior-type']) };
+ const agent = {
+ _activeProvider: () => provider, _runAbortSignal: () => null, systemOneContext: () => ({ isCurrent: () => true }),
+ _latestTaskText: () => 'Read the rules and publish a post', _originalTaskText: () => '', _progressTaskKeyHash: () => 'task',
+ _planExecutionGuards: new Map(), completionInvariants: new Map([[1, run]]), _completionSubmitStates: new Map(),
+ conversationIds: new Map(), conversations: new Map([[1, messages]]), _completionDocumentStamp: async () => 'document',
+ executeTool: async () => ({ success: true, pageContent: 'Requested post published' }), recordSystemOneVerdict() {},
+ evaluateSystemOne: async (_tab, _client, args) => { state = args.state; const result = response(JSON.stringify(state).includes('CURRENT') ? 'succeeded' : 'uncertain'); result.model = 'kev-latest'; return result; },
+ };
+ try {
+ for (const compacted of [false, true]) {
+ if (compacted) agent.conversations.set(1, structuredClone(messages.slice(1)));
+ agent._completionVerdicts?.clear();
+ assert.equal((await runtime.verifyBrowserCompletion(agent, 1)).outcome, 'succeeded');
+ assert.match(JSON.stringify(state), /CURRENT/); assert.doesNotMatch(JSON.stringify(state), /STALE/);
+ assert.deepEqual(state[0].recorded_actions, [{ tool: 'click_ax' }]);
+ }
+ delete run.historyToolCallIdsBeforeRun; agent._completionVerdicts.clear();
+ assert.equal((await runtime.verifyBrowserCompletion(agent, 1)).outcome, 'uncertain', 'unknown read provenance cannot establish task completion');
+ assert.deepEqual(state[0].recorded_reads, []); assert.deepEqual(state[0].recorded_actions, []);
+ } finally { globalThis.chrome = previousChrome; globalThis.browser = previousBrowser; }
+ });
test(`${build}: stale image verdicts cannot succeed without fresh evidence`, async () => {
const previousChrome = globalThis.chrome, previousBrowser = globalThis.browser;
const navigatorDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'navigator');
diff --git a/test/run.js b/test/run.js
index 75435c418..7ea095350 100644
--- a/test/run.js
+++ b/test/run.js
@@ -28717,8 +28717,13 @@ test('completion invariant run tokens isolate overlapping and cleared runs', ()
const agent = new AgentClass({});
const tabId = 6049;
agent.conversationModes.set(tabId, 'act');
+ agent.conversations.set(tabId, [
+ { role: 'assistant', tool_calls: [{ id: 'old-rules-read', function: { name: 'read_page' } }] },
+ { role: 'tool', tool_call_id: 'old-rules-read', content: 'Rules from the previous task' },
+ ]);
const oldToken = agent._beginCompletionInvariant(tabId);
+ assert.deepEqual([...agent.completionInvariants.get(tabId).historyToolCallIdsBeforeRun], ['old-rules-read']);
agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_1' }, { success: true });
assert.equal(agent.completionInvariants.get(tabId)?.verificationDebt, true);