Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Give the interrupted dispatch path the normal path's args and repair exemption

Emre Sokullu6177482491fe
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index b2ad121f4..1fc2a2c12 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -1843,7 +1843,27 @@ export class Agent extends LoopDetector {
}
}
- _recordCompletionToolResult(tabId, name, args, result, { detectedSubmit = null, messageRecipientContext = null } = {}) {
+ // The args the completion records judge a call by: a skill tool that needs
+ // download permission is a download action. Shared with the interrupted
+ // path in _executeToolBatch, so both judge a call the same way.
+ _completionToolArgs(tabId, name, args) {
+ return this._activeSkillToolForName(tabId, name)?.requiresDownloadPermission
+ ? { ...(args || {}), __completionDownloadAction: true }
+ : args;
+ }
+
+ // A successful click_ax the app sent itself to read a native chat, while its
+ // ownership still holds: not the model's action. Shared like the above.
+ _isOwnedNativeChatReadRepair(tabId, name, args, result, messageRecipientContext) {
+ return name === 'click_ax' && result?.success === true && !result.outcomeUnknown
+ && result.noDispatch !== true && ((result._nativeWhatsappReadRepair === true
+ && !!messageRecipientContext?.nativeWhatsappReadRepairOwner
+ && messageRecipientContext.nativeWhatsappReadRepairOwner === this._nativeChatOwnerScope(tabId))
+ || (result._nativeTelegramReadRepair === true
+ && this._nativeTelegramReadPreparationCurrent(tabId, args, messageRecipientContext)));
+ }
+
+ _recordCompletionToolResult(tabId, name, args, result, { detectedSubmit = null, messageRecipientContext = null, ownedReadRepairBatch = false } = {}) {
const state = this.completionInvariants.get(tabId);
if (!state) return null;
const terminalGuard = this._planExecutionGuards.get(tabId);
@@ -1858,9 +1878,7 @@ export class Agent extends LoopDetector {
const candidates = social.contract.actions.filter(a => a.platform === platform && eligible.includes(a.id));
if (candidates.length === 1) social.outcomes[candidates[0].id] = { status: 'failed', cause: 'unavailable' };
}
- const completionArgs = this._activeSkillToolForName(tabId, name)?.requiresDownloadPermission
- ? { ...(args || {}), __completionDownloadAction: true }
- : args;
+ const completionArgs = this._completionToolArgs(tabId, name, args);
const workflowInventory = this._rememberWorkflowInventoryObservation(tabId, name, args, result);
if (workflowInventory && result && typeof result === 'object') result.workflowInventory = workflowInventory;
const workflowReleaseAssetObservation = this._rememberWorkflowReleaseAssetObservation(
@@ -1883,17 +1901,13 @@ export class Agent extends LoopDetector {
? { nativeOpaqueBaseline: messageRecipientContext.genericChatPending.nativeOpaqueBaseline } : {}) }
: (structuredChatBinding && structuredChatBinding.threadKey === String(args.thread_key || args.threadKey || '')
&& structuredChatBinding.body === this._workflowMessageBody(args.text) ? structuredChatBinding : null);
- const nativeChatReadRepair = name === 'click_ax' && result?.success === true && !result.outcomeUnknown
- && result.noDispatch !== true && ((result._nativeWhatsappReadRepair === true
- && !!messageRecipientContext?.nativeWhatsappReadRepairOwner
- && messageRecipientContext.nativeWhatsappReadRepairOwner === this._nativeChatOwnerScope(tabId))
- || (result._nativeTelegramReadRepair === true
- && this._nativeTelegramReadPreparationCurrent(tabId, args, messageRecipientContext)));
+ const nativeChatReadRepair = this._isOwnedNativeChatReadRepair(tabId, name, args, result, messageRecipientContext);
const next = recordCompletionToolResult(state, name, completionArgs, result, { chatDeliveryBinding, nativeChatReadRepair });
this.completionInvariants.set(tabId, next);
// The completion state does not count an app-owned read repair as an
- // action either: the model did not send it.
- if (!nativeChatReadRepair) this._noteProgressActionDispatch(tabId, name, completionArgs, result);
+ // action either: the model did not send it. Nor does the dispatch record
+ // count any call of the batch the app runs as its own repair.
+ if (!nativeChatReadRepair && !ownedReadRepairBatch) this._noteProgressActionDispatch(tabId, name, completionArgs, result);
this._rememberPageFeedbackCalculation(tabId, name, args, result, next);
if (verifiedResumedDone && this._planExecutionGuards.get(tabId) === terminalGuard
&& next.sequence === state.sequence + 1 && !next.hadAction && !next.lastAction && !next.verificationDebt) {
@@ -13617,8 +13631,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// normal path judges it (a stale ref or a skipped click was not sent);
// without one, a call that may have been sent counts as sent, so it can
// back an "acted" row, and only a provably unsent one is held against one.
- if (interrupted?.name && !interrupted.dispatchNoted) {
- this._noteProgressActionDispatch(tabId, interrupted.name, interrupted.args || {}, interruptedRaw,
+ // The args and the read-repair exemption are the normal path's too; Stop
+ // clears the owner scope that exemption checks, so the app's own repair
+ // batch is also recognized by its run option.
+ if (interrupted?.name && !interrupted.dispatchNoted && runOptions[OWNED_NATIVE_CHAT_READ_REPAIR] !== true
+ && !this._isOwnedNativeChatReadRepair(tabId, interrupted.name, interrupted.args || {}, interruptedRaw,
+ interrupted.messageRecipientContext)) {
+ this._noteProgressActionDispatch(tabId, interrupted.name,
+ this._completionToolArgs(tabId, interrupted.name, interrupted.args || {}), interruptedRaw,
interruptedRaw ? undefined : !interruptedUnsent);
}
if (this._checkAbort(tabId) || error?.name === 'AbortError') {
@@ -14998,6 +15018,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
callState.args = fnArgs;
callState.name = fnName;
callState.detected = detectedSubmitAction;
+ callState.messageRecipientContext = messageRecipientExecutionContext;
onUpdate('tool_call', {
name: fnName,
args: fnArgs,
@@ -15276,7 +15297,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
fnName,
fnArgs,
toolResult,
- { detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext },
+ { detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext,
+ ownedReadRepairBatch: runOptions[OWNED_NATIVE_CHAT_READ_REPAIR] === true },
);
// That also noted the call in the dispatch record; see the catch in
// _executeToolBatch for a call interrupted before this point.
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index c9b69cfdd..3cb330ed5 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -1430,7 +1430,27 @@ export class Agent extends LoopDetector {
}
}
- _recordCompletionToolResult(tabId, name, args, result, { detectedSubmit = null, messageRecipientContext = null } = {}) {
+ // The args the completion records judge a call by: a skill tool that needs
+ // download permission is a download action. Shared with the interrupted
+ // path in _executeToolBatch, so both judge a call the same way.
+ _completionToolArgs(tabId, name, args) {
+ return this._activeSkillToolForName(tabId, name)?.requiresDownloadPermission
+ ? { ...(args || {}), __completionDownloadAction: true }
+ : args;
+ }
+
+ // A successful click_ax the app sent itself to read a native chat, while its
+ // ownership still holds: not the model's action. Shared like the above.
+ _isOwnedNativeChatReadRepair(tabId, name, args, result, messageRecipientContext) {
+ return name === 'click_ax' && result?.success === true && !result.outcomeUnknown
+ && result.noDispatch !== true && ((result._nativeWhatsappReadRepair === true
+ && !!messageRecipientContext?.nativeWhatsappReadRepairOwner
+ && messageRecipientContext.nativeWhatsappReadRepairOwner === this._nativeChatOwnerScope(tabId))
+ || (result._nativeTelegramReadRepair === true
+ && this._nativeTelegramReadPreparationCurrent(tabId, args, messageRecipientContext)));
+ }
+
+ _recordCompletionToolResult(tabId, name, args, result, { detectedSubmit = null, messageRecipientContext = null, ownedReadRepairBatch = false } = {}) {
const state = this.completionInvariants.get(tabId);
if (!state) return null;
const terminalGuard = this._planExecutionGuards.get(tabId);
@@ -1445,9 +1465,7 @@ export class Agent extends LoopDetector {
const candidates = social.contract.actions.filter(a => a.platform === platform && eligible.includes(a.id));
if (candidates.length === 1) social.outcomes[candidates[0].id] = { status: 'failed', cause: 'unavailable' };
}
- const completionArgs = this._activeSkillToolForName(tabId, name)?.requiresDownloadPermission
- ? { ...(args || {}), __completionDownloadAction: true }
- : args;
+ const completionArgs = this._completionToolArgs(tabId, name, args);
const workflowInventory = this._rememberWorkflowInventoryObservation(tabId, name, args, result);
if (workflowInventory && result && typeof result === 'object') result.workflowInventory = workflowInventory;
const workflowReleaseAssetObservation = this._rememberWorkflowReleaseAssetObservation(
@@ -1470,17 +1488,13 @@ export class Agent extends LoopDetector {
? { nativeOpaqueBaseline: messageRecipientContext.genericChatPending.nativeOpaqueBaseline } : {}) }
: (structuredChatBinding && structuredChatBinding.threadKey === String(args.thread_key || args.threadKey || '')
&& structuredChatBinding.body === this._workflowMessageBody(args.text) ? structuredChatBinding : null);
- const nativeChatReadRepair = name === 'click_ax' && result?.success === true && !result.outcomeUnknown
- && result.noDispatch !== true && ((result._nativeWhatsappReadRepair === true
- && !!messageRecipientContext?.nativeWhatsappReadRepairOwner
- && messageRecipientContext.nativeWhatsappReadRepairOwner === this._nativeChatOwnerScope(tabId))
- || (result._nativeTelegramReadRepair === true
- && this._nativeTelegramReadPreparationCurrent(tabId, args, messageRecipientContext)));
+ const nativeChatReadRepair = this._isOwnedNativeChatReadRepair(tabId, name, args, result, messageRecipientContext);
const next = recordCompletionToolResult(state, name, completionArgs, result, { chatDeliveryBinding, nativeChatReadRepair });
this.completionInvariants.set(tabId, next);
// The completion state does not count an app-owned read repair as an
- // action either: the model did not send it.
- if (!nativeChatReadRepair) this._noteProgressActionDispatch(tabId, name, completionArgs, result);
+ // action either: the model did not send it. Nor does the dispatch record
+ // count any call of the batch the app runs as its own repair.
+ if (!nativeChatReadRepair && !ownedReadRepairBatch) this._noteProgressActionDispatch(tabId, name, completionArgs, result);
this._rememberPageFeedbackCalculation(tabId, name, args, result, next);
if (verifiedResumedDone && this._planExecutionGuards.get(tabId) === terminalGuard
&& next.sequence === state.sequence + 1 && !next.hadAction && !next.lastAction && !next.verificationDebt) {
@@ -12188,8 +12202,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// normal path judges it (a stale ref or a skipped click was not sent);
// without one, a call that may have been sent counts as sent, so it can
// back an "acted" row, and only a provably unsent one is held against one.
- if (interrupted?.name && !interrupted.dispatchNoted) {
- this._noteProgressActionDispatch(tabId, interrupted.name, interrupted.args || {}, interruptedRaw,
+ // The args and the read-repair exemption are the normal path's too; Stop
+ // clears the owner scope that exemption checks, so the app's own repair
+ // batch is also recognized by its run option.
+ if (interrupted?.name && !interrupted.dispatchNoted && runOptions[OWNED_NATIVE_CHAT_READ_REPAIR] !== true
+ && !this._isOwnedNativeChatReadRepair(tabId, interrupted.name, interrupted.args || {}, interruptedRaw,
+ interrupted.messageRecipientContext)) {
+ this._noteProgressActionDispatch(tabId, interrupted.name,
+ this._completionToolArgs(tabId, interrupted.name, interrupted.args || {}), interruptedRaw,
interruptedRaw ? undefined : !interruptedUnsent);
}
if (this._checkAbort(tabId) || error?.name === 'AbortError') {
@@ -13494,6 +13514,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
callState.args = fnArgs;
callState.name = fnName;
callState.detected = detectedSubmitAction;
+ callState.messageRecipientContext = messageRecipientExecutionContext;
onUpdate('tool_call', {
name: fnName,
args: fnArgs,
@@ -13754,7 +13775,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
fnName,
fnArgs,
toolResult,
- { detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext },
+ { detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext,
+ ownedReadRepairBatch: runOptions[OWNED_NATIVE_CHAT_READ_REPAIR] === true },
);
// That also noted the call in the dispatch record; see the catch in
// _executeToolBatch for a call interrupted before this point.
diff --git a/test/improver-guards-ledger.mjs b/test/improver-guards-ledger.mjs
index ecc29073a..0e8282d4f 100644
--- a/test/improver-guards-ledger.mjs
+++ b/test/improver-guards-ledger.mjs
@@ -95,14 +95,14 @@ function allowRealBatch(agent) {
});
}
-// One click_ax call through the real _executeToolBatch, in a claimed run;
-// resolves to { value } or { error }.
-async function realBatch(agent, tabId, step) {
- const call = { id: `real_${step}`, function: { name: 'click_ax', arguments: JSON.stringify({ ref_id: `ref_${step}` }) } };
+// One call (a click_ax by default) through the real _executeToolBatch, in a
+// claimed run; resolves to { value } or { error }.
+async function realBatch(agent, tabId, step, name = 'click_ax', args = { ref_id: `ref_${step}` }) {
+ const call = { id: `real_${step}`, function: { name, arguments: JSON.stringify(args) } };
await agent._claimRunEntry(tabId, 'interactive');
try {
return await agent._executeToolBatch(tabId, [call], [{ role: 'assistant', tool_calls: [call] }], () => {},
- agent._activeProvider(tabId), null, new Set(['click_ax']), step, {}).then(value => ({ value }), error => ({ error }));
+ agent._activeProvider(tabId), null, new Set([name]), step, {}).then(value => ({ value }), error => ({ error }));
} finally {
agent._releaseRunEntry(tabId);
}
@@ -525,6 +525,86 @@ for (const build of BUILDS) {
});
});
+ test(`${build}: an interrupted app-owned chat read repair is not the model's dispatch`, async () => {
+ await withSessionStorage(async () => {
+ const tabId = 102;
+ const agent = makeAgent(Agent, tabId, 'Reply to the latest message.');
+ allowRealBatch(agent);
+ // The native chat owner scope is tied to the provider object, which a
+ // real run keeps stable.
+ const provider = agent._activeProvider(tabId);
+ agent._activeProvider = () => provider;
+ agent._beginCompletionInvariant(tabId);
+ agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_1' }, blocked); // the model's own blocked click
+ const before = { sent: 0, unsent: { name: 'click_ax' }, fence: null };
+ const repaired = { success: true, dispatched: true, _nativeWhatsappReadRepair: true };
+ // The recipient guard marks a click as the app's own read repair.
+ agent._messageRecipientGuardBlock = async (tab, _name, _args, _url, context) => {
+ context.nativeWhatsappReadRepairOwner = agent._nativeChatOwnerScope(tab);
+ assert.ok(context.nativeWhatsappReadRepairOwner, 'no native chat owner in this fixture');
+ return null;
+ };
+ agent.executeTool = async () => ({ ...repaired });
+ assert.ok((await realBatch(agent, tabId, 1)).value, 'the repair did not complete');
+ assert.deepEqual(agent.progressActionDispatches.get(tabId), before, 'the normal path counted the repair');
+ // An error after the repair returned, before its bookkeeping: the same exemption.
+ agent._captureCompletionSemanticNavigation = async () => { throw new Error('Capture failed.'); };
+ assert.equal((await realBatch(agent, tabId, 2)).error?.message, 'Capture failed.');
+ assert.deepEqual(agent.progressActionDispatches.get(tabId), before, 'an interrupted repair counted as sent');
+ delete agent._captureCompletionSemanticNavigation;
+ delete agent._messageRecipientGuardBlock;
+
+ // Stop during the batch the app runs as its own repair clears the owner
+ // scope; the batch is still the app's, whatever the click returned, and
+ // so it is when the batch ends normally.
+ const failed = { success: false, error: 'The pane closed.' };
+ for (const [step, returned, stop] of [[3, repaired, true], [4, failed, true], [5, failed, false]]) {
+ await agent._claimRunEntry(tabId, 'interactive');
+ try {
+ const owner = agent._nativeChatOwnerScope(tabId);
+ agent.chatSessions.set(tabId, { threadKey: 'thread-1' });
+ agent._captureOwnedNativeChatReadAuthority = () => ({ threadKey: 'thread-1', isCurrent: () => true });
+ agent._ownedNativeChatReadRepairs.set(tabId, { pending: { scope: { owner } },
+ controller: { isCurrent: () => true, run: (_record, dispatch) => dispatch('click_ax', { ref_id: `ref_${step}` }) } });
+ agent.executeTool = async () => { if (stop) agent.abort(tabId); return { ...returned }; };
+ const failure = await agent._runOwnedNativeChatReadRepair(tabId, [], () => {}, agent._activeProvider(tabId),
+ new Set(['click_ax']), step, {}, null).then(() => null, error => error);
+ assert.equal(failure?.name, stop ? 'AbortError' : undefined, `step ${step}: the owned repair ended unexpectedly`);
+ } finally {
+ agent._releaseRunEntry(tabId);
+ }
+ assert.deepEqual(agent.progressActionDispatches.get(tabId), before, `step ${step}: a stopped owned repair counted`);
+ }
+ });
+ });
+
+ test(`${build}: an interrupted skill download is judged as a download, like the normal path`, async () => {
+ await withSessionStorage(async () => {
+ const tabId = 103;
+ const agent = makeAgent(Agent, tabId, 'Save the monthly report.');
+ allowRealBatch(agent);
+ agent._activeSkillToolForName = (_tab, name) => (name === 'save_report' ? { name, requiresDownloadPermission: true,
+ parameters: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } } : null);
+ agent._beginCompletionInvariant(tabId);
+ const record = () => agent.progressActionDispatches.get(tabId) || { sent: 0, unsent: null, fence: null };
+ const save = step => realBatch(agent, tabId, step, 'save_report', { url: 'https://example.test/report.pdf' });
+ // The normal path counts it as a download action.
+ agent.executeTool = async () => ({ success: true, downloadId: 7 });
+ assert.ok((await save(1)).value);
+ assert.deepEqual(record(), { sent: 1, unsent: null, fence: null });
+ // Stop as it returns, and Stop with no result, count it the same way.
+ agent.executeTool = async () => { agent.abort(tabId); return { success: true, downloadId: 8 }; };
+ assert.equal((await save(2)).error?.name, 'AbortError');
+ assert.deepEqual(record(), { sent: 2, unsent: null, fence: null }, 'a stopped download was not noted');
+ agent.executeTool = async (_tab, _name, _args, _update, context) => {
+ agent.abort(tabId);
+ throw context._contentActionAbortSignal.reason;
+ };
+ assert.equal((await save(3)).error?.name, 'AbortError');
+ assert.deepEqual(record(), { sent: 3, unsent: null, fence: null }, 'a download stopped in flight was not noted');
+ });
+ });
+
test(`${build}: an interrupted upload that returned without dispatching is noted as unsent`, async () => {
await withSessionStorage(async () => {
const tabId = 101;