Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Give the interrupted dispatch path the normal path's args and repair exemption
Emre Sokullu6177482491fe
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex b2ad121f4..1fc2a2c12 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -1843,7 +1843,27 @@ export class Agent extends LoopDetector {}}- _recordCompletionToolResult(tabId, name, args, result, { detectedSubmit = null, messageRecipientContext = null } = {}) {+ // The args the completion records judge a call by: a skill tool that needs+ // download permission is a download action. Shared with the interrupted+ // path in _executeToolBatch, so both judge a call the same way.+ _completionToolArgs(tabId, name, args) {+ return this._activeSkillToolForName(tabId, name)?.requiresDownloadPermission+ ? { ...(args || {}), __completionDownloadAction: true }+ : args;+ }++ // A successful click_ax the app sent itself to read a native chat, while its+ // ownership still holds: not the model's action. Shared like the above.+ _isOwnedNativeChatReadRepair(tabId, name, args, result, messageRecipientContext) {+ return name === 'click_ax' && result?.success === true && !result.outcomeUnknown+ && result.noDispatch !== true && ((result._nativeWhatsappReadRepair === true+ && !!messageRecipientContext?.nativeWhatsappReadRepairOwner+ && messageRecipientContext.nativeWhatsappReadRepairOwner === this._nativeChatOwnerScope(tabId))+ || (result._nativeTelegramReadRepair === true+ && this._nativeTelegramReadPreparationCurrent(tabId, args, messageRecipientContext)));+ }++ _recordCompletionToolResult(tabId, name, args, result, { detectedSubmit = null, messageRecipientContext = null, ownedReadRepairBatch = false } = {}) {const state = this.completionInvariants.get(tabId);if (!state) return null;const terminalGuard = this._planExecutionGuards.get(tabId);@@ -1858,9 +1878,7 @@ export class Agent extends LoopDetector {const candidates = social.contract.actions.filter(a => a.platform === platform && eligible.includes(a.id));if (candidates.length === 1) social.outcomes[candidates[0].id] = { status: 'failed', cause: 'unavailable' };}- const completionArgs = this._activeSkillToolForName(tabId, name)?.requiresDownloadPermission- ? { ...(args || {}), __completionDownloadAction: true }- : args;+ const completionArgs = this._completionToolArgs(tabId, name, args);const workflowInventory = this._rememberWorkflowInventoryObservation(tabId, name, args, result);if (workflowInventory && result && typeof result === 'object') result.workflowInventory = workflowInventory;const workflowReleaseAssetObservation = this._rememberWorkflowReleaseAssetObservation(@@ -1883,17 +1901,13 @@ export class Agent extends LoopDetector {? { nativeOpaqueBaseline: messageRecipientContext.genericChatPending.nativeOpaqueBaseline } : {}) }: (structuredChatBinding && structuredChatBinding.threadKey === String(args.thread_key || args.threadKey || '')&& structuredChatBinding.body === this._workflowMessageBody(args.text) ? structuredChatBinding : null);- const nativeChatReadRepair = name === 'click_ax' && result?.success === true && !result.outcomeUnknown- && result.noDispatch !== true && ((result._nativeWhatsappReadRepair === true- && !!messageRecipientContext?.nativeWhatsappReadRepairOwner- && messageRecipientContext.nativeWhatsappReadRepairOwner === this._nativeChatOwnerScope(tabId))- || (result._nativeTelegramReadRepair === true- && this._nativeTelegramReadPreparationCurrent(tabId, args, messageRecipientContext)));+ const nativeChatReadRepair = this._isOwnedNativeChatReadRepair(tabId, name, args, result, messageRecipientContext);const next = recordCompletionToolResult(state, name, completionArgs, result, { chatDeliveryBinding, nativeChatReadRepair });this.completionInvariants.set(tabId, next);// The completion state does not count an app-owned read repair as an- // action either: the model did not send it.- if (!nativeChatReadRepair) this._noteProgressActionDispatch(tabId, name, completionArgs, result);+ // action either: the model did not send it. Nor does the dispatch record+ // count any call of the batch the app runs as its own repair.+ if (!nativeChatReadRepair && !ownedReadRepairBatch) this._noteProgressActionDispatch(tabId, name, completionArgs, result);this._rememberPageFeedbackCalculation(tabId, name, args, result, next);if (verifiedResumedDone && this._planExecutionGuards.get(tabId) === terminalGuard&& next.sequence === state.sequence + 1 && !next.hadAction && !next.lastAction && !next.verificationDebt) {@@ -13617,8 +13631,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// normal path judges it (a stale ref or a skipped click was not sent);// without one, a call that may have been sent counts as sent, so it can// back an "acted" row, and only a provably unsent one is held against one.- if (interrupted?.name && !interrupted.dispatchNoted) {- this._noteProgressActionDispatch(tabId, interrupted.name, interrupted.args || {}, interruptedRaw,+ // The args and the read-repair exemption are the normal path's too; Stop+ // clears the owner scope that exemption checks, so the app's own repair+ // batch is also recognized by its run option.+ if (interrupted?.name && !interrupted.dispatchNoted && runOptions[OWNED_NATIVE_CHAT_READ_REPAIR] !== true+ && !this._isOwnedNativeChatReadRepair(tabId, interrupted.name, interrupted.args || {}, interruptedRaw,+ interrupted.messageRecipientContext)) {+ this._noteProgressActionDispatch(tabId, interrupted.name,+ this._completionToolArgs(tabId, interrupted.name, interrupted.args || {}), interruptedRaw,interruptedRaw ? undefined : !interruptedUnsent);}if (this._checkAbort(tabId) || error?.name === 'AbortError') {@@ -14998,6 +15018,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dcallState.args = fnArgs;callState.name = fnName;callState.detected = detectedSubmitAction;+ callState.messageRecipientContext = messageRecipientExecutionContext;onUpdate('tool_call', {name: fnName,args: fnArgs,@@ -15276,7 +15297,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dfnName,fnArgs,toolResult,- { detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext },+ { detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext,+ ownedReadRepairBatch: runOptions[OWNED_NATIVE_CHAT_READ_REPAIR] === true },);// That also noted the call in the dispatch record; see the catch in// _executeToolBatch for a call interrupted before this point.diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex c9b69cfdd..3cb330ed5 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -1430,7 +1430,27 @@ export class Agent extends LoopDetector {}}- _recordCompletionToolResult(tabId, name, args, result, { detectedSubmit = null, messageRecipientContext = null } = {}) {+ // The args the completion records judge a call by: a skill tool that needs+ // download permission is a download action. Shared with the interrupted+ // path in _executeToolBatch, so both judge a call the same way.+ _completionToolArgs(tabId, name, args) {+ return this._activeSkillToolForName(tabId, name)?.requiresDownloadPermission+ ? { ...(args || {}), __completionDownloadAction: true }+ : args;+ }++ // A successful click_ax the app sent itself to read a native chat, while its+ // ownership still holds: not the model's action. Shared like the above.+ _isOwnedNativeChatReadRepair(tabId, name, args, result, messageRecipientContext) {+ return name === 'click_ax' && result?.success === true && !result.outcomeUnknown+ && result.noDispatch !== true && ((result._nativeWhatsappReadRepair === true+ && !!messageRecipientContext?.nativeWhatsappReadRepairOwner+ && messageRecipientContext.nativeWhatsappReadRepairOwner === this._nativeChatOwnerScope(tabId))+ || (result._nativeTelegramReadRepair === true+ && this._nativeTelegramReadPreparationCurrent(tabId, args, messageRecipientContext)));+ }++ _recordCompletionToolResult(tabId, name, args, result, { detectedSubmit = null, messageRecipientContext = null, ownedReadRepairBatch = false } = {}) {const state = this.completionInvariants.get(tabId);if (!state) return null;const terminalGuard = this._planExecutionGuards.get(tabId);@@ -1445,9 +1465,7 @@ export class Agent extends LoopDetector {const candidates = social.contract.actions.filter(a => a.platform === platform && eligible.includes(a.id));if (candidates.length === 1) social.outcomes[candidates[0].id] = { status: 'failed', cause: 'unavailable' };}- const completionArgs = this._activeSkillToolForName(tabId, name)?.requiresDownloadPermission- ? { ...(args || {}), __completionDownloadAction: true }- : args;+ const completionArgs = this._completionToolArgs(tabId, name, args);const workflowInventory = this._rememberWorkflowInventoryObservation(tabId, name, args, result);if (workflowInventory && result && typeof result === 'object') result.workflowInventory = workflowInventory;const workflowReleaseAssetObservation = this._rememberWorkflowReleaseAssetObservation(@@ -1470,17 +1488,13 @@ export class Agent extends LoopDetector {? { nativeOpaqueBaseline: messageRecipientContext.genericChatPending.nativeOpaqueBaseline } : {}) }: (structuredChatBinding && structuredChatBinding.threadKey === String(args.thread_key || args.threadKey || '')&& structuredChatBinding.body === this._workflowMessageBody(args.text) ? structuredChatBinding : null);- const nativeChatReadRepair = name === 'click_ax' && result?.success === true && !result.outcomeUnknown- && result.noDispatch !== true && ((result._nativeWhatsappReadRepair === true- && !!messageRecipientContext?.nativeWhatsappReadRepairOwner- && messageRecipientContext.nativeWhatsappReadRepairOwner === this._nativeChatOwnerScope(tabId))- || (result._nativeTelegramReadRepair === true- && this._nativeTelegramReadPreparationCurrent(tabId, args, messageRecipientContext)));+ const nativeChatReadRepair = this._isOwnedNativeChatReadRepair(tabId, name, args, result, messageRecipientContext);const next = recordCompletionToolResult(state, name, completionArgs, result, { chatDeliveryBinding, nativeChatReadRepair });this.completionInvariants.set(tabId, next);// The completion state does not count an app-owned read repair as an- // action either: the model did not send it.- if (!nativeChatReadRepair) this._noteProgressActionDispatch(tabId, name, completionArgs, result);+ // action either: the model did not send it. Nor does the dispatch record+ // count any call of the batch the app runs as its own repair.+ if (!nativeChatReadRepair && !ownedReadRepairBatch) this._noteProgressActionDispatch(tabId, name, completionArgs, result);this._rememberPageFeedbackCalculation(tabId, name, args, result, next);if (verifiedResumedDone && this._planExecutionGuards.get(tabId) === terminalGuard&& next.sequence === state.sequence + 1 && !next.hadAction && !next.lastAction && !next.verificationDebt) {@@ -12188,8 +12202,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// normal path judges it (a stale ref or a skipped click was not sent);// without one, a call that may have been sent counts as sent, so it can// back an "acted" row, and only a provably unsent one is held against one.- if (interrupted?.name && !interrupted.dispatchNoted) {- this._noteProgressActionDispatch(tabId, interrupted.name, interrupted.args || {}, interruptedRaw,+ // The args and the read-repair exemption are the normal path's too; Stop+ // clears the owner scope that exemption checks, so the app's own repair+ // batch is also recognized by its run option.+ if (interrupted?.name && !interrupted.dispatchNoted && runOptions[OWNED_NATIVE_CHAT_READ_REPAIR] !== true+ && !this._isOwnedNativeChatReadRepair(tabId, interrupted.name, interrupted.args || {}, interruptedRaw,+ interrupted.messageRecipientContext)) {+ this._noteProgressActionDispatch(tabId, interrupted.name,+ this._completionToolArgs(tabId, interrupted.name, interrupted.args || {}), interruptedRaw,interruptedRaw ? undefined : !interruptedUnsent);}if (this._checkAbort(tabId) || error?.name === 'AbortError') {@@ -13494,6 +13514,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dcallState.args = fnArgs;callState.name = fnName;callState.detected = detectedSubmitAction;+ callState.messageRecipientContext = messageRecipientExecutionContext;onUpdate('tool_call', {name: fnName,args: fnArgs,@@ -13754,7 +13775,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dfnName,fnArgs,toolResult,- { detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext },+ { detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext,+ ownedReadRepairBatch: runOptions[OWNED_NATIVE_CHAT_READ_REPAIR] === true },);// That also noted the call in the dispatch record; see the catch in// _executeToolBatch for a call interrupted before this point.diff --git a/test/improver-guards-ledger.mjs b/test/improver-guards-ledger.mjsindex ecc29073a..0e8282d4f 100644--- a/test/improver-guards-ledger.mjs+++ b/test/improver-guards-ledger.mjs@@ -95,14 +95,14 @@ function allowRealBatch(agent) {});}-// One click_ax call through the real _executeToolBatch, in a claimed run;-// resolves to { value } or { error }.-async function realBatch(agent, tabId, step) {- const call = { id: `real_${step}`, function: { name: 'click_ax', arguments: JSON.stringify({ ref_id: `ref_${step}` }) } };+// One call (a click_ax by default) through the real _executeToolBatch, in a+// claimed run; resolves to { value } or { error }.+async function realBatch(agent, tabId, step, name = 'click_ax', args = { ref_id: `ref_${step}` }) {+ const call = { id: `real_${step}`, function: { name, arguments: JSON.stringify(args) } };await agent._claimRunEntry(tabId, 'interactive');try {return await agent._executeToolBatch(tabId, [call], [{ role: 'assistant', tool_calls: [call] }], () => {},- agent._activeProvider(tabId), null, new Set(['click_ax']), step, {}).then(value => ({ value }), error => ({ error }));+ agent._activeProvider(tabId), null, new Set([name]), step, {}).then(value => ({ value }), error => ({ error }));} finally {agent._releaseRunEntry(tabId);}@@ -525,6 +525,86 @@ for (const build of BUILDS) {});});+ test(`${build}: an interrupted app-owned chat read repair is not the model's dispatch`, async () => {+ await withSessionStorage(async () => {+ const tabId = 102;+ const agent = makeAgent(Agent, tabId, 'Reply to the latest message.');+ allowRealBatch(agent);+ // The native chat owner scope is tied to the provider object, which a+ // real run keeps stable.+ const provider = agent._activeProvider(tabId);+ agent._activeProvider = () => provider;+ agent._beginCompletionInvariant(tabId);+ agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_1' }, blocked); // the model's own blocked click+ const before = { sent: 0, unsent: { name: 'click_ax' }, fence: null };+ const repaired = { success: true, dispatched: true, _nativeWhatsappReadRepair: true };+ // The recipient guard marks a click as the app's own read repair.+ agent._messageRecipientGuardBlock = async (tab, _name, _args, _url, context) => {+ context.nativeWhatsappReadRepairOwner = agent._nativeChatOwnerScope(tab);+ assert.ok(context.nativeWhatsappReadRepairOwner, 'no native chat owner in this fixture');+ return null;+ };+ agent.executeTool = async () => ({ ...repaired });+ assert.ok((await realBatch(agent, tabId, 1)).value, 'the repair did not complete');+ assert.deepEqual(agent.progressActionDispatches.get(tabId), before, 'the normal path counted the repair');+ // An error after the repair returned, before its bookkeeping: the same exemption.+ agent._captureCompletionSemanticNavigation = async () => { throw new Error('Capture failed.'); };+ assert.equal((await realBatch(agent, tabId, 2)).error?.message, 'Capture failed.');+ assert.deepEqual(agent.progressActionDispatches.get(tabId), before, 'an interrupted repair counted as sent');+ delete agent._captureCompletionSemanticNavigation;+ delete agent._messageRecipientGuardBlock;++ // Stop during the batch the app runs as its own repair clears the owner+ // scope; the batch is still the app's, whatever the click returned, and+ // so it is when the batch ends normally.+ const failed = { success: false, error: 'The pane closed.' };+ for (const [step, returned, stop] of [[3, repaired, true], [4, failed, true], [5, failed, false]]) {+ await agent._claimRunEntry(tabId, 'interactive');+ try {+ const owner = agent._nativeChatOwnerScope(tabId);+ agent.chatSessions.set(tabId, { threadKey: 'thread-1' });+ agent._captureOwnedNativeChatReadAuthority = () => ({ threadKey: 'thread-1', isCurrent: () => true });+ agent._ownedNativeChatReadRepairs.set(tabId, { pending: { scope: { owner } },+ controller: { isCurrent: () => true, run: (_record, dispatch) => dispatch('click_ax', { ref_id: `ref_${step}` }) } });+ agent.executeTool = async () => { if (stop) agent.abort(tabId); return { ...returned }; };+ const failure = await agent._runOwnedNativeChatReadRepair(tabId, [], () => {}, agent._activeProvider(tabId),+ new Set(['click_ax']), step, {}, null).then(() => null, error => error);+ assert.equal(failure?.name, stop ? 'AbortError' : undefined, `step ${step}: the owned repair ended unexpectedly`);+ } finally {+ agent._releaseRunEntry(tabId);+ }+ assert.deepEqual(agent.progressActionDispatches.get(tabId), before, `step ${step}: a stopped owned repair counted`);+ }+ });+ });++ test(`${build}: an interrupted skill download is judged as a download, like the normal path`, async () => {+ await withSessionStorage(async () => {+ const tabId = 103;+ const agent = makeAgent(Agent, tabId, 'Save the monthly report.');+ allowRealBatch(agent);+ agent._activeSkillToolForName = (_tab, name) => (name === 'save_report' ? { name, requiresDownloadPermission: true,+ parameters: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } } : null);+ agent._beginCompletionInvariant(tabId);+ const record = () => agent.progressActionDispatches.get(tabId) || { sent: 0, unsent: null, fence: null };+ const save = step => realBatch(agent, tabId, step, 'save_report', { url: 'https://example.test/report.pdf' });+ // The normal path counts it as a download action.+ agent.executeTool = async () => ({ success: true, downloadId: 7 });+ assert.ok((await save(1)).value);+ assert.deepEqual(record(), { sent: 1, unsent: null, fence: null });+ // Stop as it returns, and Stop with no result, count it the same way.+ agent.executeTool = async () => { agent.abort(tabId); return { success: true, downloadId: 8 }; };+ assert.equal((await save(2)).error?.name, 'AbortError');+ assert.deepEqual(record(), { sent: 2, unsent: null, fence: null }, 'a stopped download was not noted');+ agent.executeTool = async (_tab, _name, _args, _update, context) => {+ agent.abort(tabId);+ throw context._contentActionAbortSignal.reason;+ };+ assert.equal((await save(3)).error?.name, 'AbortError');+ assert.deepEqual(record(), { sent: 3, unsent: null, fence: null }, 'a download stopped in flight was not noted');+ });+ });+test(`${build}: an interrupted upload that returned without dispatching is noted as unsent`, async () => {await withSessionStorage(async () => {const tabId = 101;