Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Count interrupted actions in the acted-claim dispatch record

Emre Sokullufdab964c09ca
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 09840fb48..94915b235 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -10115,6 +10115,12 @@ export class Agent extends LoopDetector {
const fnName = tc?.function?.name || 'unknown_tool';
const fnArgs = this._toolCallArgsWithReplayMethod(tabId, fnName, this._toolCallArgs(tc));
const result = makeResult(fnName, fnArgs, i);
+ // An action call answered here as not sent (a steering skip, a
+ // recovery checkpoint that could not be saved) is the latest action
+ // call for the dispatch record, like any other unsent result.
+ if (result?.noDispatch === true || result?.dispatched === false) {
+ this._noteProgressActionDispatch(tabId, fnName, fnArgs, result, false);
+ }
messages.push({
role: 'tool',
tool_call_id: tc.id,
@@ -13595,17 +13601,24 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// An action that was sent, or may have been, before Stop or an error
// cut its bookkeeping short still joins the task's record of attempts.
const interrupted = cancellationState.active;
+ const interruptedRaw = interrupted?.result || interrupted?.dispatchState?.rawToolResult;
+ const interruptedUnsent = !interrupted?.invoked || interruptedRaw?.noDispatch === true || interruptedRaw?.dispatched === false
+ || (!interruptedRaw && interrupted.dispatchState?.tracksTextMutationDispatch === true && interrupted.dispatchState.started !== true);
if (interrupted && !interrupted.attemptRecorded) {
- const raw = interrupted.result || interrupted.dispatchState?.rawToolResult;
- const unsent = !interrupted.invoked || raw?.noDispatch === true || raw?.dispatched === false
- || (!raw && interrupted.dispatchState?.tracksTextMutationDispatch === true && interrupted.dispatchState.started !== true);
- if (!unsent) {
+ if (!interruptedUnsent) {
this._recordInterruptedTaskAttempt(tabId, interrupted.name, interrupted.args, interrupted.detected,
Object.hasOwn(interrupted, 'potentialAttempt') ? interrupted.potentialAttempt : undefined);
} else {
this._resaveTaskAttemptCheckpoint(tabId);
}
}
+ // So does the progress ledger's dispatch record, unless the call's own
+ // bookkeeping got that far: a call that may have been sent counts as
+ // sent, so it can back an "acted" row, and only a provably unsent one
+ // can be held against a row.
+ if (interrupted?.name && !interrupted.dispatchNoted) {
+ this._noteProgressActionDispatch(tabId, interrupted.name, interrupted.args || {}, interruptedRaw, !interruptedUnsent);
+ }
if (this._checkAbort(tabId) || error?.name === 'AbortError') {
// Repair the whole assistant tool batch before any caller persists a
// terminal cancellation. Earlier completed results must stay intact.
@@ -13649,6 +13662,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
} : {}),
};
const name = call.function?.name || 'unknown_tool';
+ // A call the batch never reached was not sent, and is noted like any
+ // other unsent result; the interrupted call was noted above.
+ if (!active) this._noteProgressActionDispatch(tabId, name, this._toolCallArgs(call), result, false);
const message = { role: 'tool', tool_call_id: call.id, content: serialize(name, result) };
messages.push(message);
notifications.push({ name, args: active?.args || this._toolCallArgs(call), result, message, active });
@@ -15260,6 +15276,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
toolResult,
{ detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext },
);
+ // That also noted the call in the dispatch record; see the catch in
+ // _executeToolBatch for a call interrupted before this point.
+ callState.dispatchNoted = true;
const readCompletenessBeforeTool = this.readCompletenessStates.get(tabId) || null;
const readCompletenessAfterTool = this._recordReadCompleteness(tabId, fnName, fnArgs, toolResult);
const requiredReadProgress = readCompletenessMadeProgress(
@@ -34252,11 +34271,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// and keeps the latest one whose result proves it did not (blocked,
// denied, never dispatched). A verified no-op, such as set_checked on a box
// already in the requested state, leaves that state in place and so is not
- // a failed send.
- _noteProgressActionDispatch(tabId, name, args, result) {
+ // a failed send. A caller that knows better than the result, such as an
+ // interrupted or a synthetic answer, passes `sent` itself.
+ _noteProgressActionDispatch(tabId, name, args, result, sent = didCompletionActionExecute(name, args, result)) {
if (!isCompletionActionTool(name, args)) return;
const record = this.progressActionDispatches.get(tabId) || { sent: 0, unsent: null, fence: null };
- if (didCompletionActionExecute(name, args, result)) {
+ if (sent) {
record.sent += 1;
record.unsent = null;
} else {
@@ -34278,22 +34298,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// progress update. A saved null means that history has no action call.
_restoredProgressActionDispatches(entry) {
const raw = entry?.progressActionDispatches;
- if (raw === null) return null;
const fence = raw?.fence ?? null;
const unsentName = raw?.unsent?.name;
- if (Number.isSafeInteger(raw?.sent) && raw.sent >= 0
+ let record = null;
+ if (raw !== null && Number.isSafeInteger(raw?.sent) && raw.sent >= 0
&& (fence === null || (Number.isSafeInteger(fence) && fence >= 0 && fence <= raw.sent))
&& (raw.unsent == null || (typeof unsentName === 'string' && unsentName))) {
- return {
+ record = {
sent: raw.sent,
unsent: raw.unsent == null ? null : { name: unsentName.slice(0, 80) },
fence,
...(raw.historyUnknown === true ? { historyUnknown: true } : {}),
};
+ } else if (raw !== null) {
+ return Array.isArray(entry?.messages) && entry.messages.length
+ ? { sent: 0, unsent: null, fence: null, historyUnknown: true }
+ : null;
}
- return Array.isArray(entry?.messages) && entry.messages.length
- ? { sent: 0, unsent: null, fence: null, historyUnknown: true }
- : null;
+ // A worker that stopped mid-action saved the call but not its outcome,
+ // so the record may not count it: that call is the latest one and is not
+ // known to be unsent.
+ return this._historyEndsWithUncountedAction(entry?.messages)
+ ? { ...(record || { sent: 0, fence: null }), unsent: null, historyUnknown: true }
+ : record;
+ }
+
+ // Whether the last tool batch of a restored history holds an action call
+ // whose result is missing or reports an unknown outcome.
+ _historyEndsWithUncountedAction(messages) {
+ if (!Array.isArray(messages)) return false;
+ const batchIndex = messages.findLastIndex(message => message?.role === 'assistant'
+ && Array.isArray(message.tool_calls) && message.tool_calls.length);
+ if (batchIndex < 0) return false;
+ const results = new Map(messages.slice(batchIndex + 1).filter(message => message?.role === 'tool')
+ .map(message => [message.tool_call_id, String(message.content ?? '')]));
+ return messages[batchIndex].tool_calls.some(call => {
+ let args = {};
+ try { args = JSON.parse(call?.function?.arguments || '{}') || {}; } catch {}
+ if (!isCompletionActionTool(call?.function?.name, args)) return false;
+ const content = results.get(call?.id);
+ return content === undefined || /"(?:outcomeUnknown|missingToolResponse|mutationMayHaveOccurred)":true/.test(content);
+ });
}
// A model-written "acted" row claims its action reached the page. When the
@@ -34322,16 +34367,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
.map(row => row.id))];
if (!ids.length) return null;
const name = record.unsent.name;
- // The refusal states what the runtime knows. It must not say this row's
- // action was never sent: that action may predate the last update.
- const guidance = record.fence == null
- ? `and no action call has been sent yet. Retry the action and mark the row acted after it goes through, keep it pending, or mark it failed or skipped with the reason.`
- : `and no action call has been sent since your last successful progress_update. If a row's action was sent before that update, do not repeat it: check the page and record what it shows (processed once the result is visible), or keep the row pending. Otherwise it has not happened yet: retry the action and mark the row acted after it goes through, keep it pending, or mark it failed or skipped with the reason.`;
+ // The refusal states only what the runtime knows and sends the model to
+ // the page first. It never asks for a retry outright: a row's action may
+ // predate the last update, and repeating a follow, send or post that did
+ // land would do it twice.
+ const since = record.fence == null ? 'yet' : 'since your last successful progress_update';
return {
success: false,
actionNotDispatched: true,
undispatchedAction: { name },
- error: `progress_update: ${ids.slice(0, 6).join(', ')} cannot be marked acted now. Your latest ${name} call was not sent (it was blocked or never dispatched), ${guidance}`,
+ error: `progress_update: ${ids.slice(0, 6).join(', ')} cannot be marked acted now. Your latest ${name} call was not sent (it was blocked or never dispatched), and no action call has been sent ${since}. Check the page first and record what it shows: mark a row processed if its result is visible there; otherwise keep it pending, or mark it failed or skipped with the reason. Send an action again only if the page shows it did not happen.`,
ids: ids.slice(0, 20),
};
}
diff --git a/src/chrome/src/agent/loop-detector.js b/src/chrome/src/agent/loop-detector.js
index ff76e02a9..7809cb462 100644
--- a/src/chrome/src/agent/loop-detector.js
+++ b/src/chrome/src/agent/loop-detector.js
@@ -62,6 +62,8 @@ export class LoopDetector {
// Calls naming a tool the run does not offer, counted per name apart from
// recentCalls: in that window a refusal would dilute or complete another
// call's loop, and as a clean call it would retire a real loop's nudges.
+ // A navigation does not make such a tool available, so only the run
+ // boundary (or two clean calls in a row) clears it.
this.unavailableToolCalls = new Map(); // tabId -> { counts: Map(name -> refusals), cleanCalls }
}
@@ -156,6 +158,7 @@ export class LoopDetector {
const reason = toolResult?.errorCode === 'retired_tool' ? 'it is retired' : 'this run does not offer it';
if (count >= 10) {
this._clearLoopState(tabId);
+ this.unavailableToolCalls.delete(tabId);
return {
kind: 'stop',
message: `Stopped: I called ${name} ${count} times, but ${reason}, so nothing was run. Please give a different instruction.`,
@@ -372,23 +375,26 @@ export class LoopDetector {
/**
* Clear everything the detector accumulated for `tabId` except the nav
- * arrival history, which must outlive intra-run resets so navigation
- * ping-pong is still catchable. See _clearRunLoopState for the run boundary.
+ * arrival history and the unavailable-tool counts, which must outlive
+ * intra-run resets so navigation ping-pong and a repeated refused tool are
+ * still catchable. See _clearRunLoopState for the run boundary.
*/
_clearLoopState(tabId) {
this.recentCalls.delete(tabId);
this.loopNudges.delete(tabId);
this.healthyCallsSinceLoop.delete(tabId);
- this.unavailableToolCalls.delete(tabId);
this._clearPageLoopState(tabId);
}
/**
* Run-boundary reset. Only here is the nav arrival history discarded, since
- * a new run may legitimately revisit URLs the previous run already saw.
+ * a new run may legitimately revisit URLs the previous run already saw, and
+ * the unavailable-tool counts, which every real navigation would otherwise
+ * reset (navigate, screenshot, navigate, screenshot... never repeats).
*/
_clearRunLoopState(tabId) {
this.recentNavUrls.delete(tabId);
+ this.unavailableToolCalls.delete(tabId);
this._clearLoopState(tabId);
this.verificationChallengeStates.delete(tabId);
this.carouselIntentStates.delete(tabId);
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 6171d091d..2fd463252 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -9835,6 +9835,12 @@ export class Agent extends LoopDetector {
const fnName = tc?.function?.name || 'unknown_tool';
const fnArgs = this._toolCallArgsWithReplayMethod(tabId, fnName, this._toolCallArgs(tc));
const result = makeResult(fnName, fnArgs, i);
+ // An action call answered here as not sent (a steering skip, a
+ // recovery checkpoint that could not be saved) is the latest action
+ // call for the dispatch record, like any other unsent result.
+ if (result?.noDispatch === true || result?.dispatched === false) {
+ this._noteProgressActionDispatch(tabId, fnName, fnArgs, result, false);
+ }
messages.push({
role: 'tool',
tool_call_id: tc.id,
@@ -12166,17 +12172,24 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// An action that was sent, or may have been, before Stop or an error
// cut its bookkeeping short still joins the task's record of attempts.
const interrupted = cancellationState.active;
+ const interruptedRaw = interrupted?.result || interrupted?.dispatchState?.rawToolResult;
+ const interruptedUnsent = !interrupted?.invoked || interruptedRaw?.noDispatch === true || interruptedRaw?.dispatched === false
+ || (!interruptedRaw && interrupted.dispatchState?.tracksTextMutationDispatch === true && interrupted.dispatchState.started !== true);
if (interrupted && !interrupted.attemptRecorded) {
- const raw = interrupted.result || interrupted.dispatchState?.rawToolResult;
- const unsent = !interrupted.invoked || raw?.noDispatch === true || raw?.dispatched === false
- || (!raw && interrupted.dispatchState?.tracksTextMutationDispatch === true && interrupted.dispatchState.started !== true);
- if (!unsent) {
+ if (!interruptedUnsent) {
this._recordInterruptedTaskAttempt(tabId, interrupted.name, interrupted.args, interrupted.detected,
Object.hasOwn(interrupted, 'potentialAttempt') ? interrupted.potentialAttempt : undefined);
} else {
this._resaveTaskAttemptCheckpoint(tabId);
}
}
+ // So does the progress ledger's dispatch record, unless the call's own
+ // bookkeeping got that far: a call that may have been sent counts as
+ // sent, so it can back an "acted" row, and only a provably unsent one
+ // can be held against a row.
+ if (interrupted?.name && !interrupted.dispatchNoted) {
+ this._noteProgressActionDispatch(tabId, interrupted.name, interrupted.args || {}, interruptedRaw, !interruptedUnsent);
+ }
if (this._checkAbort(tabId) || error?.name === 'AbortError') {
// Repair the whole assistant tool batch before any caller persists a
// terminal cancellation. Earlier completed results must stay intact.
@@ -12220,6 +12233,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
} : {}),
};
const name = call.function?.name || 'unknown_tool';
+ // A call the batch never reached was not sent, and is noted like any
+ // other unsent result; the interrupted call was noted above.
+ if (!active) this._noteProgressActionDispatch(tabId, name, this._toolCallArgs(call), result, false);
const message = { role: 'tool', tool_call_id: call.id, content: serialize(name, result) };
messages.push(message);
notifications.push({ name, args: active?.args || this._toolCallArgs(call), result, message, active });
@@ -13738,6 +13754,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
toolResult,
{ detectedSubmit: detectedSubmitAction, messageRecipientContext: messageRecipientExecutionContext },
);
+ // That also noted the call in the dispatch record; see the catch in
+ // _executeToolBatch for a call interrupted before this point.
+ callState.dispatchNoted = true;
const readCompletenessBeforeTool = this.readCompletenessStates.get(tabId) || null;
const readCompletenessAfterTool = this._recordReadCompleteness(tabId, fnName, fnArgs, toolResult);
const requiredReadProgress = readCompletenessMadeProgress(
@@ -31472,11 +31491,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// and keeps the latest one whose result proves it did not (blocked,
// denied, never dispatched). A verified no-op, such as set_checked on a box
// already in the requested state, leaves that state in place and so is not
- // a failed send.
- _noteProgressActionDispatch(tabId, name, args, result) {
+ // a failed send. A caller that knows better than the result, such as an
+ // interrupted or a synthetic answer, passes `sent` itself.
+ _noteProgressActionDispatch(tabId, name, args, result, sent = didCompletionActionExecute(name, args, result)) {
if (!isCompletionActionTool(name, args)) return;
const record = this.progressActionDispatches.get(tabId) || { sent: 0, unsent: null, fence: null };
- if (didCompletionActionExecute(name, args, result)) {
+ if (sent) {
record.sent += 1;
record.unsent = null;
} else {
@@ -31498,22 +31518,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// progress update. A saved null means that history has no action call.
_restoredProgressActionDispatches(entry) {
const raw = entry?.progressActionDispatches;
- if (raw === null) return null;
const fence = raw?.fence ?? null;
const unsentName = raw?.unsent?.name;
- if (Number.isSafeInteger(raw?.sent) && raw.sent >= 0
+ let record = null;
+ if (raw !== null && Number.isSafeInteger(raw?.sent) && raw.sent >= 0
&& (fence === null || (Number.isSafeInteger(fence) && fence >= 0 && fence <= raw.sent))
&& (raw.unsent == null || (typeof unsentName === 'string' && unsentName))) {
- return {
+ record = {
sent: raw.sent,
unsent: raw.unsent == null ? null : { name: unsentName.slice(0, 80) },
fence,
...(raw.historyUnknown === true ? { historyUnknown: true } : {}),
};
+ } else if (raw !== null) {
+ return Array.isArray(entry?.messages) && entry.messages.length
+ ? { sent: 0, unsent: null, fence: null, historyUnknown: true }
+ : null;
}
- return Array.isArray(entry?.messages) && entry.messages.length
- ? { sent: 0, unsent: null, fence: null, historyUnknown: true }
- : null;
+ // A worker that stopped mid-action saved the call but not its outcome,
+ // so the record may not count it: that call is the latest one and is not
+ // known to be unsent.
+ return this._historyEndsWithUncountedAction(entry?.messages)
+ ? { ...(record || { sent: 0, fence: null }), unsent: null, historyUnknown: true }
+ : record;
+ }
+
+ // Whether the last tool batch of a restored history holds an action call
+ // whose result is missing or reports an unknown outcome.
+ _historyEndsWithUncountedAction(messages) {
+ if (!Array.isArray(messages)) return false;
+ const batchIndex = messages.findLastIndex(message => message?.role === 'assistant'
+ && Array.isArray(message.tool_calls) && message.tool_calls.length);
+ if (batchIndex < 0) return false;
+ const results = new Map(messages.slice(batchIndex + 1).filter(message => message?.role === 'tool')
+ .map(message => [message.tool_call_id, String(message.content ?? '')]));
+ return messages[batchIndex].tool_calls.some(call => {
+ let args = {};
+ try { args = JSON.parse(call?.function?.arguments || '{}') || {}; } catch {}
+ if (!isCompletionActionTool(call?.function?.name, args)) return false;
+ const content = results.get(call?.id);
+ return content === undefined || /"(?:outcomeUnknown|missingToolResponse|mutationMayHaveOccurred)":true/.test(content);
+ });
}
// A model-written "acted" row claims its action reached the page. When the
@@ -31542,16 +31587,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
.map(row => row.id))];
if (!ids.length) return null;
const name = record.unsent.name;
- // The refusal states what the runtime knows. It must not say this row's
- // action was never sent: that action may predate the last update.
- const guidance = record.fence == null
- ? `and no action call has been sent yet. Retry the action and mark the row acted after it goes through, keep it pending, or mark it failed or skipped with the reason.`
- : `and no action call has been sent since your last successful progress_update. If a row's action was sent before that update, do not repeat it: check the page and record what it shows (processed once the result is visible), or keep the row pending. Otherwise it has not happened yet: retry the action and mark the row acted after it goes through, keep it pending, or mark it failed or skipped with the reason.`;
+ // The refusal states only what the runtime knows and sends the model to
+ // the page first. It never asks for a retry outright: a row's action may
+ // predate the last update, and repeating a follow, send or post that did
+ // land would do it twice.
+ const since = record.fence == null ? 'yet' : 'since your last successful progress_update';
return {
success: false,
actionNotDispatched: true,
undispatchedAction: { name },
- error: `progress_update: ${ids.slice(0, 6).join(', ')} cannot be marked acted now. Your latest ${name} call was not sent (it was blocked or never dispatched), ${guidance}`,
+ error: `progress_update: ${ids.slice(0, 6).join(', ')} cannot be marked acted now. Your latest ${name} call was not sent (it was blocked or never dispatched), and no action call has been sent ${since}. Check the page first and record what it shows: mark a row processed if its result is visible there; otherwise keep it pending, or mark it failed or skipped with the reason. Send an action again only if the page shows it did not happen.`,
ids: ids.slice(0, 20),
};
}
diff --git a/src/firefox/src/agent/loop-detector.js b/src/firefox/src/agent/loop-detector.js
index ff76e02a9..7809cb462 100644
--- a/src/firefox/src/agent/loop-detector.js
+++ b/src/firefox/src/agent/loop-detector.js
@@ -62,6 +62,8 @@ export class LoopDetector {
// Calls naming a tool the run does not offer, counted per name apart from
// recentCalls: in that window a refusal would dilute or complete another
// call's loop, and as a clean call it would retire a real loop's nudges.
+ // A navigation does not make such a tool available, so only the run
+ // boundary (or two clean calls in a row) clears it.
this.unavailableToolCalls = new Map(); // tabId -> { counts: Map(name -> refusals), cleanCalls }
}
@@ -156,6 +158,7 @@ export class LoopDetector {
const reason = toolResult?.errorCode === 'retired_tool' ? 'it is retired' : 'this run does not offer it';
if (count >= 10) {
this._clearLoopState(tabId);
+ this.unavailableToolCalls.delete(tabId);
return {
kind: 'stop',
message: `Stopped: I called ${name} ${count} times, but ${reason}, so nothing was run. Please give a different instruction.`,
@@ -372,23 +375,26 @@ export class LoopDetector {
/**
* Clear everything the detector accumulated for `tabId` except the nav
- * arrival history, which must outlive intra-run resets so navigation
- * ping-pong is still catchable. See _clearRunLoopState for the run boundary.
+ * arrival history and the unavailable-tool counts, which must outlive
+ * intra-run resets so navigation ping-pong and a repeated refused tool are
+ * still catchable. See _clearRunLoopState for the run boundary.
*/
_clearLoopState(tabId) {
this.recentCalls.delete(tabId);
this.loopNudges.delete(tabId);
this.healthyCallsSinceLoop.delete(tabId);
- this.unavailableToolCalls.delete(tabId);
this._clearPageLoopState(tabId);
}
/**
* Run-boundary reset. Only here is the nav arrival history discarded, since
- * a new run may legitimately revisit URLs the previous run already saw.
+ * a new run may legitimately revisit URLs the previous run already saw, and
+ * the unavailable-tool counts, which every real navigation would otherwise
+ * reset (navigate, screenshot, navigate, screenshot... never repeats).
*/
_clearRunLoopState(tabId) {
this.recentNavUrls.delete(tabId);
+ this.unavailableToolCalls.delete(tabId);
this._clearLoopState(tabId);
this.verificationChallengeStates.delete(tabId);
this.carouselIntentStates.delete(tabId);
diff --git a/test/improver-guards-ledger.mjs b/test/improver-guards-ledger.mjs
index ba6aca6d5..929a1d71b 100644
--- a/test/improver-guards-ledger.mjs
+++ b/test/improver-guards-ledger.mjs
@@ -37,7 +37,8 @@ const rowStatus = (agent, tabId, sessionId, id) => agent._rowsForProgressSession
.find(row => row.id === id)?.status;
// An in-memory storage.session (JSON round-trip, like the browser's) under
-// both globals, since each build reads its own.
+// both globals, since each build reads its own, plus the few tab and runtime
+// calls an interrupted batch's cleanup makes.
async function withSessionStorage(fn) {
const store = new Map();
const copy = value => JSON.parse(JSON.stringify(value));
@@ -48,7 +49,11 @@ async function withSessionStorage(fn) {
remove: async key => { for (const name of Array.isArray(key) ? key : [key]) store.delete(name); },
};
const saved = { chrome: globalThis.chrome, browser: globalThis.browser };
- globalThis.chrome = globalThis.browser = { storage: { session: area, local: area } };
+ globalThis.chrome = globalThis.browser = {
+ storage: { session: area, local: area },
+ runtime: { getURL: value => value, sendMessage: async () => ({}) },
+ tabs: { get: async id => ({ id, url: 'https://example.test/' }) },
+ };
try {
return await fn(store);
} finally {
@@ -66,6 +71,29 @@ async function restart(Agent, tabId) {
await agent._hydrate(tabId);
return agent;
}
+
+// One tool batch whose first call reached `active` (the batch's record of the
+// call in flight) and was then cut short, by Stop or by an ordinary error;
+// `more` are later calls of the batch it never reached.
+async function interruptBatch(agent, tabId, active, { stop, more = [] }) {
+ const messages = agent.conversations.get(tabId);
+ const [call, ...rest] = [active, ...more].map((entry, index) => ({ id: `call_${messages.length}_${index}`, type: 'function',
+ function: { name: entry.name, arguments: JSON.stringify(entry.args || {}) } }));
+ messages.push({ role: 'assistant', content: '', tool_calls: [call, ...rest] });
+ agent._executeToolBatchInner = async (...params) => {
+ params[10].active = { index: 0, consequential: true, result: null, ...active };
+ if (!stop) throw new Error('The page connection was lost after the action was sent.');
+ agent._checkAbort = () => true;
+ throw new DOMException('Stopped', 'AbortError');
+ };
+ try {
+ await assert.rejects(agent._executeToolBatch(tabId, [call, ...rest], messages, () => {}, {}, null));
+ } finally {
+ delete agent._executeToolBatchInner;
+ delete agent._checkAbort;
+ }
+ return messages.filter(message => message.tool_call_id === call.id);
+}
const blocked = { success: false, noDispatch: true, dispatched: false, error: 'Blocked before dispatch.' };
const sent = { success: true, dispatched: true };
@@ -125,8 +153,10 @@ for (const build of BUILDS) {
assert.deepEqual(rejected.ids, ['alice']);
assert.deepEqual(rejected.undispatchedAction, { name: 'click_ax' });
assert.match(rejected.error, /^progress_update: alice cannot be marked acted now\. Your latest click_ax call was not sent/);
- assert.match(rejected.error, /no action call has been sent yet/);
+ assert.match(rejected.error, /no action call has been sent yet\. Check the page first and record what it shows/);
assert.match(rejected.error, /failed or skipped/);
+ assert.match(rejected.error, /Send an action again only if the page shows it did not happen\.$/);
+ assert.doesNotMatch(rejected.error, /retry the action/i, 'a refusal must not send the model to repeat the action');
assert.equal(rowStatus(agent, tabId, sessionId, 'alice'), 'pending', 'the refused update changed nothing');
assert.equal(agent._deliveryCheckpointMadeMeaningfulProgress('progress_update', rejected), false,
'a refused update is not delivery progress');
@@ -171,8 +201,9 @@ for (const build of BUILDS) {
assert.equal(fenced.success, false, 'the earlier dispatch was already recorded by the last update');
assert.equal(fenced.actionNotDispatched, true);
assert.match(fenced.error, /^progress_update: bob cannot be marked acted now\. Your latest click_ax call was not sent/);
- assert.match(fenced.error, /no action call has been sent since your last successful progress_update/);
- assert.match(fenced.error, /If a row's action was sent before that update, do not repeat it/);
+ assert.match(fenced.error, /no action call has been sent since your last successful progress_update\. Check the page first/);
+ assert.match(fenced.error, /Send an action again only if the page shows it did not happen\.$/);
+ assert.doesNotMatch(fenced.error, /retry the action/i, 'a refusal must not send the model to repeat the action');
assert.doesNotMatch(fenced.error, /the last click_ax call|bob[^.]*was not sent/, 'the refusal says this row\'s action was never sent');
// Runtime rows (observe, auto) are not the model recording its dispatches.
@@ -336,6 +367,121 @@ for (const build of BUILDS) {
});
});
+ test(`${build}: an action cut short after it may have been sent backs an acted row`, async () => {
+ await withSessionStorage(async () => {
+ const cases = [
+ ['Stop after dispatch', true, { dispatchState: { started: true, rawToolResult: { success: true, dispatched: true } } }],
+ ['an error after dispatch', false, { dispatchState: { started: true } }],
+ ['Stop with an unknown outcome', true, { result: { success: false, outcomeUnknown: true, error: 'No response.' }, dispatchState: {} }],
+ ];
+ for (const [index, [label, stop, state]] of cases.entries()) {
+ const tabId = 80 + index;
+ const sessionId = `${build}-interrupted-${index}`;
+ const agent = makeAgent(Agent, tabId, 'Follow alice.');
+ seedRows(agent, tabId, sessionId, ['alice']);
+ agent._beginCompletionInvariant(tabId);
+ agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_1' }, blocked); // an earlier blocked attempt
+ const results = await interruptBatch(agent, tabId, {
+ name: 'click_ax', args: { ref_id: 'ref_2' }, invoked: true, ...state,
+ }, { stop });
+ if (stop) assert.equal(JSON.parse(results[0].content.replace(/^[^\n]*\n|\n[^\n]*$/g, '')).cancelled, true, label);
+ assert.deepEqual(agent.progressActionDispatches.get(tabId), { sent: 1, unsent: null, fence: null }, label);
+ agent._beginCompletionInvariant(tabId); // the next run records the row
+ const claim = agent._progressUpdate(tabId, { items: [{ id: 'alice', status: 'acted' }] }, { sessionId });
+ assert.equal(claim.success, true, `${label}: ${claim.error || ''}`);
+ }
+
+ // Stop also answers the calls the batch never reached; they were not sent.
+ const tailTab = 84;
+ const tailSession = `${build}-interrupted-tail`;
+ const tailAgent = makeAgent(Agent, tailTab, 'Follow alice and bob.');
+ seedRows(tailAgent, tailTab, tailSession, ['alice', 'bob']);
+ tailAgent._beginCompletionInvariant(tailTab);
+ const answered = await interruptBatch(tailAgent, tailTab, { name: 'click_ax', args: { ref_id: 'ref_1' }, invoked: true,
+ dispatchState: { started: true, rawToolResult: sent } }, { stop: true, more: [{ name: 'click_ax', args: { ref_id: 'ref_2' } }] });
+ assert.equal(answered.length, 1);
+ assert.deepEqual(tailAgent.progressActionDispatches.get(tailTab), { sent: 1, unsent: { name: 'click_ax' }, fence: null });
+ tailAgent._beginCompletionInvariant(tailTab);
+ assert.equal(tailAgent._progressUpdate(tailTab, { items: [{ id: 'alice', status: 'acted' }] }, { sessionId: tailSession }).success, true);
+ assert.equal(tailAgent._progressUpdate(tailTab, { items: [{ id: 'bob', status: 'acted' }] }, { sessionId: tailSession }).success, false);
+
+ // A call cut short before it was invoked was not sent; a call whose own
+ // bookkeeping already ran is not counted twice.
+ const tabId = 85;
+ const sessionId = `${build}-interrupted-unsent`;
+ const agent = makeAgent(Agent, tabId, 'Follow alice and bob.');
+ seedRows(agent, tabId, sessionId, ['alice', 'bob']);
+ agent._beginCompletionInvariant(tabId);
+ agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_1' }, sent);
+ assert.equal(agent._progressUpdate(tabId, { items: [{ id: 'alice', status: 'acted' }] }, { sessionId }).success, true);
+ await interruptBatch(agent, tabId, { name: 'click_ax', args: { ref_id: 'ref_2' }, invoked: false }, { stop: true });
+ assert.deepEqual(agent.progressActionDispatches.get(tabId), { sent: 1, unsent: { name: 'click_ax' }, fence: 1 });
+ await interruptBatch(agent, tabId, { name: 'click_ax', args: { ref_id: 'ref_3' }, invoked: true, dispatchNoted: true,
+ dispatchState: { started: true, rawToolResult: sent } }, { stop: false });
+ assert.deepEqual(agent.progressActionDispatches.get(tabId), { sent: 1, unsent: { name: 'click_ax' }, fence: 1 });
+ agent._beginCompletionInvariant(tabId);
+ const refused = agent._progressUpdate(tabId, { items: [{ id: 'bob', status: 'acted' }] }, { sessionId });
+ assert.equal(refused.success, false);
+ assert.match(refused.error, /Check the page first and record what it shows/);
+ assert.doesNotMatch(refused.error, /retry the action/i);
+ });
+ });
+
+ test(`${build}: a call answered as not sent without running is noted as unsent`, () => {
+ const tabId = 86;
+ const agent = makeAgent(Agent, tabId, 'Follow alice.');
+ agent._beginCompletionInvariant(tabId);
+ agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_1' }, sent);
+ const calls = [
+ { id: 'skip_1', function: { name: 'read_page', arguments: '{}' } },
+ { id: 'skip_2', function: { name: 'click_ax', arguments: '{"ref_id":"ref_2"}' } },
+ ];
+ const messages = [];
+ agent._appendSyntheticToolResults(tabId, calls, 0, messages, () => {}, 1, () => ({
+ success: false, dispatched: false, noDispatch: true, errorCode: 'checkpoint_not_durable', error: 'Not saved.',
+ }));
+ assert.equal(messages.length, 2);
+ assert.deepEqual(agent.progressActionDispatches.get(tabId), { sent: 1, unsent: { name: 'click_ax' }, fence: null });
+ // A synthetic answer that does not say the call went unsent changes nothing.
+ agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_3' }, sent);
+ agent._appendSyntheticToolResults(tabId, [calls[1]], 0, [], () => {}, 1, () => ({ success: false, skipped: true, error: 'skipped' }));
+ assert.deepEqual(agent.progressActionDispatches.get(tabId), { sent: 2, unsent: null, fence: null });
+ });
+
+ test(`${build}: a restored history ending in an action without a known outcome refuses nothing`, async () => {
+ await withSessionStorage(async store => {
+ const sessionId = `${build}-trailing`;
+ const call = (id, name, args) => ({ id, type: 'function', function: { name, arguments: JSON.stringify(args) } });
+ const tails = [
+ ['a missing result', [{ role: 'assistant', content: '', tool_calls: [call('t1', 'click_ax', { ref_id: 'ref_2' })] }], true],
+ ['an unknown outcome', [{ role: 'assistant', content: '', tool_calls: [call('t1', 'click_ax', { ref_id: 'ref_2' })] },
+ { role: 'tool', tool_call_id: 't1', content: JSON.stringify({ success: false, outcomeUnknown: true, cancelled: true }) }], true],
+ ['a read without a result', [{ role: 'assistant', content: '', tool_calls: [call('t1', 'read_page', {})] }], false],
+ ['an action proven unsent', [{ role: 'assistant', content: '', tool_calls: [call('t1', 'click_ax', { ref_id: 'ref_2' })] },
+ { role: 'tool', tool_call_id: 't1', content: JSON.stringify(blocked) }], false],
+ ];
+ for (const [index, [label, tail, unknown]] of tails.entries()) {
+ const tabId = 90 + index;
+ const before = makeAgent(Agent, tabId, 'Follow alice.');
+ seedRows(before, tabId, sessionId, ['alice']);
+ before._beginCompletionInvariant(tabId);
+ before._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_1' }, blocked);
+ before.conversations.get(tabId).push(...tail); // the worker stopped here
+ assert.equal((await before._persistNow(tabId)).ok, true);
+ const after = await restart(Agent, tabId);
+ assert.deepEqual(after.progressActionDispatches.get(tabId), unknown
+ ? { sent: 0, unsent: null, fence: null, historyUnknown: true }
+ : { sent: 0, unsent: { name: 'click_ax' }, fence: null }, label);
+ after._beginCompletionInvariant(tabId);
+ after._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_3' }, blocked);
+ const claim = after._progressUpdate(tabId, { items: [{ id: 'alice', status: 'acted' }] }, { sessionId });
+ assert.equal(claim.success, unknown, `${label}: ${claim.error || ''}`);
+ if (!unknown) assert.doesNotMatch(claim.error, /retry the action/i);
+ }
+ assert.ok(store.size >= tails.length);
+ });
+ });
+
test(`${build}: a refused claim creates and activates no progress session`, () => {
const tabId = 45;
const task = 'Follow alice.';
diff --git a/test/improver-guards-tools.mjs b/test/improver-guards-tools.mjs
index 42414a901..3151e74b3 100644
--- a/test/improver-guards-tools.mjs
+++ b/test/improver-guards-tools.mjs
@@ -131,7 +131,8 @@ for (const build of BUILDS) {
rejected: { args: { maxChars: 1000 }, stops: true },
executed: { args: { offset: 0 }, stops: false, executeTool: async () => ({ success: false, error: 'Element not found' }) },
};
- // Loop state after each stuck attempt, until the run stops or 14 attempts.
+ // Loop state after each stuck attempt, until the run stops (on a stuck
+ // attempt, or on the tenth refusal of one name) or 14 attempts.
async function run(tabId, loop, pattern) {
const { agent } = makeAgent(tabId);
if (loop.executeTool) agent.executeTool = loop.executeTool;
@@ -143,11 +144,12 @@ for (const build of BUILDS) {
const before = agent.loopNudges.get(tabId) || 0;
const call = await callOnce(agent, tabId, name, name === 'read_page' ? loop.args : {}, ++step, allowed);
if (name !== 'read_page') {
+ if (call.outcome?.action === 'recover') return { trace, outcome: call.outcome, stoppedBy: name, refusalNudgeChanges };
if ((agent.loopNudges.get(tabId) || 0) !== before) refusalNudgeChanges.push(step);
continue;
}
trace.push(`${agent.loopNudges.get(tabId) || 0}:${call.outcome?.status || call.outcome?.action || '-'}`);
- if (call.outcome?.action === 'recover') return { trace, outcome: call.outcome, refusalNudgeChanges };
+ if (call.outcome?.action === 'recover') return { trace, outcome: call.outcome, stoppedBy: name, refusalNudgeChanges };
if (trace.length >= 14) return { trace, outcome: null, refusalNudgeChanges };
}
}
@@ -160,9 +162,17 @@ for (const build of BUILDS) {
[89, ['read_page', 'screenshot', 'list_tabs']], // refusals between every attempt
]) {
const mixed = await run(tabId, loop, pattern);
- assert.deepEqual(mixed.trace, alone.trace, `${kind} ${pattern}: refused calls changed the loop`);
assert.deepEqual(mixed.refusalNudgeChanges, [], `${kind} ${pattern}: a refused call changed the loop's nudge count`);
- if (loop.stops) assert.match(mixed.outcome.value, /looping on the same call to read_page/);
+ if (mixed.stoppedBy === 'screenshot') {
+ // Its own budget: the tenth screenshot stops the run, which the
+ // stuck loop had not done by then; up to there the loop ran as alone.
+ assert.match(mixed.outcome.value, /^Stopped: I called screenshot 10 times/);
+ assert.deepEqual(mixed.trace, alone.trace.slice(0, mixed.trace.length), `${kind} ${pattern}: refused calls changed the loop`);
+ assert.ok(mixed.trace.length < alone.trace.length);
+ } else {
+ assert.deepEqual(mixed.trace, alone.trace, `${kind} ${pattern}: refused calls changed the loop`);
+ if (loop.stops) assert.match(mixed.outcome.value, /looping on the same call to read_page/);
+ }
}
}
});
@@ -213,6 +223,24 @@ for (const build of BUILDS) {
assert.equal(other.healthyCallsSinceLoop.get(94), 1);
});
+ test(`${build}: a navigation between refusals does not reset the unavailable-tool count`, () => {
+ const tabId = 95;
+ const { agent } = makeAgent(tabId);
+ const refusal = { success: false, denied: true, noDispatch: true, errorCode: 'retired_tool', error: 'retired' };
+ const kinds = [];
+ for (let step = 1; step <= 10; step++) {
+ const url = `https://example.test/page-${step}`;
+ agent._checkLoop(tabId, 'navigate', { url }, { success: true, pageUrlChanged: true, currentUrl: url });
+ kinds.push(agent._checkLoop(tabId, 'screenshot', {}, refusal).kind);
+ }
+ assert.deepEqual(kinds, ['none', 'none', ...Array(7).fill('nudge'), 'stop']);
+ // Only the run boundary clears it.
+ agent._checkLoop(tabId, 'screenshot', {}, refusal);
+ agent._checkLoop(tabId, 'screenshot', {}, refusal);
+ agent._clearRunLoopState(tabId);
+ assert.equal(agent._checkLoop(tabId, 'screenshot', {}, refusal).kind, 'none');
+ });
+
test(`${build}: missing required arguments are named apart from invalid ones`, () => {
const schema = name => tools.AGENT_TOOLS.find(tool => tool.function?.name === name).function.parameters;