Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix CAPTCHA preflight stalls and API fallback freshness

Emre Sokullue26cd503aed0
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 0852a1c2a..577d9b0ba 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -268,6 +268,8 @@ const DEFAULT_INPUT_COST_PER_MILLION_USD = 3;
const DEFAULT_OUTPUT_COST_PER_MILLION_USD = 15;
const VISION_SUB_CALL_TIMEOUT_MS = 90_000;
const CONTENT_ACTION_TIMEOUT_MS = 60_000;
+// Read-only CAPTCHA checks must not consume a full page-action timeout.
+const CAPTCHA_PREFLIGHT_TIMEOUT_MS = 8_000;
const RUN_STARTUP_TIMEOUT_MS = 15_000;
const CONTENT_ACTION_RESPONSE_GRACE_MS = 5_000;
// At completion the active model reads what a page's status texts say only
@@ -12238,7 +12240,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
async _readAwsWafChallenge(tabId) {
try {
const [entry] = await chrome.scripting.executeScript({
- target: { tabId, frameIds: [0] }, world: 'MAIN', func: observeAwsWafChallengeInPage,
+ target: { tabId, frameIds: [0] }, world: 'MAIN', injectImmediately: true, func: observeAwsWafChallengeInPage,
});
const observed = entry?.result;
return typeof observed?.active === 'boolean' && observed.inspectionComplete === true
@@ -12260,10 +12262,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (submitted && result?.success === true && gate?.status === 'cleared') gate.continuationDispatched = true;
}
- async _observeAwsWafGate(tabId, observed = null) {
+ async _observeAwsWafGate(tabId, observed = null, abortSignal = null) {
observed ||= await this._readAwsWafChallenge(tabId);
+ this._throwIfAborted(abortSignal);
if (!observed) return null; // Failed inspection never retires a paid lock.
- await this._refreshCaptchaGateDocument(tabId, observed.rootDocument);
+ await this._refreshCaptchaGateDocument(tabId, observed.rootDocument, abortSignal);
+ this._throwIfAborted(abortSignal);
let previous = this._captchaGateStates.get(tabId);
const answer = this._nativeCaptchaSolutions?.get(tabId);
if (!observed.active) {
@@ -12292,12 +12296,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
&& (state.responseTokenPresent === true || !oldWidgetPresent);
} catch {}
}
+ this._throwIfAborted(abortSignal);
if (oldCleared) { this._retireCompletedCaptcha(tabId); previous = null; }
else if (previous.publicGate?.solveAttempted) return previous.publicGate;
}
if (previous?.awsWaf && previous.status !== 'cleared') return previous.publicGate;
if (previous?.status === 'cleared') this._retireCompletedCaptcha(tabId);
const providers = getCaptchaProviders(await chrome.storage.local.get(CAPTCHA_SETTINGS_KEYS));
+ this._throwIfAborted(abortSignal);
const capability = describeAwsWafObservation(providers, observed);
const attempted = answer?.family === 'aws_waf' && !answer.challengeCleared && answer.pageUrl === observed.pageUrl
? answer.awsAttempts || [] : [];
@@ -12539,7 +12545,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return clearedGate;
}
- async _refreshCaptchaGateDocument(tabId, rootDocument = null) {
+ async _refreshCaptchaGateDocument(tabId, rootDocument = null, abortSignal = null) {
const gate = this._captchaGateStates.get(tabId);
const previousRoot = gate?.rootDocument;
if (!(previousRoot?.timeOrigin > 0)) return null;
@@ -12550,6 +12556,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (root) rootDocument = { url: root.url, timeOrigin: root.timeOrigin };
}
const currentUrl = await this._currentUrl(tabId);
+ this._throwIfAborted(abortSignal);
if (!(rootDocument?.timeOrigin > 0) || rootDocument.timeOrigin === previousRoot.timeOrigin
|| this._normalizeUrl(rootDocument.url) !== this._normalizeUrl(currentUrl)
|| this._captchaGateStates.get(tabId) !== gate) return null;
@@ -12600,6 +12607,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
async _detectChallengeDialogBeforeMutation(tabId, options = {}) {
+ this._throwIfAborted(options.abortSignal);
const includeStatus = options?.includeStatus === true;
const expectedFrameId = Number.isInteger(options?.expectedFrameId)
? options.expectedFrameId
@@ -12613,12 +12621,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
navigationInspectionComplete = true;
}
} catch {}
+ this._throwIfAborted(options.abortSignal);
if (!navigationFrames.length) {
navigationFrames = [{ frameId: 0, parentFrameId: -1, url: '' }];
}
// The scan takes the dialog title readings the agent holds at each run.
const execute = target => chrome.scripting.executeScript({
target,
+ injectImmediately: true,
func: detectChallengeDialogInPage,
args: [{
includeFrameContext: true,
@@ -12685,6 +12695,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
: null;
}
+ this._throwIfAborted(options.abortSignal);
let outcome = inspected(results);
// Visible dialog titles nobody has read are read, in any language, and
// the page scanned again with those readings.
@@ -12812,6 +12823,24 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
async _captchaMutationPreflight(tabId, toolName, toolArgs = {}, abortSignal = null) {
+ const linked = this._linkAbortSignals(abortSignal, this._runAbortSignal(tabId));
+ const runId = this.currentRunId.get(tabId);
+ let stage = 'start';
+ const started = Date.now();
+ try {
+ return await this._withContentActionDeadline(
+ signal => this._inspectCaptchaBeforeMutation(tabId, toolName, toolArgs, signal, value => { stage = value; }),
+ toolName, CAPTCHA_PREFLIGHT_TIMEOUT_MS, linked.signal,
+ );
+ } catch (error) {
+ if (error?.code === 'content_action_timeout' && runId) {
+ trace.recordNote(runId, 0, 'captcha_preflight_timeout', { stage, elapsedMs: Date.now() - started });
+ }
+ throw error;
+ } finally { linked.dispose(); }
+ }
+
+ async _inspectCaptchaBeforeMutation(tabId, toolName, toolArgs = {}, abortSignal = null, onStage = () => {}) {
this._throwIfAborted(abortSignal);
if (toolName === 'done' && ['failed', 'partial'].includes(normalizeDoneOutcome(toolArgs?.outcome))) return null;
const gatedCompletion = toolName === 'done' || toolName === 'done_json';
@@ -12819,12 +12848,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
|| gatedCompletion
|| isNetworkMutation(toolName, toolArgs);
if (toolName === 'solve_captcha') {
- await this._refreshCaptchaGateDocument(tabId);
+ onStage('document_identity');
+ await this._refreshCaptchaGateDocument(tabId, null, abortSignal);
this._throwIfAborted(abortSignal);
return null;
}
if (!gatedAction || Agent.NAV_TOOLS.has(toolName)) return null;
- const awsGate = await this._observeAwsWafGate(tabId);
+ onStage('aws_waf');
+ const awsGate = await this._observeAwsWafGate(tabId, null, abortSignal);
+ this._throwIfAborted(abortSignal);
if (awsGate && awsGate.status !== 'cleared') return awsGate;
const activeGate = this._captchaGateForTools(tabId, this._captchaGateStates.get(tabId));
if (
@@ -12832,7 +12864,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
&& activeGate.status !== 'cleared'
&& !this._shouldRetryCaptchaManualGate(activeGate)
) return null;
- const detectedChallenge = await this._detectChallengeDialogBeforeMutation(tabId);
+ onStage('dialog_scan');
+ const detectedChallenge = await this._detectChallengeDialogBeforeMutation(tabId, { abortSignal });
this._throwIfAborted(abortSignal);
const challenge = detectedChallenge?.label
? detectedChallenge
@@ -12843,6 +12876,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
let pageUrl = '';
try { pageUrl = await this._currentUrl(tabId); } catch {}
this._throwIfAborted(abortSignal);
+ onStage('challenge_observation');
const observation = await this._observeCaptchaChallenge(
tabId,
'get_accessibility_tree',
@@ -12856,12 +12890,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
captchaChallengeVisibilityConfirmed: true,
},
{},
+ abortSignal,
);
this._throwIfAborted(abortSignal);
return observation.gate;
}
- async _observeCaptchaChallenge(tabId, toolName, toolResult, toolArgs = {}) {
+ async _observeCaptchaChallenge(tabId, toolName, toolResult, toolArgs = {}, abortSignal = null) {
+ this._throwIfAborted(abortSignal);
if (
toolName !== 'get_accessibility_tree'
|| !toolResult
@@ -12884,7 +12920,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
&& !toolResult.truncated && !toolResult.hasMore && !toolResult.autoDegraded
&& (!toolArgs?.maxDepth || Number(toolArgs.maxDepth) >= 15)
&& toolArgs?.filter !== 'interactive') {
- const awsGate = await this._observeAwsWafGate(tabId);
+ const awsGate = await this._observeAwsWafGate(tabId, null, abortSignal);
if (awsGate) {
toolResult.captchaGate = awsGate;
return { gate: awsGate, loopCheck: { kind: 'none' } };
@@ -12941,7 +12977,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if (pageUrl && this._nativeCaptchaSolutions?.get(tabId)?.pageUrl !== pageUrl) {
try {
- if (this._retireNativeCaptchaAnswerIfPageChanged(tabId, await this._currentUrl(tabId))) {
+ const currentUrl = await this._currentUrl(tabId);
+ this._throwIfAborted(abortSignal);
+ if (this._retireNativeCaptchaAnswerIfPageChanged(tabId, currentUrl)) {
this._captchaGateStates.delete(tabId);
activeGate = null;
}
@@ -12991,8 +13029,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if ((rootReplaced || widgetGone)
&& this._normalizeUrl(await this._currentUrl(tabId)) === this._normalizeUrl(currentUrl)) {
+ this._throwIfAborted(abortSignal);
navigationClearance = rootReplaced
- ? await this._refreshCaptchaGateDocument(tabId, currentRoot)
+ ? await this._refreshCaptchaGateDocument(tabId, currentRoot, abortSignal)
: this._clearCaptchaGateAfterPageChange(tabId, gatedPageUrl, currentUrl, toolResult);
if (navigationClearance) toolResult.captchaGate = navigationClearance;
if (navigationClearance) activeGate = null;
@@ -13073,12 +13112,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
);
} else if (!frameInspection.inspectionComplete) {
const guardedState = normalizeCaptchaGateState(activeGate);
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, guardedState);
toolResult.captchaGate = guardedState.publicGate;
return { gate: guardedState.publicGate, loopCheck: { kind: 'none' } };
}
}
if (activeGate?.status === 'cleared' && activeGate.continuationDispatched && challenge) {
+ this._throwIfAborted(abortSignal);
this._retireCompletedCaptcha(tabId);
activeGate = null;
}
@@ -13118,6 +13159,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
responseTokenPresent: true,
clearedByResponseToken: true,
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(activeGate),
status: 'cleared',
@@ -13141,6 +13183,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
activeChallengeAfterSolve: true,
...(challenge?.label ? { challengeDialog: { label: challenge.label } } : {}),
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(activeGate),
status: 'manual_required',
@@ -13161,6 +13204,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
status: 'verification_pending',
...(challenge?.label ? { challengeDialog: { label: challenge.label } } : {}),
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(activeGate),
status: 'verification_pending',
@@ -13182,6 +13226,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
});
if (inspection.inspectionComplete && !inspection.challenge) {
const clearedGate = { ...activeGate.publicGate, status: 'cleared', clearedByNativeApplication: true };
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, { ...activeGate, status: 'cleared', publicGate: clearedGate });
toolResult.captchaGate = clearedGate;
return { gate: clearedGate, loopCheck };
@@ -13201,6 +13246,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
status: 'cleared',
clearedByManualCompletion: true,
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.delete(tabId);
toolResult.captchaGate = clearedGate;
return { gate: clearedGate, loopCheck };
@@ -13208,6 +13254,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if (activeGate && authoritativeRootRead && activeGate.status !== 'cleared') {
const guardedState = normalizeCaptchaGateState(activeGate);
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, guardedState);
toolResult.captchaGate = guardedState.publicGate;
return { gate: guardedState.publicGate, loopCheck };
@@ -13217,6 +13264,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if (activeGate) {
const guardedState = normalizeCaptchaGateState(activeGate);
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, guardedState);
toolResult.captchaGate = guardedState.publicGate;
return { gate: guardedState.publicGate, loopCheck };
@@ -13234,6 +13282,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
status: 'manual_required',
challengeDialog: { label: challenge.label },
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(existing),
status: 'manual_required',
@@ -13248,6 +13297,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
status: 'verification_pending',
challengeDialog: { label: challenge.label },
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(existing),
status: 'verification_pending',
@@ -13313,6 +13363,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const captchaCandidateIdentity = selectedCorrelated
? captchaGateCandidateIdentity(detection?.selected)
: null;
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
key,
pageUrl,
@@ -42271,7 +42322,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!pendingNativeAnswer && !args?.provider && !args?.method && (!args?.family || args.family === 'aws_waf')) {
const observed = await (async () => {
const [entry] = await chrome.scripting.executeScript({
- target: { tabId, frameIds: [0] }, world: 'MAIN', func: observeAwsWafChallengeInPage,
+ target: { tabId, frameIds: [0] }, world: 'MAIN', injectImmediately: true, func: observeAwsWafChallengeInPage,
});
return entry?.result || null;
})().catch(() => null);
diff --git a/src/chrome/src/agent/captcha-solution-application.js b/src/chrome/src/agent/captcha-solution-application.js
index 7ce881d4c..6c6e5f03e 100644
--- a/src/chrome/src/agent/captcha-solution-application.js
+++ b/src/chrome/src/agent/captcha-solution-application.js
@@ -75,12 +75,12 @@ export function prepareCaptchaApplication(solution, application) {
export async function captureCaptchaDocuments(tabId, frames, api = globalThis.browser || globalThis.chrome) {
const read = () => ({ url: location.href, timeOrigin: performance.timeOrigin });
if (typeof api.scripting?.executeScript === 'function') {
- const results = await api.scripting.executeScript({ target: { tabId, allFrames: true }, func: read });
+ const results = await api.scripting.executeScript({ target: { tabId, allFrames: true }, injectImmediately: true, func: read });
return results.filter(r => Number.isFinite(r.result?.timeOrigin)).map(r => ({ frameId: r.frameId, ...r.result }));
}
const identities = await Promise.all(frames.map(async frame => {
try {
- const result = (await api.tabs.executeScript(tabId, { frameId: frame.frameId, code: `(${read.toString()})()` }))?.[0];
+ const result = (await api.tabs.executeScript(tabId, { frameId: frame.frameId, runAt: 'document_start', code: `(${read.toString()})()` }))?.[0];
return Number.isFinite(result?.timeOrigin) ? { frameId: frame.frameId, ...result } : null;
} catch { return null; }
}));
diff --git a/src/chrome/src/agent/captcha-solver.js b/src/chrome/src/agent/captcha-solver.js
index 9c360f878..75044479f 100644
--- a/src/chrome/src/agent/captcha-solver.js
+++ b/src/chrome/src/agent/captcha-solver.js
@@ -313,6 +313,7 @@ export async function detectCaptcha(tabId, constraints = {}) {
const [scriptAttempt, frameTreeAttempt] = await Promise.allSettled([
chrome.scripting.executeScript({
target: { tabId, allFrames: true },
+ injectImmediately: true,
func: detectCaptchaCandidatesInPage,
args: [null, captchaChallengeMatcherOptions(constraints?.challengeReadings)],
}),
diff --git a/src/chrome/src/agent/page-feedback-policy.js b/src/chrome/src/agent/page-feedback-policy.js
index 6c3abe2f7..f675b4a40 100644
--- a/src/chrome/src/agent/page-feedback-policy.js
+++ b/src/chrome/src/agent/page-feedback-policy.js
@@ -254,7 +254,14 @@ export function pageFeedbackCallPolicy(name, args = {}, state = {}, currentPage
if (DETACHED_TOOLS.has(name)) return { kind: 'independent' };
if (['fetch_url', 'research_url'].includes(name) && httpUrl(args.url)
&& ['GET', 'HEAD'].includes(String(args.method || 'GET').toUpperCase())
- && !args.replayRequestId && !args.body) return { kind: 'independent' };
+ && !args.replayRequestId && !args.apiReplayRequestId && !args.body) return { kind: 'independent' };
+ // A concrete API request does not act on a DOM target. Retaining it only
+ // preserves the proposed call: API consent, recipient, submission and CAPTCHA
+ // checks still run before dispatch. Captured replays depend on page state.
+ if (name === 'fetch_url' && httpUrl(args.url)
+ && ['POST', 'PUT', 'PATCH', 'DELETE'].includes(String(args.method || '').toUpperCase())
+ && !args.replayRequestId && !args.apiReplayRequestId
+ && (args.body == null || typeof args.body === 'string')) return { kind: 'request' };
if (['download_files', 'download_file'].includes(name)) {
// Match the handler's singular-URL alias and precedence exactly. Page churn
// does not alter a concrete resource URL already supplied to the download.
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index 9bfa37310..6d6231624 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -1089,9 +1089,10 @@ export const pageFeedbackMethods = {
+ 'Do not repeat an unresolved download.' };
},
- _pageFeedbackStateCurrent(tabId, run, state, url = run.url) {
+ _pageFeedbackStateCurrent(tabId, run, state, url = run.url, { rootOnly = false } = {}) {
const calculationScope = state?.calculationResponseScope ? this._pageFeedbackCalculationResponseScope(tabId) : null;
return !!state && state.token === run.token && state.url === url
+ && (!rootOnly || (state.frames.has(0) && !this._hasPendingSteering?.(tabId)))
&& (!state.calculationResponseScope || (calculationScope
&& state.calculationResponseScope.owner === calculationScope.owner
&& state.calculationResponseScope.runToken === calculationScope.runToken
@@ -1100,7 +1101,7 @@ export const pageFeedbackMethods = {
&& this._pageFeedbackCalculationComplete(tabId)))
&& (state.runMode === undefined || state.runMode === (this._effectiveRunMode?.(tabId, null) || null))
&& state.steeringRevision === (this._steeringRuns?.get(tabId)?.acceptedIds.size || 0)
- && [...state.frames].every(([id, frame]) => run.frames.get(id)?.token === frame.token
+ && [...state.frames].filter(([id]) => !rootOnly || id === 0).every(([id, frame]) => run.frames.get(id)?.token === frame.token
&& run.frames.get(id)?.id === frame.id);
},
@@ -1169,8 +1170,10 @@ export const pageFeedbackMethods = {
|| !this._pageFeedbackStateCurrent(tabId, run, run.dispatchCall.modelState))) return true;
if (!run?.events.size) return false;
const events = [...run.events.values()];
+ const policy = pageFeedbackCallPolicy(name, args, run.modelState, run.latestPage, events);
// Automatic churn (a superset of relaxed-retry DOM) revalidates per policy.
- if (!isAutomaticPageFeedback(events) || !this._pageFeedbackStateCurrent(tabId, run, run.modelState)) return true;
+ if (!isAutomaticPageFeedback(events) || !this._pageFeedbackStateCurrent(tabId, run, run.modelState, run.url,
+ { rootOnly: policy.kind === 'request' })) return true;
// Unknown DOM is observed while an agent action is in flight. It is that
// call's own uncertain effect only after the call sent input (typeahead
// renders between keys). Observed during an earlier action, or during this
@@ -1179,12 +1182,12 @@ export const pageFeedbackMethods = {
.some(({ call, afterInput }) => call && call.mutation !== false && !(call === run.dispatchCall && afterInput)))) return true;
const policyEvents = this._monitoringRetryCurrent(tabId, run)
? events.map(event => event.source === 'unknown' && event.kind === 'dom' ? { ...event, source: 'page' } : event) : events;
- const policy = pageFeedbackCallPolicy(name, args, run.modelState, run.latestPage, policyEvents);
+ const currentPolicy = pageFeedbackCallPolicy(name, args, run.modelState, run.latestPage, policyEvents);
// Targets need a fresh comparison first; the content monitor then checks
// their immutable node and action context at the actual dispatch boundary.
- if (policy.kind === 'bound_target') return run.dispatchCall?.allowPassiveRebase !== true
+ if (currentPolicy.kind === 'bound_target') return run.dispatchCall?.allowPassiveRebase !== true
|| run.validatedActionBindings?.get(`${name}:${JSON.stringify(args)}`) !== run.modelState.actionBinding.snapshotToken;
- return policy.kind === 'unsafe' || (policy.kind === 'target' && !run.dispatchCall?.allowPassiveRebase);
+ return currentPolicy.kind === 'unsafe' || (currentPolicy.kind === 'target' && !run.dispatchCall?.allowPassiveRebase);
},
_compactPageFeedbackMessages(messages) {
@@ -1350,8 +1353,12 @@ export const pageFeedbackMethods = {
documentMatches = state.documentId ? documentId === state.documentId : state.frames.has(0);
}
const currentRun = this._pageFeedbackRuns.get(tabId) === run;
+ const concreteRequests = !!responseToolCalls?.length && responseToolCalls.every(call => {
+ try { return pageFeedbackCallPolicy(call.function?.name, JSON.parse(call.function.arguments)).kind === 'request'; }
+ catch { return false; }
+ });
const stateCurrent = documentMatches && currentRun
- ? this._pageFeedbackStateCurrent(tabId, run, state, url) : null;
+ ? this._pageFeedbackStateCurrent(tabId, run, state, url, { rootOnly: concreteRequests }) : null;
const sameDocument = documentMatches && currentRun && stateCurrent;
// A tool-free Ask answer or a strictly owned literal arithmetic-only task
// is independent of passive churn. Task-only scope permits normal protocol
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 02d698f96..9168060cd 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -202,6 +202,8 @@ const WORKFLOW_FIELD_KEYS = new Set(['alt_text', 'paid_promotion', 'recording_da
const STAGED_SCREENSHOT_REDACTION_MAX_REGIONS = 400;
const VISION_SUB_CALL_TIMEOUT_MS = 90_000;
const CONTENT_ACTION_TIMEOUT_MS = 60_000;
+// Read-only CAPTCHA checks must not consume a full page-action timeout.
+const CAPTCHA_PREFLIGHT_TIMEOUT_MS = 8_000;
const RUN_STARTUP_TIMEOUT_MS = 15_000;
// At completion the active model reads what a page's status texts say only
// for this long, so a slow model cannot hold a finished task; texts it did
@@ -10893,7 +10895,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
async _readAwsWafChallenge(tabId) {
try {
- const [observed] = await browser.tabs.executeScript(tabId, { frameId: 0, code: `(${observeAwsWafChallengeInPage.toString()})()` });
+ const [observed] = await browser.tabs.executeScript(tabId, { frameId: 0, runAt: 'document_start', code: `(${observeAwsWafChallengeInPage.toString()})()` });
return typeof observed?.active === 'boolean' && observed.inspectionComplete === true
&& observed.pageUrl === await this._currentUrl(tabId) ? observed : null;
} catch { return null; }
@@ -10913,10 +10915,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (submitted && result?.success === true && gate?.status === 'cleared') gate.continuationDispatched = true;
}
- async _observeAwsWafGate(tabId, observed = null) {
+ async _observeAwsWafGate(tabId, observed = null, abortSignal = null) {
observed ||= await this._readAwsWafChallenge(tabId);
+ this._throwIfAborted(abortSignal);
if (!observed) return null; // Failed inspection never retires a paid lock.
- await this._refreshCaptchaGateDocument(tabId, observed.rootDocument);
+ await this._refreshCaptchaGateDocument(tabId, observed.rootDocument, abortSignal);
+ this._throwIfAborted(abortSignal);
let previous = this._captchaGateStates.get(tabId);
const answer = this._nativeCaptchaSolutions?.get(tabId);
if (!observed.active) {
@@ -10945,12 +10949,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
&& (state.responseTokenPresent === true || !oldWidgetPresent);
} catch {}
}
+ this._throwIfAborted(abortSignal);
if (oldCleared) { this._retireCompletedCaptcha(tabId); previous = null; }
else if (previous.publicGate?.solveAttempted) return previous.publicGate;
}
if (previous?.awsWaf && previous.status !== 'cleared') return previous.publicGate;
if (previous?.status === 'cleared') this._retireCompletedCaptcha(tabId);
const providers = getCaptchaProviders(await browser.storage.local.get(CAPTCHA_SETTINGS_KEYS));
+ this._throwIfAborted(abortSignal);
const capability = describeAwsWafObservation(providers, observed);
const attempted = answer?.family === 'aws_waf' && !answer.challengeCleared && answer.pageUrl === observed.pageUrl
? answer.awsAttempts || [] : [];
@@ -11192,7 +11198,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return clearedGate;
}
- async _refreshCaptchaGateDocument(tabId, rootDocument = null) {
+ async _refreshCaptchaGateDocument(tabId, rootDocument = null, abortSignal = null) {
const gate = this._captchaGateStates.get(tabId);
const previousRoot = gate?.rootDocument;
if (!(previousRoot?.timeOrigin > 0)) return null;
@@ -11203,6 +11209,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (root) rootDocument = { url: root.url, timeOrigin: root.timeOrigin };
}
const currentUrl = await this._currentUrl(tabId);
+ this._throwIfAborted(abortSignal);
if (!(rootDocument?.timeOrigin > 0) || rootDocument.timeOrigin === previousRoot.timeOrigin
|| this._normalizeUrl(rootDocument.url) !== this._normalizeUrl(currentUrl)
|| this._captchaGateStates.get(tabId) !== gate) return null;
@@ -11253,6 +11260,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
async _detectChallengeDialogBeforeMutation(tabId, options = {}) {
+ this._throwIfAborted(options.abortSignal);
const includeStatus = options?.includeStatus === true;
const expectedFrameId = Number.isInteger(options?.expectedFrameId)
? options.expectedFrameId
@@ -11266,6 +11274,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
navigationInspectionComplete = true;
}
} catch {}
+ this._throwIfAborted(options.abortSignal);
if (!navigationFrames.length) {
navigationFrames = [{ frameId: 0, parentFrameId: -1, url: '' }];
}
@@ -11282,6 +11291,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
code,
frameId: frame.frameId,
matchAboutBlank: true,
+ runAt: 'document_start',
});
return {
frameId: frame.frameId,
@@ -11296,6 +11306,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
);
};
let successfulEntries = await scan();
+ this._throwIfAborted(options.abortSignal);
// Visible dialog titles nobody has read are read, in any language, and
// the page scanned again with those readings.
const unread = successfulEntries.flatMap(entry => (Array.isArray(entry.payload?.unreadTitles) ? entry.payload.unreadTitles : []));
@@ -11456,6 +11467,24 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
async _captchaMutationPreflight(tabId, toolName, toolArgs = {}, abortSignal = null) {
+ const linked = this._linkAbortSignals(abortSignal, this._runAbortSignal(tabId));
+ const runId = this.currentRunId.get(tabId);
+ let stage = 'start';
+ const started = Date.now();
+ try {
+ return await this._withContentActionDeadline(
+ signal => this._inspectCaptchaBeforeMutation(tabId, toolName, toolArgs, signal, value => { stage = value; }),
+ toolName, CAPTCHA_PREFLIGHT_TIMEOUT_MS, linked.signal,
+ );
+ } catch (error) {
+ if (error?.code === 'content_action_timeout' && runId) {
+ trace.recordNote(runId, 0, 'captcha_preflight_timeout', { stage, elapsedMs: Date.now() - started });
+ }
+ throw error;
+ } finally { linked.dispose(); }
+ }
+
+ async _inspectCaptchaBeforeMutation(tabId, toolName, toolArgs = {}, abortSignal = null, onStage = () => {}) {
this._throwIfAborted(abortSignal);
if (toolName === 'done' && ['failed', 'partial'].includes(normalizeDoneOutcome(toolArgs?.outcome))) return null;
const gatedCompletion = toolName === 'done' || toolName === 'done_json';
@@ -11463,12 +11492,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
|| gatedCompletion
|| isNetworkMutation(toolName, toolArgs);
if (toolName === 'solve_captcha') {
- await this._refreshCaptchaGateDocument(tabId);
+ onStage('document_identity');
+ await this._refreshCaptchaGateDocument(tabId, null, abortSignal);
this._throwIfAborted(abortSignal);
return null;
}
if (!gatedAction || Agent.NAV_TOOLS.has(toolName)) return null;
- const awsGate = await this._observeAwsWafGate(tabId);
+ onStage('aws_waf');
+ const awsGate = await this._observeAwsWafGate(tabId, null, abortSignal);
+ this._throwIfAborted(abortSignal);
if (awsGate && awsGate.status !== 'cleared') return awsGate;
const activeGate = this._captchaGateForTools(tabId, this._captchaGateStates.get(tabId));
if (
@@ -11476,7 +11508,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
&& activeGate.status !== 'cleared'
&& !this._shouldRetryCaptchaManualGate(activeGate)
) return null;
- const detectedChallenge = await this._detectChallengeDialogBeforeMutation(tabId);
+ onStage('dialog_scan');
+ const detectedChallenge = await this._detectChallengeDialogBeforeMutation(tabId, { abortSignal });
this._throwIfAborted(abortSignal);
const challenge = detectedChallenge?.label
? detectedChallenge
@@ -11487,6 +11520,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
let pageUrl = '';
try { pageUrl = await this._currentUrl(tabId); } catch {}
this._throwIfAborted(abortSignal);
+ onStage('challenge_observation');
const observation = await this._observeCaptchaChallenge(
tabId,
'get_accessibility_tree',
@@ -11500,12 +11534,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
captchaChallengeVisibilityConfirmed: true,
},
{},
+ abortSignal,
);
this._throwIfAborted(abortSignal);
return observation.gate;
}
- async _observeCaptchaChallenge(tabId, toolName, toolResult, toolArgs = {}) {
+ async _observeCaptchaChallenge(tabId, toolName, toolResult, toolArgs = {}, abortSignal = null) {
+ this._throwIfAborted(abortSignal);
if (
toolName !== 'get_accessibility_tree'
|| !toolResult
@@ -11528,7 +11564,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
&& !toolResult.truncated && !toolResult.hasMore && !toolResult.autoDegraded
&& (!toolArgs?.maxDepth || Number(toolArgs.maxDepth) >= 15)
&& toolArgs?.filter !== 'interactive') {
- const awsGate = await this._observeAwsWafGate(tabId);
+ const awsGate = await this._observeAwsWafGate(tabId, null, abortSignal);
if (awsGate) {
toolResult.captchaGate = awsGate;
return { gate: awsGate, loopCheck: { kind: 'none' } };
@@ -11585,7 +11621,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if (pageUrl && this._nativeCaptchaSolutions?.get(tabId)?.pageUrl !== pageUrl) {
try {
- if (this._retireNativeCaptchaAnswerIfPageChanged(tabId, await this._currentUrl(tabId))) {
+ const currentUrl = await this._currentUrl(tabId);
+ this._throwIfAborted(abortSignal);
+ if (this._retireNativeCaptchaAnswerIfPageChanged(tabId, currentUrl)) {
this._captchaGateStates.delete(tabId);
activeGate = null;
}
@@ -11635,8 +11673,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if ((rootReplaced || widgetGone)
&& this._normalizeUrl(await this._currentUrl(tabId)) === this._normalizeUrl(currentUrl)) {
+ this._throwIfAborted(abortSignal);
navigationClearance = rootReplaced
- ? await this._refreshCaptchaGateDocument(tabId, currentRoot)
+ ? await this._refreshCaptchaGateDocument(tabId, currentRoot, abortSignal)
: this._clearCaptchaGateAfterPageChange(tabId, gatedPageUrl, currentUrl, toolResult);
if (navigationClearance) toolResult.captchaGate = navigationClearance;
if (navigationClearance) activeGate = null;
@@ -11717,12 +11756,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
);
} else if (!frameInspection.inspectionComplete) {
const guardedState = normalizeCaptchaGateState(activeGate);
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, guardedState);
toolResult.captchaGate = guardedState.publicGate;
return { gate: guardedState.publicGate, loopCheck: { kind: 'none' } };
}
}
if (activeGate?.status === 'cleared' && activeGate.continuationDispatched && challenge) {
+ this._throwIfAborted(abortSignal);
this._retireCompletedCaptcha(tabId);
activeGate = null;
}
@@ -11762,6 +11803,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
responseTokenPresent: true,
clearedByResponseToken: true,
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(activeGate),
status: 'cleared',
@@ -11785,6 +11827,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
activeChallengeAfterSolve: true,
...(challenge?.label ? { challengeDialog: { label: challenge.label } } : {}),
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(activeGate),
status: 'manual_required',
@@ -11805,6 +11848,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
status: 'verification_pending',
...(challenge?.label ? { challengeDialog: { label: challenge.label } } : {}),
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(activeGate),
status: 'verification_pending',
@@ -11826,6 +11870,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
});
if (inspection.inspectionComplete && !inspection.challenge) {
const clearedGate = { ...activeGate.publicGate, status: 'cleared', clearedByNativeApplication: true };
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, { ...activeGate, status: 'cleared', publicGate: clearedGate });
toolResult.captchaGate = clearedGate;
return { gate: clearedGate, loopCheck };
@@ -11845,6 +11890,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
status: 'cleared',
clearedByManualCompletion: true,
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.delete(tabId);
toolResult.captchaGate = clearedGate;
return { gate: clearedGate, loopCheck };
@@ -11852,6 +11898,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if (activeGate && authoritativeRootRead && activeGate.status !== 'cleared') {
const guardedState = normalizeCaptchaGateState(activeGate);
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, guardedState);
toolResult.captchaGate = guardedState.publicGate;
return { gate: guardedState.publicGate, loopCheck };
@@ -11861,6 +11908,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if (activeGate) {
const guardedState = normalizeCaptchaGateState(activeGate);
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, guardedState);
toolResult.captchaGate = guardedState.publicGate;
return { gate: guardedState.publicGate, loopCheck };
@@ -11878,6 +11926,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
status: 'manual_required',
challengeDialog: { label: challenge.label },
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(existing),
status: 'manual_required',
@@ -11892,6 +11941,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
status: 'verification_pending',
challengeDialog: { label: challenge.label },
};
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
...withoutLegacyCaptchaVerificationState(existing),
status: 'verification_pending',
@@ -11957,6 +12007,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const captchaCandidateIdentity = selectedCorrelated
? captchaGateCandidateIdentity(detection?.selected)
: null;
+ this._throwIfAborted(abortSignal);
this._captchaGateStates.set(tabId, {
key,
pageUrl,
@@ -38953,7 +39004,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!pendingNativeAnswer && !args?.provider && !args?.method && (!args?.family || args.family === 'aws_waf')) {
const observed = await (async () => {
const [result] = await browser.tabs.executeScript(tabId, {
- frameId: 0, code: `(${observeAwsWafChallengeInPage.toString()})()`,
+ frameId: 0, runAt: 'document_start', code: `(${observeAwsWafChallengeInPage.toString()})()`,
});
return result || null;
})().catch(() => null);
diff --git a/src/firefox/src/agent/captcha-solution-application.js b/src/firefox/src/agent/captcha-solution-application.js
index 7ce881d4c..6c6e5f03e 100644
--- a/src/firefox/src/agent/captcha-solution-application.js
+++ b/src/firefox/src/agent/captcha-solution-application.js
@@ -75,12 +75,12 @@ export function prepareCaptchaApplication(solution, application) {
export async function captureCaptchaDocuments(tabId, frames, api = globalThis.browser || globalThis.chrome) {
const read = () => ({ url: location.href, timeOrigin: performance.timeOrigin });
if (typeof api.scripting?.executeScript === 'function') {
- const results = await api.scripting.executeScript({ target: { tabId, allFrames: true }, func: read });
+ const results = await api.scripting.executeScript({ target: { tabId, allFrames: true }, injectImmediately: true, func: read });
return results.filter(r => Number.isFinite(r.result?.timeOrigin)).map(r => ({ frameId: r.frameId, ...r.result }));
}
const identities = await Promise.all(frames.map(async frame => {
try {
- const result = (await api.tabs.executeScript(tabId, { frameId: frame.frameId, code: `(${read.toString()})()` }))?.[0];
+ const result = (await api.tabs.executeScript(tabId, { frameId: frame.frameId, runAt: 'document_start', code: `(${read.toString()})()` }))?.[0];
return Number.isFinite(result?.timeOrigin) ? { frameId: frame.frameId, ...result } : null;
} catch { return null; }
}));
diff --git a/src/firefox/src/agent/captcha-solver.js b/src/firefox/src/agent/captcha-solver.js
index 84408b0c0..73f584fdd 100644
--- a/src/firefox/src/agent/captcha-solver.js
+++ b/src/firefox/src/agent/captcha-solver.js
@@ -380,6 +380,7 @@ export async function detectCaptcha(tabId, constraints = {}) {
code,
frameId: frame.frameId,
matchAboutBlank: true,
+ runAt: 'document_start',
});
const response = results?.[0];
const payload = response?.direct || response;
diff --git a/src/firefox/src/agent/page-feedback-policy.js b/src/firefox/src/agent/page-feedback-policy.js
index 6c3abe2f7..f675b4a40 100644
--- a/src/firefox/src/agent/page-feedback-policy.js
+++ b/src/firefox/src/agent/page-feedback-policy.js
@@ -254,7 +254,14 @@ export function pageFeedbackCallPolicy(name, args = {}, state = {}, currentPage
if (DETACHED_TOOLS.has(name)) return { kind: 'independent' };
if (['fetch_url', 'research_url'].includes(name) && httpUrl(args.url)
&& ['GET', 'HEAD'].includes(String(args.method || 'GET').toUpperCase())
- && !args.replayRequestId && !args.body) return { kind: 'independent' };
+ && !args.replayRequestId && !args.apiReplayRequestId && !args.body) return { kind: 'independent' };
+ // A concrete API request does not act on a DOM target. Retaining it only
+ // preserves the proposed call: API consent, recipient, submission and CAPTCHA
+ // checks still run before dispatch. Captured replays depend on page state.
+ if (name === 'fetch_url' && httpUrl(args.url)
+ && ['POST', 'PUT', 'PATCH', 'DELETE'].includes(String(args.method || '').toUpperCase())
+ && !args.replayRequestId && !args.apiReplayRequestId
+ && (args.body == null || typeof args.body === 'string')) return { kind: 'request' };
if (['download_files', 'download_file'].includes(name)) {
// Match the handler's singular-URL alias and precedence exactly. Page churn
// does not alter a concrete resource URL already supplied to the download.
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index 9bfa37310..6d6231624 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -1089,9 +1089,10 @@ export const pageFeedbackMethods = {
+ 'Do not repeat an unresolved download.' };
},
- _pageFeedbackStateCurrent(tabId, run, state, url = run.url) {
+ _pageFeedbackStateCurrent(tabId, run, state, url = run.url, { rootOnly = false } = {}) {
const calculationScope = state?.calculationResponseScope ? this._pageFeedbackCalculationResponseScope(tabId) : null;
return !!state && state.token === run.token && state.url === url
+ && (!rootOnly || (state.frames.has(0) && !this._hasPendingSteering?.(tabId)))
&& (!state.calculationResponseScope || (calculationScope
&& state.calculationResponseScope.owner === calculationScope.owner
&& state.calculationResponseScope.runToken === calculationScope.runToken
@@ -1100,7 +1101,7 @@ export const pageFeedbackMethods = {
&& this._pageFeedbackCalculationComplete(tabId)))
&& (state.runMode === undefined || state.runMode === (this._effectiveRunMode?.(tabId, null) || null))
&& state.steeringRevision === (this._steeringRuns?.get(tabId)?.acceptedIds.size || 0)
- && [...state.frames].every(([id, frame]) => run.frames.get(id)?.token === frame.token
+ && [...state.frames].filter(([id]) => !rootOnly || id === 0).every(([id, frame]) => run.frames.get(id)?.token === frame.token
&& run.frames.get(id)?.id === frame.id);
},
@@ -1169,8 +1170,10 @@ export const pageFeedbackMethods = {
|| !this._pageFeedbackStateCurrent(tabId, run, run.dispatchCall.modelState))) return true;
if (!run?.events.size) return false;
const events = [...run.events.values()];
+ const policy = pageFeedbackCallPolicy(name, args, run.modelState, run.latestPage, events);
// Automatic churn (a superset of relaxed-retry DOM) revalidates per policy.
- if (!isAutomaticPageFeedback(events) || !this._pageFeedbackStateCurrent(tabId, run, run.modelState)) return true;
+ if (!isAutomaticPageFeedback(events) || !this._pageFeedbackStateCurrent(tabId, run, run.modelState, run.url,
+ { rootOnly: policy.kind === 'request' })) return true;
// Unknown DOM is observed while an agent action is in flight. It is that
// call's own uncertain effect only after the call sent input (typeahead
// renders between keys). Observed during an earlier action, or during this
@@ -1179,12 +1182,12 @@ export const pageFeedbackMethods = {
.some(({ call, afterInput }) => call && call.mutation !== false && !(call === run.dispatchCall && afterInput)))) return true;
const policyEvents = this._monitoringRetryCurrent(tabId, run)
? events.map(event => event.source === 'unknown' && event.kind === 'dom' ? { ...event, source: 'page' } : event) : events;
- const policy = pageFeedbackCallPolicy(name, args, run.modelState, run.latestPage, policyEvents);
+ const currentPolicy = pageFeedbackCallPolicy(name, args, run.modelState, run.latestPage, policyEvents);
// Targets need a fresh comparison first; the content monitor then checks
// their immutable node and action context at the actual dispatch boundary.
- if (policy.kind === 'bound_target') return run.dispatchCall?.allowPassiveRebase !== true
+ if (currentPolicy.kind === 'bound_target') return run.dispatchCall?.allowPassiveRebase !== true
|| run.validatedActionBindings?.get(`${name}:${JSON.stringify(args)}`) !== run.modelState.actionBinding.snapshotToken;
- return policy.kind === 'unsafe' || (policy.kind === 'target' && !run.dispatchCall?.allowPassiveRebase);
+ return currentPolicy.kind === 'unsafe' || (currentPolicy.kind === 'target' && !run.dispatchCall?.allowPassiveRebase);
},
_compactPageFeedbackMessages(messages) {
@@ -1350,8 +1353,12 @@ export const pageFeedbackMethods = {
documentMatches = state.documentId ? documentId === state.documentId : state.frames.has(0);
}
const currentRun = this._pageFeedbackRuns.get(tabId) === run;
+ const concreteRequests = !!responseToolCalls?.length && responseToolCalls.every(call => {
+ try { return pageFeedbackCallPolicy(call.function?.name, JSON.parse(call.function.arguments)).kind === 'request'; }
+ catch { return false; }
+ });
const stateCurrent = documentMatches && currentRun
- ? this._pageFeedbackStateCurrent(tabId, run, state, url) : null;
+ ? this._pageFeedbackStateCurrent(tabId, run, state, url, { rootOnly: concreteRequests }) : null;
const sameDocument = documentMatches && currentRun && stateCurrent;
// A tool-free Ask answer or a strictly owned literal arithmetic-only task
// is independent of passive churn. Task-only scope permits normal protocol
diff --git a/test/captcha-runtime-policy.mjs b/test/captcha-runtime-policy.mjs
index 8d867e765..9ca7b1cbd 100644
--- a/test/captcha-runtime-policy.mjs
+++ b/test/captcha-runtime-policy.mjs
@@ -5,6 +5,102 @@ import vm from 'node:vm';
import { fixturePageBlockerReading, stubPageTextReadings } from './semantic-decision-stub.mjs';
const captchaNames = ['get_captcha_capabilities', 'solve_captcha', 'apply_captcha_solution'];
const solverInstructions = /get_captcha_capabilities|solve_captcha|apply_captcha_solution|\[CAPTCHA SOLVER/;
+
+for (const build of ['chrome', 'firefox']) {
+ const { Agent } = await import(`../src/${build}/src/agent/agent.js`);
+ test(`${build}: CAPTCHA preflight scans loading pages and ad frames without waiting for document_idle`, { timeout: 2000 }, async t => {
+ const url = 'https://mail.test/inbox', scans = [];
+ const payload = { challenge: null, frameContext: { frameUrl: url, childFrames: [] } };
+ const probe = async (name, immediate, frameId = 0) => {
+ scans.push({ name, immediate, frameId });
+ // Models the browser waiting forever for a loading ad's document_idle.
+ if (!immediate) return new Promise(() => {});
+ return name.includes('observeAwsWafChallengeInPage')
+ ? { active: false, inspectionComplete: true, pageUrl: url }
+ : { ...payload, frameContext: { ...payload.frameContext, frameUrl: frameId ? 'https://ads.test/loading' : url } };
+ };
+ const apiName = build === 'chrome' ? 'chrome' : 'browser', previous = globalThis[apiName];
+ globalThis[apiName] = {
+ tabs: { get: async () => ({ url }), executeScript: async (_tab, opts) => [await probe(opts.code, opts.runAt === 'document_start', opts.frameId)] },
+ scripting: { executeScript: async opts => {
+ const ids = opts.target.allFrames ? [0, 7] : opts.target.frameIds || [0];
+ return Promise.all(ids.map(async frameId => ({ frameId, result: await probe(opts.func.name, opts.injectImmediately === true, frameId) })));
+ } },
+ webNavigation: { getAllFrames: async () => [
+ { frameId: 0, parentFrameId: -1, url }, { frameId: 7, parentFrameId: 0, url: 'https://ads.test/loading' },
+ ] },
+ };
+ t.after(() => { globalThis[apiName] = previous; });
+ const agent = new Agent({ getActive: () => ({ promptTier: 'full' }) });
+ agent.conversationModes.set(1, 'act');
+ assert.equal(await agent._captchaMutationPreflight(1, 'click_ax', { ref_id: 'ref_signout' }), null);
+ assert.ok(scans.some(scan => scan.name.includes('observeAwsWafChallengeInPage')));
+ assert.ok(scans.some(scan => scan.frameId === 7));
+ assert.ok(scans.every(scan => scan.immediate));
+
+ // Immediate injection must still recognize an actual visible challenge.
+ payload.challenge = { label: 'CAPTCHA', normalizedLabel: 'captcha' };
+ let observed = 0;
+ agent._observeCaptchaChallenge = async (_tab, _tool, result) => {
+ observed++;
+ assert.equal(result.captchaChallengeVisibilityConfirmed, true);
+ assert.equal(result.captchaChallengeFrameId, 0);
+ return { gate: { status: 'solve_required' } };
+ };
+ assert.equal((await agent._captchaMutationPreflight(1, 'click_ax')).status, 'solve_required');
+ assert.equal(observed, 1);
+ });
+
+ test(`${build}: a stalled CAPTCHA probe fails closed after eight seconds and ignores its late result`, async t => {
+ t.mock.timers.enable({ apis: ['setTimeout'] });
+ const agent = new Agent({ getActive: () => ({ promptTier: 'full' }) });
+ let resolveProbe;
+ agent._readAwsWafChallenge = () => new Promise(resolve => { resolveProbe = resolve; });
+ const work = agent._captchaMutationPreflight(1, 'click_ax');
+ const rejected = assert.rejects(work, error => error.code === 'content_action_timeout' && /8 seconds/.test(error.message));
+ await Promise.resolve(); await Promise.resolve();
+ assert.equal(typeof resolveProbe, 'function');
+ t.mock.timers.tick(8000);
+ await rejected;
+ resolveProbe({ active: true, inspectionComplete: true, pageUrl: 'https://mail.test/inbox' });
+ await Promise.resolve(); await Promise.resolve();
+ assert.equal(agent._captchaGateStates.has(1), false, 'An expired preflight cannot arm or retire a later gate');
+ });
+
+ test(`${build}: Stop interrupts a stalled CAPTCHA probe immediately`, async () => {
+ const agent = new Agent({});
+ agent._readAwsWafChallenge = () => new Promise(() => {});
+ const controller = new AbortController();
+ agent._runAbortSignal = () => controller.signal;
+ const work = agent._captchaMutationPreflight(1, 'click_ax', {}, new AbortController().signal);
+ const error = new Error('Stopped fixture preflight'); error.name = 'AbortError';
+ controller.abort(error);
+ await assert.rejects(work, value => value === error);
+ });
+
+ test(`${build}: a late document identity check cannot retire a paid CAPTCHA answer after preflight expiry`, async t => {
+ t.mock.timers.enable({ apis: ['setTimeout'] });
+ const agent = new Agent({});
+ const url = 'https://mail.test/inbox';
+ const gate = { pageUrl: url, rootDocument: { url, timeOrigin: 1000 }, status: 'solve_required' };
+ const answer = { solution: 'paid-answer', documents: [{ frameId: 0, url, timeOrigin: 1000 }] };
+ agent._captchaGateStates.set(1, gate); agent._nativeCaptchaSolutions = new Map([[1, answer]]);
+ agent._readAwsWafChallenge = async () => ({ active: false, rootDocument: { url, timeOrigin: 2000 } });
+ let resolveUrl;
+ agent._currentUrl = () => new Promise(resolve => { resolveUrl = resolve; });
+ const rejected = assert.rejects(agent._captchaMutationPreflight(1, 'click_ax'), error => error.code === 'content_action_timeout');
+ for (let turn = 0; turn < 5; turn++) await Promise.resolve();
+ assert.equal(typeof resolveUrl, 'function');
+ t.mock.timers.tick(8000);
+ await rejected;
+ resolveUrl(url);
+ for (let turn = 0; turn < 5; turn++) await Promise.resolve();
+ assert.equal(agent._captchaGateStates.get(1), gate);
+ assert.equal(answer.solution, 'paid-answer');
+ assert.equal(answer.documentRetired, undefined);
+ });
+}
+
test('CAPTCHA tool matrix matches Ask, Compact, Mid, Full and Dev availability', async () => {
const rows = (await readFile(new URL('../docs/agent-tools.md', import.meta.url), 'utf8')).split('\n');
for (const name of captchaNames) {
diff --git a/test/page-feedback-policy.mjs b/test/page-feedback-policy.mjs
index e6e260b17..8c2980487 100644
--- a/test/page-feedback-policy.mjs
+++ b/test/page-feedback-policy.mjs
@@ -104,12 +104,15 @@ for (const build of ['chrome', 'firefox']) {
for (const name of ['fetch_url', 'research_url']) {
for (const method of [undefined, 'GET', 'HEAD', 'get', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']) {
- const expected = method == null || ['GET', 'HEAD'].includes(method.toUpperCase());
+ const expected = method == null || ['GET', 'HEAD'].includes(method.toUpperCase())
+ || (name === 'fetch_url' && ['POST', 'PUT', 'PATCH', 'DELETE'].includes(method.toUpperCase()));
test(`${build}: passive churn ${expected ? 'retains' : 'supersedes'} explicit ${name} ${method || 'default GET'}`, () => {
assert.equal(keep(name, { url: 'https://example.com/resource', ...(method ? { method } : {}) }), expected);
});
}
- for (const extra of [{ replayRequestId: 'captured-post' }, { method: 'GET', replayRequestId: 'captured-post' },
+ for (const extra of [{ replayRequestId: 'captured-post' }, { apiReplayRequestId: 'captured-post' },
+ { method: 'POST', body: '{}', replayRequestId: 'captured-post' },
+ { method: 'POST', body: '{}', apiReplayRequestId: 'captured-post' }, { method: 'GET', replayRequestId: 'captured-post' },
{ body: 'changed-server-data' }, { method: 'GET', body: '{}' }]) {
test(`${build}: passive churn supersedes ${name} ${Object.keys(extra).join('+')} payload`, () => {
assert.equal(keep(name, { url: 'https://example.com/resource', ...extra }), false,
@@ -118,6 +121,25 @@ for (const build of ['chrome', 'firefox']) {
}
}
+ test(`${build}: explicit API writes survive counters and ad frames but retain intervention fences`, () => {
+ const args = { url: 'https://api.mail.test/accounts', method: 'POST',
+ headers: { 'Content-Type': 'application/json' }, body: '{"address":"[email protected]","password":"fixture"}' };
+ assert.equal(keep('fetch_url', args), true);
+ assert.equal(keep('fetch_url', args, page, [
+ { kind: 'navigation', source: 'unknown', frameId: 23 },
+ { kind: 'dom', source: 'page', frameId: 0, target: 'span#counter' },
+ ]), true);
+ for (const event of [
+ { kind: 'input', source: 'user', frameId: 0 },
+ { kind: 'click', source: 'user', frameId: 23 },
+ { kind: 'navigation', source: 'unknown', frameId: 0 },
+ { kind: 'dom', source: 'page', frameId: 0, target: 'base' },
+ ]) assert.equal(keep('fetch_url', args, page, [event]), false, JSON.stringify(event));
+ for (const extra of [{ url: '/accounts' }, { url: 'https://user:[email protected]/accounts' },
+ { body: { address: 'not-a-serialized-payload' } }, { replayRequestId: 'old-request' },
+ { apiReplayRequestId: 'old-request' }]) assert.equal(keep('fetch_url', { ...args, ...extra }), false);
+ });
+
for (const [name, args] of [
['click_ax', { ref_id: 'ref_6' }],
['type_ax', { ref_id: 'ref_4', text: 'Updated caption' }],
diff --git a/test/page-feedback-progress.mjs b/test/page-feedback-progress.mjs
index c7a8c7005..5c6502a4f 100644
--- a/test/page-feedback-progress.mjs
+++ b/test/page-feedback-progress.mjs
@@ -187,6 +187,40 @@ for (const build of ['chrome', 'firefox']) {
});
for (const streaming of [false, true]) {
+ for (const allowed of [true, false]) {
+ test(`${build}: ${streaming ? 'stream' : 'chat'} API fallback survives continuous counters and still requires API consent (${allowed})`, async () => {
+ const tab = nextTab++, updates = [], sent = [];
+ const args = { url: 'https://api.mail.test/accounts', method: 'POST',
+ headers: { 'Content-Type': 'application/json' }, body: '{"address":"[email protected]","password":"fixture"}' };
+ let send, requests = 0, reads = 0;
+ const agent = makeAgent(Agent, async () => {
+ requests++;
+ assert.ok(requests <= 2, 'Unrelated counters cannot restart the API decision');
+ send({}); send({});
+ return requests === 1 ? call('create-mailbox', 'fetch_url', args)
+ : allowed ? done() : call('blocked-api', 'done', { summary: 'API consent is required.', outcome: 'partial' });
+ });
+ agent.isApiMutationsAllowed = () => allowed;
+ agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => {
+ send = bind(agent, tab); return { role: 'user', content };
+ };
+ agent.executeTool = async (_tab, name, actual) => {
+ if (name === 'get_accessibility_tree') {
+ reads++; send({});
+ return { success: true, pageContent: `heading "Inbox" [ref_inbox]\ntext "${reads} seconds ago"` };
+ }
+ if (name === 'done') return { success: true, done: true, summary: actual.summary, outcome: actual.outcome };
+ assert.equal(name, 'fetch_url');
+ assert.equal(allowed, true, 'The freshness exception must not authorize API mutations');
+ sent.push(structuredClone(actual));
+ return { success: true, status: 201, json: '{"id":"fixture-mailbox"}' };
+ };
+ await finish(agent, tab, streaming, updates, { detachedRequestId: 'api-fallback-' + tab, askStreamingEnabled: false });
+ assert.equal(requests, 2);
+ assert.deepEqual(sent, allowed ? [args] : []);
+ if (!allowed) assert.ok(updates.some(update => update.type === 'warning' && /API mutation blocked/.test(update.data.message)));
+ });
+ }
for (const condition of ['unsupported', 'targeted_recovery', 'target_drift', 'stale_document']) {
test(build + ': ' + (streaming ? 'stream' : 'chat') + ' ' + condition + ' action-context diagnostics preserve no-dispatch and fresh-decision fences', async () => {
const tab = nextTab++, updates = [], modelMessages = [], captures = [];
diff --git a/test/page-feedback-transport-races.mjs b/test/page-feedback-transport-races.mjs
index a9c337b15..c3f9bc6ed 100644
--- a/test/page-feedback-transport-races.mjs
+++ b/test/page-feedback-transport-races.mjs
@@ -15,6 +15,52 @@ for (const build of ['chrome', 'firefox']) {
};
const noDispatch = { success: false, pageFeedbackPending: true, noDispatch: true, dispatched: false };
+ test(`${build}: a concrete API request ignores replaced ad documents but keeps root, steering and replay fences`, () => {
+ const args = { url: 'https://api.mail.test/accounts', method: 'POST', body: '{"address":"[email protected]"}' };
+ const { agent, run } = makeAgent();
+ run.modelState.frames.set(7, { token: 'old-ad', id: 'old-ad' });
+ run.frames.set(7, { token: 'new-ad', id: 'new-ad' });
+ run.events.set('ad', { kind: 'navigation', source: 'unknown', frameId: 7 });
+ assert.equal(agent._pageFeedbackBlocksCall(tab, 'fetch_url', args), false);
+ for (const extra of [{ replayRequestId: 'captured-post' }, { apiReplayRequestId: 'captured-post' }])
+ assert.equal(agent._pageFeedbackBlocksCall(tab, 'fetch_url', { ...args, ...extra }), true);
+ agent._hasPendingSteering = () => true;
+ assert.equal(agent._pageFeedbackBlocksCall(tab, 'fetch_url', args), true);
+ agent._hasPendingSteering = () => false;
+ run.events.set('human', { kind: 'click', source: 'user', frameId: 7 });
+ assert.equal(agent._pageFeedbackBlocksCall(tab, 'fetch_url', args), true);
+ run.events.delete('human');
+ run.frames.set(0, { token: 'new-root', id: 'new-root' });
+ assert.equal(agent._pageFeedbackBlocksCall(tab, 'fetch_url', args), true);
+ });
+
+ test(`${build}: response refresh retains an API fallback across ad reloads and counter changes during the refresh`, async t => {
+ const { agent, run } = makeAgent();
+ const priorChrome = globalThis.chrome, priorBrowser = globalThis.browser;
+ const api = { tabs: { get: async () => ({ url: run.url }) }, webNavigation: { getAllFrames: async () => [
+ { frameId: 0, documentId: 'document-id' }, { frameId: 7, documentId: 'new-ad' },
+ ] } };
+ globalThis.chrome = api; globalThis.browser = api;
+ t.after(() => { globalThis.chrome = priorChrome; globalThis.browser = priorBrowser; });
+ run.documents = new Map([[0, { id: 'document-id' }]]);
+ run.modelState.documentId = 'document-id';
+ run.modelState.frames.set(7, { token: 'old-ad', id: 'old-ad' });
+ run.frames.set(7, { token: 'new-ad', id: 'new-ad' });
+ run.events.set('ad', { kind: 'navigation', source: 'unknown', frameId: 7 });
+ Object.assign(agent, { _checkAbort: () => false, _waitForPageFeedbackIdle: async () => {},
+ _runAbortSignal: () => null, _throwIfAborted: () => {}, _monitoringRetryCurrent: () => null,
+ _wrapUntrusted: (_name, value) => value, _persist: () => {}, _shouldAutoScreenshot: () => false,
+ executeTool: async () => {
+ run.events.set('counter', { kind: 'dom', source: 'page', frameId: 0, target: 'span#counter' });
+ return { success: true, pageContent: 'heading "Inbox"' };
+ } });
+ const request = { function: { name: 'fetch_url', arguments: JSON.stringify({
+ url: 'https://api.mail.test/accounts', method: 'POST', body: '{"address":"[email protected]"}',
+ }) } };
+ assert.equal(await agent._applyPendingPageFeedback(tab, [], () => {}, { workflow: true, responseToolCalls: [request] }), false);
+ assert.equal(run.passiveSupersessionStreak || 0, 0);
+ });
+
test(`${build}: preparation failures consume the retry budget before delayed DOM feedback arrives`, () => {
const { agent, run } = makeAgent();
for (let attempt = 0; attempt < 5; attempt++) {