Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Keep recoverable CAPTCHA application failures pending

Emre Sokullucf0fa84ea2bd
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index b1b662a30..aff48a169 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -10881,6 +10881,31 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return resultContent;
}
+ _captchaSolveGateAfterTool(tabId, toolName, toolResult) {
+ const activeGate = this._captchaGateStates.get(tabId);
+ if (!['solve_captcha', 'apply_captcha_solution'].includes(toolName) || !activeGate
+ || !toolResult || typeof toolResult !== 'object' || toolResult.noDispatch === true) return null;
+ // A failed binding/preflight has not consumed the paid answer. Keep its
+ // gate pending so the model can inspect the page and correct the binding.
+ const pending = (toolResult.success === true && (toolResult.injected === true || toolResult.applicationRequired === true))
+ || (toolName === 'apply_captcha_solution' && toolResult.applicationRetryable === true);
+ const publicGate = pending ? {
+ ...withoutLegacyCaptchaVerificationState(activeGate.publicGate),
+ status: 'verification_pending',
+ solveAttempted: true,
+ } : {
+ ...activeGate.publicGate,
+ status: 'manual_required',
+ solveFailed: true,
+ };
+ this._captchaGateStates.set(tabId, {
+ ...(pending ? withoutLegacyCaptchaVerificationState(activeGate) : activeGate),
+ status: publicGate.status,
+ publicGate,
+ });
+ return publicGate;
+ }
+
_captchaGateBlockResult(tabId, toolName, toolArgs = {}) {
const gate = this._captchaGateForTools(tabId, this._captchaGateStates.get(tabId));
const gatedCompletion = toolName === 'done' || toolName === 'done_json';
@@ -13109,34 +13134,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
}
- let captchaSolveOutcome = null;
- const activeCaptchaGate = this._captchaGateStates.get(tabId);
- if (['solve_captcha', 'apply_captcha_solution'].includes(fnName) && activeCaptchaGate && toolResult && typeof toolResult === 'object' && toolResult.noDispatch !== true) {
- if (toolResult.success === true && (toolResult.injected === true || toolResult.applicationRequired === true)) {
- const verificationGate = {
- ...withoutLegacyCaptchaVerificationState(activeCaptchaGate.publicGate),
- status: 'verification_pending',
- solveAttempted: true,
- };
- captchaSolveOutcome = verificationGate;
- this._captchaGateStates.set(tabId, {
- ...withoutLegacyCaptchaVerificationState(activeCaptchaGate),
- status: 'verification_pending',
- publicGate: verificationGate,
- });
- } else {
- const manualGate = {
- ...activeCaptchaGate.publicGate,
- status: 'manual_required',
- solveFailed: true,
- };
- captchaSolveOutcome = manualGate;
- this._captchaGateStates.set(tabId, {
- ...activeCaptchaGate,
- status: 'manual_required',
- publicGate: manualGate,
- });
- }
+ const captchaSolveOutcome = this._captchaSolveGateAfterTool(tabId, fnName, toolResult);
+ if (captchaSolveOutcome) {
toolResult.captchaGate = captchaSolveOutcome;
onUpdate('captcha_gate', captchaSolveOutcome);
}
@@ -35726,13 +35725,19 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
return getCaptchaCapabilities(getCaptchaProviders(stored), args || {});
}
if (name === 'apply_captcha_solution') {
+ const record = this._nativeCaptchaSolutions?.get(tabId);
+ const applicationRetryable = async () => {
+ if (record?.solution === undefined || record.applied || Date.now() - record.createdAt > 180_000) return false;
+ try { return (await chrome.tabs.get(tabId))?.url === record.pageUrl; } catch { return false; }
+ };
try {
- const record = this._nativeCaptchaSolutions?.get(tabId);
const result = await applyNativeCaptchaSolution(tabId, record, args, chrome);
if (record && result.success === true) record.applicationSucceeded = true;
- return { ...result, injected: result.success === true, dispatched: record?.applied === true };
+ return { ...result, injected: result.success === true, dispatched: record?.applied === true,
+ applicationRetryable: result.success !== true && await applicationRetryable() };
} catch (error) {
- return { success: false, error: error.message, dispatched: this._nativeCaptchaSolutions?.get(tabId)?.applied === true };
+ return { success: false, error: error.message, dispatched: record?.applied === true,
+ applicationRetryable: await applicationRetryable() };
}
}
if (name === 'solve_captcha') {
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 823e9ed4c..23404d45a 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -9495,6 +9495,31 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return resultContent;
}
+ _captchaSolveGateAfterTool(tabId, toolName, toolResult) {
+ const activeGate = this._captchaGateStates.get(tabId);
+ if (!['solve_captcha', 'apply_captcha_solution'].includes(toolName) || !activeGate
+ || !toolResult || typeof toolResult !== 'object' || toolResult.noDispatch === true) return null;
+ // A failed binding/preflight has not consumed the paid answer. Keep its
+ // gate pending so the model can inspect the page and correct the binding.
+ const pending = (toolResult.success === true && (toolResult.injected === true || toolResult.applicationRequired === true))
+ || (toolName === 'apply_captcha_solution' && toolResult.applicationRetryable === true);
+ const publicGate = pending ? {
+ ...withoutLegacyCaptchaVerificationState(activeGate.publicGate),
+ status: 'verification_pending',
+ solveAttempted: true,
+ } : {
+ ...activeGate.publicGate,
+ status: 'manual_required',
+ solveFailed: true,
+ };
+ this._captchaGateStates.set(tabId, {
+ ...(pending ? withoutLegacyCaptchaVerificationState(activeGate) : activeGate),
+ status: publicGate.status,
+ publicGate,
+ });
+ return publicGate;
+ }
+
_captchaGateBlockResult(tabId, toolName, toolArgs = {}) {
const gate = this._captchaGateForTools(tabId, this._captchaGateStates.get(tabId));
const gatedCompletion = toolName === 'done' || toolName === 'done_json';
@@ -11626,34 +11651,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
}
- let captchaSolveOutcome = null;
- const activeCaptchaGate = this._captchaGateStates.get(tabId);
- if (['solve_captcha', 'apply_captcha_solution'].includes(fnName) && activeCaptchaGate && toolResult && typeof toolResult === 'object' && toolResult.noDispatch !== true) {
- if (toolResult.success === true && (toolResult.injected === true || toolResult.applicationRequired === true)) {
- const verificationGate = {
- ...withoutLegacyCaptchaVerificationState(activeCaptchaGate.publicGate),
- status: 'verification_pending',
- solveAttempted: true,
- };
- captchaSolveOutcome = verificationGate;
- this._captchaGateStates.set(tabId, {
- ...withoutLegacyCaptchaVerificationState(activeCaptchaGate),
- status: 'verification_pending',
- publicGate: verificationGate,
- });
- } else {
- const manualGate = {
- ...activeCaptchaGate.publicGate,
- status: 'manual_required',
- solveFailed: true,
- };
- captchaSolveOutcome = manualGate;
- this._captchaGateStates.set(tabId, {
- ...activeCaptchaGate,
- status: 'manual_required',
- publicGate: manualGate,
- });
- }
+ const captchaSolveOutcome = this._captchaSolveGateAfterTool(tabId, fnName, toolResult);
+ if (captchaSolveOutcome) {
toolResult.captchaGate = captchaSolveOutcome;
onUpdate('captcha_gate', captchaSolveOutcome);
}
@@ -32712,13 +32711,19 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return getCaptchaCapabilities(getCaptchaProviders(stored), args || {});
}
if (name === 'apply_captcha_solution') {
+ const record = this._nativeCaptchaSolutions?.get(tabId);
+ const applicationRetryable = async () => {
+ if (record?.solution === undefined || record.applied || Date.now() - record.createdAt > 180_000) return false;
+ try { return (await browser.tabs.get(tabId))?.url === record.pageUrl; } catch { return false; }
+ };
try {
- const record = this._nativeCaptchaSolutions?.get(tabId);
const result = await applyNativeCaptchaSolution(tabId, record, args, browser);
if (record && result.success === true) record.applicationSucceeded = true;
- return { ...result, injected: result.success === true, dispatched: record?.applied === true };
+ return { ...result, injected: result.success === true, dispatched: record?.applied === true,
+ applicationRetryable: result.success !== true && await applicationRetryable() };
} catch (error) {
- return { success: false, error: error.message, dispatched: this._nativeCaptchaSolutions?.get(tabId)?.applied === true };
+ return { success: false, error: error.message, dispatched: record?.applied === true,
+ applicationRetryable: await applicationRetryable() };
}
}
if (name === 'solve_captcha') {
diff --git a/test/run.js b/test/run.js
index 653016ae9..fb252f1d8 100644
--- a/test/run.js
+++ b/test/run.js
@@ -122013,10 +122013,21 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structure
assert.equal(agent._captchaGateBlockResult(1, 'apply_captcha_solution', {}), null, build);
assert.equal(agent._captchaGateBlockResult(1, 'solve_captcha', args)?.denied, true, build);
assert.equal(agent._captchaGateBlockResult(1, 'click', {})?.denied, true, build);
+ if (build === 'chrome') api.scripting.executeScript = async () => [{ frameId: 0, result: { success: false, error: 'Observed target is ambiguous.' } }];
+ else api.tabs.executeScript = async () => [{ success: false, error: 'Observed target is ambiguous.' }];
+ const preflightFailure = await agent._executeToolImpl(1, 'apply_captcha_solution', { frameId: 0, frameUrl: 'https://example.test/form', callback: { name: 'captcha.done', path: '' } });
+ assert.equal(preflightFailure.applicationRetryable, true, build);
+ assert.equal(agent._nativeCaptchaSolutions.get(1).applied, false, build);
+ const pendingAfterPreflight = agent._captchaSolveGateAfterTool(1, 'apply_captcha_solution', preflightFailure);
+ assert.equal(pendingAfterPreflight.status, 'verification_pending', build);
+ assert.equal(pendingAfterPreflight.solveFailed, undefined, build);
+ assert.equal(agent._captchaGateBlockResult(1, 'apply_captcha_solution', {})?.denied, undefined, build);
+ assert.equal(calls.length, 1, build);
if (build === 'chrome') api.scripting.executeScript = async () => [{ frameId: 0, result: { success: true } }];
else api.tabs.executeScript = async () => [{ success: true }];
const applied = await agent._executeToolImpl(1, 'apply_captcha_solution', { frameId: 0, frameUrl: 'https://example.test/form', callback: { name: 'captcha.done', path: '' } });
assert.equal(applied.success, true, build);
+ assert.equal(applied.applicationRetryable, false, build);
assert.equal(agent._nativeCaptchaSolutions.get(1).applicationSucceeded, true, build);
agent._detectChallengeDialogBeforeMutation = async () => ({ inspectionComplete: true, challenge: null });
const cleared = await agent._observeCaptchaChallenge(1, 'get_accessibility_tree', { pageUrl: 'https://example.test/form', pageContent: 'heading "Verification complete"' });
@@ -122040,6 +122051,10 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structure
const restored = await agent._executeToolImpl(1, 'solve_captcha', args);
assert.equal(restored.dispatched, false, `${build}: a restored document cannot incur a second charge`);
assert.equal(calls.length, 2, build);
+ agent._nativeCaptchaSolutions.get(1).applied = true;
+ const consumed = await agent._executeToolImpl(1, 'apply_captcha_solution', { frameId: 0, frameUrl: 'https://example.test/form' });
+ assert.equal(consumed.applicationRetryable, false, build);
+ assert.equal(agent._captchaSolveGateAfterTool(1, 'apply_captcha_solution', consumed).status, 'manual_required', build);
} finally { globalThis.fetch = previousFetch; }
});
}