Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Keep recoverable CAPTCHA application failures pending
Emre Sokullucf0fa84ea2bd
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex b1b662a30..aff48a169 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -10881,6 +10881,31 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn resultContent;}+ _captchaSolveGateAfterTool(tabId, toolName, toolResult) {+ const activeGate = this._captchaGateStates.get(tabId);+ if (!['solve_captcha', 'apply_captcha_solution'].includes(toolName) || !activeGate+ || !toolResult || typeof toolResult !== 'object' || toolResult.noDispatch === true) return null;+ // A failed binding/preflight has not consumed the paid answer. Keep its+ // gate pending so the model can inspect the page and correct the binding.+ const pending = (toolResult.success === true && (toolResult.injected === true || toolResult.applicationRequired === true))+ || (toolName === 'apply_captcha_solution' && toolResult.applicationRetryable === true);+ const publicGate = pending ? {+ ...withoutLegacyCaptchaVerificationState(activeGate.publicGate),+ status: 'verification_pending',+ solveAttempted: true,+ } : {+ ...activeGate.publicGate,+ status: 'manual_required',+ solveFailed: true,+ };+ this._captchaGateStates.set(tabId, {+ ...(pending ? withoutLegacyCaptchaVerificationState(activeGate) : activeGate),+ status: publicGate.status,+ publicGate,+ });+ return publicGate;+ }+_captchaGateBlockResult(tabId, toolName, toolArgs = {}) {const gate = this._captchaGateForTools(tabId, this._captchaGateStates.get(tabId));const gatedCompletion = toolName === 'done' || toolName === 'done_json';@@ -13109,34 +13134,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}}- let captchaSolveOutcome = null;- const activeCaptchaGate = this._captchaGateStates.get(tabId);- if (['solve_captcha', 'apply_captcha_solution'].includes(fnName) && activeCaptchaGate && toolResult && typeof toolResult === 'object' && toolResult.noDispatch !== true) {- if (toolResult.success === true && (toolResult.injected === true || toolResult.applicationRequired === true)) {- const verificationGate = {- ...withoutLegacyCaptchaVerificationState(activeCaptchaGate.publicGate),- status: 'verification_pending',- solveAttempted: true,- };- captchaSolveOutcome = verificationGate;- this._captchaGateStates.set(tabId, {- ...withoutLegacyCaptchaVerificationState(activeCaptchaGate),- status: 'verification_pending',- publicGate: verificationGate,- });- } else {- const manualGate = {- ...activeCaptchaGate.publicGate,- status: 'manual_required',- solveFailed: true,- };- captchaSolveOutcome = manualGate;- this._captchaGateStates.set(tabId, {- ...activeCaptchaGate,- status: 'manual_required',- publicGate: manualGate,- });- }+ const captchaSolveOutcome = this._captchaSolveGateAfterTool(tabId, fnName, toolResult);+ if (captchaSolveOutcome) {toolResult.captchaGate = captchaSolveOutcome;onUpdate('captcha_gate', captchaSolveOutcome);}@@ -35726,13 +35725,19 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaireturn getCaptchaCapabilities(getCaptchaProviders(stored), args || {});}if (name === 'apply_captcha_solution') {+ const record = this._nativeCaptchaSolutions?.get(tabId);+ const applicationRetryable = async () => {+ if (record?.solution === undefined || record.applied || Date.now() - record.createdAt > 180_000) return false;+ try { return (await chrome.tabs.get(tabId))?.url === record.pageUrl; } catch { return false; }+ };try {- const record = this._nativeCaptchaSolutions?.get(tabId);const result = await applyNativeCaptchaSolution(tabId, record, args, chrome);if (record && result.success === true) record.applicationSucceeded = true;- return { ...result, injected: result.success === true, dispatched: record?.applied === true };+ return { ...result, injected: result.success === true, dispatched: record?.applied === true,+ applicationRetryable: result.success !== true && await applicationRetryable() };} catch (error) {- return { success: false, error: error.message, dispatched: this._nativeCaptchaSolutions?.get(tabId)?.applied === true };+ return { success: false, error: error.message, dispatched: record?.applied === true,+ applicationRetryable: await applicationRetryable() };}}if (name === 'solve_captcha') {diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 823e9ed4c..23404d45a 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -9495,6 +9495,31 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn resultContent;}+ _captchaSolveGateAfterTool(tabId, toolName, toolResult) {+ const activeGate = this._captchaGateStates.get(tabId);+ if (!['solve_captcha', 'apply_captcha_solution'].includes(toolName) || !activeGate+ || !toolResult || typeof toolResult !== 'object' || toolResult.noDispatch === true) return null;+ // A failed binding/preflight has not consumed the paid answer. Keep its+ // gate pending so the model can inspect the page and correct the binding.+ const pending = (toolResult.success === true && (toolResult.injected === true || toolResult.applicationRequired === true))+ || (toolName === 'apply_captcha_solution' && toolResult.applicationRetryable === true);+ const publicGate = pending ? {+ ...withoutLegacyCaptchaVerificationState(activeGate.publicGate),+ status: 'verification_pending',+ solveAttempted: true,+ } : {+ ...activeGate.publicGate,+ status: 'manual_required',+ solveFailed: true,+ };+ this._captchaGateStates.set(tabId, {+ ...(pending ? withoutLegacyCaptchaVerificationState(activeGate) : activeGate),+ status: publicGate.status,+ publicGate,+ });+ return publicGate;+ }+_captchaGateBlockResult(tabId, toolName, toolArgs = {}) {const gate = this._captchaGateForTools(tabId, this._captchaGateStates.get(tabId));const gatedCompletion = toolName === 'done' || toolName === 'done_json';@@ -11626,34 +11651,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}}- let captchaSolveOutcome = null;- const activeCaptchaGate = this._captchaGateStates.get(tabId);- if (['solve_captcha', 'apply_captcha_solution'].includes(fnName) && activeCaptchaGate && toolResult && typeof toolResult === 'object' && toolResult.noDispatch !== true) {- if (toolResult.success === true && (toolResult.injected === true || toolResult.applicationRequired === true)) {- const verificationGate = {- ...withoutLegacyCaptchaVerificationState(activeCaptchaGate.publicGate),- status: 'verification_pending',- solveAttempted: true,- };- captchaSolveOutcome = verificationGate;- this._captchaGateStates.set(tabId, {- ...withoutLegacyCaptchaVerificationState(activeCaptchaGate),- status: 'verification_pending',- publicGate: verificationGate,- });- } else {- const manualGate = {- ...activeCaptchaGate.publicGate,- status: 'manual_required',- solveFailed: true,- };- captchaSolveOutcome = manualGate;- this._captchaGateStates.set(tabId, {- ...activeCaptchaGate,- status: 'manual_required',- publicGate: manualGate,- });- }+ const captchaSolveOutcome = this._captchaSolveGateAfterTool(tabId, fnName, toolResult);+ if (captchaSolveOutcome) {toolResult.captchaGate = captchaSolveOutcome;onUpdate('captcha_gate', captchaSolveOutcome);}@@ -32712,13 +32711,19 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn getCaptchaCapabilities(getCaptchaProviders(stored), args || {});}if (name === 'apply_captcha_solution') {+ const record = this._nativeCaptchaSolutions?.get(tabId);+ const applicationRetryable = async () => {+ if (record?.solution === undefined || record.applied || Date.now() - record.createdAt > 180_000) return false;+ try { return (await browser.tabs.get(tabId))?.url === record.pageUrl; } catch { return false; }+ };try {- const record = this._nativeCaptchaSolutions?.get(tabId);const result = await applyNativeCaptchaSolution(tabId, record, args, browser);if (record && result.success === true) record.applicationSucceeded = true;- return { ...result, injected: result.success === true, dispatched: record?.applied === true };+ return { ...result, injected: result.success === true, dispatched: record?.applied === true,+ applicationRetryable: result.success !== true && await applicationRetryable() };} catch (error) {- return { success: false, error: error.message, dispatched: this._nativeCaptchaSolutions?.get(tabId)?.applied === true };+ return { success: false, error: error.message, dispatched: record?.applied === true,+ applicationRetryable: await applicationRetryable() };}}if (name === 'solve_captcha') {diff --git a/test/run.js b/test/run.jsindex 653016ae9..fb252f1d8 100644--- a/test/run.js+++ b/test/run.js@@ -122013,10 +122013,21 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structureassert.equal(agent._captchaGateBlockResult(1, 'apply_captcha_solution', {}), null, build);assert.equal(agent._captchaGateBlockResult(1, 'solve_captcha', args)?.denied, true, build);assert.equal(agent._captchaGateBlockResult(1, 'click', {})?.denied, true, build);+ if (build === 'chrome') api.scripting.executeScript = async () => [{ frameId: 0, result: { success: false, error: 'Observed target is ambiguous.' } }];+ else api.tabs.executeScript = async () => [{ success: false, error: 'Observed target is ambiguous.' }];+ const preflightFailure = await agent._executeToolImpl(1, 'apply_captcha_solution', { frameId: 0, frameUrl: 'https://example.test/form', callback: { name: 'captcha.done', path: '' } });+ assert.equal(preflightFailure.applicationRetryable, true, build);+ assert.equal(agent._nativeCaptchaSolutions.get(1).applied, false, build);+ const pendingAfterPreflight = agent._captchaSolveGateAfterTool(1, 'apply_captcha_solution', preflightFailure);+ assert.equal(pendingAfterPreflight.status, 'verification_pending', build);+ assert.equal(pendingAfterPreflight.solveFailed, undefined, build);+ assert.equal(agent._captchaGateBlockResult(1, 'apply_captcha_solution', {})?.denied, undefined, build);+ assert.equal(calls.length, 1, build);if (build === 'chrome') api.scripting.executeScript = async () => [{ frameId: 0, result: { success: true } }];else api.tabs.executeScript = async () => [{ success: true }];const applied = await agent._executeToolImpl(1, 'apply_captcha_solution', { frameId: 0, frameUrl: 'https://example.test/form', callback: { name: 'captcha.done', path: '' } });assert.equal(applied.success, true, build);+ assert.equal(applied.applicationRetryable, false, build);assert.equal(agent._nativeCaptchaSolutions.get(1).applicationSucceeded, true, build);agent._detectChallengeDialogBeforeMutation = async () => ({ inspectionComplete: true, challenge: null });const cleared = await agent._observeCaptchaChallenge(1, 'get_accessibility_tree', { pageUrl: 'https://example.test/form', pageContent: 'heading "Verification complete"' });@@ -122040,6 +122051,10 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structureconst restored = await agent._executeToolImpl(1, 'solve_captcha', args);assert.equal(restored.dispatched, false, `${build}: a restored document cannot incur a second charge`);assert.equal(calls.length, 2, build);+ agent._nativeCaptchaSolutions.get(1).applied = true;+ const consumed = await agent._executeToolImpl(1, 'apply_captcha_solution', { frameId: 0, frameUrl: 'https://example.test/form' });+ assert.equal(consumed.applicationRetryable, false, build);+ assert.equal(agent._captchaSolveGateAfterTool(1, 'apply_captcha_solution', consumed).status, 'manual_required', build);} finally { globalThis.fetch = previousFetch; }});}