Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Persist native CAPTCHA dispatch locks before provider calls
Emre Sokulludae5e9e89c57
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 542e6d6f3..1e08a83bb 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -10865,7 +10865,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dresultContent += '\n[TRUSTED CAPTCHA GATE: A supported verification challenge is active. Call solve_captcha once now. Do not dismiss or close the dialog, click Continue/Submit, or use another page-changing tool until solve_captcha returns.]';onUpdate('warning', { message: 'Supported verification challenge detected; solve_captcha is required.' });} else if (captchaGateDecision?.status === 'manual_required') {- resultContent += captchaGateDecision.cloudflareManagedChallenge === true+ resultContent += captchaGateDecision.nativeDispatchInterrupted === true+ ? '\n[TRUSTED CAPTCHA GATE: A native solve was already dispatched for this document, but its answer is unavailable after restart. Do not send another paid request or submit the page; ask the user to complete the challenge manually.]'+ : captchaGateDecision.cloudflareManagedChallenge === true? '\n[TRUSTED CAPTCHA GATE: A response-backed full-page Cloudflare managed challenge is active. Inspect get_captcha_capabilities for a Cloudflare native method; use one solve_captcha providerTasks dispatch only with all required observed inputs, otherwise ask for manual completion. Do not submit; the network/navigation monitor will clear the gate when Cloudflare resumes the destination.]': captchaGateDecision.activeChallengeAfterSolve === true? '\n[TRUSTED CAPTCHA GATE: The active challenge frame is visible again after the one automatic solve. The site may have rejected the token. Do not call solve_captcha again; stop automation and ask the user to complete the challenge manually.]'@@ -16502,19 +16504,32 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d});}this._restorePersistedRichTextToolbarAudit(tabId, entry.richTextToolbarAudit);+ const savedDispatch = entry.nativeCaptchaDispatch;+ const rootDocument = Array.isArray(savedDispatch?.documents)+ ? savedDispatch.documents.find(document => document?.frameId === 0) : null;+ const nativeDispatchRestored = typeof savedDispatch?.pageUrl === 'string'+ && savedDispatch.pageUrl && rootDocument?.url === savedDispatch.pageUrl+ && Number.isFinite(rootDocument.timeOrigin)+ && Array.isArray(savedDispatch.dispatchedTimeOrigins)+ && savedDispatch.dispatchedTimeOrigins.includes(rootDocument.timeOrigin);const savedAnswer = entry.nativeCaptchaAnswer;const nativeAnswerRestored = savedAnswer && typeof savedAnswer.pageUrl === 'string'&& Number.isFinite(savedAnswer.createdAt)&& Date.now() >= savedAnswer.createdAt && Date.now() - savedAnswer.createdAt <= 180_000&& savedAnswer.applied === false && savedAnswer.solution !== undefined&& Array.isArray(savedAnswer.documents)- && savedAnswer.documents.some(document => document?.frameId === 0);- if (nativeAnswerRestored) {+ && savedAnswer.documents.some(document => document?.frameId === 0)+ && (!nativeDispatchRestored || (savedAnswer.pageUrl === savedDispatch.pageUrl+ && savedAnswer.documents.some(document => document?.frameId === 0+ && document.timeOrigin === rootDocument.timeOrigin)));+ if (nativeDispatchRestored || nativeAnswerRestored) {+ const record = nativeDispatchRestored ? savedDispatch : savedAnswer;this._nativeCaptchaSolutions ||= new Map();this._nativeCaptchaSolutions.set(tabId, {- ...savedAnswer,- dispatchedTimeOrigins: new Set(Array.isArray(savedAnswer.dispatchedTimeOrigins)- ? savedAnswer.dispatchedTimeOrigins.filter(Number.isFinite) : []),+ ...record,+ ...(nativeAnswerRestored ? { solution: savedAnswer.solution, createdAt: savedAnswer.createdAt } : {}),+ dispatchedTimeOrigins: new Set(Array.isArray(record.dispatchedTimeOrigins)+ ? record.dispatchedTimeOrigins.filter(Number.isFinite) : []),});}const captchaGateState = entry.captchaGateState;@@ -16530,8 +16545,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d&& captchaGateState.cloudflareManagedChallenge !== true&& captchaGateState.publicGate.cloudflareManagedChallenge !== true) {- if (!cloudflareSignal && (!captchaGateState.publicGate.nativeAnswerPending || nativeAnswerRestored)) {- this._captchaGateStates.set(tabId, normalizeCaptchaGateState(captchaGateState));+ if (!cloudflareSignal) {+ const gate = normalizeCaptchaGateState(captchaGateState);+ if (nativeDispatchRestored && !nativeAnswerRestored && !savedDispatch.applied+ && ['solve_required', 'verification_pending'].includes(gate.status)) {+ this._captchaGateStates.set(tabId, { ...gate, status: 'manual_required',+ publicGate: { ...gate.publicGate, status: 'manual_required', solveAttempted: true,+ solveFailed: true, nativeAnswerPending: false, nativeDispatchInterrupted: true } });+ } else if (!gate.publicGate.nativeAnswerPending || nativeAnswerRestored || savedDispatch?.applied) {+ this._captchaGateStates.set(tabId, gate);+ }}}}@@ -16575,6 +16598,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d: { messages: [], compacted: false, bytes: 2 };const captchaGateState = this._captchaGateStates.get(tabId) || null;const nativeAnswer = this._nativeCaptchaSolutions?.get(tabId);+ const nativeCaptchaDispatch = nativeAnswer?.pageUrl && Array.isArray(nativeAnswer.documents)+ && nativeAnswer.dispatchedTimeOrigins instanceof Set+ ? { pageUrl: nativeAnswer.pageUrl, createdAt: nativeAnswer.createdAt,+ applied: nativeAnswer.applied === true, applicationSucceeded: nativeAnswer.applicationSucceeded === true,+ documents: nativeAnswer.documents,+ dispatchedTimeOrigins: [...nativeAnswer.dispatchedTimeOrigins] }+ : null;const savedNativeAnswer = nativeAnswer?.solution !== undefined && nativeAnswer.applied === false&& Date.now() - nativeAnswer.createdAt <= 180_000? { pageUrl: nativeAnswer.pageUrl, createdAt: nativeAnswer.createdAt, applied: false,@@ -16604,6 +16634,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d? null: captchaGateState,nativeCaptchaAnswer,+ nativeCaptchaDispatch,};}@@ -35860,7 +35891,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai}}const previous = this._nativeCaptchaSolutions.get(tabId);- const dispatchedTimeOrigins = previous?.dispatchedTimeOrigins || new Set(previous?.documents+ const dispatchedTimeOrigins = new Set(previous?.dispatchedTimeOrigins || previous?.documents?.filter(d => d.frameId === 0).map(d => d.timeOrigin) || []);if (dispatchedTimeOrigins.has(rootDocument.timeOrigin)) {return noDispatchFailure('A native solve was already dispatched for this page document. Use its result or ask for manual completion; do not spend again.');@@ -35868,6 +35899,12 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaidispatchedTimeOrigins.add(rootDocument.timeOrigin);const record = { pageUrl: websiteURL, createdAt: Date.now(), applied: false, documents, dispatchedTimeOrigins };this._nativeCaptchaSolutions.set(tabId, record);+ const lockPersistence = await this._persistNow(tabId);+ if (!lockPersistence.ok) {+ if (previous) this._nativeCaptchaSolutions.set(tabId, previous);+ else this._nativeCaptchaSolutions.delete(tabId);+ return noDispatchFailure('Could not save the native CAPTCHA dispatch lock. No provider was contacted; retry after session storage is available.');+ }dispatched = true;const result = await solveNativeCaptchaTasks(prepared);record.solution = result.solution;diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 32fd0ecd1..c0d2e202d 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -5951,19 +5951,32 @@ export class Agent extends LoopDetector {});}this._restorePersistedRichTextToolbarAudit(tabId, entry.richTextToolbarAudit);+ const savedDispatch = entry.nativeCaptchaDispatch;+ const rootDocument = Array.isArray(savedDispatch?.documents)+ ? savedDispatch.documents.find(document => document?.frameId === 0) : null;+ const nativeDispatchRestored = typeof savedDispatch?.pageUrl === 'string'+ && savedDispatch.pageUrl && rootDocument?.url === savedDispatch.pageUrl+ && Number.isFinite(rootDocument.timeOrigin)+ && Array.isArray(savedDispatch.dispatchedTimeOrigins)+ && savedDispatch.dispatchedTimeOrigins.includes(rootDocument.timeOrigin);const savedAnswer = entry.nativeCaptchaAnswer;const nativeAnswerRestored = savedAnswer && typeof savedAnswer.pageUrl === 'string'&& Number.isFinite(savedAnswer.createdAt)&& Date.now() >= savedAnswer.createdAt && Date.now() - savedAnswer.createdAt <= 180_000&& savedAnswer.applied === false && savedAnswer.solution !== undefined&& Array.isArray(savedAnswer.documents)- && savedAnswer.documents.some(document => document?.frameId === 0);- if (nativeAnswerRestored) {+ && savedAnswer.documents.some(document => document?.frameId === 0)+ && (!nativeDispatchRestored || (savedAnswer.pageUrl === savedDispatch.pageUrl+ && savedAnswer.documents.some(document => document?.frameId === 0+ && document.timeOrigin === rootDocument.timeOrigin)));+ if (nativeDispatchRestored || nativeAnswerRestored) {+ const record = nativeDispatchRestored ? savedDispatch : savedAnswer;this._nativeCaptchaSolutions ||= new Map();this._nativeCaptchaSolutions.set(tabId, {- ...savedAnswer,- dispatchedTimeOrigins: new Set(Array.isArray(savedAnswer.dispatchedTimeOrigins)- ? savedAnswer.dispatchedTimeOrigins.filter(Number.isFinite) : []),+ ...record,+ ...(nativeAnswerRestored ? { solution: savedAnswer.solution, createdAt: savedAnswer.createdAt } : {}),+ dispatchedTimeOrigins: new Set(Array.isArray(record.dispatchedTimeOrigins)+ ? record.dispatchedTimeOrigins.filter(Number.isFinite) : []),});}const captchaGateState = entry.captchaGateState;@@ -5979,8 +5992,16 @@ export class Agent extends LoopDetector {&& captchaGateState.cloudflareManagedChallenge !== true&& captchaGateState.publicGate.cloudflareManagedChallenge !== true) {- if (!cloudflareSignal && (!captchaGateState.publicGate.nativeAnswerPending || nativeAnswerRestored)) {- this._captchaGateStates.set(tabId, normalizeCaptchaGateState(captchaGateState));+ if (!cloudflareSignal) {+ const gate = normalizeCaptchaGateState(captchaGateState);+ if (nativeDispatchRestored && !nativeAnswerRestored && !savedDispatch.applied+ && ['solve_required', 'verification_pending'].includes(gate.status)) {+ this._captchaGateStates.set(tabId, { ...gate, status: 'manual_required',+ publicGate: { ...gate.publicGate, status: 'manual_required', solveAttempted: true,+ solveFailed: true, nativeAnswerPending: false, nativeDispatchInterrupted: true } });+ } else if (!gate.publicGate.nativeAnswerPending || nativeAnswerRestored || savedDispatch?.applied) {+ this._captchaGateStates.set(tabId, gate);+ }}}}@@ -6024,6 +6045,13 @@ export class Agent extends LoopDetector {: { messages: [], compacted: false, bytes: 2 };const captchaGateState = this._captchaGateStates.get(tabId) || null;const nativeAnswer = this._nativeCaptchaSolutions?.get(tabId);+ const nativeCaptchaDispatch = nativeAnswer?.pageUrl && Array.isArray(nativeAnswer.documents)+ && nativeAnswer.dispatchedTimeOrigins instanceof Set+ ? { pageUrl: nativeAnswer.pageUrl, createdAt: nativeAnswer.createdAt,+ applied: nativeAnswer.applied === true, applicationSucceeded: nativeAnswer.applicationSucceeded === true,+ documents: nativeAnswer.documents,+ dispatchedTimeOrigins: [...nativeAnswer.dispatchedTimeOrigins] }+ : null;const savedNativeAnswer = nativeAnswer?.solution !== undefined && nativeAnswer.applied === false&& Date.now() - nativeAnswer.createdAt <= 180_000? { pageUrl: nativeAnswer.pageUrl, createdAt: nativeAnswer.createdAt, applied: false,@@ -6053,6 +6081,7 @@ export class Agent extends LoopDetector {? null: captchaGateState,nativeCaptchaAnswer,+ nativeCaptchaDispatch,};}@@ -9504,7 +9533,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dresultContent += '\n[TRUSTED CAPTCHA GATE: A supported verification challenge is active. Call solve_captcha once now. Do not dismiss or close the dialog, click Continue/Submit, or use another page-changing tool until solve_captcha returns.]';onUpdate('warning', { message: 'Supported verification challenge detected; solve_captcha is required.' });} else if (captchaGateDecision?.status === 'manual_required') {- resultContent += captchaGateDecision.cloudflareManagedChallenge === true+ resultContent += captchaGateDecision.nativeDispatchInterrupted === true+ ? '\n[TRUSTED CAPTCHA GATE: A native solve was already dispatched for this document, but its answer is unavailable after restart. Do not send another paid request or submit the page; ask the user to complete the challenge manually.]'+ : captchaGateDecision.cloudflareManagedChallenge === true? '\n[TRUSTED CAPTCHA GATE: A response-backed full-page Cloudflare managed challenge is active. Inspect get_captcha_capabilities for a Cloudflare native method; use one solve_captcha providerTasks dispatch only with all required observed inputs, otherwise ask for manual completion. Do not submit; the network/navigation monitor will clear the gate when Cloudflare resumes the destination.]': captchaGateDecision.activeChallengeAfterSolve === true? '\n[TRUSTED CAPTCHA GATE: The active challenge frame is visible again after the one automatic solve. The site may have rejected the token. Do not call solve_captcha again; stop automation and ask the user to complete the challenge manually.]'@@ -32843,7 +32874,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}}const previous = this._nativeCaptchaSolutions.get(tabId);- const dispatchedTimeOrigins = previous?.dispatchedTimeOrigins || new Set(previous?.documents+ const dispatchedTimeOrigins = new Set(previous?.dispatchedTimeOrigins || previous?.documents?.filter(d => d.frameId === 0).map(d => d.timeOrigin) || []);if (dispatchedTimeOrigins.has(rootDocument.timeOrigin)) {return noDispatchFailure('A native solve was already dispatched for this page document. Use its result or ask for manual completion; do not spend again.');@@ -32851,6 +32882,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout ddispatchedTimeOrigins.add(rootDocument.timeOrigin);const record = { pageUrl: websiteURL, createdAt: Date.now(), applied: false, documents, dispatchedTimeOrigins };this._nativeCaptchaSolutions.set(tabId, record);+ const lockPersistence = await this._persistNow(tabId);+ if (!lockPersistence.ok) {+ if (previous) this._nativeCaptchaSolutions.set(tabId, previous);+ else this._nativeCaptchaSolutions.delete(tabId);+ return noDispatchFailure('Could not save the native CAPTCHA dispatch lock. No provider was contacted; retry after session storage is available.');+ }dispatched = true;const result = await solveNativeCaptchaTasks(prepared);record.solution = result.solution;diff --git a/test/captcha-runtime-policy.mjs b/test/captcha-runtime-policy.mjsindex c9280dcdb..eaba5c0ba 100644--- a/test/captcha-runtime-policy.mjs+++ b/test/captcha-runtime-policy.mjs@@ -189,6 +189,7 @@ for (const build of ['chrome', 'firefox']) {const entry = structuredClone(agent._conversationStorageEntry(1));const savedAnswer = entry.nativeCaptchaAnswer;assert.deepEqual(entry.nativeCaptchaAnswer.solution, { token: 'paid-answer' });+ assert.deepEqual(entry.nativeCaptchaDispatch.dispatchedTimeOrigins, [1000]);agent._nativeCaptchaSolutions.get(1).applied = true;agent._captchaSolveGateAfterTool(1, 'apply_captcha_solution', { success: true, injected: true });assert.equal(agent._conversationStorageEntry(1).nativeCaptchaAnswer, null);@@ -208,11 +209,20 @@ for (const build of ['chrome', 'firefox']) {const missing = agentFor('act', 'full');delete entry.nativeCaptchaAnswer;await missing._hydrate(1);- assert.equal(missing._captchaGateStates.has(1), false);+ assert.equal(missing._captchaGateStates.get(1).status, 'manual_required');+ assert.equal(missing._nativeCaptchaSolutions.get(1).dispatchedTimeOrigins.has(1000), true);+ assert.equal(missing._captchaGateBlockResult(1, 'solve_captcha', {}).denied, true);+ assert.equal(missing._captchaGateBlockResult(1, 'done', { outcome: 'partial' }), null);const expired = agentFor('act', 'full');entry.nativeCaptchaAnswer = { ...savedAnswer, createdAt: Date.now() - 180_001 };await expired._hydrate(1);- assert.equal(expired._captchaGateStates.has(1), false);+ assert.equal(expired._captchaGateStates.get(1).status, 'manual_required');+ const interrupted = agentFor('act', 'full');+ delete entry.nativeCaptchaAnswer;+ entry.captchaGateState = { ...entry.captchaGateState, status: 'solve_required',+ publicGate: { status: 'solve_required' } };+ await interrupted._hydrate(1);+ assert.equal(interrupted._captchaGateStates.get(1).publicGate.nativeDispatchInterrupted, true);} finally {if (previous === undefined) delete globalThis[apiName]; else globalThis[apiName] = previous;}diff --git a/test/run.js b/test/run.jsindex afb9d22ca..c17b7512e 100644--- a/test/run.js+++ b/test/run.js@@ -122013,7 +122013,13 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structurefor (const [build, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) {await withCaptchaFakePage(build, [], async () => {const api = build === 'chrome' ? globalThis.chrome : globalThis.browser;- api.storage = { local: { get: async () => ({ twoCaptchaEnabled: true, twoCaptchaApiKey: 'a'.repeat(32) }) } };+ const persisted = [];+ let storageUnavailable = false;+ api.storage = { local: { get: async () => ({ twoCaptchaEnabled: true, twoCaptchaApiKey: 'a'.repeat(32) }) },+ session: { set: async value => {+ if (storageUnavailable) throw new Error('Session storage unavailable');+ persisted.push(structuredClone(Object.values(value)[0]));+ } } };api.tabs = { ...api.tabs, get: async () => ({ url: 'https://example.test/form' }) };api.webNavigation = { getAllFrames: async () => [{ frameId: 0, url: 'https://example.test/form' }] };if (build === 'chrome') api.scripting = { executeScript: async () => [{ frameId: 0, result: { url: 'https://example.test/form', timeOrigin: 1000 } }] };@@ -122021,11 +122027,14 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structureconst previousFetch = globalThis.fetch;const calls = [];globalThis.fetch = async (url, options) => {+ assert.ok(persisted.at(-1)?.nativeCaptchaDispatch?.dispatchedTimeOrigins.length,+ `${build}: provider was contacted before its dispatch lock was saved`);calls.push({ url, body: JSON.parse(options.body) });return Response.json({ taskId: 42, status: 'ready', solution: { lot_number: 'lot', pass_token: 'pass', captcha_output: 'out' } });};try {const agent = new AgentClass({});+ agent.conversations.set(1, [{ role: 'system', content: 'test' }]);const catalog = await agent._executeToolImpl(1, 'get_captcha_capabilities', { family: 'geetest' });assert.ok(catalog.methods.some(method => method.method === 'GeeTestTaskProxyless'), build);const args = { inject: false, providerTasks: [{ provider: '2captcha', method: 'GeeTestTaskProxyless', parameters: { websiteURL: 'https://example.test/form', version: 4, initParameters: { captcha_id: 'observed-id' } } }] };@@ -122039,6 +122048,11 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structureconst unobserved = await agent._executeToolImpl(1, 'solve_captcha', wrongPath);assert.equal(unobserved.dispatched, false, build); assert.match(unobserved.error, /observed frames/);assert.equal(calls.length, 0, build);+ storageUnavailable = true;+ const unpersisted = await agent._executeToolImpl(1, 'solve_captcha', args);+ assert.equal(unpersisted.noDispatch, true, `${build}: a paid task started without a persisted lock`);+ assert.equal(calls.length, 0, build);+ storageUnavailable = false;const result = await agent._executeToolImpl(1, 'solve_captcha', args);assert.equal(result.success, true, `${build}: ${result.error}`);assert.equal(result.applicationRequired, true, build); assert.equal(result.solution.lot_number, 'lot', build);