Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Persist native CAPTCHA dispatch locks before provider calls

Emre Sokulludae5e9e89c57
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 542e6d6f3..1e08a83bb 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -10865,7 +10865,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
resultContent += '\n[TRUSTED CAPTCHA GATE: A supported verification challenge is active. Call solve_captcha once now. Do not dismiss or close the dialog, click Continue/Submit, or use another page-changing tool until solve_captcha returns.]';
onUpdate('warning', { message: 'Supported verification challenge detected; solve_captcha is required.' });
} else if (captchaGateDecision?.status === 'manual_required') {
- resultContent += captchaGateDecision.cloudflareManagedChallenge === true
+ resultContent += captchaGateDecision.nativeDispatchInterrupted === true
+ ? '\n[TRUSTED CAPTCHA GATE: A native solve was already dispatched for this document, but its answer is unavailable after restart. Do not send another paid request or submit the page; ask the user to complete the challenge manually.]'
+ : captchaGateDecision.cloudflareManagedChallenge === true
? '\n[TRUSTED CAPTCHA GATE: A response-backed full-page Cloudflare managed challenge is active. Inspect get_captcha_capabilities for a Cloudflare native method; use one solve_captcha providerTasks dispatch only with all required observed inputs, otherwise ask for manual completion. Do not submit; the network/navigation monitor will clear the gate when Cloudflare resumes the destination.]'
: captchaGateDecision.activeChallengeAfterSolve === true
? '\n[TRUSTED CAPTCHA GATE: The active challenge frame is visible again after the one automatic solve. The site may have rejected the token. Do not call solve_captcha again; stop automation and ask the user to complete the challenge manually.]'
@@ -16502,19 +16504,32 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
});
}
this._restorePersistedRichTextToolbarAudit(tabId, entry.richTextToolbarAudit);
+ const savedDispatch = entry.nativeCaptchaDispatch;
+ const rootDocument = Array.isArray(savedDispatch?.documents)
+ ? savedDispatch.documents.find(document => document?.frameId === 0) : null;
+ const nativeDispatchRestored = typeof savedDispatch?.pageUrl === 'string'
+ && savedDispatch.pageUrl && rootDocument?.url === savedDispatch.pageUrl
+ && Number.isFinite(rootDocument.timeOrigin)
+ && Array.isArray(savedDispatch.dispatchedTimeOrigins)
+ && savedDispatch.dispatchedTimeOrigins.includes(rootDocument.timeOrigin);
const savedAnswer = entry.nativeCaptchaAnswer;
const nativeAnswerRestored = savedAnswer && typeof savedAnswer.pageUrl === 'string'
&& Number.isFinite(savedAnswer.createdAt)
&& Date.now() >= savedAnswer.createdAt && Date.now() - savedAnswer.createdAt <= 180_000
&& savedAnswer.applied === false && savedAnswer.solution !== undefined
&& Array.isArray(savedAnswer.documents)
- && savedAnswer.documents.some(document => document?.frameId === 0);
- if (nativeAnswerRestored) {
+ && savedAnswer.documents.some(document => document?.frameId === 0)
+ && (!nativeDispatchRestored || (savedAnswer.pageUrl === savedDispatch.pageUrl
+ && savedAnswer.documents.some(document => document?.frameId === 0
+ && document.timeOrigin === rootDocument.timeOrigin)));
+ if (nativeDispatchRestored || nativeAnswerRestored) {
+ const record = nativeDispatchRestored ? savedDispatch : savedAnswer;
this._nativeCaptchaSolutions ||= new Map();
this._nativeCaptchaSolutions.set(tabId, {
- ...savedAnswer,
- dispatchedTimeOrigins: new Set(Array.isArray(savedAnswer.dispatchedTimeOrigins)
- ? savedAnswer.dispatchedTimeOrigins.filter(Number.isFinite) : []),
+ ...record,
+ ...(nativeAnswerRestored ? { solution: savedAnswer.solution, createdAt: savedAnswer.createdAt } : {}),
+ dispatchedTimeOrigins: new Set(Array.isArray(record.dispatchedTimeOrigins)
+ ? record.dispatchedTimeOrigins.filter(Number.isFinite) : []),
});
}
const captchaGateState = entry.captchaGateState;
@@ -16530,8 +16545,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
&& captchaGateState.cloudflareManagedChallenge !== true
&& captchaGateState.publicGate.cloudflareManagedChallenge !== true
) {
- if (!cloudflareSignal && (!captchaGateState.publicGate.nativeAnswerPending || nativeAnswerRestored)) {
- this._captchaGateStates.set(tabId, normalizeCaptchaGateState(captchaGateState));
+ if (!cloudflareSignal) {
+ const gate = normalizeCaptchaGateState(captchaGateState);
+ if (nativeDispatchRestored && !nativeAnswerRestored && !savedDispatch.applied
+ && ['solve_required', 'verification_pending'].includes(gate.status)) {
+ this._captchaGateStates.set(tabId, { ...gate, status: 'manual_required',
+ publicGate: { ...gate.publicGate, status: 'manual_required', solveAttempted: true,
+ solveFailed: true, nativeAnswerPending: false, nativeDispatchInterrupted: true } });
+ } else if (!gate.publicGate.nativeAnswerPending || nativeAnswerRestored || savedDispatch?.applied) {
+ this._captchaGateStates.set(tabId, gate);
+ }
}
}
}
@@ -16575,6 +16598,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
: { messages: [], compacted: false, bytes: 2 };
const captchaGateState = this._captchaGateStates.get(tabId) || null;
const nativeAnswer = this._nativeCaptchaSolutions?.get(tabId);
+ const nativeCaptchaDispatch = nativeAnswer?.pageUrl && Array.isArray(nativeAnswer.documents)
+ && nativeAnswer.dispatchedTimeOrigins instanceof Set
+ ? { pageUrl: nativeAnswer.pageUrl, createdAt: nativeAnswer.createdAt,
+ applied: nativeAnswer.applied === true, applicationSucceeded: nativeAnswer.applicationSucceeded === true,
+ documents: nativeAnswer.documents,
+ dispatchedTimeOrigins: [...nativeAnswer.dispatchedTimeOrigins] }
+ : null;
const savedNativeAnswer = nativeAnswer?.solution !== undefined && nativeAnswer.applied === false
&& Date.now() - nativeAnswer.createdAt <= 180_000
? { pageUrl: nativeAnswer.pageUrl, createdAt: nativeAnswer.createdAt, applied: false,
@@ -16604,6 +16634,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
? null
: captchaGateState,
nativeCaptchaAnswer,
+ nativeCaptchaDispatch,
};
}
@@ -35860,7 +35891,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
}
}
const previous = this._nativeCaptchaSolutions.get(tabId);
- const dispatchedTimeOrigins = previous?.dispatchedTimeOrigins || new Set(previous?.documents
+ const dispatchedTimeOrigins = new Set(previous?.dispatchedTimeOrigins || previous?.documents
?.filter(d => d.frameId === 0).map(d => d.timeOrigin) || []);
if (dispatchedTimeOrigins.has(rootDocument.timeOrigin)) {
return noDispatchFailure('A native solve was already dispatched for this page document. Use its result or ask for manual completion; do not spend again.');
@@ -35868,6 +35899,12 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
dispatchedTimeOrigins.add(rootDocument.timeOrigin);
const record = { pageUrl: websiteURL, createdAt: Date.now(), applied: false, documents, dispatchedTimeOrigins };
this._nativeCaptchaSolutions.set(tabId, record);
+ const lockPersistence = await this._persistNow(tabId);
+ if (!lockPersistence.ok) {
+ if (previous) this._nativeCaptchaSolutions.set(tabId, previous);
+ else this._nativeCaptchaSolutions.delete(tabId);
+ return noDispatchFailure('Could not save the native CAPTCHA dispatch lock. No provider was contacted; retry after session storage is available.');
+ }
dispatched = true;
const result = await solveNativeCaptchaTasks(prepared);
record.solution = result.solution;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 32fd0ecd1..c0d2e202d 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -5951,19 +5951,32 @@ export class Agent extends LoopDetector {
});
}
this._restorePersistedRichTextToolbarAudit(tabId, entry.richTextToolbarAudit);
+ const savedDispatch = entry.nativeCaptchaDispatch;
+ const rootDocument = Array.isArray(savedDispatch?.documents)
+ ? savedDispatch.documents.find(document => document?.frameId === 0) : null;
+ const nativeDispatchRestored = typeof savedDispatch?.pageUrl === 'string'
+ && savedDispatch.pageUrl && rootDocument?.url === savedDispatch.pageUrl
+ && Number.isFinite(rootDocument.timeOrigin)
+ && Array.isArray(savedDispatch.dispatchedTimeOrigins)
+ && savedDispatch.dispatchedTimeOrigins.includes(rootDocument.timeOrigin);
const savedAnswer = entry.nativeCaptchaAnswer;
const nativeAnswerRestored = savedAnswer && typeof savedAnswer.pageUrl === 'string'
&& Number.isFinite(savedAnswer.createdAt)
&& Date.now() >= savedAnswer.createdAt && Date.now() - savedAnswer.createdAt <= 180_000
&& savedAnswer.applied === false && savedAnswer.solution !== undefined
&& Array.isArray(savedAnswer.documents)
- && savedAnswer.documents.some(document => document?.frameId === 0);
- if (nativeAnswerRestored) {
+ && savedAnswer.documents.some(document => document?.frameId === 0)
+ && (!nativeDispatchRestored || (savedAnswer.pageUrl === savedDispatch.pageUrl
+ && savedAnswer.documents.some(document => document?.frameId === 0
+ && document.timeOrigin === rootDocument.timeOrigin)));
+ if (nativeDispatchRestored || nativeAnswerRestored) {
+ const record = nativeDispatchRestored ? savedDispatch : savedAnswer;
this._nativeCaptchaSolutions ||= new Map();
this._nativeCaptchaSolutions.set(tabId, {
- ...savedAnswer,
- dispatchedTimeOrigins: new Set(Array.isArray(savedAnswer.dispatchedTimeOrigins)
- ? savedAnswer.dispatchedTimeOrigins.filter(Number.isFinite) : []),
+ ...record,
+ ...(nativeAnswerRestored ? { solution: savedAnswer.solution, createdAt: savedAnswer.createdAt } : {}),
+ dispatchedTimeOrigins: new Set(Array.isArray(record.dispatchedTimeOrigins)
+ ? record.dispatchedTimeOrigins.filter(Number.isFinite) : []),
});
}
const captchaGateState = entry.captchaGateState;
@@ -5979,8 +5992,16 @@ export class Agent extends LoopDetector {
&& captchaGateState.cloudflareManagedChallenge !== true
&& captchaGateState.publicGate.cloudflareManagedChallenge !== true
) {
- if (!cloudflareSignal && (!captchaGateState.publicGate.nativeAnswerPending || nativeAnswerRestored)) {
- this._captchaGateStates.set(tabId, normalizeCaptchaGateState(captchaGateState));
+ if (!cloudflareSignal) {
+ const gate = normalizeCaptchaGateState(captchaGateState);
+ if (nativeDispatchRestored && !nativeAnswerRestored && !savedDispatch.applied
+ && ['solve_required', 'verification_pending'].includes(gate.status)) {
+ this._captchaGateStates.set(tabId, { ...gate, status: 'manual_required',
+ publicGate: { ...gate.publicGate, status: 'manual_required', solveAttempted: true,
+ solveFailed: true, nativeAnswerPending: false, nativeDispatchInterrupted: true } });
+ } else if (!gate.publicGate.nativeAnswerPending || nativeAnswerRestored || savedDispatch?.applied) {
+ this._captchaGateStates.set(tabId, gate);
+ }
}
}
}
@@ -6024,6 +6045,13 @@ export class Agent extends LoopDetector {
: { messages: [], compacted: false, bytes: 2 };
const captchaGateState = this._captchaGateStates.get(tabId) || null;
const nativeAnswer = this._nativeCaptchaSolutions?.get(tabId);
+ const nativeCaptchaDispatch = nativeAnswer?.pageUrl && Array.isArray(nativeAnswer.documents)
+ && nativeAnswer.dispatchedTimeOrigins instanceof Set
+ ? { pageUrl: nativeAnswer.pageUrl, createdAt: nativeAnswer.createdAt,
+ applied: nativeAnswer.applied === true, applicationSucceeded: nativeAnswer.applicationSucceeded === true,
+ documents: nativeAnswer.documents,
+ dispatchedTimeOrigins: [...nativeAnswer.dispatchedTimeOrigins] }
+ : null;
const savedNativeAnswer = nativeAnswer?.solution !== undefined && nativeAnswer.applied === false
&& Date.now() - nativeAnswer.createdAt <= 180_000
? { pageUrl: nativeAnswer.pageUrl, createdAt: nativeAnswer.createdAt, applied: false,
@@ -6053,6 +6081,7 @@ export class Agent extends LoopDetector {
? null
: captchaGateState,
nativeCaptchaAnswer,
+ nativeCaptchaDispatch,
};
}
@@ -9504,7 +9533,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
resultContent += '\n[TRUSTED CAPTCHA GATE: A supported verification challenge is active. Call solve_captcha once now. Do not dismiss or close the dialog, click Continue/Submit, or use another page-changing tool until solve_captcha returns.]';
onUpdate('warning', { message: 'Supported verification challenge detected; solve_captcha is required.' });
} else if (captchaGateDecision?.status === 'manual_required') {
- resultContent += captchaGateDecision.cloudflareManagedChallenge === true
+ resultContent += captchaGateDecision.nativeDispatchInterrupted === true
+ ? '\n[TRUSTED CAPTCHA GATE: A native solve was already dispatched for this document, but its answer is unavailable after restart. Do not send another paid request or submit the page; ask the user to complete the challenge manually.]'
+ : captchaGateDecision.cloudflareManagedChallenge === true
? '\n[TRUSTED CAPTCHA GATE: A response-backed full-page Cloudflare managed challenge is active. Inspect get_captcha_capabilities for a Cloudflare native method; use one solve_captcha providerTasks dispatch only with all required observed inputs, otherwise ask for manual completion. Do not submit; the network/navigation monitor will clear the gate when Cloudflare resumes the destination.]'
: captchaGateDecision.activeChallengeAfterSolve === true
? '\n[TRUSTED CAPTCHA GATE: The active challenge frame is visible again after the one automatic solve. The site may have rejected the token. Do not call solve_captcha again; stop automation and ask the user to complete the challenge manually.]'
@@ -32843,7 +32874,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
}
const previous = this._nativeCaptchaSolutions.get(tabId);
- const dispatchedTimeOrigins = previous?.dispatchedTimeOrigins || new Set(previous?.documents
+ const dispatchedTimeOrigins = new Set(previous?.dispatchedTimeOrigins || previous?.documents
?.filter(d => d.frameId === 0).map(d => d.timeOrigin) || []);
if (dispatchedTimeOrigins.has(rootDocument.timeOrigin)) {
return noDispatchFailure('A native solve was already dispatched for this page document. Use its result or ask for manual completion; do not spend again.');
@@ -32851,6 +32882,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
dispatchedTimeOrigins.add(rootDocument.timeOrigin);
const record = { pageUrl: websiteURL, createdAt: Date.now(), applied: false, documents, dispatchedTimeOrigins };
this._nativeCaptchaSolutions.set(tabId, record);
+ const lockPersistence = await this._persistNow(tabId);
+ if (!lockPersistence.ok) {
+ if (previous) this._nativeCaptchaSolutions.set(tabId, previous);
+ else this._nativeCaptchaSolutions.delete(tabId);
+ return noDispatchFailure('Could not save the native CAPTCHA dispatch lock. No provider was contacted; retry after session storage is available.');
+ }
dispatched = true;
const result = await solveNativeCaptchaTasks(prepared);
record.solution = result.solution;
diff --git a/test/captcha-runtime-policy.mjs b/test/captcha-runtime-policy.mjs
index c9280dcdb..eaba5c0ba 100644
--- a/test/captcha-runtime-policy.mjs
+++ b/test/captcha-runtime-policy.mjs
@@ -189,6 +189,7 @@ for (const build of ['chrome', 'firefox']) {
const entry = structuredClone(agent._conversationStorageEntry(1));
const savedAnswer = entry.nativeCaptchaAnswer;
assert.deepEqual(entry.nativeCaptchaAnswer.solution, { token: 'paid-answer' });
+ assert.deepEqual(entry.nativeCaptchaDispatch.dispatchedTimeOrigins, [1000]);
agent._nativeCaptchaSolutions.get(1).applied = true;
agent._captchaSolveGateAfterTool(1, 'apply_captcha_solution', { success: true, injected: true });
assert.equal(agent._conversationStorageEntry(1).nativeCaptchaAnswer, null);
@@ -208,11 +209,20 @@ for (const build of ['chrome', 'firefox']) {
const missing = agentFor('act', 'full');
delete entry.nativeCaptchaAnswer;
await missing._hydrate(1);
- assert.equal(missing._captchaGateStates.has(1), false);
+ assert.equal(missing._captchaGateStates.get(1).status, 'manual_required');
+ assert.equal(missing._nativeCaptchaSolutions.get(1).dispatchedTimeOrigins.has(1000), true);
+ assert.equal(missing._captchaGateBlockResult(1, 'solve_captcha', {}).denied, true);
+ assert.equal(missing._captchaGateBlockResult(1, 'done', { outcome: 'partial' }), null);
const expired = agentFor('act', 'full');
entry.nativeCaptchaAnswer = { ...savedAnswer, createdAt: Date.now() - 180_001 };
await expired._hydrate(1);
- assert.equal(expired._captchaGateStates.has(1), false);
+ assert.equal(expired._captchaGateStates.get(1).status, 'manual_required');
+ const interrupted = agentFor('act', 'full');
+ delete entry.nativeCaptchaAnswer;
+ entry.captchaGateState = { ...entry.captchaGateState, status: 'solve_required',
+ publicGate: { status: 'solve_required' } };
+ await interrupted._hydrate(1);
+ assert.equal(interrupted._captchaGateStates.get(1).publicGate.nativeDispatchInterrupted, true);
} finally {
if (previous === undefined) delete globalThis[apiName]; else globalThis[apiName] = previous;
}
diff --git a/test/run.js b/test/run.js
index afb9d22ca..c17b7512e 100644
--- a/test/run.js
+++ b/test/run.js
@@ -122013,7 +122013,13 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structure
for (const [build, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) {
await withCaptchaFakePage(build, [], async () => {
const api = build === 'chrome' ? globalThis.chrome : globalThis.browser;
- api.storage = { local: { get: async () => ({ twoCaptchaEnabled: true, twoCaptchaApiKey: 'a'.repeat(32) }) } };
+ const persisted = [];
+ let storageUnavailable = false;
+ api.storage = { local: { get: async () => ({ twoCaptchaEnabled: true, twoCaptchaApiKey: 'a'.repeat(32) }) },
+ session: { set: async value => {
+ if (storageUnavailable) throw new Error('Session storage unavailable');
+ persisted.push(structuredClone(Object.values(value)[0]));
+ } } };
api.tabs = { ...api.tabs, get: async () => ({ url: 'https://example.test/form' }) };
api.webNavigation = { getAllFrames: async () => [{ frameId: 0, url: 'https://example.test/form' }] };
if (build === 'chrome') api.scripting = { executeScript: async () => [{ frameId: 0, result: { url: 'https://example.test/form', timeOrigin: 1000 } }] };
@@ -122021,11 +122027,14 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structure
const previousFetch = globalThis.fetch;
const calls = [];
globalThis.fetch = async (url, options) => {
+ assert.ok(persisted.at(-1)?.nativeCaptchaDispatch?.dispatchedTimeOrigins.length,
+ `${build}: provider was contacted before its dispatch lock was saved`);
calls.push({ url, body: JSON.parse(options.body) });
return Response.json({ taskId: 42, status: 'ready', solution: { lot_number: 'lot', pass_token: 'pass', captcha_output: 'out' } });
};
try {
const agent = new AgentClass({});
+ agent.conversations.set(1, [{ role: 'system', content: 'test' }]);
const catalog = await agent._executeToolImpl(1, 'get_captcha_capabilities', { family: 'geetest' });
assert.ok(catalog.methods.some(method => method.method === 'GeeTestTaskProxyless'), build);
const args = { inject: false, providerTasks: [{ provider: '2captcha', method: 'GeeTestTaskProxyless', parameters: { websiteURL: 'https://example.test/form', version: 4, initParameters: { captcha_id: 'observed-id' } } }] };
@@ -122039,6 +122048,11 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structure
const unobserved = await agent._executeToolImpl(1, 'solve_captcha', wrongPath);
assert.equal(unobserved.dispatched, false, build); assert.match(unobserved.error, /observed frames/);
assert.equal(calls.length, 0, build);
+ storageUnavailable = true;
+ const unpersisted = await agent._executeToolImpl(1, 'solve_captcha', args);
+ assert.equal(unpersisted.noDispatch, true, `${build}: a paid task started without a persisted lock`);
+ assert.equal(calls.length, 0, build);
+ storageUnavailable = false;
const result = await agent._executeToolImpl(1, 'solve_captcha', args);
assert.equal(result.success, true, `${build}: ${result.error}`);
assert.equal(result.applicationRequired, true, build); assert.equal(result.solution.lot_number, 'lot', build);