Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Gate CAPTCHA application and match fallback proxy identity

Emre Sokulluaf4e716aa35a
diff --git a/docs/captcha-provider-coverage.md b/docs/captcha-provider-coverage.md
index a402201d8..58d03b134 100644
--- a/docs/captcha-provider-coverage.md
+++ b/docs/captcha-provider-coverage.md
@@ -148,7 +148,7 @@ Provider-specific identifiers (for example `appId`/`app_id` and `miseryKey`/`mis
Supported bindings:
- `fields`: bind a solution path to one observed input/textarea selector; structured values require `encoding: "json"`.
-- `callback`: pass a solution value or object to an observed named page callback. Native/global code evaluators and unsafe property paths are rejected. Provider-returned code is never evaluated.
+- `callback`: pass a solution value or object to an observed named page callback. Application uses the frame host's normal permissions: Type for response fields, Click for image/grid clicks, and JavaScript for callbacks or cookies. Native/global code evaluators and unsafe property paths are rejected. Provider-returned code is never evaluated.
- `cookies`: bind explicit cookie names to returned values. Complete Set-Cookie strings are reduced to the explicitly named cookie value; returned attributes are ignored. Cookies are host-only, scoped to the active top-level page and its browser cookie store. Provider-supplied domains/URLs are not adopted. Both extension manifests declare the `cookies` permission for this behavior.
- `clicks`: apply coordinate arrays, numbered grid cells, or boolean grids to one observed challenge image/grid. Original image dimensions are converted once into CSS coordinates; out-of-bounds, ambiguous, obscured, or invisible targets fail. SolveCaptcha `click:` grid answers (including letter-numbered cells) and `coordinate:` answers are decoded into these arrays. These are synthetic page events; some challenges require trusted input and will reject them.
@@ -177,6 +177,7 @@ A proxyless method means the solver chooses its network route. It does not guara
- **CapSolver Cloudflare and DataDome** require the proxy inputs documented for their native methods.
- **NoneCap** accepts optional string/object proxies. Its returned User-Agent and response key can matter for acceptance.
- Proxied JSON APIs use provider-specific `proxyType`, `proxyAddress`, `proxyPort`, and optional authentication fields. SolveCaptcha uses form-encoded proxy fields.
+- Weighted native fallbacks compare supplied proxy scheme, host, port, and credentials across provider-specific formats before any paid request. Identity-bound fallbacks require a proxy on every attempt; supplied User-Agent values must also agree.
WebBrain does not silently change the browser's proxy or User-Agent, transplant a complete remote session, or infer an IP match. Satisfy those prerequisites separately. Returned cookies can be explicitly bound; returned JavaScript, arbitrary storage dumps, and arbitrary navigation instructions cannot.
diff --git a/src/chrome/src/agent/captcha-native-providers.js b/src/chrome/src/agent/captcha-native-providers.js
index a2dc5e9e1..6736dcb03 100644
--- a/src/chrome/src/agent/captcha-native-providers.js
+++ b/src/chrome/src/agent/captcha-native-providers.js
@@ -144,8 +144,55 @@ const RECOGNITION_FAMILIES = new Set([
'geetest_recognition', 'lemin_recognition',
]);
const RECOGNITION_MEDIA_FIELDS = ['body', 'image', 'images', 'image_data', 'imagesBase64', 'imageUrls', 'audio_data'];
+function proxyIdentity(task) {
+ const raw = task.proxy;
+ if (!raw && !task.proxyAddress) return null;
+ let scheme = String(task.proxyType || task.proxytype || (object(raw) && raw.scheme) || '').toLowerCase();
+ let host, port, login = '', password = '';
+ if (task.proxyAddress) {
+ host = task.proxyAddress;
+ port = task.proxyPort;
+ login = task.proxyLogin || '';
+ password = task.proxyPassword || '';
+ } else if (object(raw)) {
+ host = raw.host;
+ port = raw.port;
+ login = raw.username || raw.login || '';
+ password = raw.password || '';
+ } else if (typeof raw === 'string') {
+ // CapSolver accepts scheme:host:port:login:password; other providers use
+ // scheme://login:password@host:port or login:password@host:port.
+ const colonForm = raw.match(/^(https?|socks[45]):([^:/@]+):(\d+)(?::([^:]*):(.+))?$/i);
+ if (colonForm) {
+ [, scheme, host, port, login = '', password = ''] = colonForm;
+ } else {
+ let parsed;
+ try { parsed = new URL(raw.includes('://') ? raw : `${scheme || 'http'}://${raw}`); }
+ catch { throw new Error('Fallback tasks must use comparable proxy identities.'); }
+ const parsedScheme = parsed.protocol.slice(0, -1).toLowerCase();
+ if (scheme && scheme !== parsedScheme) throw new Error('Fallback tasks must use the same proxy identity.');
+ scheme = parsedScheme;
+ host = parsed.hostname;
+ port = parsed.port;
+ login = decodeURIComponent(parsed.username);
+ password = decodeURIComponent(parsed.password);
+ }
+ }
+ if (!scheme || !host || !Number.isInteger(Number(port)) || Number(port) < 1 || Number(port) > 65535) {
+ throw new Error('Fallback tasks must use comparable proxy identities.');
+ }
+ return JSON.stringify([scheme.toLowerCase(), String(host).toLowerCase(), Number(port), String(login), String(password)]);
+}
function validateFallbackIdentifiers(built) {
const family = built[0].contract.family;
+ if (built.length > 1) {
+ const proxies = built.map(({ task }) => proxyIdentity(task));
+ const present = proxies.filter(Boolean);
+ if (new Set(present).size > 1 || (present.length && present.length !== built.length
+ && ['datadome', 'captchafox', 'cybersiara', 'cloudflare_challenge', 'cloudflare_waiting_room', 'imperva', 'tspd'].includes(family))) {
+ throw new Error('Fallback tasks must use the same proxy identity.');
+ }
+ }
if (built.length > 1 && RECOGNITION_FAMILIES.has(family)) {
const signatures = built.map(({ task }) => {
const media = RECOGNITION_MEDIA_FIELDS.flatMap(path => {
diff --git a/src/chrome/src/agent/permission-gate.js b/src/chrome/src/agent/permission-gate.js
index 25e04fb3c..10d97d787 100644
--- a/src/chrome/src/agent/permission-gate.js
+++ b/src/chrome/src/agent/permission-gate.js
@@ -492,6 +492,14 @@ export function capabilityFor(name, args) {
*/
export function capabilitiesFor(name, args) {
args = args || {};
+ // CAPTCHA application mutates the selected page. A named callback executes
+ // page JavaScript; cookies likewise change browser state. Page-authored
+ // bindings must not bypass the ordinary host-specific permission gate.
+ if (name === 'apply_captcha_solution') return [
+ ...(args.fields?.length ? [Capability.TYPE] : []),
+ ...(args.clicks?.length ? [Capability.CLICK] : []),
+ ...(args.callback || args.cookies?.length ? [Capability.EXECUTE_JS] : []),
+ ];
if (name === 'chat_send') return [Capability.TYPE, Capability.CLICK];
if (name === 'delegate_research') {
// agent.js substitutes explicit one-use research authorization for these
@@ -561,6 +569,9 @@ function resolveHostAgainst(url, base) {
*/
export function hostForCapability(capability, args, currentUrlOrHost, toolName) {
args = args || {};
+ if (toolName === 'apply_captcha_solution') {
+ return normalizeHost(args.frameUrl);
+ }
if (toolName === 'read_email_verification_message' && capability === Capability.CLICK) {
// agent.js supplies this from its opaque inspected-mailbox session only to
// the permission check; it never comes from model arguments.
diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js
index bd3ad8106..3aeb41af1 100644
--- a/src/chrome/src/agent/tools.js
+++ b/src/chrome/src/agent/tools.js
@@ -1243,7 +1243,7 @@ export const AGENT_TOOLS = [
"type": "function",
"function": {
"name": "apply_captcha_solution",
- "description": "Apply the stored answer from the one native CAPTCHA solve to its original page. Bind solution paths to observed response fields, host-only cookies, image/grid clicks, or an observed named CAPTCHA callback. Never invent selectors/callbacks or execute returned scripts. Requires exact observed frameId and frameUrl; cookie bindings require frame 0. Match any required proxy/User-Agent first. Then verify fresh page state. This never requests a paid solve.",
+ "description": "Apply the stored answer from the one native CAPTCHA solve to its original page. Bind solution paths to observed response fields, host-only cookies, image/grid clicks, or an observed named CAPTCHA callback. The frame host's Type, Click, or JavaScript permission is required for the corresponding binding. Never invent selectors/callbacks or execute returned scripts. Requires exact observed frameId and frameUrl; cookie bindings require frame 0. Match any required proxy/User-Agent first. Then verify fresh page state. This never requests a paid solve.",
"parameters": {
"type": "object",
"properties": {
diff --git a/src/firefox/src/agent/captcha-native-providers.js b/src/firefox/src/agent/captcha-native-providers.js
index a2dc5e9e1..6736dcb03 100644
--- a/src/firefox/src/agent/captcha-native-providers.js
+++ b/src/firefox/src/agent/captcha-native-providers.js
@@ -144,8 +144,55 @@ const RECOGNITION_FAMILIES = new Set([
'geetest_recognition', 'lemin_recognition',
]);
const RECOGNITION_MEDIA_FIELDS = ['body', 'image', 'images', 'image_data', 'imagesBase64', 'imageUrls', 'audio_data'];
+function proxyIdentity(task) {
+ const raw = task.proxy;
+ if (!raw && !task.proxyAddress) return null;
+ let scheme = String(task.proxyType || task.proxytype || (object(raw) && raw.scheme) || '').toLowerCase();
+ let host, port, login = '', password = '';
+ if (task.proxyAddress) {
+ host = task.proxyAddress;
+ port = task.proxyPort;
+ login = task.proxyLogin || '';
+ password = task.proxyPassword || '';
+ } else if (object(raw)) {
+ host = raw.host;
+ port = raw.port;
+ login = raw.username || raw.login || '';
+ password = raw.password || '';
+ } else if (typeof raw === 'string') {
+ // CapSolver accepts scheme:host:port:login:password; other providers use
+ // scheme://login:password@host:port or login:password@host:port.
+ const colonForm = raw.match(/^(https?|socks[45]):([^:/@]+):(\d+)(?::([^:]*):(.+))?$/i);
+ if (colonForm) {
+ [, scheme, host, port, login = '', password = ''] = colonForm;
+ } else {
+ let parsed;
+ try { parsed = new URL(raw.includes('://') ? raw : `${scheme || 'http'}://${raw}`); }
+ catch { throw new Error('Fallback tasks must use comparable proxy identities.'); }
+ const parsedScheme = parsed.protocol.slice(0, -1).toLowerCase();
+ if (scheme && scheme !== parsedScheme) throw new Error('Fallback tasks must use the same proxy identity.');
+ scheme = parsedScheme;
+ host = parsed.hostname;
+ port = parsed.port;
+ login = decodeURIComponent(parsed.username);
+ password = decodeURIComponent(parsed.password);
+ }
+ }
+ if (!scheme || !host || !Number.isInteger(Number(port)) || Number(port) < 1 || Number(port) > 65535) {
+ throw new Error('Fallback tasks must use comparable proxy identities.');
+ }
+ return JSON.stringify([scheme.toLowerCase(), String(host).toLowerCase(), Number(port), String(login), String(password)]);
+}
function validateFallbackIdentifiers(built) {
const family = built[0].contract.family;
+ if (built.length > 1) {
+ const proxies = built.map(({ task }) => proxyIdentity(task));
+ const present = proxies.filter(Boolean);
+ if (new Set(present).size > 1 || (present.length && present.length !== built.length
+ && ['datadome', 'captchafox', 'cybersiara', 'cloudflare_challenge', 'cloudflare_waiting_room', 'imperva', 'tspd'].includes(family))) {
+ throw new Error('Fallback tasks must use the same proxy identity.');
+ }
+ }
if (built.length > 1 && RECOGNITION_FAMILIES.has(family)) {
const signatures = built.map(({ task }) => {
const media = RECOGNITION_MEDIA_FIELDS.flatMap(path => {
diff --git a/src/firefox/src/agent/permission-gate.js b/src/firefox/src/agent/permission-gate.js
index 639179fa9..1b172c176 100644
--- a/src/firefox/src/agent/permission-gate.js
+++ b/src/firefox/src/agent/permission-gate.js
@@ -477,6 +477,14 @@ export function capabilityFor(name, args) {
*/
export function capabilitiesFor(name, args) {
args = args || {};
+ // CAPTCHA application mutates the selected page. A named callback executes
+ // page JavaScript; cookies likewise change browser state. Page-authored
+ // bindings must not bypass the ordinary host-specific permission gate.
+ if (name === 'apply_captcha_solution') return [
+ ...(args.fields?.length ? [Capability.TYPE] : []),
+ ...(args.clicks?.length ? [Capability.CLICK] : []),
+ ...(args.callback || args.cookies?.length ? [Capability.EXECUTE_JS] : []),
+ ];
if (name === 'chat_send') return [Capability.TYPE, Capability.CLICK];
if (name === 'delegate_research') {
// agent.js substitutes explicit one-use research authorization for these
@@ -546,6 +554,9 @@ function resolveHostAgainst(url, base) {
*/
export function hostForCapability(capability, args, currentUrlOrHost, toolName) {
args = args || {};
+ if (toolName === 'apply_captcha_solution') {
+ return normalizeHost(args.frameUrl);
+ }
if (toolName === 'read_email_verification_message' && capability === Capability.CLICK) {
// agent.js supplies this from its opaque inspected-mailbox session only to
// the permission check; it never comes from model arguments.
diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js
index 7993b8e9c..868d25bde 100644
--- a/src/firefox/src/agent/tools.js
+++ b/src/firefox/src/agent/tools.js
@@ -1097,7 +1097,7 @@ export const AGENT_TOOLS = [
"type": "function",
"function": {
"name": "apply_captcha_solution",
- "description": "Apply the stored answer from the one native CAPTCHA solve to its original page. Bind solution paths to observed response fields, host-only cookies, image/grid clicks, or an observed named CAPTCHA callback. Never invent selectors/callbacks or execute returned scripts. Requires exact observed frameId and frameUrl; cookie bindings require frame 0. Match any required proxy/User-Agent first. Then verify fresh page state. This never requests a paid solve.",
+ "description": "Apply the stored answer from the one native CAPTCHA solve to its original page. Bind solution paths to observed response fields, host-only cookies, image/grid clicks, or an observed named CAPTCHA callback. The frame host's Type, Click, or JavaScript permission is required for the corresponding binding. Never invent selectors/callbacks or execute returned scripts. Requires exact observed frameId and frameUrl; cookie bindings require frame 0. Match any required proxy/User-Agent first. Then verify fresh page state. This never requests a paid solve.",
"parameters": {
"type": "object",
"properties": {
diff --git a/test/captcha-native-providers.mjs b/test/captcha-native-providers.mjs
index e17d42840..dd3b50952 100644
--- a/test/captcha-native-providers.mjs
+++ b/test/captcha-native-providers.mjs
@@ -244,6 +244,7 @@ for (const browser of ['chrome', 'firefox']) {
});
const actionBoundPairs = [
['DataDome User-Agent', ['2captcha', 'DataDomeSliderTask', { websiteURL: url, captchaUrl: 'https://example.test/captcha', userAgent: 'browser-UA', proxyType: 'http', proxyAddress: 'proxy.test', proxyPort: 8080 }], ['solvecaptcha', 'datadome', { pageurl: url, captcha_url: 'https://example.test/captcha', userAgent: 'browser-UA', proxy: 'proxy.test:8080', proxytype: 'http' }], 'userAgent'],
+ ['DataDome proxy', ['2captcha', 'DataDomeSliderTask', { websiteURL: url, captchaUrl: 'https://example.test/captcha', userAgent: 'browser-UA', proxyType: 'http', proxyAddress: 'proxy.test', proxyPort: 8080 }], ['solvecaptcha', 'datadome', { pageurl: url, captcha_url: 'https://example.test/captcha', userAgent: 'browser-UA', proxy: 'proxy.test:8080', proxytype: 'http' }], 'proxy'],
['reCAPTCHA v2 data-s', ['2captcha', 'RecaptchaV2TaskProxyless', { websiteURL: url, websiteKey: 'site', recaptchaDataSValue: 'observed' }], ['solvecaptcha', 'recaptcha_v2', { pageurl: url, googlekey: 'site', 'data-s': 'observed' }], 'data-s'],
['reCAPTCHA v2 enterprise data-s', ['2captcha', 'RecaptchaV2EnterpriseTaskProxyless', { websiteURL: url, websiteKey: 'site', enterprisePayload: { s: 'observed' } }], ['solvecaptcha', 'recaptcha_v2_enterprise', { pageurl: url, googlekey: 'site', 'data-s': 'observed' }], 'data-s'],
['reCAPTCHA v2 nested data-s', ['2captcha', 'RecaptchaV2TaskProxyless', { websiteURL: url, websiteKey: 'site', recaptchaDataSValue: 'observed' }], ['nopecha', 'token/recaptcha2', { url, sitekey: 'site', data: { s: 'observed' } }], 'data.s'],
@@ -262,11 +263,15 @@ for (const browser of ['chrome', 'firefox']) {
const path = changed.split('.');
let target = entries[1].parameters;
for (const part of path.slice(0, -1)) target = target[part];
- target[path.at(-1)] = 'different';
- assert.throws(() => native.prepareNativeCaptchaTasks(enabled, entries), /same observed challenge/);
+ target[path.at(-1)] = changed === 'proxy' ? 'other.test:8080' : 'different';
+ assert.throws(() => native.prepareNativeCaptchaTasks(enabled, entries), /same observed challenge|same proxy identity/);
assert.equal(calls.length, 0);
});
for (const [label, entries] of [
+ ['DataDome proxy credentials mismatch', [
+ { provider: '2captcha', method: 'DataDomeSliderTask', parameters: { websiteURL: url, captchaUrl: 'https://example.test/captcha', userAgent: 'browser-UA', proxyType: 'http', proxyAddress: 'proxy.test', proxyPort: 8080, proxyLogin: 'alice' } },
+ { provider: 'capmonster', method: 'CustomTask:DataDome', parameters: { websiteURL: url, metadata: { datadomeCookie: 'observed', captchaUrl: 'https://example.test/captcha' }, userAgent: 'browser-UA', proxyType: 'http', proxyAddress: 'proxy.test', proxyPort: 8080, proxyLogin: 'bob' } },
+ ]],
['reCAPTCHA v2 omitted User-Agent', [
{ provider: '2captcha', method: 'RecaptchaV2TaskProxyless', parameters: { websiteURL: url, websiteKey: 'site', userAgent: 'browser-UA' } },
{ provider: 'solvecaptcha', method: 'recaptcha_v2', parameters: { pageurl: url, googlekey: 'site' } },
@@ -289,7 +294,7 @@ for (const browser of ['chrome', 'firefox']) {
]],
]) test(`${browser}: ${label} cannot spend`, t => {
const calls = mockApi(t, () => { throw new Error('Must not dispatch'); });
- assert.throws(() => native.prepareNativeCaptchaTasks(entries.map(entry => ({ id: entry.provider, apiKey: 'key' })), entries), /same observed challenge/);
+ assert.throws(() => native.prepareNativeCaptchaTasks(entries.map(entry => ({ id: entry.provider, apiKey: 'key' })), entries), /same observed challenge|same proxy identity/);
assert.equal(calls.length, 0);
});
const recognitionPairs = [
diff --git a/test/captcha-runtime-policy.mjs b/test/captcha-runtime-policy.mjs
index c017d82e5..54d9430fa 100644
--- a/test/captcha-runtime-policy.mjs
+++ b/test/captcha-runtime-policy.mjs
@@ -14,6 +14,19 @@ test('CAPTCHA tool matrix matches Ask, Compact, Mid, Full and Dev availability',
for (const build of ['chrome', 'firefox']) {
const { Agent } = await import(`../src/${build}/src/agent/agent.js`);
const { getToolsForMode } = await import(`../src/${build}/src/agent/tools.js`);
+ const { Capability, capabilitiesFor, requiredHosts } = await import(`../src/${build}/src/agent/permission-gate.js`);
+ test(`${build}: CAPTCHA bindings require their mutation permissions for the selected frame host`, () => {
+ const args = { frameId: 3, frameUrl: 'https://captcha.example.test/challenge', callback: { name: 'app.deleteAccount', path: 'token' } };
+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', args), [Capability.EXECUTE_JS]);
+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, callback: undefined, fields: [{ selector: '#answer', path: 'token' }] }), [Capability.TYPE]);
+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, callback: undefined, clicks: [{ selector: '#grid', path: 'points' }] }), [Capability.CLICK]);
+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, callback: undefined, cookies: [{ name: 'clearance', path: 'cookie' }] }), [Capability.EXECUTE_JS]);
+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, fields: [{}], clicks: [{}], cookies: [{}] }), [Capability.TYPE, Capability.CLICK, Capability.EXECUTE_JS]);
+ assert.deepEqual(requiredHosts(Capability.EXECUTE_JS, args, 'https://main.example.test/', 'apply_captcha_solution'), ['captcha.example.test']);
+ assert.deepEqual(requiredHosts(Capability.CLICK, args, 'https://main.example.test/', 'apply_captcha_solution'), ['captcha.example.test']);
+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, callback: undefined }), []);
+ assert.deepEqual(requiredHosts(Capability.EXECUTE_JS, { ...args, frameUrl: '' }, 'https://main.example.test/', 'apply_captcha_solution'), []);
+ });
test(`${build}: hCaptcha Enterprise rqdata is optional in the model-visible tool schema`, () => {
const solve = getToolsForMode('act', { tier: 'full' }).find(tool => tool.function.name === 'solve_captcha');
assert.match(solve.function.parameters.properties.rqdata.description, /optional.*when the widget exposes it/i);