Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Gate CAPTCHA application and match fallback proxy identity
Emre Sokulluaf4e716aa35a
diff --git a/docs/captcha-provider-coverage.md b/docs/captcha-provider-coverage.mdindex a402201d8..58d03b134 100644--- a/docs/captcha-provider-coverage.md+++ b/docs/captcha-provider-coverage.md@@ -148,7 +148,7 @@ Provider-specific identifiers (for example `appId`/`app_id` and `miseryKey`/`misSupported bindings:- `fields`: bind a solution path to one observed input/textarea selector; structured values require `encoding: "json"`.-- `callback`: pass a solution value or object to an observed named page callback. Native/global code evaluators and unsafe property paths are rejected. Provider-returned code is never evaluated.+- `callback`: pass a solution value or object to an observed named page callback. Application uses the frame host's normal permissions: Type for response fields, Click for image/grid clicks, and JavaScript for callbacks or cookies. Native/global code evaluators and unsafe property paths are rejected. Provider-returned code is never evaluated.- `cookies`: bind explicit cookie names to returned values. Complete Set-Cookie strings are reduced to the explicitly named cookie value; returned attributes are ignored. Cookies are host-only, scoped to the active top-level page and its browser cookie store. Provider-supplied domains/URLs are not adopted. Both extension manifests declare the `cookies` permission for this behavior.- `clicks`: apply coordinate arrays, numbered grid cells, or boolean grids to one observed challenge image/grid. Original image dimensions are converted once into CSS coordinates; out-of-bounds, ambiguous, obscured, or invisible targets fail. SolveCaptcha `click:` grid answers (including letter-numbered cells) and `coordinate:` answers are decoded into these arrays. These are synthetic page events; some challenges require trusted input and will reject them.@@ -177,6 +177,7 @@ A proxyless method means the solver chooses its network route. It does not guara- **CapSolver Cloudflare and DataDome** require the proxy inputs documented for their native methods.- **NoneCap** accepts optional string/object proxies. Its returned User-Agent and response key can matter for acceptance.- Proxied JSON APIs use provider-specific `proxyType`, `proxyAddress`, `proxyPort`, and optional authentication fields. SolveCaptcha uses form-encoded proxy fields.+- Weighted native fallbacks compare supplied proxy scheme, host, port, and credentials across provider-specific formats before any paid request. Identity-bound fallbacks require a proxy on every attempt; supplied User-Agent values must also agree.WebBrain does not silently change the browser's proxy or User-Agent, transplant a complete remote session, or infer an IP match. Satisfy those prerequisites separately. Returned cookies can be explicitly bound; returned JavaScript, arbitrary storage dumps, and arbitrary navigation instructions cannot.diff --git a/src/chrome/src/agent/captcha-native-providers.js b/src/chrome/src/agent/captcha-native-providers.jsindex a2dc5e9e1..6736dcb03 100644--- a/src/chrome/src/agent/captcha-native-providers.js+++ b/src/chrome/src/agent/captcha-native-providers.js@@ -144,8 +144,55 @@ const RECOGNITION_FAMILIES = new Set(['geetest_recognition', 'lemin_recognition',]);const RECOGNITION_MEDIA_FIELDS = ['body', 'image', 'images', 'image_data', 'imagesBase64', 'imageUrls', 'audio_data'];+function proxyIdentity(task) {+ const raw = task.proxy;+ if (!raw && !task.proxyAddress) return null;+ let scheme = String(task.proxyType || task.proxytype || (object(raw) && raw.scheme) || '').toLowerCase();+ let host, port, login = '', password = '';+ if (task.proxyAddress) {+ host = task.proxyAddress;+ port = task.proxyPort;+ login = task.proxyLogin || '';+ password = task.proxyPassword || '';+ } else if (object(raw)) {+ host = raw.host;+ port = raw.port;+ login = raw.username || raw.login || '';+ password = raw.password || '';+ } else if (typeof raw === 'string') {+ // CapSolver accepts scheme:host:port:login:password; other providers use+ // scheme://login:password@host:port or login:password@host:port.+ const colonForm = raw.match(/^(https?|socks[45]):([^:/@]+):(\d+)(?::([^:]*):(.+))?$/i);+ if (colonForm) {+ [, scheme, host, port, login = '', password = ''] = colonForm;+ } else {+ let parsed;+ try { parsed = new URL(raw.includes('://') ? raw : `${scheme || 'http'}://${raw}`); }+ catch { throw new Error('Fallback tasks must use comparable proxy identities.'); }+ const parsedScheme = parsed.protocol.slice(0, -1).toLowerCase();+ if (scheme && scheme !== parsedScheme) throw new Error('Fallback tasks must use the same proxy identity.');+ scheme = parsedScheme;+ host = parsed.hostname;+ port = parsed.port;+ login = decodeURIComponent(parsed.username);+ password = decodeURIComponent(parsed.password);+ }+ }+ if (!scheme || !host || !Number.isInteger(Number(port)) || Number(port) < 1 || Number(port) > 65535) {+ throw new Error('Fallback tasks must use comparable proxy identities.');+ }+ return JSON.stringify([scheme.toLowerCase(), String(host).toLowerCase(), Number(port), String(login), String(password)]);+}function validateFallbackIdentifiers(built) {const family = built[0].contract.family;+ if (built.length > 1) {+ const proxies = built.map(({ task }) => proxyIdentity(task));+ const present = proxies.filter(Boolean);+ if (new Set(present).size > 1 || (present.length && present.length !== built.length+ && ['datadome', 'captchafox', 'cybersiara', 'cloudflare_challenge', 'cloudflare_waiting_room', 'imperva', 'tspd'].includes(family))) {+ throw new Error('Fallback tasks must use the same proxy identity.');+ }+ }if (built.length > 1 && RECOGNITION_FAMILIES.has(family)) {const signatures = built.map(({ task }) => {const media = RECOGNITION_MEDIA_FIELDS.flatMap(path => {diff --git a/src/chrome/src/agent/permission-gate.js b/src/chrome/src/agent/permission-gate.jsindex 25e04fb3c..10d97d787 100644--- a/src/chrome/src/agent/permission-gate.js+++ b/src/chrome/src/agent/permission-gate.js@@ -492,6 +492,14 @@ export function capabilityFor(name, args) {*/export function capabilitiesFor(name, args) {args = args || {};+ // CAPTCHA application mutates the selected page. A named callback executes+ // page JavaScript; cookies likewise change browser state. Page-authored+ // bindings must not bypass the ordinary host-specific permission gate.+ if (name === 'apply_captcha_solution') return [+ ...(args.fields?.length ? [Capability.TYPE] : []),+ ...(args.clicks?.length ? [Capability.CLICK] : []),+ ...(args.callback || args.cookies?.length ? [Capability.EXECUTE_JS] : []),+ ];if (name === 'chat_send') return [Capability.TYPE, Capability.CLICK];if (name === 'delegate_research') {// agent.js substitutes explicit one-use research authorization for these@@ -561,6 +569,9 @@ function resolveHostAgainst(url, base) {*/export function hostForCapability(capability, args, currentUrlOrHost, toolName) {args = args || {};+ if (toolName === 'apply_captcha_solution') {+ return normalizeHost(args.frameUrl);+ }if (toolName === 'read_email_verification_message' && capability === Capability.CLICK) {// agent.js supplies this from its opaque inspected-mailbox session only to// the permission check; it never comes from model arguments.diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.jsindex bd3ad8106..3aeb41af1 100644--- a/src/chrome/src/agent/tools.js+++ b/src/chrome/src/agent/tools.js@@ -1243,7 +1243,7 @@ export const AGENT_TOOLS = ["type": "function","function": {"name": "apply_captcha_solution",- "description": "Apply the stored answer from the one native CAPTCHA solve to its original page. Bind solution paths to observed response fields, host-only cookies, image/grid clicks, or an observed named CAPTCHA callback. Never invent selectors/callbacks or execute returned scripts. Requires exact observed frameId and frameUrl; cookie bindings require frame 0. Match any required proxy/User-Agent first. Then verify fresh page state. This never requests a paid solve.",+ "description": "Apply the stored answer from the one native CAPTCHA solve to its original page. Bind solution paths to observed response fields, host-only cookies, image/grid clicks, or an observed named CAPTCHA callback. The frame host's Type, Click, or JavaScript permission is required for the corresponding binding. Never invent selectors/callbacks or execute returned scripts. Requires exact observed frameId and frameUrl; cookie bindings require frame 0. Match any required proxy/User-Agent first. Then verify fresh page state. This never requests a paid solve.","parameters": {"type": "object","properties": {diff --git a/src/firefox/src/agent/captcha-native-providers.js b/src/firefox/src/agent/captcha-native-providers.jsindex a2dc5e9e1..6736dcb03 100644--- a/src/firefox/src/agent/captcha-native-providers.js+++ b/src/firefox/src/agent/captcha-native-providers.js@@ -144,8 +144,55 @@ const RECOGNITION_FAMILIES = new Set(['geetest_recognition', 'lemin_recognition',]);const RECOGNITION_MEDIA_FIELDS = ['body', 'image', 'images', 'image_data', 'imagesBase64', 'imageUrls', 'audio_data'];+function proxyIdentity(task) {+ const raw = task.proxy;+ if (!raw && !task.proxyAddress) return null;+ let scheme = String(task.proxyType || task.proxytype || (object(raw) && raw.scheme) || '').toLowerCase();+ let host, port, login = '', password = '';+ if (task.proxyAddress) {+ host = task.proxyAddress;+ port = task.proxyPort;+ login = task.proxyLogin || '';+ password = task.proxyPassword || '';+ } else if (object(raw)) {+ host = raw.host;+ port = raw.port;+ login = raw.username || raw.login || '';+ password = raw.password || '';+ } else if (typeof raw === 'string') {+ // CapSolver accepts scheme:host:port:login:password; other providers use+ // scheme://login:password@host:port or login:password@host:port.+ const colonForm = raw.match(/^(https?|socks[45]):([^:/@]+):(\d+)(?::([^:]*):(.+))?$/i);+ if (colonForm) {+ [, scheme, host, port, login = '', password = ''] = colonForm;+ } else {+ let parsed;+ try { parsed = new URL(raw.includes('://') ? raw : `${scheme || 'http'}://${raw}`); }+ catch { throw new Error('Fallback tasks must use comparable proxy identities.'); }+ const parsedScheme = parsed.protocol.slice(0, -1).toLowerCase();+ if (scheme && scheme !== parsedScheme) throw new Error('Fallback tasks must use the same proxy identity.');+ scheme = parsedScheme;+ host = parsed.hostname;+ port = parsed.port;+ login = decodeURIComponent(parsed.username);+ password = decodeURIComponent(parsed.password);+ }+ }+ if (!scheme || !host || !Number.isInteger(Number(port)) || Number(port) < 1 || Number(port) > 65535) {+ throw new Error('Fallback tasks must use comparable proxy identities.');+ }+ return JSON.stringify([scheme.toLowerCase(), String(host).toLowerCase(), Number(port), String(login), String(password)]);+}function validateFallbackIdentifiers(built) {const family = built[0].contract.family;+ if (built.length > 1) {+ const proxies = built.map(({ task }) => proxyIdentity(task));+ const present = proxies.filter(Boolean);+ if (new Set(present).size > 1 || (present.length && present.length !== built.length+ && ['datadome', 'captchafox', 'cybersiara', 'cloudflare_challenge', 'cloudflare_waiting_room', 'imperva', 'tspd'].includes(family))) {+ throw new Error('Fallback tasks must use the same proxy identity.');+ }+ }if (built.length > 1 && RECOGNITION_FAMILIES.has(family)) {const signatures = built.map(({ task }) => {const media = RECOGNITION_MEDIA_FIELDS.flatMap(path => {diff --git a/src/firefox/src/agent/permission-gate.js b/src/firefox/src/agent/permission-gate.jsindex 639179fa9..1b172c176 100644--- a/src/firefox/src/agent/permission-gate.js+++ b/src/firefox/src/agent/permission-gate.js@@ -477,6 +477,14 @@ export function capabilityFor(name, args) {*/export function capabilitiesFor(name, args) {args = args || {};+ // CAPTCHA application mutates the selected page. A named callback executes+ // page JavaScript; cookies likewise change browser state. Page-authored+ // bindings must not bypass the ordinary host-specific permission gate.+ if (name === 'apply_captcha_solution') return [+ ...(args.fields?.length ? [Capability.TYPE] : []),+ ...(args.clicks?.length ? [Capability.CLICK] : []),+ ...(args.callback || args.cookies?.length ? [Capability.EXECUTE_JS] : []),+ ];if (name === 'chat_send') return [Capability.TYPE, Capability.CLICK];if (name === 'delegate_research') {// agent.js substitutes explicit one-use research authorization for these@@ -546,6 +554,9 @@ function resolveHostAgainst(url, base) {*/export function hostForCapability(capability, args, currentUrlOrHost, toolName) {args = args || {};+ if (toolName === 'apply_captcha_solution') {+ return normalizeHost(args.frameUrl);+ }if (toolName === 'read_email_verification_message' && capability === Capability.CLICK) {// agent.js supplies this from its opaque inspected-mailbox session only to// the permission check; it never comes from model arguments.diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.jsindex 7993b8e9c..868d25bde 100644--- a/src/firefox/src/agent/tools.js+++ b/src/firefox/src/agent/tools.js@@ -1097,7 +1097,7 @@ export const AGENT_TOOLS = ["type": "function","function": {"name": "apply_captcha_solution",- "description": "Apply the stored answer from the one native CAPTCHA solve to its original page. Bind solution paths to observed response fields, host-only cookies, image/grid clicks, or an observed named CAPTCHA callback. Never invent selectors/callbacks or execute returned scripts. Requires exact observed frameId and frameUrl; cookie bindings require frame 0. Match any required proxy/User-Agent first. Then verify fresh page state. This never requests a paid solve.",+ "description": "Apply the stored answer from the one native CAPTCHA solve to its original page. Bind solution paths to observed response fields, host-only cookies, image/grid clicks, or an observed named CAPTCHA callback. The frame host's Type, Click, or JavaScript permission is required for the corresponding binding. Never invent selectors/callbacks or execute returned scripts. Requires exact observed frameId and frameUrl; cookie bindings require frame 0. Match any required proxy/User-Agent first. Then verify fresh page state. This never requests a paid solve.","parameters": {"type": "object","properties": {diff --git a/test/captcha-native-providers.mjs b/test/captcha-native-providers.mjsindex e17d42840..dd3b50952 100644--- a/test/captcha-native-providers.mjs+++ b/test/captcha-native-providers.mjs@@ -244,6 +244,7 @@ for (const browser of ['chrome', 'firefox']) {});const actionBoundPairs = [['DataDome User-Agent', ['2captcha', 'DataDomeSliderTask', { websiteURL: url, captchaUrl: 'https://example.test/captcha', userAgent: 'browser-UA', proxyType: 'http', proxyAddress: 'proxy.test', proxyPort: 8080 }], ['solvecaptcha', 'datadome', { pageurl: url, captcha_url: 'https://example.test/captcha', userAgent: 'browser-UA', proxy: 'proxy.test:8080', proxytype: 'http' }], 'userAgent'],+ ['DataDome proxy', ['2captcha', 'DataDomeSliderTask', { websiteURL: url, captchaUrl: 'https://example.test/captcha', userAgent: 'browser-UA', proxyType: 'http', proxyAddress: 'proxy.test', proxyPort: 8080 }], ['solvecaptcha', 'datadome', { pageurl: url, captcha_url: 'https://example.test/captcha', userAgent: 'browser-UA', proxy: 'proxy.test:8080', proxytype: 'http' }], 'proxy'],['reCAPTCHA v2 data-s', ['2captcha', 'RecaptchaV2TaskProxyless', { websiteURL: url, websiteKey: 'site', recaptchaDataSValue: 'observed' }], ['solvecaptcha', 'recaptcha_v2', { pageurl: url, googlekey: 'site', 'data-s': 'observed' }], 'data-s'],['reCAPTCHA v2 enterprise data-s', ['2captcha', 'RecaptchaV2EnterpriseTaskProxyless', { websiteURL: url, websiteKey: 'site', enterprisePayload: { s: 'observed' } }], ['solvecaptcha', 'recaptcha_v2_enterprise', { pageurl: url, googlekey: 'site', 'data-s': 'observed' }], 'data-s'],['reCAPTCHA v2 nested data-s', ['2captcha', 'RecaptchaV2TaskProxyless', { websiteURL: url, websiteKey: 'site', recaptchaDataSValue: 'observed' }], ['nopecha', 'token/recaptcha2', { url, sitekey: 'site', data: { s: 'observed' } }], 'data.s'],@@ -262,11 +263,15 @@ for (const browser of ['chrome', 'firefox']) {const path = changed.split('.');let target = entries[1].parameters;for (const part of path.slice(0, -1)) target = target[part];- target[path.at(-1)] = 'different';- assert.throws(() => native.prepareNativeCaptchaTasks(enabled, entries), /same observed challenge/);+ target[path.at(-1)] = changed === 'proxy' ? 'other.test:8080' : 'different';+ assert.throws(() => native.prepareNativeCaptchaTasks(enabled, entries), /same observed challenge|same proxy identity/);assert.equal(calls.length, 0);});for (const [label, entries] of [+ ['DataDome proxy credentials mismatch', [+ { provider: '2captcha', method: 'DataDomeSliderTask', parameters: { websiteURL: url, captchaUrl: 'https://example.test/captcha', userAgent: 'browser-UA', proxyType: 'http', proxyAddress: 'proxy.test', proxyPort: 8080, proxyLogin: 'alice' } },+ { provider: 'capmonster', method: 'CustomTask:DataDome', parameters: { websiteURL: url, metadata: { datadomeCookie: 'observed', captchaUrl: 'https://example.test/captcha' }, userAgent: 'browser-UA', proxyType: 'http', proxyAddress: 'proxy.test', proxyPort: 8080, proxyLogin: 'bob' } },+ ]],['reCAPTCHA v2 omitted User-Agent', [{ provider: '2captcha', method: 'RecaptchaV2TaskProxyless', parameters: { websiteURL: url, websiteKey: 'site', userAgent: 'browser-UA' } },{ provider: 'solvecaptcha', method: 'recaptcha_v2', parameters: { pageurl: url, googlekey: 'site' } },@@ -289,7 +294,7 @@ for (const browser of ['chrome', 'firefox']) {]],]) test(`${browser}: ${label} cannot spend`, t => {const calls = mockApi(t, () => { throw new Error('Must not dispatch'); });- assert.throws(() => native.prepareNativeCaptchaTasks(entries.map(entry => ({ id: entry.provider, apiKey: 'key' })), entries), /same observed challenge/);+ assert.throws(() => native.prepareNativeCaptchaTasks(entries.map(entry => ({ id: entry.provider, apiKey: 'key' })), entries), /same observed challenge|same proxy identity/);assert.equal(calls.length, 0);});const recognitionPairs = [diff --git a/test/captcha-runtime-policy.mjs b/test/captcha-runtime-policy.mjsindex c017d82e5..54d9430fa 100644--- a/test/captcha-runtime-policy.mjs+++ b/test/captcha-runtime-policy.mjs@@ -14,6 +14,19 @@ test('CAPTCHA tool matrix matches Ask, Compact, Mid, Full and Dev availability',for (const build of ['chrome', 'firefox']) {const { Agent } = await import(`../src/${build}/src/agent/agent.js`);const { getToolsForMode } = await import(`../src/${build}/src/agent/tools.js`);+ const { Capability, capabilitiesFor, requiredHosts } = await import(`../src/${build}/src/agent/permission-gate.js`);+ test(`${build}: CAPTCHA bindings require their mutation permissions for the selected frame host`, () => {+ const args = { frameId: 3, frameUrl: 'https://captcha.example.test/challenge', callback: { name: 'app.deleteAccount', path: 'token' } };+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', args), [Capability.EXECUTE_JS]);+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, callback: undefined, fields: [{ selector: '#answer', path: 'token' }] }), [Capability.TYPE]);+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, callback: undefined, clicks: [{ selector: '#grid', path: 'points' }] }), [Capability.CLICK]);+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, callback: undefined, cookies: [{ name: 'clearance', path: 'cookie' }] }), [Capability.EXECUTE_JS]);+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, fields: [{}], clicks: [{}], cookies: [{}] }), [Capability.TYPE, Capability.CLICK, Capability.EXECUTE_JS]);+ assert.deepEqual(requiredHosts(Capability.EXECUTE_JS, args, 'https://main.example.test/', 'apply_captcha_solution'), ['captcha.example.test']);+ assert.deepEqual(requiredHosts(Capability.CLICK, args, 'https://main.example.test/', 'apply_captcha_solution'), ['captcha.example.test']);+ assert.deepEqual(capabilitiesFor('apply_captcha_solution', { ...args, callback: undefined }), []);+ assert.deepEqual(requiredHosts(Capability.EXECUTE_JS, { ...args, frameUrl: '' }, 'https://main.example.test/', 'apply_captcha_solution'), []);+ });test(`${build}: hCaptcha Enterprise rqdata is optional in the model-visible tool schema`, () => {const solve = getToolsForMode('act', { tier: 'full' }).find(tool => tool.function.name === 'solve_captcha');assert.match(solve.function.parameters.properties.rqdata.description, /optional.*when the widget exposes it/i);