Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Bind native CAPTCHA fallbacks to observed media and frame URLs
Emre Sokullu6d7670976d15
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex e9ef67e45..f825e1a4a 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -35770,18 +35770,21 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaiif (args.inject !== false) return noDispatchFailure('Native CAPTCHA methods require inject: false. Apply the returned structured answer with apply_captcha_solution after inspecting the page.');const prepared = prepareNativeCaptchaTasks(providers, args.providerTasks);const frames = typeof chrome.webNavigation?.getAllFrames === 'function' ? await chrome.webNavigation.getAllFrames({ tabId }) : [];- const origins = new Set([websiteURL, ...frames.map(frame => frame.url)].flatMap(url => { try { return [new URL(url).origin]; } catch { return []; } }));- for (const { task } of prepared) {- for (const key of ['websiteURL', 'pageurl', 'url']) {- if (task[key] && (!/^https?:/.test(task[key]) || !origins.has(new URL(task[key]).origin))) return noDispatchFailure('Native CAPTCHA page URLs must belong to the active tab or one of its observed frames.');- }- }this._nativeCaptchaSolutions ||= new Map();// Record dispatch before sending. Even timeout/failure cannot trigger// a second call for this document; fallback stays inside one call.const documents = await captureCaptchaDocuments(tabId, frames, chrome);const rootDocument = documents.find(d => d.frameId === 0 && d.url === websiteURL);if (!rootDocument) return noDispatchFailure('Could not bind this page document before the native CAPTCHA request.');+ const normalizedPageUrl = value => {+ try { const parsed = new URL(value); return ['http:', 'https:'].includes(parsed.protocol) ? parsed.href : null; } catch { return null; }+ };+ const observedUrls = new Set(documents.map(d => normalizedPageUrl(d.url)).filter(Boolean));+ for (const { task } of prepared) {+ for (const key of ['websiteURL', 'pageurl', 'url']) {+ if (task[key] && !observedUrls.has(normalizedPageUrl(task[key]))) return noDispatchFailure('Native CAPTCHA page URLs must match the active tab or one of its observed frames.');+ }+ }const previous = this._nativeCaptchaSolutions.get(tabId);const dispatchedTimeOrigins = previous?.dispatchedTimeOrigins || new Set(previous?.documents?.filter(d => d.frameId === 0).map(d => d.timeOrigin) || []);diff --git a/src/chrome/src/agent/captcha-native-providers.js b/src/chrome/src/agent/captcha-native-providers.jsindex 337e112df..10a735eae 100644--- a/src/chrome/src/agent/captcha-native-providers.js+++ b/src/chrome/src/agent/captcha-native-providers.js@@ -127,9 +127,35 @@ const FAMILY_IDENTIFIERS = {['data', 'cData', 'metadata.cdata', 'data.cdata', 'data.cData'],['pagedata', 'pageData', 'chlPageData', 'data.pagedata', 'data.pageData', 'data.chlPageData'],],+ vk_recognition: [['steps']],+ grid: [['rows', 'recaptcharows'], ['columns', 'recaptchacols']],};+const RECOGNITION_FAMILIES = new Set([+ 'audio', 'bounding_box', 'coordinates', 'drag_and_drop', 'draw_around',+ 'funcaptcha_recognition', 'grid', 'image_to_text', 'recaptcha_recognition',+ 'rotate', 'temu_recognition', 'vk_recognition', 'yandex_recognition',+ 'aws_recognition', 'vision_engine', 'complex_image', 'funcaptcha_match',+ 'geetest_recognition', 'lemin_recognition',+]);+const RECOGNITION_MEDIA_FIELDS = ['body', 'image', 'images', 'image_data', 'imagesBase64', 'imageUrls', 'audio_data'];function validateFallbackIdentifiers(built) {const family = built[0].contract.family;+ if (built.length > 1 && RECOGNITION_FAMILIES.has(family)) {+ const signatures = built.map(({ task }) => {+ const media = RECOGNITION_MEDIA_FIELDS.flatMap(path => {+ const value = at(task, path);+ if (!usable(value)) return [];+ const kind = path === 'audio_data' ? 'audio' : path === 'imageUrls' ? 'image-url' : 'image';+ return [JSON.stringify([kind, Array.isArray(value) ? value : [value]])];+ });+ if (!media.length || new Set(media).size !== 1) throw new Error('Fallback recognition tasks must use the same observed challenge media.');+ const parts = family === 'temu_recognition'+ ? task.parts || [task.part1, task.part2, task.part3]+ : null;+ return JSON.stringify([media[0], parts]);+ });+ if (new Set(signatures).size > 1) throw new Error('Fallback recognition tasks must use the same observed challenge media.');+ }const groups = [['websiteURL', 'pageurl', 'url'],['websiteKey', 'sitekey', 'googlekey', 'websitePublicKey', 'publickey'],diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex fcf03d766..d6e8cd75b 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -32753,18 +32753,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (args.inject !== false) return noDispatchFailure('Native CAPTCHA methods require inject: false. Apply the returned structured answer with apply_captcha_solution after inspecting the page.');const prepared = prepareNativeCaptchaTasks(providers, args.providerTasks);const frames = typeof browser.webNavigation?.getAllFrames === 'function' ? await browser.webNavigation.getAllFrames({ tabId }) : [];- const origins = new Set([websiteURL, ...frames.map(frame => frame.url)].flatMap(url => { try { return [new URL(url).origin]; } catch { return []; } }));- for (const { task } of prepared) {- for (const key of ['websiteURL', 'pageurl', 'url']) {- if (task[key] && (!/^https?:/.test(task[key]) || !origins.has(new URL(task[key]).origin))) return noDispatchFailure('Native CAPTCHA page URLs must belong to the active tab or one of its observed frames.');- }- }this._nativeCaptchaSolutions ||= new Map();// Record dispatch before sending. Even timeout/failure cannot trigger// a second call for this document; fallback stays inside one call.const documents = await captureCaptchaDocuments(tabId, frames, browser);const rootDocument = documents.find(d => d.frameId === 0 && d.url === websiteURL);if (!rootDocument) return noDispatchFailure('Could not bind this page document before the native CAPTCHA request.');+ const normalizedPageUrl = value => {+ try { const parsed = new URL(value); return ['http:', 'https:'].includes(parsed.protocol) ? parsed.href : null; } catch { return null; }+ };+ const observedUrls = new Set(documents.map(d => normalizedPageUrl(d.url)).filter(Boolean));+ for (const { task } of prepared) {+ for (const key of ['websiteURL', 'pageurl', 'url']) {+ if (task[key] && !observedUrls.has(normalizedPageUrl(task[key]))) return noDispatchFailure('Native CAPTCHA page URLs must match the active tab or one of its observed frames.');+ }+ }const previous = this._nativeCaptchaSolutions.get(tabId);const dispatchedTimeOrigins = previous?.dispatchedTimeOrigins || new Set(previous?.documents?.filter(d => d.frameId === 0).map(d => d.timeOrigin) || []);diff --git a/src/firefox/src/agent/captcha-native-providers.js b/src/firefox/src/agent/captcha-native-providers.jsindex 337e112df..10a735eae 100644--- a/src/firefox/src/agent/captcha-native-providers.js+++ b/src/firefox/src/agent/captcha-native-providers.js@@ -127,9 +127,35 @@ const FAMILY_IDENTIFIERS = {['data', 'cData', 'metadata.cdata', 'data.cdata', 'data.cData'],['pagedata', 'pageData', 'chlPageData', 'data.pagedata', 'data.pageData', 'data.chlPageData'],],+ vk_recognition: [['steps']],+ grid: [['rows', 'recaptcharows'], ['columns', 'recaptchacols']],};+const RECOGNITION_FAMILIES = new Set([+ 'audio', 'bounding_box', 'coordinates', 'drag_and_drop', 'draw_around',+ 'funcaptcha_recognition', 'grid', 'image_to_text', 'recaptcha_recognition',+ 'rotate', 'temu_recognition', 'vk_recognition', 'yandex_recognition',+ 'aws_recognition', 'vision_engine', 'complex_image', 'funcaptcha_match',+ 'geetest_recognition', 'lemin_recognition',+]);+const RECOGNITION_MEDIA_FIELDS = ['body', 'image', 'images', 'image_data', 'imagesBase64', 'imageUrls', 'audio_data'];function validateFallbackIdentifiers(built) {const family = built[0].contract.family;+ if (built.length > 1 && RECOGNITION_FAMILIES.has(family)) {+ const signatures = built.map(({ task }) => {+ const media = RECOGNITION_MEDIA_FIELDS.flatMap(path => {+ const value = at(task, path);+ if (!usable(value)) return [];+ const kind = path === 'audio_data' ? 'audio' : path === 'imageUrls' ? 'image-url' : 'image';+ return [JSON.stringify([kind, Array.isArray(value) ? value : [value]])];+ });+ if (!media.length || new Set(media).size !== 1) throw new Error('Fallback recognition tasks must use the same observed challenge media.');+ const parts = family === 'temu_recognition'+ ? task.parts || [task.part1, task.part2, task.part3]+ : null;+ return JSON.stringify([media[0], parts]);+ });+ if (new Set(signatures).size > 1) throw new Error('Fallback recognition tasks must use the same observed challenge media.');+ }const groups = [['websiteURL', 'pageurl', 'url'],['websiteKey', 'sitekey', 'googlekey', 'websitePublicKey', 'publickey'],diff --git a/test/captcha-native-providers.mjs b/test/captcha-native-providers.mjsindex 061405010..bfa3d4c18 100644--- a/test/captcha-native-providers.mjs+++ b/test/captcha-native-providers.mjs@@ -226,4 +226,27 @@ for (const browser of ['chrome', 'firefox']) {assert.throws(() => native.prepareNativeCaptchaTasks(enabled, entries), /same observed challenge/);assert.equal(calls.length, 0);});+ const recognitionPairs = [+ ['coordinates body', ['2captcha', 'CoordinatesTask', { body: 'image-A' }], ['solvecaptcha', 'coordinates', { body: 'image-A' }], task => { task.body = 'image-B'; }],+ ['image-to-text array alias', ['2captcha', 'ImageToTextTask', { body: 'image-A' }], ['nopecha', 'recognition/textcaptcha', { image_data: ['image-A'] }], task => { task.image_data = ['image-B']; }],+ ['Temu parts', ['2captcha', 'TemuImageTask', { image: 'background', parts: ['one', 'two', 'three'] }], ['solvecaptcha', 'temuimage', { body: 'background', part1: 'one', part2: 'two', part3: 'three' }], task => { task.part2 = 'different'; }],+ ];+ for (const [label, a, b, change] of recognitionPairs) test(`${browser}: ${label} must match before paid fallback`, t => {+ const calls = mockApi(t, () => { throw new Error('Must not dispatch'); });+ const entries = [a, b].map(([provider, method, parameters]) => ({ provider, method, parameters: structuredClone(parameters) }));+ const enabled = entries.map(entry => ({ id: entry.provider, apiKey: 'key' }));+ assert.equal(native.prepareNativeCaptchaTasks(enabled, entries).length, 2);+ change(entries[1].parameters);+ assert.throws(() => native.prepareNativeCaptchaTasks(enabled, entries), /same observed challenge media/);+ assert.equal(calls.length, 0);+ });+ test(`${browser}: image and audio AWS recognition cannot share a fallback`, () => {+ assert.throws(() => native.prepareNativeCaptchaTasks(+ [{ id: 'capsolver', apiKey: 'key' }, { id: 'nopecha', apiKey: 'key' }],+ [+ { provider: 'capsolver', method: 'AwsWafClassification', parameters: { images: ['same-bytes'], question: 'cars' } },+ { provider: 'nopecha', method: 'recognition/awscaptcha', parameters: { audio_data: ['same-bytes'] } },+ ],+ ), /same observed challenge media/);+ });}diff --git a/test/run.js b/test/run.jsindex ea1f88ba7..a552ae8e2 100644--- a/test/run.js+++ b/test/run.js@@ -122002,6 +122002,10 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structureconst wrongPage = structuredClone(args); wrongPage.providerTasks[0].parameters.websiteURL = 'https://other.test/';const rejected = await agent._executeToolImpl(1, 'solve_captcha', wrongPage);assert.equal(rejected.dispatched, false, build); assert.equal(calls.length, 0, build);+ const wrongPath = structuredClone(args); wrongPath.providerTasks[0].parameters.websiteURL = 'https://example.test/admin';+ const unobserved = await agent._executeToolImpl(1, 'solve_captcha', wrongPath);+ assert.equal(unobserved.dispatched, false, build); assert.match(unobserved.error, /observed frames/);+ assert.equal(calls.length, 0, build);const result = await agent._executeToolImpl(1, 'solve_captcha', args);assert.equal(result.success, true, `${build}: ${result.error}`);assert.equal(result.applicationRequired, true, build); assert.equal(result.solution.lot_number, 'lot', build);