Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Bind native CAPTCHA fallbacks to observed media and frame URLs

Emre Sokullu6d7670976d15
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index e9ef67e45..f825e1a4a 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -35770,18 +35770,21 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
if (args.inject !== false) return noDispatchFailure('Native CAPTCHA methods require inject: false. Apply the returned structured answer with apply_captcha_solution after inspecting the page.');
const prepared = prepareNativeCaptchaTasks(providers, args.providerTasks);
const frames = typeof chrome.webNavigation?.getAllFrames === 'function' ? await chrome.webNavigation.getAllFrames({ tabId }) : [];
- const origins = new Set([websiteURL, ...frames.map(frame => frame.url)].flatMap(url => { try { return [new URL(url).origin]; } catch { return []; } }));
- for (const { task } of prepared) {
- for (const key of ['websiteURL', 'pageurl', 'url']) {
- if (task[key] && (!/^https?:/.test(task[key]) || !origins.has(new URL(task[key]).origin))) return noDispatchFailure('Native CAPTCHA page URLs must belong to the active tab or one of its observed frames.');
- }
- }
this._nativeCaptchaSolutions ||= new Map();
// Record dispatch before sending. Even timeout/failure cannot trigger
// a second call for this document; fallback stays inside one call.
const documents = await captureCaptchaDocuments(tabId, frames, chrome);
const rootDocument = documents.find(d => d.frameId === 0 && d.url === websiteURL);
if (!rootDocument) return noDispatchFailure('Could not bind this page document before the native CAPTCHA request.');
+ const normalizedPageUrl = value => {
+ try { const parsed = new URL(value); return ['http:', 'https:'].includes(parsed.protocol) ? parsed.href : null; } catch { return null; }
+ };
+ const observedUrls = new Set(documents.map(d => normalizedPageUrl(d.url)).filter(Boolean));
+ for (const { task } of prepared) {
+ for (const key of ['websiteURL', 'pageurl', 'url']) {
+ if (task[key] && !observedUrls.has(normalizedPageUrl(task[key]))) return noDispatchFailure('Native CAPTCHA page URLs must match the active tab or one of its observed frames.');
+ }
+ }
const previous = this._nativeCaptchaSolutions.get(tabId);
const dispatchedTimeOrigins = previous?.dispatchedTimeOrigins || new Set(previous?.documents
?.filter(d => d.frameId === 0).map(d => d.timeOrigin) || []);
diff --git a/src/chrome/src/agent/captcha-native-providers.js b/src/chrome/src/agent/captcha-native-providers.js
index 337e112df..10a735eae 100644
--- a/src/chrome/src/agent/captcha-native-providers.js
+++ b/src/chrome/src/agent/captcha-native-providers.js
@@ -127,9 +127,35 @@ const FAMILY_IDENTIFIERS = {
['data', 'cData', 'metadata.cdata', 'data.cdata', 'data.cData'],
['pagedata', 'pageData', 'chlPageData', 'data.pagedata', 'data.pageData', 'data.chlPageData'],
],
+ vk_recognition: [['steps']],
+ grid: [['rows', 'recaptcharows'], ['columns', 'recaptchacols']],
};
+const RECOGNITION_FAMILIES = new Set([
+ 'audio', 'bounding_box', 'coordinates', 'drag_and_drop', 'draw_around',
+ 'funcaptcha_recognition', 'grid', 'image_to_text', 'recaptcha_recognition',
+ 'rotate', 'temu_recognition', 'vk_recognition', 'yandex_recognition',
+ 'aws_recognition', 'vision_engine', 'complex_image', 'funcaptcha_match',
+ 'geetest_recognition', 'lemin_recognition',
+]);
+const RECOGNITION_MEDIA_FIELDS = ['body', 'image', 'images', 'image_data', 'imagesBase64', 'imageUrls', 'audio_data'];
function validateFallbackIdentifiers(built) {
const family = built[0].contract.family;
+ if (built.length > 1 && RECOGNITION_FAMILIES.has(family)) {
+ const signatures = built.map(({ task }) => {
+ const media = RECOGNITION_MEDIA_FIELDS.flatMap(path => {
+ const value = at(task, path);
+ if (!usable(value)) return [];
+ const kind = path === 'audio_data' ? 'audio' : path === 'imageUrls' ? 'image-url' : 'image';
+ return [JSON.stringify([kind, Array.isArray(value) ? value : [value]])];
+ });
+ if (!media.length || new Set(media).size !== 1) throw new Error('Fallback recognition tasks must use the same observed challenge media.');
+ const parts = family === 'temu_recognition'
+ ? task.parts || [task.part1, task.part2, task.part3]
+ : null;
+ return JSON.stringify([media[0], parts]);
+ });
+ if (new Set(signatures).size > 1) throw new Error('Fallback recognition tasks must use the same observed challenge media.');
+ }
const groups = [
['websiteURL', 'pageurl', 'url'],
['websiteKey', 'sitekey', 'googlekey', 'websitePublicKey', 'publickey'],
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index fcf03d766..d6e8cd75b 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -32753,18 +32753,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (args.inject !== false) return noDispatchFailure('Native CAPTCHA methods require inject: false. Apply the returned structured answer with apply_captcha_solution after inspecting the page.');
const prepared = prepareNativeCaptchaTasks(providers, args.providerTasks);
const frames = typeof browser.webNavigation?.getAllFrames === 'function' ? await browser.webNavigation.getAllFrames({ tabId }) : [];
- const origins = new Set([websiteURL, ...frames.map(frame => frame.url)].flatMap(url => { try { return [new URL(url).origin]; } catch { return []; } }));
- for (const { task } of prepared) {
- for (const key of ['websiteURL', 'pageurl', 'url']) {
- if (task[key] && (!/^https?:/.test(task[key]) || !origins.has(new URL(task[key]).origin))) return noDispatchFailure('Native CAPTCHA page URLs must belong to the active tab or one of its observed frames.');
- }
- }
this._nativeCaptchaSolutions ||= new Map();
// Record dispatch before sending. Even timeout/failure cannot trigger
// a second call for this document; fallback stays inside one call.
const documents = await captureCaptchaDocuments(tabId, frames, browser);
const rootDocument = documents.find(d => d.frameId === 0 && d.url === websiteURL);
if (!rootDocument) return noDispatchFailure('Could not bind this page document before the native CAPTCHA request.');
+ const normalizedPageUrl = value => {
+ try { const parsed = new URL(value); return ['http:', 'https:'].includes(parsed.protocol) ? parsed.href : null; } catch { return null; }
+ };
+ const observedUrls = new Set(documents.map(d => normalizedPageUrl(d.url)).filter(Boolean));
+ for (const { task } of prepared) {
+ for (const key of ['websiteURL', 'pageurl', 'url']) {
+ if (task[key] && !observedUrls.has(normalizedPageUrl(task[key]))) return noDispatchFailure('Native CAPTCHA page URLs must match the active tab or one of its observed frames.');
+ }
+ }
const previous = this._nativeCaptchaSolutions.get(tabId);
const dispatchedTimeOrigins = previous?.dispatchedTimeOrigins || new Set(previous?.documents
?.filter(d => d.frameId === 0).map(d => d.timeOrigin) || []);
diff --git a/src/firefox/src/agent/captcha-native-providers.js b/src/firefox/src/agent/captcha-native-providers.js
index 337e112df..10a735eae 100644
--- a/src/firefox/src/agent/captcha-native-providers.js
+++ b/src/firefox/src/agent/captcha-native-providers.js
@@ -127,9 +127,35 @@ const FAMILY_IDENTIFIERS = {
['data', 'cData', 'metadata.cdata', 'data.cdata', 'data.cData'],
['pagedata', 'pageData', 'chlPageData', 'data.pagedata', 'data.pageData', 'data.chlPageData'],
],
+ vk_recognition: [['steps']],
+ grid: [['rows', 'recaptcharows'], ['columns', 'recaptchacols']],
};
+const RECOGNITION_FAMILIES = new Set([
+ 'audio', 'bounding_box', 'coordinates', 'drag_and_drop', 'draw_around',
+ 'funcaptcha_recognition', 'grid', 'image_to_text', 'recaptcha_recognition',
+ 'rotate', 'temu_recognition', 'vk_recognition', 'yandex_recognition',
+ 'aws_recognition', 'vision_engine', 'complex_image', 'funcaptcha_match',
+ 'geetest_recognition', 'lemin_recognition',
+]);
+const RECOGNITION_MEDIA_FIELDS = ['body', 'image', 'images', 'image_data', 'imagesBase64', 'imageUrls', 'audio_data'];
function validateFallbackIdentifiers(built) {
const family = built[0].contract.family;
+ if (built.length > 1 && RECOGNITION_FAMILIES.has(family)) {
+ const signatures = built.map(({ task }) => {
+ const media = RECOGNITION_MEDIA_FIELDS.flatMap(path => {
+ const value = at(task, path);
+ if (!usable(value)) return [];
+ const kind = path === 'audio_data' ? 'audio' : path === 'imageUrls' ? 'image-url' : 'image';
+ return [JSON.stringify([kind, Array.isArray(value) ? value : [value]])];
+ });
+ if (!media.length || new Set(media).size !== 1) throw new Error('Fallback recognition tasks must use the same observed challenge media.');
+ const parts = family === 'temu_recognition'
+ ? task.parts || [task.part1, task.part2, task.part3]
+ : null;
+ return JSON.stringify([media[0], parts]);
+ });
+ if (new Set(signatures).size > 1) throw new Error('Fallback recognition tasks must use the same observed challenge media.');
+ }
const groups = [
['websiteURL', 'pageurl', 'url'],
['websiteKey', 'sitekey', 'googlekey', 'websitePublicKey', 'publickey'],
diff --git a/test/captcha-native-providers.mjs b/test/captcha-native-providers.mjs
index 061405010..bfa3d4c18 100644
--- a/test/captcha-native-providers.mjs
+++ b/test/captcha-native-providers.mjs
@@ -226,4 +226,27 @@ for (const browser of ['chrome', 'firefox']) {
assert.throws(() => native.prepareNativeCaptchaTasks(enabled, entries), /same observed challenge/);
assert.equal(calls.length, 0);
});
+ const recognitionPairs = [
+ ['coordinates body', ['2captcha', 'CoordinatesTask', { body: 'image-A' }], ['solvecaptcha', 'coordinates', { body: 'image-A' }], task => { task.body = 'image-B'; }],
+ ['image-to-text array alias', ['2captcha', 'ImageToTextTask', { body: 'image-A' }], ['nopecha', 'recognition/textcaptcha', { image_data: ['image-A'] }], task => { task.image_data = ['image-B']; }],
+ ['Temu parts', ['2captcha', 'TemuImageTask', { image: 'background', parts: ['one', 'two', 'three'] }], ['solvecaptcha', 'temuimage', { body: 'background', part1: 'one', part2: 'two', part3: 'three' }], task => { task.part2 = 'different'; }],
+ ];
+ for (const [label, a, b, change] of recognitionPairs) test(`${browser}: ${label} must match before paid fallback`, t => {
+ const calls = mockApi(t, () => { throw new Error('Must not dispatch'); });
+ const entries = [a, b].map(([provider, method, parameters]) => ({ provider, method, parameters: structuredClone(parameters) }));
+ const enabled = entries.map(entry => ({ id: entry.provider, apiKey: 'key' }));
+ assert.equal(native.prepareNativeCaptchaTasks(enabled, entries).length, 2);
+ change(entries[1].parameters);
+ assert.throws(() => native.prepareNativeCaptchaTasks(enabled, entries), /same observed challenge media/);
+ assert.equal(calls.length, 0);
+ });
+ test(`${browser}: image and audio AWS recognition cannot share a fallback`, () => {
+ assert.throws(() => native.prepareNativeCaptchaTasks(
+ [{ id: 'capsolver', apiKey: 'key' }, { id: 'nopecha', apiKey: 'key' }],
+ [
+ { provider: 'capsolver', method: 'AwsWafClassification', parameters: { images: ['same-bytes'], question: 'cars' } },
+ { provider: 'nopecha', method: 'recognition/awscaptcha', parameters: { audio_data: ['same-bytes'] } },
+ ],
+ ), /same observed challenge media/);
+ });
}
diff --git a/test/run.js b/test/run.js
index ea1f88ba7..a552ae8e2 100644
--- a/test/run.js
+++ b/test/run.js
@@ -122002,6 +122002,10 @@ test('CAPTCHA native methods dispatch through the real agent, preserve structure
const wrongPage = structuredClone(args); wrongPage.providerTasks[0].parameters.websiteURL = 'https://other.test/';
const rejected = await agent._executeToolImpl(1, 'solve_captcha', wrongPage);
assert.equal(rejected.dispatched, false, build); assert.equal(calls.length, 0, build);
+ const wrongPath = structuredClone(args); wrongPath.providerTasks[0].parameters.websiteURL = 'https://example.test/admin';
+ const unobserved = await agent._executeToolImpl(1, 'solve_captcha', wrongPath);
+ assert.equal(unobserved.dispatched, false, build); assert.match(unobserved.error, /observed frames/);
+ assert.equal(calls.length, 0, build);
const result = await agent._executeToolImpl(1, 'solve_captcha', args);
assert.equal(result.success, true, `${build}: ${result.error}`);
assert.equal(result.applicationRequired, true, build); assert.equal(result.solution.lot_number, 'lot', build);