Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Validate CAPTCHA instruction aliases and child frame freshness
Emre Sokulluf852335439d5
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 663897f89..15621974b 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -85,7 +85,7 @@ import { normalizeRuntimeTraceConfig } from '../trace/runtime-config.js';import { tracesToMarkdown } from './trace-export.js';import { hcaptchaParamError } from './captcha-hcaptcha-providers.js';import { getCaptchaCapabilities, prepareNativeCaptchaTasks, solveNativeCaptchaTasks } from './captcha-native-providers.js';-import { applyNativeCaptchaSolution, captureCaptchaDocuments, captchaAnswerDocumentCurrent } from './captcha-solution-application.js';+import { applyNativeCaptchaSolution, captureCaptchaDocuments, captchaAnswerDocumentStatus } from './captcha-solution-application.js';import { solveCaptchaWithProviders, detectCaptcha, injectToken, captchaParamError, captchaTypesMatch, captchaWebsiteUrl } from './captcha-solver.js';import { CAPTCHA_SETTINGS_KEYS, getCaptchaProviders, captchaProviderSupportsType } from './captcha-provider-config.js';import { captchaChallengeKey, captchaChallengeMatcherOptions, detectChallengeDialog, detectChallengeDialogInPage } from './captcha-gate.js';@@ -35749,10 +35749,12 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai// A same-URL reload also invalidates the answer. Only a confirmed// document change retires it; transient inspection failures do not.try {- const current = await captchaAnswerDocumentCurrent(tabId, record, chrome);- if (current === false) {+ const documentStatus = await captchaAnswerDocumentStatus(tabId, record, args, chrome);+ if (documentStatus === 'root_changed' || documentStatus === 'frame_changed') {delete record.solution;- this._captchaGateStates.delete(tabId);+ // A new root may expose a new CAPTCHA. A reloaded child frame in+ // the same root has already used this document's paid dispatch.+ if (documentStatus === 'root_changed') this._captchaGateStates.delete(tabId);return false;}return true;diff --git a/src/chrome/src/agent/captcha-native-providers.js b/src/chrome/src/agent/captcha-native-providers.jsindex c08320363..43b6b2a01 100644--- a/src/chrome/src/agent/captcha-native-providers.js+++ b/src/chrome/src/agent/captcha-native-providers.js@@ -224,7 +224,8 @@ function validateFallbackIdentifiers(built) {const parts = family === 'temu_recognition'? task.parts || [task.part1, task.part2, task.part3]: null;- const instructions = ['comment', 'task', 'question', 'metadata.Task', 'metadata.TaskDefinition', 'metadata.TaskArgument']+ const instructions = ['comment', 'task', 'question', 'imgInstructions', 'imginstructions', 'textinstructions',+ 'metadata.Task', 'metadata.TaskDefinition', 'metadata.TaskArgument'].map(path => at(task, path)).filter(usable);if (new Set(instructions.map(String)).size > 1) throw new Error('Fallback recognition tasks must use the same observed challenge instructions.');return JSON.stringify([media[0], parts, instructions[0] ?? null]);diff --git a/src/chrome/src/agent/captcha-solution-application.js b/src/chrome/src/agent/captcha-solution-application.jsindex 2afcbb7a9..06095ab64 100644--- a/src/chrome/src/agent/captcha-solution-application.js+++ b/src/chrome/src/agent/captcha-solution-application.js@@ -77,18 +77,31 @@ export async function captureCaptchaDocuments(tabId, frames, api = globalThis.brreturn identities.filter(Boolean);}-// A same-URL reload is a different document and cannot reuse its paid answer.-// Null means the current document could not be read, so a transient API error-// does not discard an answer that may still be valid.-export async function captchaAnswerDocumentCurrent(tabId, record, api = globalThis.browser || globalThis.chrome) {+// A same-URL reload of either the root or selected challenge frame invalidates+// its paid answer. Null means inspection was inconclusive, not a confirmed+// reload; transient API errors must not discard a usable answer.+export async function captchaAnswerDocumentStatus(tabId, record, application, api = globalThis.browser || globalThis.chrome) {const tab = await api.tabs.get(tabId);- if (tab?.url !== record.pageUrl) return false;- const expected = record.documents?.find(d => d.frameId === 0 && d.url === record.pageUrl);- if (!expected) return null;- const documents = await captureCaptchaDocuments(tabId, [{ frameId: 0, url: record.pageUrl }], api);- const current = documents.find(d => d.frameId === 0);- if (!current) return null;- return current.url === expected.url && current.timeOrigin === expected.timeOrigin;+ if (tab?.url !== record.pageUrl) return 'root_changed';+ const expectedRoot = record.documents?.find(d => d.frameId === 0 && d.url === record.pageUrl);+ if (!expectedRoot) return null;+ const expectedFrame = record.documents?.find(d => d.frameId === application?.frameId && d.url === application?.frameUrl);+ const selectedFrame = expectedFrame && expectedFrame.frameId !== 0 ? expectedFrame : null;+ const frames = typeof api.webNavigation?.getAllFrames === 'function'+ ? await api.webNavigation.getAllFrames({ tabId }) : null;+ const selectedFrameMissing = selectedFrame && frames+ && !frames.some(f => f.frameId === selectedFrame.frameId && f.url === selectedFrame.url);+ const documents = await captureCaptchaDocuments(tabId, [expectedRoot, ...(selectedFrame ? [selectedFrame] : [])], api);+ const root = documents.find(d => d.frameId === 0);+ if (!root) return null;+ if (root.url !== expectedRoot.url || root.timeOrigin !== expectedRoot.timeOrigin) return 'root_changed';+ if (selectedFrame) {+ if (selectedFrameMissing) return 'frame_changed';+ const currentFrame = documents.find(d => d.frameId === selectedFrame.frameId);+ if (!currentFrame) return null;+ if (currentFrame.url !== selectedFrame.url || currentFrame.timeOrigin !== selectedFrame.timeOrigin) return 'frame_changed';+ }+ return 'current';}// Self-contained for MAIN-world execution in one explicitly selected frame.diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 45503e5ae..c5c658fe5 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -80,7 +80,7 @@ import { normalizeRuntimeTraceConfig } from '../trace/runtime-config.js';import { tracesToMarkdown } from './trace-export.js';import { hcaptchaParamError } from './captcha-hcaptcha-providers.js';import { getCaptchaCapabilities, prepareNativeCaptchaTasks, solveNativeCaptchaTasks } from './captcha-native-providers.js';-import { applyNativeCaptchaSolution, captureCaptchaDocuments, captchaAnswerDocumentCurrent } from './captcha-solution-application.js';+import { applyNativeCaptchaSolution, captureCaptchaDocuments, captchaAnswerDocumentStatus } from './captcha-solution-application.js';import { solveCaptchaWithProviders, detectCaptcha, injectToken, captchaParamError, captchaTypesMatch, captchaWebsiteUrl } from './captcha-solver.js';import { CAPTCHA_SETTINGS_KEYS, getCaptchaProviders, captchaProviderSupportsType } from './captcha-provider-config.js';import { captchaChallengeKey, captchaChallengeMatcherOptions, detectChallengeDialog, detectChallengeDialogInPage } from './captcha-gate.js';@@ -32735,10 +32735,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// A same-URL reload also invalidates the answer. Only a confirmed// document change retires it; transient inspection failures do not.try {- const current = await captchaAnswerDocumentCurrent(tabId, record, browser);- if (current === false) {+ const documentStatus = await captchaAnswerDocumentStatus(tabId, record, args, browser);+ if (documentStatus === 'root_changed' || documentStatus === 'frame_changed') {delete record.solution;- this._captchaGateStates.delete(tabId);+ // A new root may expose a new CAPTCHA. A reloaded child frame in+ // the same root has already used this document's paid dispatch.+ if (documentStatus === 'root_changed') this._captchaGateStates.delete(tabId);return false;}return true;diff --git a/src/firefox/src/agent/captcha-native-providers.js b/src/firefox/src/agent/captcha-native-providers.jsindex c08320363..43b6b2a01 100644--- a/src/firefox/src/agent/captcha-native-providers.js+++ b/src/firefox/src/agent/captcha-native-providers.js@@ -224,7 +224,8 @@ function validateFallbackIdentifiers(built) {const parts = family === 'temu_recognition'? task.parts || [task.part1, task.part2, task.part3]: null;- const instructions = ['comment', 'task', 'question', 'metadata.Task', 'metadata.TaskDefinition', 'metadata.TaskArgument']+ const instructions = ['comment', 'task', 'question', 'imgInstructions', 'imginstructions', 'textinstructions',+ 'metadata.Task', 'metadata.TaskDefinition', 'metadata.TaskArgument'].map(path => at(task, path)).filter(usable);if (new Set(instructions.map(String)).size > 1) throw new Error('Fallback recognition tasks must use the same observed challenge instructions.');return JSON.stringify([media[0], parts, instructions[0] ?? null]);diff --git a/src/firefox/src/agent/captcha-solution-application.js b/src/firefox/src/agent/captcha-solution-application.jsindex 2afcbb7a9..06095ab64 100644--- a/src/firefox/src/agent/captcha-solution-application.js+++ b/src/firefox/src/agent/captcha-solution-application.js@@ -77,18 +77,31 @@ export async function captureCaptchaDocuments(tabId, frames, api = globalThis.brreturn identities.filter(Boolean);}-// A same-URL reload is a different document and cannot reuse its paid answer.-// Null means the current document could not be read, so a transient API error-// does not discard an answer that may still be valid.-export async function captchaAnswerDocumentCurrent(tabId, record, api = globalThis.browser || globalThis.chrome) {+// A same-URL reload of either the root or selected challenge frame invalidates+// its paid answer. Null means inspection was inconclusive, not a confirmed+// reload; transient API errors must not discard a usable answer.+export async function captchaAnswerDocumentStatus(tabId, record, application, api = globalThis.browser || globalThis.chrome) {const tab = await api.tabs.get(tabId);- if (tab?.url !== record.pageUrl) return false;- const expected = record.documents?.find(d => d.frameId === 0 && d.url === record.pageUrl);- if (!expected) return null;- const documents = await captureCaptchaDocuments(tabId, [{ frameId: 0, url: record.pageUrl }], api);- const current = documents.find(d => d.frameId === 0);- if (!current) return null;- return current.url === expected.url && current.timeOrigin === expected.timeOrigin;+ if (tab?.url !== record.pageUrl) return 'root_changed';+ const expectedRoot = record.documents?.find(d => d.frameId === 0 && d.url === record.pageUrl);+ if (!expectedRoot) return null;+ const expectedFrame = record.documents?.find(d => d.frameId === application?.frameId && d.url === application?.frameUrl);+ const selectedFrame = expectedFrame && expectedFrame.frameId !== 0 ? expectedFrame : null;+ const frames = typeof api.webNavigation?.getAllFrames === 'function'+ ? await api.webNavigation.getAllFrames({ tabId }) : null;+ const selectedFrameMissing = selectedFrame && frames+ && !frames.some(f => f.frameId === selectedFrame.frameId && f.url === selectedFrame.url);+ const documents = await captureCaptchaDocuments(tabId, [expectedRoot, ...(selectedFrame ? [selectedFrame] : [])], api);+ const root = documents.find(d => d.frameId === 0);+ if (!root) return null;+ if (root.url !== expectedRoot.url || root.timeOrigin !== expectedRoot.timeOrigin) return 'root_changed';+ if (selectedFrame) {+ if (selectedFrameMissing) return 'frame_changed';+ const currentFrame = documents.find(d => d.frameId === selectedFrame.frameId);+ if (!currentFrame) return null;+ if (currentFrame.url !== selectedFrame.url || currentFrame.timeOrigin !== selectedFrame.timeOrigin) return 'frame_changed';+ }+ return 'current';}// Self-contained for MAIN-world execution in one explicitly selected frame.diff --git a/test/captcha-native-providers.mjs b/test/captcha-native-providers.mjsindex 13cb6fcf1..eda288a8b 100644--- a/test/captcha-native-providers.mjs+++ b/test/captcha-native-providers.mjs@@ -338,6 +338,8 @@ for (const browser of ['chrome', 'firefox']) {['Temu parts', ['2captcha', 'TemuImageTask', { image: 'background', parts: ['one', 'two', 'three'] }], ['solvecaptcha', 'temuimage', { body: 'background', part1: 'one', part2: 'two', part3: 'three' }], task => { task.part2 = 'different'; }],['FunCaptcha instruction', ['2captcha', 'GridTask:funcaptcha_recognition', { body: 'image-A', comment: 'cars' }], ['nopecha', 'recognition/funcaptcha', { image_data: ['image-A'], task: 'cars' }], task => { task.task = 'bicycles'; }],['reCAPTCHA recognition instruction', ['capsolver', 'ReCaptchaV2Classification', { image: 'image-A', question: 'cars' }], ['nopecha', 'recognition/recaptcha', { image_data: ['image-A'], grid: '3x3', task: 'cars' }], task => { task.task = 'bicycles'; }],+ ['coordinate image instruction', ['2captcha', 'CoordinatesTask', { body: 'image-A', imgInstructions: 'cars' }], ['solvecaptcha', 'coordinates', { body: 'image-A', textinstructions: 'cars' }], task => { task.textinstructions = 'bicycles'; }],+ ['coordinate lowercase image instruction', ['2captcha', 'CoordinatesTask', { body: 'image-A', imgInstructions: 'cars' }], ['solvecaptcha', 'coordinates', { body: 'image-A', imginstructions: 'cars' }], task => { task.imginstructions = 'bicycles'; }],];for (const [label, a, b, change] of recognitionPairs) test(`${browser}: ${label} must match before paid fallback`, t => {const calls = mockApi(t, () => { throw new Error('Must not dispatch'); });diff --git a/test/captcha-runtime-policy.mjs b/test/captcha-runtime-policy.mjsindex 54d9430fa..f514b2d42 100644--- a/test/captcha-runtime-policy.mjs+++ b/test/captcha-runtime-policy.mjs@@ -172,4 +172,39 @@ for (const build of ['chrome', 'firefox']) {if (previous === undefined) delete globalThis[key]; else globalThis[key] = previous;}});+ test(`${build}: a same-URL challenge-frame reload retires its answer and requires manual completion`, async () => {+ const agent = agentFor('act', 'full');+ const pageUrl = 'https://example.test/challenge';+ const frameUrl = 'https://captcha.example.test/widget';+ const record = { pageUrl, createdAt: Date.now(), applied: false, solution: { token: 'answer' },+ documents: [{ frameId: 0, url: pageUrl, timeOrigin: 1000 }, { frameId: 2, url: frameUrl, timeOrigin: 2000 }],+ dispatchedTimeOrigins: new Set([1000]) };+ agent._nativeCaptchaSolutions = new Map([[1, record]]);+ agent._captchaGateStates.set(1, { status: 'verification_pending', publicGate: { status: 'verification_pending' } });+ const api = {+ tabs: {+ get: async () => ({ url: pageUrl }),+ executeScript: async (_tabId, options) => [options.code.includes('applyCaptchaValuesInPage')+ ? { success: false, error: 'CAPTCHA frame navigated before application.' }+ : options.frameId === 2 ? { url: frameUrl, timeOrigin: 3000 } : { url: pageUrl, timeOrigin: 1000 }],+ },+ scripting: build === 'chrome' ? { executeScript: async options => options.func.name === 'read'+ ? [{ frameId: 0, result: { url: pageUrl, timeOrigin: 1000 } }, { frameId: 2, result: { url: frameUrl, timeOrigin: 3000 } }]+ : [{ frameId: 2, result: { success: false, error: 'CAPTCHA frame navigated before application.' } }] } : undefined,+ webNavigation: { getAllFrames: async () => [{ frameId: 0, url: pageUrl }, { frameId: 2, url: frameUrl }] },+ };+ const key = build === 'chrome' ? 'chrome' : 'browser';+ const previous = globalThis[key];+ globalThis[key] = api;+ try {+ const result = await agent._executeToolImpl(1, 'apply_captcha_solution', { frameId: 2, frameUrl,+ fields: [{ selector: '#response', path: 'token' }] });+ assert.equal(result.applicationRetryable, false);+ assert.equal(record.solution, undefined);+ assert.equal(record.dispatchedTimeOrigins.has(1000), true);+ assert.equal(agent._captchaSolveGateAfterTool(1, 'apply_captcha_solution', result).status, 'manual_required');+ } finally {+ if (previous === undefined) delete globalThis[key]; else globalThis[key] = previous;+ }+ });}