Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Validate CAPTCHA instruction aliases and child frame freshness

Emre Sokulluf852335439d5
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 663897f89..15621974b 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -85,7 +85,7 @@ import { normalizeRuntimeTraceConfig } from '../trace/runtime-config.js';
import { tracesToMarkdown } from './trace-export.js';
import { hcaptchaParamError } from './captcha-hcaptcha-providers.js';
import { getCaptchaCapabilities, prepareNativeCaptchaTasks, solveNativeCaptchaTasks } from './captcha-native-providers.js';
-import { applyNativeCaptchaSolution, captureCaptchaDocuments, captchaAnswerDocumentCurrent } from './captcha-solution-application.js';
+import { applyNativeCaptchaSolution, captureCaptchaDocuments, captchaAnswerDocumentStatus } from './captcha-solution-application.js';
import { solveCaptchaWithProviders, detectCaptcha, injectToken, captchaParamError, captchaTypesMatch, captchaWebsiteUrl } from './captcha-solver.js';
import { CAPTCHA_SETTINGS_KEYS, getCaptchaProviders, captchaProviderSupportsType } from './captcha-provider-config.js';
import { captchaChallengeKey, captchaChallengeMatcherOptions, detectChallengeDialog, detectChallengeDialogInPage } from './captcha-gate.js';
@@ -35749,10 +35749,12 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
// A same-URL reload also invalidates the answer. Only a confirmed
// document change retires it; transient inspection failures do not.
try {
- const current = await captchaAnswerDocumentCurrent(tabId, record, chrome);
- if (current === false) {
+ const documentStatus = await captchaAnswerDocumentStatus(tabId, record, args, chrome);
+ if (documentStatus === 'root_changed' || documentStatus === 'frame_changed') {
delete record.solution;
- this._captchaGateStates.delete(tabId);
+ // A new root may expose a new CAPTCHA. A reloaded child frame in
+ // the same root has already used this document's paid dispatch.
+ if (documentStatus === 'root_changed') this._captchaGateStates.delete(tabId);
return false;
}
return true;
diff --git a/src/chrome/src/agent/captcha-native-providers.js b/src/chrome/src/agent/captcha-native-providers.js
index c08320363..43b6b2a01 100644
--- a/src/chrome/src/agent/captcha-native-providers.js
+++ b/src/chrome/src/agent/captcha-native-providers.js
@@ -224,7 +224,8 @@ function validateFallbackIdentifiers(built) {
const parts = family === 'temu_recognition'
? task.parts || [task.part1, task.part2, task.part3]
: null;
- const instructions = ['comment', 'task', 'question', 'metadata.Task', 'metadata.TaskDefinition', 'metadata.TaskArgument']
+ const instructions = ['comment', 'task', 'question', 'imgInstructions', 'imginstructions', 'textinstructions',
+ 'metadata.Task', 'metadata.TaskDefinition', 'metadata.TaskArgument']
.map(path => at(task, path)).filter(usable);
if (new Set(instructions.map(String)).size > 1) throw new Error('Fallback recognition tasks must use the same observed challenge instructions.');
return JSON.stringify([media[0], parts, instructions[0] ?? null]);
diff --git a/src/chrome/src/agent/captcha-solution-application.js b/src/chrome/src/agent/captcha-solution-application.js
index 2afcbb7a9..06095ab64 100644
--- a/src/chrome/src/agent/captcha-solution-application.js
+++ b/src/chrome/src/agent/captcha-solution-application.js
@@ -77,18 +77,31 @@ export async function captureCaptchaDocuments(tabId, frames, api = globalThis.br
return identities.filter(Boolean);
}
-// A same-URL reload is a different document and cannot reuse its paid answer.
-// Null means the current document could not be read, so a transient API error
-// does not discard an answer that may still be valid.
-export async function captchaAnswerDocumentCurrent(tabId, record, api = globalThis.browser || globalThis.chrome) {
+// A same-URL reload of either the root or selected challenge frame invalidates
+// its paid answer. Null means inspection was inconclusive, not a confirmed
+// reload; transient API errors must not discard a usable answer.
+export async function captchaAnswerDocumentStatus(tabId, record, application, api = globalThis.browser || globalThis.chrome) {
const tab = await api.tabs.get(tabId);
- if (tab?.url !== record.pageUrl) return false;
- const expected = record.documents?.find(d => d.frameId === 0 && d.url === record.pageUrl);
- if (!expected) return null;
- const documents = await captureCaptchaDocuments(tabId, [{ frameId: 0, url: record.pageUrl }], api);
- const current = documents.find(d => d.frameId === 0);
- if (!current) return null;
- return current.url === expected.url && current.timeOrigin === expected.timeOrigin;
+ if (tab?.url !== record.pageUrl) return 'root_changed';
+ const expectedRoot = record.documents?.find(d => d.frameId === 0 && d.url === record.pageUrl);
+ if (!expectedRoot) return null;
+ const expectedFrame = record.documents?.find(d => d.frameId === application?.frameId && d.url === application?.frameUrl);
+ const selectedFrame = expectedFrame && expectedFrame.frameId !== 0 ? expectedFrame : null;
+ const frames = typeof api.webNavigation?.getAllFrames === 'function'
+ ? await api.webNavigation.getAllFrames({ tabId }) : null;
+ const selectedFrameMissing = selectedFrame && frames
+ && !frames.some(f => f.frameId === selectedFrame.frameId && f.url === selectedFrame.url);
+ const documents = await captureCaptchaDocuments(tabId, [expectedRoot, ...(selectedFrame ? [selectedFrame] : [])], api);
+ const root = documents.find(d => d.frameId === 0);
+ if (!root) return null;
+ if (root.url !== expectedRoot.url || root.timeOrigin !== expectedRoot.timeOrigin) return 'root_changed';
+ if (selectedFrame) {
+ if (selectedFrameMissing) return 'frame_changed';
+ const currentFrame = documents.find(d => d.frameId === selectedFrame.frameId);
+ if (!currentFrame) return null;
+ if (currentFrame.url !== selectedFrame.url || currentFrame.timeOrigin !== selectedFrame.timeOrigin) return 'frame_changed';
+ }
+ return 'current';
}
// Self-contained for MAIN-world execution in one explicitly selected frame.
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 45503e5ae..c5c658fe5 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -80,7 +80,7 @@ import { normalizeRuntimeTraceConfig } from '../trace/runtime-config.js';
import { tracesToMarkdown } from './trace-export.js';
import { hcaptchaParamError } from './captcha-hcaptcha-providers.js';
import { getCaptchaCapabilities, prepareNativeCaptchaTasks, solveNativeCaptchaTasks } from './captcha-native-providers.js';
-import { applyNativeCaptchaSolution, captureCaptchaDocuments, captchaAnswerDocumentCurrent } from './captcha-solution-application.js';
+import { applyNativeCaptchaSolution, captureCaptchaDocuments, captchaAnswerDocumentStatus } from './captcha-solution-application.js';
import { solveCaptchaWithProviders, detectCaptcha, injectToken, captchaParamError, captchaTypesMatch, captchaWebsiteUrl } from './captcha-solver.js';
import { CAPTCHA_SETTINGS_KEYS, getCaptchaProviders, captchaProviderSupportsType } from './captcha-provider-config.js';
import { captchaChallengeKey, captchaChallengeMatcherOptions, detectChallengeDialog, detectChallengeDialogInPage } from './captcha-gate.js';
@@ -32735,10 +32735,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// A same-URL reload also invalidates the answer. Only a confirmed
// document change retires it; transient inspection failures do not.
try {
- const current = await captchaAnswerDocumentCurrent(tabId, record, browser);
- if (current === false) {
+ const documentStatus = await captchaAnswerDocumentStatus(tabId, record, args, browser);
+ if (documentStatus === 'root_changed' || documentStatus === 'frame_changed') {
delete record.solution;
- this._captchaGateStates.delete(tabId);
+ // A new root may expose a new CAPTCHA. A reloaded child frame in
+ // the same root has already used this document's paid dispatch.
+ if (documentStatus === 'root_changed') this._captchaGateStates.delete(tabId);
return false;
}
return true;
diff --git a/src/firefox/src/agent/captcha-native-providers.js b/src/firefox/src/agent/captcha-native-providers.js
index c08320363..43b6b2a01 100644
--- a/src/firefox/src/agent/captcha-native-providers.js
+++ b/src/firefox/src/agent/captcha-native-providers.js
@@ -224,7 +224,8 @@ function validateFallbackIdentifiers(built) {
const parts = family === 'temu_recognition'
? task.parts || [task.part1, task.part2, task.part3]
: null;
- const instructions = ['comment', 'task', 'question', 'metadata.Task', 'metadata.TaskDefinition', 'metadata.TaskArgument']
+ const instructions = ['comment', 'task', 'question', 'imgInstructions', 'imginstructions', 'textinstructions',
+ 'metadata.Task', 'metadata.TaskDefinition', 'metadata.TaskArgument']
.map(path => at(task, path)).filter(usable);
if (new Set(instructions.map(String)).size > 1) throw new Error('Fallback recognition tasks must use the same observed challenge instructions.');
return JSON.stringify([media[0], parts, instructions[0] ?? null]);
diff --git a/src/firefox/src/agent/captcha-solution-application.js b/src/firefox/src/agent/captcha-solution-application.js
index 2afcbb7a9..06095ab64 100644
--- a/src/firefox/src/agent/captcha-solution-application.js
+++ b/src/firefox/src/agent/captcha-solution-application.js
@@ -77,18 +77,31 @@ export async function captureCaptchaDocuments(tabId, frames, api = globalThis.br
return identities.filter(Boolean);
}
-// A same-URL reload is a different document and cannot reuse its paid answer.
-// Null means the current document could not be read, so a transient API error
-// does not discard an answer that may still be valid.
-export async function captchaAnswerDocumentCurrent(tabId, record, api = globalThis.browser || globalThis.chrome) {
+// A same-URL reload of either the root or selected challenge frame invalidates
+// its paid answer. Null means inspection was inconclusive, not a confirmed
+// reload; transient API errors must not discard a usable answer.
+export async function captchaAnswerDocumentStatus(tabId, record, application, api = globalThis.browser || globalThis.chrome) {
const tab = await api.tabs.get(tabId);
- if (tab?.url !== record.pageUrl) return false;
- const expected = record.documents?.find(d => d.frameId === 0 && d.url === record.pageUrl);
- if (!expected) return null;
- const documents = await captureCaptchaDocuments(tabId, [{ frameId: 0, url: record.pageUrl }], api);
- const current = documents.find(d => d.frameId === 0);
- if (!current) return null;
- return current.url === expected.url && current.timeOrigin === expected.timeOrigin;
+ if (tab?.url !== record.pageUrl) return 'root_changed';
+ const expectedRoot = record.documents?.find(d => d.frameId === 0 && d.url === record.pageUrl);
+ if (!expectedRoot) return null;
+ const expectedFrame = record.documents?.find(d => d.frameId === application?.frameId && d.url === application?.frameUrl);
+ const selectedFrame = expectedFrame && expectedFrame.frameId !== 0 ? expectedFrame : null;
+ const frames = typeof api.webNavigation?.getAllFrames === 'function'
+ ? await api.webNavigation.getAllFrames({ tabId }) : null;
+ const selectedFrameMissing = selectedFrame && frames
+ && !frames.some(f => f.frameId === selectedFrame.frameId && f.url === selectedFrame.url);
+ const documents = await captureCaptchaDocuments(tabId, [expectedRoot, ...(selectedFrame ? [selectedFrame] : [])], api);
+ const root = documents.find(d => d.frameId === 0);
+ if (!root) return null;
+ if (root.url !== expectedRoot.url || root.timeOrigin !== expectedRoot.timeOrigin) return 'root_changed';
+ if (selectedFrame) {
+ if (selectedFrameMissing) return 'frame_changed';
+ const currentFrame = documents.find(d => d.frameId === selectedFrame.frameId);
+ if (!currentFrame) return null;
+ if (currentFrame.url !== selectedFrame.url || currentFrame.timeOrigin !== selectedFrame.timeOrigin) return 'frame_changed';
+ }
+ return 'current';
}
// Self-contained for MAIN-world execution in one explicitly selected frame.
diff --git a/test/captcha-native-providers.mjs b/test/captcha-native-providers.mjs
index 13cb6fcf1..eda288a8b 100644
--- a/test/captcha-native-providers.mjs
+++ b/test/captcha-native-providers.mjs
@@ -338,6 +338,8 @@ for (const browser of ['chrome', 'firefox']) {
['Temu parts', ['2captcha', 'TemuImageTask', { image: 'background', parts: ['one', 'two', 'three'] }], ['solvecaptcha', 'temuimage', { body: 'background', part1: 'one', part2: 'two', part3: 'three' }], task => { task.part2 = 'different'; }],
['FunCaptcha instruction', ['2captcha', 'GridTask:funcaptcha_recognition', { body: 'image-A', comment: 'cars' }], ['nopecha', 'recognition/funcaptcha', { image_data: ['image-A'], task: 'cars' }], task => { task.task = 'bicycles'; }],
['reCAPTCHA recognition instruction', ['capsolver', 'ReCaptchaV2Classification', { image: 'image-A', question: 'cars' }], ['nopecha', 'recognition/recaptcha', { image_data: ['image-A'], grid: '3x3', task: 'cars' }], task => { task.task = 'bicycles'; }],
+ ['coordinate image instruction', ['2captcha', 'CoordinatesTask', { body: 'image-A', imgInstructions: 'cars' }], ['solvecaptcha', 'coordinates', { body: 'image-A', textinstructions: 'cars' }], task => { task.textinstructions = 'bicycles'; }],
+ ['coordinate lowercase image instruction', ['2captcha', 'CoordinatesTask', { body: 'image-A', imgInstructions: 'cars' }], ['solvecaptcha', 'coordinates', { body: 'image-A', imginstructions: 'cars' }], task => { task.imginstructions = 'bicycles'; }],
];
for (const [label, a, b, change] of recognitionPairs) test(`${browser}: ${label} must match before paid fallback`, t => {
const calls = mockApi(t, () => { throw new Error('Must not dispatch'); });
diff --git a/test/captcha-runtime-policy.mjs b/test/captcha-runtime-policy.mjs
index 54d9430fa..f514b2d42 100644
--- a/test/captcha-runtime-policy.mjs
+++ b/test/captcha-runtime-policy.mjs
@@ -172,4 +172,39 @@ for (const build of ['chrome', 'firefox']) {
if (previous === undefined) delete globalThis[key]; else globalThis[key] = previous;
}
});
+ test(`${build}: a same-URL challenge-frame reload retires its answer and requires manual completion`, async () => {
+ const agent = agentFor('act', 'full');
+ const pageUrl = 'https://example.test/challenge';
+ const frameUrl = 'https://captcha.example.test/widget';
+ const record = { pageUrl, createdAt: Date.now(), applied: false, solution: { token: 'answer' },
+ documents: [{ frameId: 0, url: pageUrl, timeOrigin: 1000 }, { frameId: 2, url: frameUrl, timeOrigin: 2000 }],
+ dispatchedTimeOrigins: new Set([1000]) };
+ agent._nativeCaptchaSolutions = new Map([[1, record]]);
+ agent._captchaGateStates.set(1, { status: 'verification_pending', publicGate: { status: 'verification_pending' } });
+ const api = {
+ tabs: {
+ get: async () => ({ url: pageUrl }),
+ executeScript: async (_tabId, options) => [options.code.includes('applyCaptchaValuesInPage')
+ ? { success: false, error: 'CAPTCHA frame navigated before application.' }
+ : options.frameId === 2 ? { url: frameUrl, timeOrigin: 3000 } : { url: pageUrl, timeOrigin: 1000 }],
+ },
+ scripting: build === 'chrome' ? { executeScript: async options => options.func.name === 'read'
+ ? [{ frameId: 0, result: { url: pageUrl, timeOrigin: 1000 } }, { frameId: 2, result: { url: frameUrl, timeOrigin: 3000 } }]
+ : [{ frameId: 2, result: { success: false, error: 'CAPTCHA frame navigated before application.' } }] } : undefined,
+ webNavigation: { getAllFrames: async () => [{ frameId: 0, url: pageUrl }, { frameId: 2, url: frameUrl }] },
+ };
+ const key = build === 'chrome' ? 'chrome' : 'browser';
+ const previous = globalThis[key];
+ globalThis[key] = api;
+ try {
+ const result = await agent._executeToolImpl(1, 'apply_captcha_solution', { frameId: 2, frameUrl,
+ fields: [{ selector: '#response', path: 'token' }] });
+ assert.equal(result.applicationRetryable, false);
+ assert.equal(record.solution, undefined);
+ assert.equal(record.dispatchedTimeOrigins.has(1000), true);
+ assert.equal(agent._captchaSolveGateAfterTool(1, 'apply_captcha_solution', result).status, 'manual_required');
+ } finally {
+ if (previous === undefined) delete globalThis[key]; else globalThis[key] = previous;
+ }
+ });
}