Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Vision mode: CAPTCHA runtime notes end at the user, not at removed tools

Emre Sokullu62dc97677dce
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index b9332659b..8ddfe71de 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -22,7 +22,7 @@ import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationCon
import { mastodonPublicationIntentMessages, normalizeMastodonPublicationIntent, captureNativeMastodonPublication, mastodonPublicationBinding, mastodonPublicationReceipt, mastodonPublicationNavigation } from './mastodon-publication.js';
import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, RETIRED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_BATCHING, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_LEAN_NOTE, SYSTEM_PROMPT_GENERATIVE_MEDIA, SYSTEM_PROMPT_DEV_APPENDIX, SYSTEM_PROMPT_WEBMCP_ASK, SYSTEM_PROMPT_WEBMCP_ACT, getVisionActPrompt } from './tools.js';
import { installVisionRuntime } from './vision-runtime.js';
-import { VISION_NATIVE_TOOLS, VISION_SITE_NOTES_PREFACE, visionCaptchaSolverNote, visionFilePickerBlockedResult, visionUniversalPreamble } from './vision-tools.js';
+import { VISION_CAPTCHA_MANUAL_NOTE, VISION_CAPTCHA_UNAPPLIED_NOTE, VISION_NATIVE_TOOLS, VISION_SITE_NOTES_PREFACE, visionCaptchaSolverNote, visionFilePickerBlockedResult, visionUniversalPreamble } from './vision-tools.js';
import { visionDomProbeExpression } from './vision-dom-probe.js';
import { normalizeVisionModeSetting } from './vision-mode.js';
import { rejectedCompletionRecovery } from './completion-recovery.js';
@@ -14569,6 +14569,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
_captchaRoutingMessage(tabId, captchaGateDecision, captchaSolveOutcome, toolResult = {}, onUpdate = () => {}) {
+ // A vision run cannot apply a native answer: its notes end at the user.
+ const visionRun = this._visionRunState?.(tabId)?.active === true;
let resultContent = '';
const gate = captchaGateDecision || captchaSolveOutcome;
if (gate && gate.status !== 'cleared' && !this._captchaToolsAvailable(tabId)) {
@@ -14582,12 +14584,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
: captchaGateDecision.nativeDispatchInterrupted === true
? '\n[TRUSTED CAPTCHA GATE: A native solve was already dispatched for this document, but its answer is unavailable after restart. Do not send another paid request or submit the page; ask the user to complete the challenge manually.]'
: captchaGateDecision.cloudflareManagedChallenge === true
- ? '\n[TRUSTED CAPTCHA GATE: A response-backed full-page Cloudflare managed challenge is active. Inspect get_captcha_capabilities for a Cloudflare native method; use one solve_captcha providerTasks dispatch only with all required observed inputs, otherwise ask for manual completion. Do not submit; the network/navigation monitor will clear the gate when Cloudflare resumes the destination.]'
+ ? (visionRun ? VISION_CAPTCHA_MANUAL_NOTE : '\n[TRUSTED CAPTCHA GATE: A response-backed full-page Cloudflare managed challenge is active. Inspect get_captcha_capabilities for a Cloudflare native method; use one solve_captcha providerTasks dispatch only with all required observed inputs, otherwise ask for manual completion. Do not submit; the network/navigation monitor will clear the gate when Cloudflare resumes the destination.]')
: captchaGateDecision.awsWaf === true
? '\n[TRUSTED CAPTCHA GATE: AWS verification requires manual completion because no untried compatible provider or complete observed input set is available. Do not buy another answer for this unresolved challenge. After manual completion, read the page so the runtime can resume.]'
: captchaGateDecision.activeChallengeAfterSolve === true
? '\n[TRUSTED CAPTCHA GATE: The active challenge frame is visible again after the one automatic solve. The site may have rejected the token. Do not call solve_captcha again; stop automation and ask the user to complete the challenge manually.]'
- : '\n[TRUSTED CAPTCHA GATE: A verification challenge is active, but no safely selectable automatic widget was detected. Inspect get_captcha_capabilities and use one native providerTasks solve only if its required parameters can be observed. Otherwise ask for manual completion. Do not dismiss, close, or resubmit the challenge.]';
+ : (visionRun ? VISION_CAPTCHA_MANUAL_NOTE : '\n[TRUSTED CAPTCHA GATE: A verification challenge is active, but no safely selectable automatic widget was detected. Inspect get_captcha_capabilities and use one native providerTasks solve only if its required parameters can be observed. Otherwise ask for manual completion. Do not dismiss, close, or resubmit the challenge.]');
onUpdate('warning', { message: 'Verification challenge requires manual completion.' });
} else if (captchaGateDecision?.status === 'cleared') {
if (captchaGateDecision.clearedByNavigation === true) {
@@ -14603,13 +14605,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
} else if ((captchaGateDecision || captchaSolveOutcome)?.status === 'verification_pending'
&& this._hasUnappliedNativeCaptchaSolution(tabId)) {
- resultContent += '\n[TRUSTED CAPTCHA GATE: The native answer is ready but has not been applied. If the prior tool result is no longer in this chat, call get_captcha_capabilities to retrieve the pending paid answer without another solve. Inspect the page and use apply_captcha_solution, then verify fresh page state.]';
+ resultContent += visionRun ? VISION_CAPTCHA_UNAPPLIED_NOTE : '\n[TRUSTED CAPTCHA GATE: The native answer is ready but has not been applied. If the prior tool result is no longer in this chat, call get_captcha_capabilities to retrieve the pending paid answer without another solve. Inspect the page and use apply_captcha_solution, then verify fresh page state.]';
} else if (toolResult?.type === 'aws_waf' && toolResult?.injected === true) {
resultContent += '\n[TRUSTED CAPTCHA GATE: The AWS cookie was applied. Reload the observed page once with navigate, then read it to verify clearance. Do not call solve_captcha again for this unresolved challenge.]';
} else if (captchaGateDecision?.status === 'verification_pending') {
resultContent += '\n[TRUSTED CAPTCHA GATE: Verification is still pending because the exact widget has no response token or its frame state could not be inspected conclusively. Wait briefly, then read the page again. Do not submit, dismiss, or call solve_captcha again.]';
} else if (toolResult?.applicationRequired === true && captchaSolveOutcome?.status === 'verification_pending') {
- resultContent += '\n[TRUSTED CAPTCHA GATE: The native answer is ready but has not been applied. Inspect the page and use apply_captcha_solution, then verify fresh page state.]';
+ resultContent += visionRun ? VISION_CAPTCHA_UNAPPLIED_NOTE : '\n[TRUSTED CAPTCHA GATE: The native answer is ready but has not been applied. Inspect the page and use apply_captcha_solution, then verify fresh page state.]';
} else if (captchaSolveOutcome?.status === 'verification_pending') {
resultContent += '\n[TRUSTED CAPTCHA GATE: The supported CAPTCHA token was injected, but clearance must be confirmed from the exact widget token and active challenge-frame state. Wait briefly, then read the page again. Do not submit, dismiss, or call solve_captcha again.]';
} else if (captchaSolveOutcome?.status === 'manual_required') {
@@ -14686,7 +14688,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
manualCompletionRequired: true,
cloudflareManagedChallenge: true,
captchaDiagnostics: gate.publicGate?.diagnostics || null,
- error: 'A full-page Cloudflare managed challenge is active. Use get_captcha_capabilities to inspect a compatible Cloudflare native method. A single solve_captcha call with observed providerTasks is allowed before any solve attempt; otherwise ask for manual completion. Navigate only to abandon the challenged page. Read the page again after Cloudflare resumes the destination.',
+ error: this._visionRunState?.(tabId)?.active === true
+ ? 'A full-page Cloudflare managed challenge is active and cannot be solved automatically in this mode. Ask the user to complete it manually; do not dismiss it or submit the page.'
+ : 'A full-page Cloudflare managed challenge is active. Use get_captcha_capabilities to inspect a compatible Cloudflare native method. A single solve_captcha call with observed providerTasks is allowed before any solve attempt; otherwise ask for manual completion. Navigate only to abandon the challenged page. Read the page again after Cloudflare resumes the destination.',
};
}
return {
@@ -49489,7 +49493,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
...(result.solution?.userAgent ? { solverUserAgent: result.solution.userAgent } : {}),
note: application.success
? 'AWS cookie applied. Reload this page once with navigate, then read it to verify clearance. A provider answer is not site acceptance; do not buy another solve for the same unresolved challenge.'
- : 'AWS answer received but could not be applied. Inspect the binding and use apply_captcha_solution; do not buy another answer.' };
+ : (this._visionRunState?.(tabId)?.active === true ? 'AWS answer received but could not be applied, and this mode has no tool to apply it. Ask the user to complete verification manually; do not buy another answer.' : 'AWS answer received but could not be applied. Inspect the binding and use apply_captcha_solution; do not buy another answer.') };
}
return { success: true, dispatched: true, type: result.family, provider: result.provider, method: result.method, taskId: result.taskId,
solution: result.solution, ...((result.userAgent || result.solution?.userAgent) ? { solverUserAgent: result.userAgent || result.solution.userAgent } : {}),
@@ -49817,7 +49821,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
...(pendingAutomaticAnswer ? { applicationRequired: true } : {}),
injection,
note: pendingAutomaticAnswer
- ? 'Token returned without injection. Inspect the page and call apply_captcha_solution with observed frame/field or callback bindings and path: token. The paid answer is saved; do not request another solve.'
+ ? (this._visionRunState?.(tabId)?.active === true ? 'Token returned without injection, and this mode has no tool to place it in the page. The paid answer is saved; do not request another solve. Ask the user to complete verification manually.' : 'Token returned without injection. Inspect the page and call apply_captcha_solution with observed frame/field or callback bindings and path: token. The paid answer is saved; do not request another solve.')
: !wantInject ? 'Recognition answer returned without page injection.'
: injection?.success
? (injection.calledCallback
diff --git a/src/chrome/src/agent/vision-tools.js b/src/chrome/src/agent/vision-tools.js
index d0baf20f4..65f580aca 100644
--- a/src/chrome/src/agent/vision-tools.js
+++ b/src/chrome/src/agent/vision-tools.js
@@ -198,9 +198,15 @@ export function visionTypeTextTool(baseTool) {
const VISION_SOLVE_CAPTCHA_DESCRIPTION = 'Solve the CAPTCHA blocking this page with the providers enabled in Settings β†’ CAPTCHA, in descending weight order; one call may charge several providers on fallback. Call it once, normally with no arguments: WebBrain finds the widget in the page itself (reCAPTCHA v2/v3, hCaptcha with NopeCHA or NoneCap, Turnstile, AWS WAF, image CAPTCHAs) and checks that the challenge cleared before the next page-changing action; look at the next screen. If the result lists ambiguous candidates, call it again with the exact frameUrl, frameId or framePath it names. Never call it again for the same unresolved challenge: if it fails or the challenge stays, ask the user to complete it.';
-/** solve_captcha for a vision run: the automatic route, without native provider tasks. */
+/**
+ * solve_captcha for a vision run: the automatic route, without native
+ * provider tasks or a token-only answer (both need apply_captcha_solution).
+ */
export function visionSolveCaptchaTool(baseTool) {
- const { providerTasks, ...properties } = baseTool?.function?.parameters?.properties || {};
+ const { providerTasks, inject, ...properties } = baseTool?.function?.parameters?.properties || {};
+ if (properties.metadata) {
+ properties.metadata = { ...properties.metadata, description: 'Turnstile only. Values observed in the selected widget configuration; never invent them. action and cdata are detected from widget data attributes when present.' };
+ }
return {
...baseTool,
function: {
@@ -358,6 +364,12 @@ export function visionUniversalPreamble(preamble) {
* come with this line, so the model keeps their facts about the site and
* does not reach for tools it does not have.
*/
+// A vision run has no apply_captcha_solution or native methods: when WebBrain
+// cannot finish a CAPTCHA itself, the model asks the user (agent
+// _captchaRoutingMessage, _captchaGateBlockResult, solve_captcha).
+export const VISION_CAPTCHA_MANUAL_NOTE = '\n[TRUSTED CAPTCHA GATE: This verification challenge cannot be solved automatically in this mode. Do not dismiss it, submit the page or call solve_captcha again; ask the user to complete it manually, and report blocked work with done({outcome:"partial"}) if they cannot.]';
+export const VISION_CAPTCHA_UNAPPLIED_NOTE = '\n[TRUSTED CAPTCHA GATE: The paid answer could not be placed in the page in this mode. Do not call solve_captcha again or submit the page; ask the user to complete the verification manually, and report blocked work with done({outcome:"partial"}) if they cannot.]';
+
/** The [CAPTCHA SOLVER] system-prompt note for a vision run (agent _buildSystemPrompt). */
export function visionCaptchaSolverNote(coverage) {
return `[CAPTCHA SOLVER β€” the user has enabled a CAPTCHA solver. Enabled providers run in descending weight order; failure or timeout tries the next compatible provider. Each attempt may charge. This fallback is internal to one tool call, not permission to call the tool again. Enabled coverage: ${coverage} When a CAPTCHA or verification dialog blocks a step, do not dismiss it: call solve_captcha once, normally with no arguments. WebBrain reads the page itself to find the widget, and checks that the dialog cleared before any page-changing action; look at the next screen. If the family is not covered, the solve fails, or the dialog remains, stop and ask the user to complete it manually; never dismiss and resubmit or retry solve_captcha. These limits apply per challenge: a new CAPTCHA that appears later, such as after a submit or on a newly loaded page, is a separate challenge. Solve it once with solve_captcha before asking the user.]`;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index b10c1191b..a175c03d3 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -21,7 +21,7 @@ import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationCon
import { mastodonPublicationIntentMessages, normalizeMastodonPublicationIntent, captureNativeMastodonPublication, mastodonPublicationBinding, mastodonPublicationReceipt, mastodonPublicationNavigation } from './mastodon-publication.js';
import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, RETIRED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_BATCHING, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_LEAN_NOTE, SYSTEM_PROMPT_GENERATIVE_MEDIA, SYSTEM_PROMPT_DEV_APPENDIX, getVisionActPrompt } from './tools.js';
import { installVisionRuntime } from './vision-runtime.js';
-import { VISION_NATIVE_TOOLS, VISION_SITE_NOTES_PREFACE, visionCaptchaSolverNote, visionFilePickerBlockedResult, visionUniversalPreamble } from './vision-tools.js';
+import { VISION_CAPTCHA_MANUAL_NOTE, VISION_CAPTCHA_UNAPPLIED_NOTE, VISION_NATIVE_TOOLS, VISION_SITE_NOTES_PREFACE, visionCaptchaSolverNote, visionFilePickerBlockedResult, visionUniversalPreamble } from './vision-tools.js';
import { visionDomProbeExpression } from './vision-dom-probe.js';
import { normalizeVisionModeSetting } from './vision-mode.js';
import { rejectedCompletionRecovery } from './completion-recovery.js';
@@ -12793,6 +12793,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
_captchaRoutingMessage(tabId, captchaGateDecision, captchaSolveOutcome, toolResult = {}, onUpdate = () => {}) {
+ // A vision run cannot apply a native answer: its notes end at the user.
+ const visionRun = this._visionRunState?.(tabId)?.active === true;
let resultContent = '';
const gate = captchaGateDecision || captchaSolveOutcome;
if (gate && gate.status !== 'cleared' && !this._captchaToolsAvailable(tabId)) {
@@ -12806,12 +12808,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
: captchaGateDecision.nativeDispatchInterrupted === true
? '\n[TRUSTED CAPTCHA GATE: A native solve was already dispatched for this document, but its answer is unavailable after restart. Do not send another paid request or submit the page; ask the user to complete the challenge manually.]'
: captchaGateDecision.cloudflareManagedChallenge === true
- ? '\n[TRUSTED CAPTCHA GATE: A response-backed full-page Cloudflare managed challenge is active. Inspect get_captcha_capabilities for a Cloudflare native method; use one solve_captcha providerTasks dispatch only with all required observed inputs, otherwise ask for manual completion. Do not submit; the network/navigation monitor will clear the gate when Cloudflare resumes the destination.]'
+ ? (visionRun ? VISION_CAPTCHA_MANUAL_NOTE : '\n[TRUSTED CAPTCHA GATE: A response-backed full-page Cloudflare managed challenge is active. Inspect get_captcha_capabilities for a Cloudflare native method; use one solve_captcha providerTasks dispatch only with all required observed inputs, otherwise ask for manual completion. Do not submit; the network/navigation monitor will clear the gate when Cloudflare resumes the destination.]')
: captchaGateDecision.awsWaf === true
? '\n[TRUSTED CAPTCHA GATE: AWS verification requires manual completion because no untried compatible provider or complete observed input set is available. Do not buy another answer for this unresolved challenge. After manual completion, read the page so the runtime can resume.]'
: captchaGateDecision.activeChallengeAfterSolve === true
? '\n[TRUSTED CAPTCHA GATE: The active challenge frame is visible again after the one automatic solve. The site may have rejected the token. Do not call solve_captcha again; stop automation and ask the user to complete the challenge manually.]'
- : '\n[TRUSTED CAPTCHA GATE: A verification challenge is active, but no safely selectable automatic widget was detected. Inspect get_captcha_capabilities and use one native providerTasks solve only if its required parameters can be observed. Otherwise ask for manual completion. Do not dismiss, close, or resubmit the challenge.]';
+ : (visionRun ? VISION_CAPTCHA_MANUAL_NOTE : '\n[TRUSTED CAPTCHA GATE: A verification challenge is active, but no safely selectable automatic widget was detected. Inspect get_captcha_capabilities and use one native providerTasks solve only if its required parameters can be observed. Otherwise ask for manual completion. Do not dismiss, close, or resubmit the challenge.]');
onUpdate('warning', { message: 'Verification challenge requires manual completion.' });
} else if (captchaGateDecision?.status === 'cleared') {
if (captchaGateDecision.clearedByNavigation === true) {
@@ -12827,13 +12829,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
} else if ((captchaGateDecision || captchaSolveOutcome)?.status === 'verification_pending'
&& this._hasUnappliedNativeCaptchaSolution(tabId)) {
- resultContent += '\n[TRUSTED CAPTCHA GATE: The native answer is ready but has not been applied. If the prior tool result is no longer in this chat, call get_captcha_capabilities to retrieve the pending paid answer without another solve. Inspect the page and use apply_captcha_solution, then verify fresh page state.]';
+ resultContent += visionRun ? VISION_CAPTCHA_UNAPPLIED_NOTE : '\n[TRUSTED CAPTCHA GATE: The native answer is ready but has not been applied. If the prior tool result is no longer in this chat, call get_captcha_capabilities to retrieve the pending paid answer without another solve. Inspect the page and use apply_captcha_solution, then verify fresh page state.]';
} else if (toolResult?.type === 'aws_waf' && toolResult?.injected === true) {
resultContent += '\n[TRUSTED CAPTCHA GATE: The AWS cookie was applied. Reload the observed page once with navigate, then read it to verify clearance. Do not call solve_captcha again for this unresolved challenge.]';
} else if (captchaGateDecision?.status === 'verification_pending') {
resultContent += '\n[TRUSTED CAPTCHA GATE: Verification is still pending because the exact widget has no response token or its frame state could not be inspected conclusively. Wait briefly, then read the page again. Do not submit, dismiss, or call solve_captcha again.]';
} else if (toolResult?.applicationRequired === true && captchaSolveOutcome?.status === 'verification_pending') {
- resultContent += '\n[TRUSTED CAPTCHA GATE: The native answer is ready but has not been applied. Inspect the page and use apply_captcha_solution, then verify fresh page state.]';
+ resultContent += visionRun ? VISION_CAPTCHA_UNAPPLIED_NOTE : '\n[TRUSTED CAPTCHA GATE: The native answer is ready but has not been applied. Inspect the page and use apply_captcha_solution, then verify fresh page state.]';
} else if (captchaSolveOutcome?.status === 'verification_pending') {
resultContent += '\n[TRUSTED CAPTCHA GATE: The supported CAPTCHA token was injected, but clearance must be confirmed from the exact widget token and active challenge-frame state. Wait briefly, then read the page again. Do not submit, dismiss, or call solve_captcha again.]';
} else if (captchaSolveOutcome?.status === 'manual_required') {
@@ -12910,7 +12912,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
manualCompletionRequired: true,
cloudflareManagedChallenge: true,
captchaDiagnostics: gate.publicGate?.diagnostics || null,
- error: 'A full-page Cloudflare managed challenge is active. Use get_captcha_capabilities to inspect a compatible Cloudflare native method. A single solve_captcha call with observed providerTasks is allowed before any solve attempt; otherwise ask for manual completion. Navigate only to abandon the challenged page. Read the page again after Cloudflare resumes the destination.',
+ error: this._visionRunState?.(tabId)?.active === true
+ ? 'A full-page Cloudflare managed challenge is active and cannot be solved automatically in this mode. Ask the user to complete it manually; do not dismiss it or submit the page.'
+ : 'A full-page Cloudflare managed challenge is active. Use get_captcha_capabilities to inspect a compatible Cloudflare native method. A single solve_captcha call with observed providerTasks is allowed before any solve attempt; otherwise ask for manual completion. Navigate only to abandon the challenged page. Read the page again after Cloudflare resumes the destination.',
};
}
return {
@@ -45255,7 +45259,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
...(result.solution?.userAgent ? { solverUserAgent: result.solution.userAgent } : {}),
note: application.success
? 'AWS cookie applied. Reload this page once with navigate, then read it to verify clearance. A provider answer is not site acceptance; do not buy another solve for the same unresolved challenge.'
- : 'AWS answer received but could not be applied. Inspect the binding and use apply_captcha_solution; do not buy another answer.' };
+ : (this._visionRunState?.(tabId)?.active === true ? 'AWS answer received but could not be applied, and this mode has no tool to apply it. Ask the user to complete verification manually; do not buy another answer.' : 'AWS answer received but could not be applied. Inspect the binding and use apply_captcha_solution; do not buy another answer.') };
}
return { success: true, dispatched: true, type: result.family, provider: result.provider, method: result.method, taskId: result.taskId,
solution: result.solution, ...((result.userAgent || result.solution?.userAgent) ? { solverUserAgent: result.userAgent || result.solution.userAgent } : {}),
@@ -45581,7 +45585,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
...(pendingAutomaticAnswer ? { applicationRequired: true } : {}),
injection,
note: pendingAutomaticAnswer
- ? 'Token returned without injection. Inspect the page and call apply_captcha_solution with observed frame/field or callback bindings and path: token. The paid answer is saved; do not request another solve.'
+ ? (this._visionRunState?.(tabId)?.active === true ? 'Token returned without injection, and this mode has no tool to place it in the page. The paid answer is saved; do not request another solve. Ask the user to complete verification manually.' : 'Token returned without injection. Inspect the page and call apply_captcha_solution with observed frame/field or callback bindings and path: token. The paid answer is saved; do not request another solve.')
: !wantInject ? 'Recognition answer returned without page injection.'
: injection?.success
? (injection.calledCallback
diff --git a/src/firefox/src/agent/vision-tools.js b/src/firefox/src/agent/vision-tools.js
index d0baf20f4..65f580aca 100644
--- a/src/firefox/src/agent/vision-tools.js
+++ b/src/firefox/src/agent/vision-tools.js
@@ -198,9 +198,15 @@ export function visionTypeTextTool(baseTool) {
const VISION_SOLVE_CAPTCHA_DESCRIPTION = 'Solve the CAPTCHA blocking this page with the providers enabled in Settings β†’ CAPTCHA, in descending weight order; one call may charge several providers on fallback. Call it once, normally with no arguments: WebBrain finds the widget in the page itself (reCAPTCHA v2/v3, hCaptcha with NopeCHA or NoneCap, Turnstile, AWS WAF, image CAPTCHAs) and checks that the challenge cleared before the next page-changing action; look at the next screen. If the result lists ambiguous candidates, call it again with the exact frameUrl, frameId or framePath it names. Never call it again for the same unresolved challenge: if it fails or the challenge stays, ask the user to complete it.';
-/** solve_captcha for a vision run: the automatic route, without native provider tasks. */
+/**
+ * solve_captcha for a vision run: the automatic route, without native
+ * provider tasks or a token-only answer (both need apply_captcha_solution).
+ */
export function visionSolveCaptchaTool(baseTool) {
- const { providerTasks, ...properties } = baseTool?.function?.parameters?.properties || {};
+ const { providerTasks, inject, ...properties } = baseTool?.function?.parameters?.properties || {};
+ if (properties.metadata) {
+ properties.metadata = { ...properties.metadata, description: 'Turnstile only. Values observed in the selected widget configuration; never invent them. action and cdata are detected from widget data attributes when present.' };
+ }
return {
...baseTool,
function: {
@@ -358,6 +364,12 @@ export function visionUniversalPreamble(preamble) {
* come with this line, so the model keeps their facts about the site and
* does not reach for tools it does not have.
*/
+// A vision run has no apply_captcha_solution or native methods: when WebBrain
+// cannot finish a CAPTCHA itself, the model asks the user (agent
+// _captchaRoutingMessage, _captchaGateBlockResult, solve_captcha).
+export const VISION_CAPTCHA_MANUAL_NOTE = '\n[TRUSTED CAPTCHA GATE: This verification challenge cannot be solved automatically in this mode. Do not dismiss it, submit the page or call solve_captcha again; ask the user to complete it manually, and report blocked work with done({outcome:"partial"}) if they cannot.]';
+export const VISION_CAPTCHA_UNAPPLIED_NOTE = '\n[TRUSTED CAPTCHA GATE: The paid answer could not be placed in the page in this mode. Do not call solve_captcha again or submit the page; ask the user to complete the verification manually, and report blocked work with done({outcome:"partial"}) if they cannot.]';
+
/** The [CAPTCHA SOLVER] system-prompt note for a vision run (agent _buildSystemPrompt). */
export function visionCaptchaSolverNote(coverage) {
return `[CAPTCHA SOLVER β€” the user has enabled a CAPTCHA solver. Enabled providers run in descending weight order; failure or timeout tries the next compatible provider. Each attempt may charge. This fallback is internal to one tool call, not permission to call the tool again. Enabled coverage: ${coverage} When a CAPTCHA or verification dialog blocks a step, do not dismiss it: call solve_captcha once, normally with no arguments. WebBrain reads the page itself to find the widget, and checks that the dialog cleared before any page-changing action; look at the next screen. If the family is not covered, the solve fails, or the dialog remains, stop and ask the user to complete it manually; never dismiss and resubmit or retry solve_captcha. These limits apply per challenge: a new CAPTCHA that appears later, such as after a submit or on a newly loaded page, is a separate challenge. Solve it once with solve_captcha before asking the user.]`;
diff --git a/test/vision-mode-guards.mjs b/test/vision-mode-guards.mjs
index f7474134f..c6603ef35 100644
--- a/test/vision-mode-guards.mjs
+++ b/test/vision-mode-guards.mjs
@@ -624,6 +624,30 @@ for (const build of ['chrome', 'firefox']) {
assert.doesNotMatch(visionPrompt.slice(visionPrompt.indexOf('[CAPTCHA SOLVER')), /get_captcha_capabilities|apply_captcha_solution|accessibility-tree/);
agent.visionMode = 'off';
assert.match(agent._buildSystemPrompt('act', 1), /call get_captcha_capabilities/);
+ // No token-only answer either, and no field description that points at the removed route.
+ assert.equal('inject' in solve.function.parameters.properties, false);
+ assert.doesNotMatch(JSON.stringify(solve.function.parameters), /providerTasks|apply_captcha_solution|get_captcha_capabilities/);
+ // The runtime notes for an answer WebBrain could not place, or a challenge it cannot solve, end at the user.
+ const removed = /get_captcha_capabilities|apply_captcha_solution|providerTasks/;
+ for (const vision of [true, false]) {
+ agent._visionRunState = () => ({ active: vision });
+ agent._hasUnappliedNativeCaptchaSolution = () => true;
+ const unapplied = agent._captchaRoutingMessage(1, { status: 'verification_pending', publicGate: { status: 'verification_pending' } }, null, {});
+ const cloudflare = agent._captchaRoutingMessage(1, { status: 'manual_required', cloudflareManagedChallenge: true, publicGate: { status: 'manual_required' } }, null, {});
+ agent._hasUnappliedNativeCaptchaSolution = () => false;
+ agent._captchaGateStates.set(1, { status: 'manual_required', publicGate: { status: 'manual_required', cloudflareManagedChallenge: true } });
+ const refused = agent._captchaGateBlockResult(1, 'click', {});
+ agent._captchaGateStates.delete(1);
+ assert.match(refused?.error || '', /Cloudflare managed challenge is active/, 'the gate refuses the click');
+ if (vision) {
+ assert.match(unapplied, /could not be placed in the page in this mode/);
+ assert.match(cloudflare, /cannot be solved automatically in this mode/);
+ for (const text of [unapplied, cloudflare, refused?.error || '']) assert.doesNotMatch(text, removed, text);
+ } else {
+ assert.match(unapplied, /get_captcha_capabilities/);
+ assert.match(cloudflare, /get_captcha_capabilities/);
+ }
+ }
});
if (build === 'firefox') test('firefox: upload_file({downloadId}) uses the saved bytes when the companion can read them', async () => {