Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Hide unconfigured media generation tools and guidance

Emre Sokulluce1ad7916654
diff --git a/package.json b/package.json
index 73b512f76..50f444e95 100644
--- a/package.json
+++ b/package.json
@@ -69,7 +69,7 @@
"test:cms:browser": "node --test test/cms-browser-smoke.mjs",
"test:compass-quota": "node --test test/compass-quota.mjs",
"test:compass-quota:ui": "node test/compass-quota-ui.mjs",
- "test:generative-media": "node --test test/generative-media.mjs",
+ "test:generative-media": "node --test test/generative-media.mjs test/generative-media-availability.mjs",
"test:generative-media:ui": "node test/generative-media-ui.mjs",
"test:generative-media:extension": "node test/generative-media-extension.mjs"
},
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 65b4c9a77..496938228 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -2,7 +2,7 @@ import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confide
import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js';
import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationContract, publicationProgress, exactPublicationText, publicationMediaMatches, publicationContractMessages, publicationAuditMessages, publicationAuditAccepted } from './social-publish-contract.js';
-import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_DEV_APPENDIX, SYSTEM_PROMPT_WEBMCP_ASK, SYSTEM_PROMPT_WEBMCP_ACT } from './tools.js';
+import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_GENERATIVE_MEDIA, SYSTEM_PROMPT_DEV_APPENDIX, SYSTEM_PROMPT_WEBMCP_ASK, SYSTEM_PROMPT_WEBMCP_ACT } from './tools.js';
import { validateToolArguments } from './tool-arguments.js';
import { isSessionQuotaError, serializeConversationForSession, SESSION_CONVERSATION_BUDGET_BYTES, SESSION_CONVERSATION_RETRY_BUDGET_BYTES } from './conversation-persistence.js';
import { formatErrorMessage } from '../error-format.js';
@@ -129,7 +129,7 @@ import {
} from '../providers/provider-compatibility.js';
import { resolveMaxOutputTokens } from '../providers/context-windows.js';
import { generateImage, readMediaConfig } from './generative-media.js';
-import { mediaPermissionUrl, validateMediaConfig } from './media-config.js';
+import { GENERATIVE_MEDIA_SETUP_NOTE, isImageGenConfigured, mediaPermissionUrl, validateMediaConfig } from './media-config.js';
import { extractFirstJsonObject } from './json-extract.js';
import { repairAssistantDisplayText, sanitizeText as sanitizePlannerText } from './text-sanitize.js';
import { emptyOutputFailureMessage, modelOutputDiagnostics } from './model-output-diagnostics.js';
@@ -1051,6 +1051,9 @@ export class Agent extends LoopDetector {
// at call time so rotating the key doesn't require a restart.
this.captchaSolverEnabled = false;
this.captchaProviderIds = [];
+ // Only advertise media generation after valid settings have hydrated.
+ // Keep credentials in storage; tool execution re-reads and validates them.
+ this.imageGenConfigured = false;
this._captchaGateStates = new Map(); // tabId -> { key, status, publicGate, challengeFrameId? }
this._cloudflareManagedChallenges = new Map(); // tabId -> sanitized response-backed interstitial state
this._cloudflareManagedChallengeTransitions = new Map(); // tabId -> serialized transition promise
@@ -21585,6 +21588,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const provider = this._activeProvider(tabId);
const plannerMessages = buildPlannerIntentMessages(enriched, tabUrl, tabTitle, historyDigest, {
noThink: this._plannerPrefersNoThinkPrompt(provider),
+ imageGenConfigured: this.imageGenConfigured,
+ tier: this._resolvePromptTier(provider),
scheduledResume: runOptions?.scheduledResume === true,
locale,
priorUserTask: followUpContext.priorUserTask,
@@ -21815,6 +21820,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const skillCatalog = this._skillCatalog(conversationMode, tier);
const plannerMessages = buildPlannerMessages(enriched, tabUrl, tabTitle, historyDigest, {
noThink: this._plannerPrefersNoThinkPrompt(provider),
+ imageGenConfigured: this.imageGenConfigured,
+ tier,
scheduledResume: runOptions?.scheduledResume === true,
allowApi: this.isApiMutationsAllowed(tabId),
skillCatalog,
@@ -25562,7 +25569,14 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
const tier = this._resolvePromptTier();
if (tier === 'compact') return SYSTEM_PROMPT_ACT_COMPACT;
if (tier === 'mid') return SYSTEM_PROMPT_ACT_MID;
- return SYSTEM_PROMPT_ACT;
+ return `${SYSTEM_PROMPT_ACT}\n\n${this.imageGenConfigured ? SYSTEM_PROMPT_GENERATIVE_MEDIA : GENERATIVE_MEDIA_SETUP_NOTE}`;
+ }
+
+ setImageGenConfig(config) {
+ const next = isImageGenConfigured(config);
+ if (this.imageGenConfigured === next) return;
+ this.imageGenConfigured = next;
+ this._refreshSystemPrompts();
}
setWebMCPEnabled(enabled) {
@@ -42944,7 +42958,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
const sourceBoundAttachments = selectionOnly ? [] : attachments;
if (sourceBoundAttachments && sourceBoundAttachments.length) {
const attachmentToolNames = new Set(
- getToolsForMode(mode, { tier: provider.promptTier })
+ getToolsForMode(mode, { tier: provider.promptTier, imageGenConfigured: this.imageGenConfigured })
.map(tool => tool?.function?.name)
.filter(Boolean),
);
@@ -43054,6 +43068,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
let tools = getToolsForMode(mode, {
strictSecretMode: this.strictSecretMode,
tier,
+ imageGenConfigured: this.imageGenConfigured,
accessibilityTreeMaxChars: readWindow.treePageChars,
webMcpAvailable: this.webMcpEnabled === true,
skillLoaderTool: this._skillLoaderDefinition(mode, tier),
@@ -43298,6 +43313,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
tools = getToolsForMode(mode, {
strictSecretMode: this.strictSecretMode,
tier,
+ imageGenConfigured: this.imageGenConfigured,
accessibilityTreeMaxChars: readWindow.treePageChars,
webMcpAvailable: this.webMcpEnabled === true,
skillLoaderTool: this._skillLoaderDefinition(mode, tier),
@@ -44387,6 +44403,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
let tools = getToolsForMode(mode, {
strictSecretMode: this.strictSecretMode,
tier,
+ imageGenConfigured: this.imageGenConfigured,
accessibilityTreeMaxChars: readWindow.treePageChars,
webMcpAvailable: this.webMcpEnabled === true,
skillLoaderTool: this._skillLoaderDefinition(mode, tier),
@@ -44467,6 +44484,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
tools = getToolsForMode(mode, {
strictSecretMode: this.strictSecretMode,
tier,
+ imageGenConfigured: this.imageGenConfigured,
accessibilityTreeMaxChars: readWindow.treePageChars,
webMcpAvailable: this.webMcpEnabled === true,
skillLoaderTool: this._skillLoaderDefinition(mode, tier),
diff --git a/src/chrome/src/agent/media-config.js b/src/chrome/src/agent/media-config.js
index 4b548f2ad..e434710fb 100644
--- a/src/chrome/src/agent/media-config.js
+++ b/src/chrome/src/agent/media-config.js
@@ -1,5 +1,7 @@
// Provider settings are user-authored. Model tool arguments never select a host or key.
export const IMAGE_GEN_MODEL_KEY = 'imageGenModel';
+export const GENERATIVE_MEDIA_SETUP_NOTE = 'Built-in media generation requires setup in Settings β†’ Assistive Models β†’ Generative Media.';
+export const GENERATIVE_MEDIA_TIER_NOTE = 'Built-in media generation requires Full-tier Act mode.';
export const MEDIA_PROVIDERS = Object.freeze({
fal: { label: 'fal.ai', model: 'fal-ai/flux/schnell', keyPlaceholder: 'FAL_KEY', docs: 'https://fal.ai/dashboard/keys' },
openrouter: { label: 'OpenRouter', model: 'google/gemini-2.5-flash-image', keyPlaceholder: 'sk-or-…', docs: 'https://openrouter.ai/docs/guides/overview/multimodal/image-generation' },
diff --git a/src/chrome/src/agent/planner.js b/src/chrome/src/agent/planner.js
index c6b484655..c3909a09d 100644
--- a/src/chrome/src/agent/planner.js
+++ b/src/chrome/src/agent/planner.js
@@ -8,6 +8,7 @@ import { normalizeMessageTarget } from './message-recipient-guard.js';
import { normalizeReadScope } from './read-completeness.js';
import { normalizeProgressAction } from './progress-intent.js';
import { sanitizeText } from './text-sanitize.js';
+import { GENERATIVE_MEDIA_SETUP_NOTE, GENERATIVE_MEDIA_TIER_NOTE } from './media-config.js';
const UNTRUSTED_PAGE_CONTENT_TAG_RE = /<\/?untrusted_page_content\b[^>]*>/gi;
const REQUEST_KINDS = new Set(['execute', 'respond', 'plan_only', 'clarify']);
@@ -378,7 +379,6 @@ ${PLANNER_RESPONSE_LANGUAGE_RULES}
read: get_accessibility_tree, read_page, extract_data, fetch_url, research_url
interact: click_ax, set_checked, type_ax, set_field, find_text, press_keys, scroll, navigate, gmail_count_results, carousel_navigate, promote_iframe
wait: wait_for_element, wait_for_stable
- media: generate_image (create an image/video/audio directly from a text prompt via the user's configured generative-media provider β€” use when the user asks to GENERATE media, not to browse an image site)
memory: scratchpad_write, progress_update, progress_read
schedule: schedule_task (future/recurring work the user explicitly asked for), schedule_resume (pause CURRENT run blocked on external event)
user input: clarify (pause and ask one concise question when a required value remains missing after relevant inspection)
@@ -468,7 +468,6 @@ ${PLANNER_RESPONSE_ONLY_RULES}
- Canonical summary, steps, and risks must be English. localized fields must use the requested wbLocale.
${PLANNER_RESPONSE_LANGUAGE_RULES}
- For execute, keep the compact plan to 1–4 steps. For plan_only, provide 2–8 useful steps. For respond and clarify, steps may be empty.
-- When the user asks to generate an image/video/audio, plan one generate_image step (WebBrain's built-in media tool). Do not plan steps to visit image-generation sites or to check whether the current page supports image generation.
- clarify pauses execution to ask one concise question for a required value. done is terminal and must never be used to request information needed to continue.
- press_keys supports only unmodified Escape, Tab, Enter, arrow keys, and ; (semicolon, for page shortcuts such as Gmail Expand all). Never plan modifier combinations or browser UI shortcuts; use find_text to select one page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection and cannot create simultaneous highlights or browser Find UI.
- For Instagram /p/<id>/ carousel enumeration, use strictly increasing carousel_navigate indexes unless the latest user request explicitly asks for reverse traversal, in which case use strictly decreasing indexes; never use arrow keys, coordinate clicks, Previous/Next, or go_back to traverse slides.
@@ -636,6 +635,11 @@ const PLANNER_RESUME_RULE = '\n- This run is an app-owned scheduled continuation
export function buildPlannerSystemPrompt(opts = {}) {
let prompt = opts.allowApi ? `${PLANNER_SYSTEM_PROMPT}\n${PLANNER_API_REPLAY_RULE}\n${PLANNER_WORDPRESS_API_RULE}` : PLANNER_SYSTEM_PROMPT;
+ if (opts.imageGenConfigured === true && (opts.tier || 'full') === 'full') {
+ prompt += "\n\nGenerative media tool: generate_image (create an image/video/audio directly from a text prompt via the user's configured generative-media provider).\n- When the user asks to generate an image/video/audio, plan one generate_image step. Do not plan steps to visit image-generation sites or to check whether the current page supports image generation.";
+ } else {
+ prompt += `\n\n${opts.imageGenConfigured === true ? GENERATIVE_MEDIA_TIER_NOTE : GENERATIVE_MEDIA_SETUP_NOTE}`;
+ }
if (opts.scheduledResume === true) prompt += PLANNER_RESUME_RULE;
prompt += `\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.`;
if (opts.researchEscalationEnabled === true) {
@@ -666,7 +670,10 @@ export function buildPlannerIntentSystemPrompt(opts = {}) {
? '\n- A complex read-only research subtask may use explicit clarify consent followed by delegate_research; never delegate private data or consequential actions.'
: '';
const workflowRouting = formatSiteWorkflowRouting(opts.siteWorkflow);
- return `${PLANNER_INTENT_SYSTEM_PROMPT}${opts.scheduledResume === true ? PLANNER_RESUME_RULE : ''}\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.${researchRule}${workflowRouting ? `\n\n${workflowRouting}` : ''}`;
+ const mediaStatus = opts.imageGenConfigured !== true ? GENERATIVE_MEDIA_SETUP_NOTE
+ : (opts.tier || 'full') !== 'full' ? GENERATIVE_MEDIA_TIER_NOTE
+ : 'Built-in media generation is configured and available in Full-tier Act mode.';
+ return `${PLANNER_INTENT_SYSTEM_PROMPT}${opts.scheduledResume === true ? PLANNER_RESUME_RULE : ''}\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.${researchRule}${workflowRouting ? `\n\n${workflowRouting}` : ''}\n\n${mediaStatus}`;
}
export function formatSiteWorkflowRouting(value) {
diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js
index 1596a2239..4025be6e0 100644
--- a/src/chrome/src/agent/tools.js
+++ b/src/chrome/src/agent/tools.js
@@ -17,6 +17,9 @@ const DONE_OUTCOME_PROPERTY = {
const DONE_REQUIRED = ['summary'];
const DONE_REQUIRED_WITH_OUTCOME = ['summary', 'outcome'];
+export const SYSTEM_PROMPT_GENERATIVE_MEDIA = `GENERATIVE MEDIA:
+- generate_image: Create media (usually an image, sometimes video/audio) directly from a text prompt through the user's configured generative-media provider. When the user asks to GENERATE media ("generate an image of a red apple", "make a logo", "create a video clip"), call this tool β€” do NOT navigate to third-party image sites (Midjourney, DALLΒ·E, Bing Images, etc.). Not available in Ask mode.`;
+
export const AGENT_TOOLS = [
{
type: 'function',
@@ -1872,6 +1875,9 @@ export function getToolsForMode(mode, opts = {}) {
&& !(normalizedMode === 'ask' && t.function.name === 'clarify'))
.map(t => (t.function.name === 'clarify' ? ordinaryClarifyTool(t) : t));
}
+ if (opts.imageGenConfigured !== true) {
+ base = base.filter(tool => tool.function?.name !== 'generate_image');
+ }
const requestedTreePageChars = tier !== 'compact'
&& Number(opts.accessibilityTreeMaxChars) === EXPANDED_TREE_PAGE_CHARS
? EXPANDED_TREE_PAGE_CHARS
@@ -2118,7 +2124,6 @@ ${BROWSER_TAB_LIMITATION}
- scratchpad_write: Pin a note in context that survives summarization (use on long tasks to remember download IDs, file paths, plans)
- progress_update / progress_read: Structured app-owned ledger for the active repeated item/action task. Use it for per-user/per-item status and collected fields; close pending/acted rows before done.
- download_public_media (if enabled by a skill) / download_social_media: One-shot image/video download from public social sites. Prefer the enabled skill tool for public media URLs; otherwise use download_social_media. Single call β€” no need to inspect the DOM yourself.
-- generate_image: Create media (usually an image, sometimes video/audio) directly from a text prompt through the user's configured generative-media provider. When the user asks to GENERATE media ("generate an image of a red apple", "make a logo", "create a video clip"), call this tool β€” do NOT navigate to third-party image sites (Midjourney, DALLΒ·E, Bing Images, etc.). Requires Settings β†’ Assistive Models β†’ Generative Media. Not available in Ask mode.
- Recording is user-driven only. If the user asks to record, do NOT call tools; tell them to type \`/record\` for current-tab recording or \`/record --full-screen\` for screen/window recording; add \`--transcribe\` to either form if they want a Whisper transcript after stop. If they ask to stop a recording, tell them to press Escape twice in WebBrain/browser surfaces or use Chrome's Stop sharing control.
- hover: CDP-trusted hover over a ref_id. Use ONLY for menus/tooltips that REVEAL on hover (GitHub three-dot menus, Linear card actions, nav menus with reveal-on-hover children). Re-read the tree after to find the newly-visible items. Do NOT call hover before every click β€” most things are clickable directly.
- drag_drop: Drag one ref_id onto another via CDP-trusted pointer events. Use for Trello/Linear/Notion-style card reordering, file-tree node moves, image-crop handles, slider thumbs. Pass \`steps: 15–20\` if the first attempt doesn't trigger the drop indicator on momentum-tracking dnd. Verify by re-reading the tree.
diff --git a/src/chrome/src/background.js b/src/chrome/src/background.js
index efecb64a0..c7d63513b 100644
--- a/src/chrome/src/background.js
+++ b/src/chrome/src/background.js
@@ -42,6 +42,7 @@ import {
import { getBalance as capsolverGetBalance } from './agent/captcha-solver.js';
import { isCapsolverEnabled } from './agent/capsolver-config.js';
import { testImageGenProvider } from './agent/fal-media.js';
+import { IMAGE_GEN_MODEL_KEY } from './agent/media-config.js';
import { CAPTCHA_SETTINGS_KEYS, getCaptchaProviders } from './agent/captcha-provider-config.js';
import { getAdditionalCaptchaBalance } from './agent/captcha-additional-providers.js';
import { getTwoCaptchaBalance } from './agent/two-captcha.js';
@@ -269,6 +270,7 @@ const scheduler = new ScheduledJobManager({
await customSkillsReady;
await alwaysAllowApiMutationsReady;
await strictSecretModeReady;
+ await imageGenConfigReady;
await webMcpEnabledReady;
if (providerManager.providers.size === 0) await providerManager.load();
},
@@ -598,6 +600,12 @@ async function loadStrictSecretMode() {
}
const strictSecretModeReady = loadStrictSecretMode().catch(() => {});
+async function loadImageGenConfig() {
+ const stored = await chrome.storage.local.get(IMAGE_GEN_MODEL_KEY).catch(() => ({}));
+ agent.setImageGenConfig(stored?.[IMAGE_GEN_MODEL_KEY]);
+}
+const imageGenConfigReady = loadImageGenConfig();
+
async function loadWebMCPEnabled() {
const stored = await chrome.storage.local.get('webMcpEnabled');
agent.setWebMCPEnabled(stored.webMcpEnabled !== false);
@@ -1233,6 +1241,9 @@ chrome.storage.onChanged.addListener((changes, areaName) => {
// already-open conversations so the next turn sees the update β€” without
// wiping the chat history.
let refreshPrompts = false;
+ if (areaName === 'local' && changes[IMAGE_GEN_MODEL_KEY]) {
+ agent.setImageGenConfig(changes[IMAGE_GEN_MODEL_KEY].newValue);
+ }
if (changes[ALWAYS_ALLOW_API_MUTATIONS_KEY]) {
const value = changes[ALWAYS_ALLOW_API_MUTATIONS_KEY].newValue;
agent.setAlwaysAllowApiMutations(value === undefined || value === true);
@@ -2860,7 +2871,7 @@ async function handleMessage(msg, sender) {
// Agent toggles and prompt add-ons hydrate once at SW boot β€” await those
// promises so the first chat can't race ahead of hydration, without a
// storage round-trip on every message.
- await Promise.all([planBeforeActReady, planReviewReady, customSkillsReady, userMemoryReady]);
+ await Promise.all([planBeforeActReady, planReviewReady, customSkillsReady, userMemoryReady, imageGenConfigReady]);
await alwaysAllowApiMutationsReady;
await strictSecretModeReady;
await webMcpEnabledReady;
@@ -3786,6 +3797,7 @@ async function handleMessage(msg, sender) {
loadResearchEscalation(),
loadScreenshotRedaction(),
loadStrictSecretMode(),
+ loadImageGenConfig(),
loadWebMCPEnabled(),
loadProfile(),
syncAgentUserMemoryFromStorage(),
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 6994e310e..70ead9434 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -3,7 +3,7 @@ import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js'
import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
import { firefoxBidi } from '../bidi/client.js';
import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationContract, publicationProgress, exactPublicationText, publicationMediaMatches, publicationContractMessages, publicationAuditMessages, publicationAuditAccepted } from './social-publish-contract.js';
-import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_DEV_APPENDIX } from './tools.js';
+import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_GENERATIVE_MEDIA, SYSTEM_PROMPT_DEV_APPENDIX } from './tools.js';
import { validateToolArguments } from './tool-arguments.js';
import { isSessionQuotaError, serializeConversationForSession, SESSION_CONVERSATION_BUDGET_BYTES, SESSION_CONVERSATION_RETRY_BUDGET_BYTES } from './conversation-persistence.js';
import { formatErrorMessage } from '../error-format.js';
@@ -123,7 +123,7 @@ import {
} from '../providers/provider-compatibility.js';
import { resolveMaxOutputTokens } from '../providers/context-windows.js';
import { generateImage, readMediaConfig } from './generative-media.js';
-import { mediaPermissionUrl, validateMediaConfig } from './media-config.js';
+import { GENERATIVE_MEDIA_SETUP_NOTE, isImageGenConfigured, mediaPermissionUrl, validateMediaConfig } from './media-config.js';
import { extractFirstJsonObject } from './json-extract.js';
import { repairAssistantDisplayText, sanitizeText as sanitizePlannerText } from './text-sanitize.js';
import { emptyOutputFailureMessage, modelOutputDiagnostics } from './model-output-diagnostics.js';
@@ -870,6 +870,9 @@ export class Agent extends LoopDetector {
// the user. The API key is read at call time from browser.storage.
this.captchaSolverEnabled = false;
this.captchaProviderIds = [];
+ // Only advertise media generation after valid settings have hydrated.
+ // Keep credentials in storage; tool execution re-reads and validates them.
+ this.imageGenConfigured = false;
this._captchaGateStates = new Map(); // tabId -> { key, status, publicGate, challengeFrameId? }
this._cloudflareManagedChallenges = new Map(); // tabId -> sanitized response-backed interstitial state
this._cloudflareManagedChallengeTransitions = new Map(); // tabId -> serialized transition promise
@@ -19421,6 +19424,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const provider = this.providerManager.getActive();
const plannerMessages = buildPlannerIntentMessages(enriched, tabUrl, tabTitle, historyDigest, {
noThink: this._plannerPrefersNoThinkPrompt(provider),
+ imageGenConfigured: this.imageGenConfigured,
+ tier: this._resolvePromptTier(provider),
scheduledResume: runOptions?.scheduledResume === true,
locale,
priorUserTask: followUpContext.priorUserTask,
@@ -19647,6 +19652,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const skillCatalog = this._skillCatalog(conversationMode, tier);
const plannerMessages = buildPlannerMessages(enriched, tabUrl, tabTitle, historyDigest, {
noThink: this._plannerPrefersNoThinkPrompt(provider),
+ imageGenConfigured: this.imageGenConfigured,
+ tier,
scheduledResume: runOptions?.scheduledResume === true,
allowApi: this.isApiMutationsAllowed(tabId),
skillCatalog,
@@ -23194,7 +23201,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const tier = this._resolvePromptTier();
if (tier === 'compact') return SYSTEM_PROMPT_ACT_COMPACT;
if (tier === 'mid') return SYSTEM_PROMPT_ACT_MID;
- return SYSTEM_PROMPT_ACT;
+ return `${SYSTEM_PROMPT_ACT}\n\n${this.imageGenConfigured ? SYSTEM_PROMPT_GENERATIVE_MEDIA : GENERATIVE_MEDIA_SETUP_NOTE}`;
+ }
+
+ setImageGenConfig(config) {
+ const next = isImageGenConfigured(config);
+ if (this.imageGenConfigured === next) return;
+ this.imageGenConfigured = next;
+ this._refreshSystemPrompts();
}
/**
@@ -35908,7 +35922,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const sourceBoundAttachments = selectionOnly ? [] : attachments;
if (sourceBoundAttachments && sourceBoundAttachments.length) {
const attachmentToolNames = new Set(
- getToolsForMode(mode, { tier: provider.promptTier })
+ getToolsForMode(mode, { tier: provider.promptTier, imageGenConfigured: this.imageGenConfigured })
.map(tool => tool?.function?.name)
.filter(Boolean),
);
@@ -36018,6 +36032,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
let tools = getToolsForMode(mode, {
strictSecretMode: this.strictSecretMode,
tier,
+ imageGenConfigured: this.imageGenConfigured,
accessibilityTreeMaxChars: readWindow.treePageChars,
skillLoaderTool: this._skillLoaderDefinition(mode, tier),
skillTools,
@@ -36247,6 +36262,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
tools = getToolsForMode(mode, {
strictSecretMode: this.strictSecretMode,
tier,
+ imageGenConfigured: this.imageGenConfigured,
accessibilityTreeMaxChars: readWindow.treePageChars,
skillLoaderTool: this._skillLoaderDefinition(mode, tier),
skillTools,
@@ -37173,6 +37189,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
let tools = getToolsForMode(mode, {
strictSecretMode: this.strictSecretMode,
tier,
+ imageGenConfigured: this.imageGenConfigured,
accessibilityTreeMaxChars: readWindow.treePageChars,
skillLoaderTool: this._skillLoaderDefinition(mode, tier),
skillTools,
@@ -37249,6 +37266,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
tools = getToolsForMode(mode, {
strictSecretMode: this.strictSecretMode,
tier,
+ imageGenConfigured: this.imageGenConfigured,
accessibilityTreeMaxChars: readWindow.treePageChars,
skillLoaderTool: this._skillLoaderDefinition(mode, tier),
skillTools,
diff --git a/src/firefox/src/agent/media-config.js b/src/firefox/src/agent/media-config.js
index 4b548f2ad..e434710fb 100644
--- a/src/firefox/src/agent/media-config.js
+++ b/src/firefox/src/agent/media-config.js
@@ -1,5 +1,7 @@
// Provider settings are user-authored. Model tool arguments never select a host or key.
export const IMAGE_GEN_MODEL_KEY = 'imageGenModel';
+export const GENERATIVE_MEDIA_SETUP_NOTE = 'Built-in media generation requires setup in Settings β†’ Assistive Models β†’ Generative Media.';
+export const GENERATIVE_MEDIA_TIER_NOTE = 'Built-in media generation requires Full-tier Act mode.';
export const MEDIA_PROVIDERS = Object.freeze({
fal: { label: 'fal.ai', model: 'fal-ai/flux/schnell', keyPlaceholder: 'FAL_KEY', docs: 'https://fal.ai/dashboard/keys' },
openrouter: { label: 'OpenRouter', model: 'google/gemini-2.5-flash-image', keyPlaceholder: 'sk-or-…', docs: 'https://openrouter.ai/docs/guides/overview/multimodal/image-generation' },
diff --git a/src/firefox/src/agent/planner.js b/src/firefox/src/agent/planner.js
index c6b484655..c3909a09d 100644
--- a/src/firefox/src/agent/planner.js
+++ b/src/firefox/src/agent/planner.js
@@ -8,6 +8,7 @@ import { normalizeMessageTarget } from './message-recipient-guard.js';
import { normalizeReadScope } from './read-completeness.js';
import { normalizeProgressAction } from './progress-intent.js';
import { sanitizeText } from './text-sanitize.js';
+import { GENERATIVE_MEDIA_SETUP_NOTE, GENERATIVE_MEDIA_TIER_NOTE } from './media-config.js';
const UNTRUSTED_PAGE_CONTENT_TAG_RE = /<\/?untrusted_page_content\b[^>]*>/gi;
const REQUEST_KINDS = new Set(['execute', 'respond', 'plan_only', 'clarify']);
@@ -378,7 +379,6 @@ ${PLANNER_RESPONSE_LANGUAGE_RULES}
read: get_accessibility_tree, read_page, extract_data, fetch_url, research_url
interact: click_ax, set_checked, type_ax, set_field, find_text, press_keys, scroll, navigate, gmail_count_results, carousel_navigate, promote_iframe
wait: wait_for_element, wait_for_stable
- media: generate_image (create an image/video/audio directly from a text prompt via the user's configured generative-media provider β€” use when the user asks to GENERATE media, not to browse an image site)
memory: scratchpad_write, progress_update, progress_read
schedule: schedule_task (future/recurring work the user explicitly asked for), schedule_resume (pause CURRENT run blocked on external event)
user input: clarify (pause and ask one concise question when a required value remains missing after relevant inspection)
@@ -468,7 +468,6 @@ ${PLANNER_RESPONSE_ONLY_RULES}
- Canonical summary, steps, and risks must be English. localized fields must use the requested wbLocale.
${PLANNER_RESPONSE_LANGUAGE_RULES}
- For execute, keep the compact plan to 1–4 steps. For plan_only, provide 2–8 useful steps. For respond and clarify, steps may be empty.
-- When the user asks to generate an image/video/audio, plan one generate_image step (WebBrain's built-in media tool). Do not plan steps to visit image-generation sites or to check whether the current page supports image generation.
- clarify pauses execution to ask one concise question for a required value. done is terminal and must never be used to request information needed to continue.
- press_keys supports only unmodified Escape, Tab, Enter, arrow keys, and ; (semicolon, for page shortcuts such as Gmail Expand all). Never plan modifier combinations or browser UI shortcuts; use find_text to select one page-text match instead of Ctrl/Cmd+F. Each call replaces the previous selection and cannot create simultaneous highlights or browser Find UI.
- For Instagram /p/<id>/ carousel enumeration, use strictly increasing carousel_navigate indexes unless the latest user request explicitly asks for reverse traversal, in which case use strictly decreasing indexes; never use arrow keys, coordinate clicks, Previous/Next, or go_back to traverse slides.
@@ -636,6 +635,11 @@ const PLANNER_RESUME_RULE = '\n- This run is an app-owned scheduled continuation
export function buildPlannerSystemPrompt(opts = {}) {
let prompt = opts.allowApi ? `${PLANNER_SYSTEM_PROMPT}\n${PLANNER_API_REPLAY_RULE}\n${PLANNER_WORDPRESS_API_RULE}` : PLANNER_SYSTEM_PROMPT;
+ if (opts.imageGenConfigured === true && (opts.tier || 'full') === 'full') {
+ prompt += "\n\nGenerative media tool: generate_image (create an image/video/audio directly from a text prompt via the user's configured generative-media provider).\n- When the user asks to generate an image/video/audio, plan one generate_image step. Do not plan steps to visit image-generation sites or to check whether the current page supports image generation.";
+ } else {
+ prompt += `\n\n${opts.imageGenConfigured === true ? GENERATIVE_MEDIA_TIER_NOTE : GENERATIVE_MEDIA_SETUP_NOTE}`;
+ }
if (opts.scheduledResume === true) prompt += PLANNER_RESUME_RULE;
prompt += `\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.`;
if (opts.researchEscalationEnabled === true) {
@@ -666,7 +670,10 @@ export function buildPlannerIntentSystemPrompt(opts = {}) {
? '\n- A complex read-only research subtask may use explicit clarify consent followed by delegate_research; never delegate private data or consequential actions.'
: '';
const workflowRouting = formatSiteWorkflowRouting(opts.siteWorkflow);
- return `${PLANNER_INTENT_SYSTEM_PROMPT}${opts.scheduledResume === true ? PLANNER_RESUME_RULE : ''}\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.${researchRule}${workflowRouting ? `\n\n${workflowRouting}` : ''}`;
+ const mediaStatus = opts.imageGenConfigured !== true ? GENERATIVE_MEDIA_SETUP_NOTE
+ : (opts.tier || 'full') !== 'full' ? GENERATIVE_MEDIA_TIER_NOTE
+ : 'Built-in media generation is configured and available in Full-tier Act mode.';
+ return `${PLANNER_INTENT_SYSTEM_PROMPT}${opts.scheduledResume === true ? PLANNER_RESUME_RULE : ''}\n- Requested wbLocale for localized display fields: ${normalizePlannerLocale(opts.locale)}.${researchRule}${workflowRouting ? `\n\n${workflowRouting}` : ''}\n\n${mediaStatus}`;
}
export function formatSiteWorkflowRouting(value) {
diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js
index 9546c4a98..9dc3a9bae 100644
--- a/src/firefox/src/agent/tools.js
+++ b/src/firefox/src/agent/tools.js
@@ -17,6 +17,9 @@ const DONE_OUTCOME_PROPERTY = {
const DONE_REQUIRED = ['summary'];
const DONE_REQUIRED_WITH_OUTCOME = ['summary', 'outcome'];
+export const SYSTEM_PROMPT_GENERATIVE_MEDIA = `GENERATIVE MEDIA:
+- generate_image: Create media (usually an image, sometimes video/audio) directly from a text prompt through the user's configured generative-media provider. When the user asks to GENERATE media ("generate an image of a red apple", "make a logo", "create a video clip"), call this tool β€” do NOT navigate to third-party image sites (Midjourney, DALLΒ·E, Bing Images, etc.). Not available in Ask mode.`;
+
export const AGENT_TOOLS = [
{
type: 'function',
@@ -1718,6 +1721,9 @@ export function getToolsForMode(mode, opts = {}) {
&& !(normalizedMode === 'ask' && t.function.name === 'clarify'))
.map(t => (t.function.name === 'clarify' ? ordinaryClarifyTool(t) : t));
}
+ if (opts.imageGenConfigured !== true) {
+ base = base.filter(tool => tool.function?.name !== 'generate_image');
+ }
const requestedTreePageChars = tier !== 'compact'
&& Number(opts.accessibilityTreeMaxChars) === EXPANDED_TREE_PAGE_CHARS
? EXPANDED_TREE_PAGE_CHARS
@@ -1994,7 +2000,6 @@ ${BROWSER_TAB_LIMITATION}
- scratchpad_write: Pin a note in context that survives summarization (use on long tasks to remember download IDs, file paths, plans)
- progress_update / progress_read: Structured app-owned ledger for the active repeated item/action task. Use it for per-user/per-item status and collected fields; close pending/acted rows before done.
- download_public_media (if enabled by a skill) / download_social_media: One-shot image/video download from public social sites. Prefer the enabled skill tool for public media URLs; otherwise use download_social_media. Single call β€” no need to inspect the DOM yourself.
-- generate_image: Create media (usually an image, sometimes video/audio) directly from a text prompt through the user's configured generative-media provider. When the user asks to GENERATE media ("generate an image of a red apple", "make a logo", "create a video clip"), call this tool β€” do NOT navigate to third-party image sites (Midjourney, DALLΒ·E, Bing Images, etc.). Requires Settings β†’ Assistive Models β†’ Generative Media. Not available in Ask mode.
- hover: Synthetic hover over a ref_id (Firefox MV2 β€” no CDP). Use ONLY for menus/tooltips that REVEAL on hover (GitHub three-dot menus, Linear card actions). Re-read the tree after to find the newly-visible items. isTrusted=false, so sites with strict event-trust gating won't respond β€” fall back to clicking the explicit "..." button if hover doesn't reveal a menu.
- drag_drop: Synthetic drag from one ref_id to another (pointerdown/move/up + HTML5 dragstart/drop). Use for Trello/Linear/Notion-style card reordering, image-crop handles. Less reliable than Chrome's CDP path β€” verify by re-reading the tree.
- wait_for_stable: Wait until the page is quiet (no DOM mutations + no in-flight network) for \`quietMs\` ms. Use AFTER navigate / set_field({submit:true}) / a click that fires async work, BEFORE re-reading the tree. Different from wait_for_element: wait_for_element answers "did X appear", wait_for_stable answers "is the page done shuffling".
diff --git a/src/firefox/src/background.js b/src/firefox/src/background.js
index 97ec36c61..589cb6724 100644
--- a/src/firefox/src/background.js
+++ b/src/firefox/src/background.js
@@ -43,6 +43,7 @@ import {
import { getBalance as capsolverGetBalance } from './agent/captcha-solver.js';
import { isCapsolverEnabled } from './agent/capsolver-config.js';
import { testImageGenProvider } from './agent/fal-media.js';
+import { IMAGE_GEN_MODEL_KEY } from './agent/media-config.js';
import { CAPTCHA_SETTINGS_KEYS, getCaptchaProviders } from './agent/captcha-provider-config.js';
import { getAdditionalCaptchaBalance } from './agent/captcha-additional-providers.js';
import { getTwoCaptchaBalance } from './agent/two-captcha.js';
@@ -198,6 +199,7 @@ const scheduler = new ScheduledJobManager({
await customSkillsReady;
await alwaysAllowApiMutationsReady;
await strictSecretModeReady;
+ await imageGenConfigReady;
if (providerManager.providers.size === 0) await providerManager.load();
},
sendUpdate: (tabId, type, data) => {
@@ -486,6 +488,12 @@ async function loadStrictSecretMode() {
}
const strictSecretModeReady = loadStrictSecretMode().catch(() => {});
+async function loadImageGenConfig() {
+ const stored = await browser.storage.local.get(IMAGE_GEN_MODEL_KEY).catch(() => ({}));
+ agent.setImageGenConfig(stored?.[IMAGE_GEN_MODEL_KEY]);
+}
+const imageGenConfigReady = loadImageGenConfig();
+
async function loadProfile() {
const stored = await browser.storage.local.get(['profileEnabled', 'profileText']);
if (stored.profileEnabled != null) agent.profileEnabled = !!stored.profileEnabled;
@@ -1082,7 +1090,7 @@ browser.runtime.onStartup?.addListener?.(async () => {
});
// Listen for setting changes
-browser.storage.onChanged.addListener((changes) => {
+browser.storage.onChanged.addListener((changes, areaName) => {
if (changes.wbLocale) {
selectionShortcutLocale = normalizeSelectionShortcutLocale(changes.wbLocale.newValue);
createContextMenus().catch(() => {});
@@ -1099,6 +1107,9 @@ browser.storage.onChanged.addListener((changes) => {
agent.autoScreenshot = changes.autoScreenshot.newValue;
}
let refreshPrompts = false;
+ if (areaName === 'local' && changes[IMAGE_GEN_MODEL_KEY]) {
+ agent.setImageGenConfig(changes[IMAGE_GEN_MODEL_KEY].newValue);
+ }
if (changes[ALWAYS_ALLOW_API_MUTATIONS_KEY]) {
const value = changes[ALWAYS_ALLOW_API_MUTATIONS_KEY].newValue;
agent.setAlwaysAllowApiMutations(value === undefined || value === true);
@@ -2500,7 +2511,7 @@ async function handleMessage(msg, sender) {
}
// Hydrate agent toggles and prompt add-ons once at boot (not per message);
// onChanged keeps them in sync afterward.
- await Promise.all([planBeforeActReady, planReviewReady, customSkillsReady, userMemoryReady]);
+ await Promise.all([planBeforeActReady, planReviewReady, customSkillsReady, userMemoryReady, imageGenConfigReady]);
await alwaysAllowApiMutationsReady;
await strictSecretModeReady;
await screenshotRedactionReady;
@@ -3248,6 +3259,7 @@ async function handleMessage(msg, sender) {
loadResearchEscalation(),
loadScreenshotRedaction(),
loadStrictSecretMode(),
+ loadImageGenConfig(),
loadProfile(),
syncAgentUserMemoryFromStorage(),
loadCustomSkills(),
diff --git a/test/generative-media-availability.mjs b/test/generative-media-availability.mjs
new file mode 100644
index 000000000..d67aa91cf
--- /dev/null
+++ b/test/generative-media-availability.mjs
@@ -0,0 +1,179 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { runInNewContext } from 'node:vm';
+
+const api = {
+ storage: { local: { get: async () => ({}), set: async () => {} } },
+ runtime: { getURL: path => `chrome-extension://test/${path}`, sendMessage: async () => ({}) },
+ tabs: { get: async id => ({ id, url: 'https://example.com/', title: 'Example' }) },
+};
+globalThis.chrome = globalThis.browser = api;
+
+const fal = { apiKey: 'synthetic-key', model: 'fal-ai/flux/schnell' };
+const workflow = { '6': { class_type: 'CLIPTextEncode', inputs: { text: '{{prompt}}' } } };
+const configurations = [
+ undefined, {}, { ...fal, apiKey: '' }, { ...fal, provider: 'unknown' },
+ { provider: 'comfyui', workflow: {} },
+ fal, { ...fal, provider: 'openrouter' },
+ { ...fal, provider: 'comfyrouter', model: 'bfl/flux-2-pro' },
+ { provider: 'comfyui', workflow },
+];
+
+function makeAgent(Agent, overrides = {}) {
+ const provider = { promptTier: 'full', supportsTools: true, supportsVision: false, contextWindow: 128000, ...overrides };
+ const agent = new Agent({ getActive: () => provider, getProvider: () => provider, getVisionProvider: async () => null });
+ agent._hydrate = async () => {};
+ agent._persist = () => {};
+ agent._persistNow = async () => ({ ok: true });
+ agent._startTraceRun = async () => null;
+ agent._endTraceRun = async () => {};
+ agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => ({ role: 'user', content });
+ agent._manageContext = async () => {};
+ agent._checkCostAllowance = async () => null;
+ agent._recordCostUsage = async () => null;
+ agent._currentUrl = async () => 'https://example.com/';
+ agent._beginReadCompleteness = async () => null;
+ return agent;
+}
+
+for (const build of ['chrome', 'firefox']) {
+ const { Agent } = await import(`../src/${build}/src/agent/agent.js`);
+ const tools = await import(`../src/${build}/src/agent/tools.js`);
+ const planner = await import(`../src/${build}/src/agent/planner.js`);
+ const { GENERATIVE_MEDIA_SETUP_NOTE, GENERATIVE_MEDIA_TIER_NOTE } = await import(`../src/${build}/src/agent/media-config.js`);
+
+ test(`${build}: media tools and planner routing require valid settings and the Full tier`, () => {
+ const agent = makeAgent(Agent);
+ assert.equal(agent.imageGenConfigured, false);
+ assert.equal(tools.getToolsForMode('act').some(tool => tool.function.name === 'generate_image'), false);
+ for (const [index, config] of configurations.entries()) {
+ agent.setImageGenConfig(config);
+ const configured = index >= 5;
+ assert.equal(agent.imageGenConfigured, configured);
+ for (const mode of ['ask', 'act', 'dev']) for (const tier of ['compact', 'mid', 'full']) {
+ const opts = { tier, imageGenConfigured: agent.imageGenConfigured };
+ assert.equal(tools.getToolsForMode(mode, opts).some(tool => tool.function.name === 'generate_image'),
+ configured && mode !== 'ask' && tier === 'full', `${index}: ${mode}/${tier}`);
+ }
+ for (const tier of ['compact', 'mid', 'full']) {
+ const opts = { tier, imageGenConfigured: agent.imageGenConfigured };
+ const prompt = planner.buildPlannerMessages('Generate an image', 'https://example.com/', '', '', opts)[0].content;
+ assert.equal(prompt.includes('generate_image'), configured && tier === 'full');
+ if (!configured) assert.ok(prompt.includes(GENERATIVE_MEDIA_SETUP_NOTE));
+ else if (tier !== 'full') assert.ok(prompt.includes(GENERATIVE_MEDIA_TIER_NOTE));
+ const intent = planner.buildPlannerIntentMessages('Generate an image', '', '', '', opts)[0].content;
+ assert.equal(intent.includes(GENERATIVE_MEDIA_SETUP_NOTE), !configured);
+ }
+ }
+ });
+
+ test(`${build}: saving, invalidating, and clearing media settings refresh existing prompts without losing history`, () => {
+ const agent = makeAgent(Agent);
+ const messages = agent.getConversation(11, 'act');
+ const history = { role: 'user', content: 'Earlier task' };
+ messages.push(history);
+ const ask = agent.getConversation(12, 'ask');
+ for (const [config, available] of [[fal, true], [{ ...fal, model: '' }, false], [fal, true], [undefined, false]]) {
+ agent.setImageGenConfig(config);
+ assert.equal(messages[0].content.includes('generate_image'), available);
+ assert.equal(messages[0].content.includes(GENERATIVE_MEDIA_SETUP_NOTE), !available);
+ assert.equal(messages[1], history);
+ assert.equal(ask[0].content.includes('generate_image'), false);
+ }
+ });
+
+ test(`${build}: background hydration hides media until storage resolves and fails closed on read errors`, async () => {
+ const agent = makeAgent(Agent);
+ const source = await readFile(`src/${build}/src/background.js`, 'utf8');
+ const start = source.indexOf('async function loadImageGenConfig()');
+ const end = source.indexOf('\nconst imageGenConfigReady', start);
+ assert.ok(start >= 0 && end > start);
+ let resolveRead;
+ const storageApi = { storage: { local: { get: () => new Promise(resolve => { resolveRead = resolve; }) } } };
+ const load = new Function('chrome', 'browser', 'agent', 'IMAGE_GEN_MODEL_KEY', `return (${source.slice(start, end)});`)(
+ storageApi, storageApi, agent, 'imageGenModel',
+ );
+ const loading = load();
+ assert.equal(agent.imageGenConfigured, false);
+ resolveRead({ imageGenModel: fal });
+ await loading;
+ assert.equal(agent.imageGenConfigured, true);
+ storageApi.storage.local.get = async () => { throw new Error('Unavailable'); };
+ await load();
+ assert.equal(agent.imageGenConfigured, false);
+ });
+
+ test(`${build}: the background settings listener applies local saves and clears to live conversations`, async () => {
+ const agent = makeAgent(Agent);
+ const messages = agent.getConversation(13, 'act');
+ const source = await readFile(`src/${build}/src/background.js`, 'utf8');
+ const start = source.indexOf(`${build === 'chrome' ? 'chrome' : 'browser'}.storage.onChanged.addListener((changes, areaName)`);
+ const end = source.indexOf('\n});', start);
+ assert.ok(start >= 0 && end > start);
+ let listener;
+ const storageApi = { storage: { onChanged: { addListener: callback => { listener = callback; } } } };
+ runInNewContext(source.slice(start, end + 4), {
+ chrome: storageApi, browser: storageApi, agent, IMAGE_GEN_MODEL_KEY: 'imageGenModel',
+ PDF_VIEWER_ENABLED_KEY: 'pdfViewerEnabled', PROFILE_SYNC_DATA_KEYS: [], CAPTCHA_SETTINGS_KEYS: [],
+ ALWAYS_ALLOW_API_MUTATIONS_KEY: 'alwaysAllowApiMutations', API_MUTATION_OBSERVER_KEY: 'apiMutationObserver',
+ USER_MEMORY_ENABLED_KEY: 'userMemoryEnabled', USER_MEMORY_MAX_PROMPT_CHARS_KEY: 'userMemoryMaxPromptChars',
+ USER_MEMORY_STORAGE_KEY: 'userMemory', CUSTOM_SKILLS_STORAGE_KEY: 'customSkills',
+ shouldClearUserMemoryExtractionQueueForChanges: () => false,
+ });
+ listener({ imageGenModel: { newValue: fal } }, 'sync');
+ assert.equal(agent.imageGenConfigured, false, 'Ignore other storage areas');
+ listener({ imageGenModel: { newValue: fal } }, 'local');
+ assert.equal(agent.imageGenConfigured, true);
+ assert.ok(messages[0].content.includes('generate_image'));
+ listener({ imageGenModel: { oldValue: fal } }, 'local');
+ assert.equal(agent.imageGenConfigured, false);
+ assert.ok(messages[0].content.includes(GENERATIVE_MEDIA_SETUP_NOTE));
+ });
+
+ test(`${build}: actual planner and intent requests receive current media availability`, async () => {
+ const provider = { promptTier: 'full' };
+ const agent = makeAgent(Agent, provider);
+ agent._checkAbort = () => true;
+ let captured;
+ agent._chatWithCostAllowance = async (_provider, messages) => {
+ captured = messages[0].content;
+ throw new Error('Cancel after observing the request');
+ };
+ for (const method of ['_runPlannerGate', '_runPlannerIntentGate']) {
+ for (const [config, available] of [[undefined, false], [fal, true], [undefined, false]]) {
+ agent.setImageGenConfig(config);
+ captured = null;
+ await agent[method](15, { role: 'user', content: 'Generate an image' }, () => {}, null);
+ assert.ok(captured, 'A planner request must be observed');
+ assert.equal(captured.includes(GENERATIVE_MEDIA_SETUP_NOTE), !available);
+ if (method === '_runPlannerGate') assert.equal(captured.includes('generate_image'), available);
+ }
+ }
+ });
+
+ for (const streaming of [false, true]) {
+ test(`${build}: ${streaming ? 'streaming' : 'ordinary'} Act requests track configured and cleared media settings`, async () => {
+ let captured;
+ let agent;
+ const capture = (messages, options) => {
+ captured = { prompt: messages[0].content, tools: options.tools };
+ agent.abort(14);
+ };
+ agent = makeAgent(Agent, {
+ chat: async (messages, options) => { capture(messages, options); return { content: 'Cancelled' }; },
+ async *chatStream(messages, options) { capture(messages, options); yield { type: 'done' }; },
+ });
+ agent._maybeRunPlannerGate = async () => ({ proceed: true });
+ for (const [config, available] of [[undefined, false], [fal, true], [undefined, false]]) {
+ agent.setImageGenConfig(config);
+ captured = null;
+ if (streaming) await agent.processMessageStream(14, 'Generate an image', () => {}, 'act', { askStreamingEnabled: false });
+ else await agent.processMessage(14, 'Generate an image', () => {}, 'act', [], { askStreamingEnabled: false });
+ assert.ok(captured, 'A model request must be observed');
+ assert.equal(captured.prompt.includes('generate_image'), available);
+ assert.equal(captured.tools.some(tool => tool.function.name === 'generate_image'), available);
+ }
+ });
+ }
+}
diff --git a/test/run.js b/test/run.js
index d633a77ab..6699c881f 100644
--- a/test/run.js
+++ b/test/run.js
@@ -4288,7 +4288,7 @@ test('user memory browser wiring is mirrored and non-blocking', () => {
assert.match(background, /USER_MEMORY_STORAGE_KEY/, `${label}: background should import memory constants`);
assert.match(background, /USER_MEMORY_FORM_CAPTURE_KEY/, `${label}: background should import the form-capture setting key`);
assert.match(background, /const userMemoryReady = syncAgentUserMemoryFromStorage\(\)\.catch\(\(\) => \{\}\);/, `${label}: first message should await memory hydration`);
- assert.match(background, /Promise\.all\(\[planBeforeActReady, planReviewReady, customSkillsReady, userMemoryReady\]\)/, `${label}: handleMessage should await memory hydration`);
+ assert.match(background, /Promise\.all\(\[[^\]]*\buserMemoryReady\b[^\]]*\]\)/, `${label}: handleMessage should await memory hydration`);
for (const action of ['get_user_memory', 'add_user_memory', 'update_user_memory', 'delete_user_memory', 'clear_user_memory', 'export_user_memory', 'import_user_memory', 'enqueue_user_memory_extraction']) {
assert.match(background, new RegExp(`case '${action}'`), `${label}: ${action} route missing`);
}
@@ -16283,7 +16283,7 @@ test('import_config_patch background handler merges against live provider storag
parseConfigImport, parseConfigPatchImport, mergeConfigPatchSettings,
providerManager, agent,
loadMaxSteps, loadClarifyTimeout, loadAutoScreenshot, loadSiteAdapters, loadResearchEscalation,
- loadScreenshotRedaction, loadStrictSecretMode, loadWebMCPEnabled, loadProfile,
+ loadScreenshotRedaction, loadStrictSecretMode, loadImageGenConfig, loadWebMCPEnabled, loadProfile,
syncAgentUserMemoryFromStorage, loadCustomSkills, loadCaptchaSolver,
loadPlanBeforeAct, loadPlanReviewSettings, loadApiMutationObserverSetting,
} = helpers;
@@ -16334,6 +16334,7 @@ test('import_config_patch background handler merges against live provider storag
loadResearchEscalation: noop,
loadScreenshotRedaction: noop,
loadStrictSecretMode: noop,
+ loadImageGenConfig: noop,
loadWebMCPEnabled: noop,
loadProfile: noop,
syncAgentUserMemoryFromStorage: noop,
@@ -29192,7 +29193,9 @@ test('getToolsForMode: mode/tier redesign exposes the intended normal and Dev to
const researchOptions = { researchEscalationEnabled: true };
assert.equal(getTools('act', { tier: 'compact', ...researchOptions }).length, 25, `[${label}] Compact should expose 25 tools after tab-tool removal`);
assert.equal(getTools('act', { tier: 'mid', ...researchOptions }).length, 48, `[${label}] Mid should expose 48 tools including CAPTCHA discovery and answer application`);
- assert.equal(getTools('act', researchOptions).length, label === 'chrome' ? 56 : 55, `[${label}] Full tool count should include opt-in MemCode recall, chat workflow tools, and fal.ai generation`);
+ assert.equal(getTools('act', researchOptions).length, label === 'chrome' ? 55 : 54, `[${label}] Full tool count should omit unconfigured media generation`);
+ assert.equal(getTools('act', { ...researchOptions, imageGenConfigured: true }).length, label === 'chrome' ? 56 : 55,
+ `[${label}] Configured Full tier should include media generation`);
assert.equal(ask.includes('recall_memcode'), true, `[${label}] Ask can read connected MemCode memories`);
assert.equal(compact.includes('recall_memcode'), false, `[${label}] Compact does not expose remote recall`);
assert.equal(compact.includes('research_url'), false, `[${label}] Compact must not gain research_url as a tab-tool replacement`);
@@ -119401,7 +119404,7 @@ test('attachments: text attachment scratchpad path never writes raw textContent'
);
assert.match(
source,
- /const attachmentToolNames = new Set\([\s\S]*?getToolsForMode\(mode, \{ tier: provider\.promptTier \}\)[\s\S]*?const canUseScratchpadTool = attachmentToolNames\.has\('scratchpad_write'\);[\s\S]*?const canUseUploadTool = attachmentToolNames\.has\('upload_file'\);[\s\S]*?(?:await )?this\._applyAttachments\(enriched, sourceBoundAttachments, provider, \{[\s\S]*?canUseScratchpadTool,[\s\S]*?canUseUploadTool,[\s\S]*?tabId,[\s\S]*?messages,[\s\S]*?\}\);[\s\S]*?_pinTextAttachmentMetadata\(tabId, sourceBoundAttachments, \{ canUseScratchpadTool \}\);/,
+ /const attachmentToolNames = new Set\([\s\S]*?getToolsForMode\(mode, \{ tier: provider\.promptTier(?:, [^}]*)? \}\)[\s\S]*?const canUseScratchpadTool = attachmentToolNames\.has\('scratchpad_write'\);[\s\S]*?const canUseUploadTool = attachmentToolNames\.has\('upload_file'\);[\s\S]*?(?:await )?this\._applyAttachments\(enriched, sourceBoundAttachments, provider, \{[\s\S]*?canUseScratchpadTool,[\s\S]*?canUseUploadTool,[\s\S]*?tabId,[\s\S]*?messages,[\s\S]*?\}\);[\s\S]*?_pinTextAttachmentMetadata\(tabId, sourceBoundAttachments, \{ canUseScratchpadTool \}\);/,
`${label} should derive attachment guidance from the active run tool catalog`,
);
}