Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Tiny Ask screenshots: only for questions about the page; never with an attached image
Claude1dd02436687f
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.mdindex ccef3154c..a2a387fcf 100644--- a/docs/webgpu-models.md+++ b/docs/webgpu-models.md@@ -173,12 +173,15 @@ Use the tools in `scripts/models/bonsai/`:turn reads the page again. Offline Wikipedia and Emergency Box referencesare added only when there is no page text.- When the model reads images (Ternary Bonsai 2 2B Vision), a visual- question also gets a screenshot of the visible page. Visual means an- image, photo, chart, colour, logo, "can you see" and similar, in English- and Turkish (`tinyAskWantsScreenshot`). A question about a page with- almost no text ("what is on this page?") gets one as well, with a note to- answer from the image. A general question asked there ("who was Ada- Lovelace?") does not, and keeps its offline references.+ question about the page also gets a screenshot of the visible page+ (`tinyAskWantsScreenshot`). Examples: "what color is the button?", "what+ does the chart show?", "can you see …", "fotoğraftaki kupa ne renk?".+ - A question about a page with almost no text ("what is on this page?")+ gets one as well, with a note to answer from the image.+ - General questions take none, even on an image page: "what does+ melanoma look like?", "who was Ada Lovelace?". A screenshot alone never+ suppresses offline references.+ - Neither does a turn with an attached image: that image is the subject.- The capture is the same as automatic screenshots: redaction, sizebudget and the per-turn limit. It is ephemeral like page text, and itnever goes to a separate vision model.diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 454e35192..20c655408 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -11970,7 +11970,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// A tiny Ask turn carries a one-line trusted clock and the page (or PDF)// text and selection, read by WebBrain before the single model call. Those// blocks are ephemeral: the persisted turn keeps only the user's question.- async _enrichTinyAskUserMessage(tabId, userMessage, userAuthoredMetadata = {}, onUpdate = null) {+ async _enrichTinyAskUserMessage(tabId, userMessage, userAuthoredMetadata = {}, onUpdate = null, { imageAttachment = false } = {}) {const { tabUrl: url, tabTitle: title } = await this._getTabUrlTitle(tabId);let page = null;let selection = '';@@ -12007,7 +12007,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d: { url: linkedUrl, text, incomplete: fetched.hasMore === true };if (linked.unavailable) onUpdate?.('thinking', { step: 0, note: 'Could not read the linked page.' });}- const screenshot = tinyAskWantsScreenshot({ question: questionText, page })+ const screenshot = tinyAskWantsScreenshot({ question: questionText, page, imageAttachment })? await this._tinyAskScreenshot(tabId, onUpdate) : null;let modelLabel = '';try {@@ -12047,7 +12047,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dcontent: [block, ...screenshotBlocks, ...question],...userAuthoredMetadata,};- if (screenshot || String(page?.text || '').trim() || String(selection || '').trim() || String(linked?.text || '').trim()) {+ // A screenshot alone does not ground the turn: offline references are+ // still looked up when the question asks for them.+ if (String(page?.text || '').trim() || String(selection || '').trim() || String(linked?.text || '').trim()) {this._tinyAskPageGrounded.add(enriched);}return enriched;@@ -12083,14 +12085,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn null;}- async _enrichUserMessageWithCurrentPage(tabId, messages, userMessage, costState = null, runOptions = {}, onUpdate = null) {+ async _enrichUserMessageWithCurrentPage(tabId, messages, userMessage, costState = null, runOptions = {}, onUpdate = null, { imageAttachment = false } = {}) {const hasPriorUserTurn = messages.some(m => m.role === 'user');const selectionScoped = isSelectionSourceGrounding(runOptions?.sourceGrounding);const standaloneChat = runOptions?.standaloneChat === true;const userAuthoredMetadata = runOptions?.scheduledResume === true? {} : { webbrainUserAuthoredText: this._messageText(userMessage) };if (this._isTinyAskRun(runOptions)) {- return this._enrichTinyAskUserMessage(tabId, userMessage, userAuthoredMetadata, onUpdate);+ return this._enrichTinyAskUserMessage(tabId, userMessage, userAuthoredMetadata, onUpdate, { imageAttachment });}// Standalone is intentionally plain chat regardless of provider.// Do not add runtime state, recording/API guidance, tab metadata, adapter@@ -54131,6 +54133,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout duserTurnKind.catch(() => {});const enriched = await this._enrichUserMessageWithCurrentPage(tabId, selectionOnly ? [] : messages, userMessage, costState, runOptions, onUpdate,+ { imageAttachment: !selectionOnly && Array.isArray(attachments) && attachments.some(attachment => attachment?.kind === 'image') },);if (autoModeRoute) {({ mode, runOptions } = await this._settleAutoModeRun(tabId, autoModeRoute, enriched, onUpdate, runOptions));diff --git a/src/chrome/src/agent/tiny-ask.js b/src/chrome/src/agent/tiny-ask.jsindex 511191925..6caf844a0 100644--- a/src/chrome/src/agent/tiny-ask.js+++ b/src/chrome/src/agent/tiny-ask.js@@ -65,9 +65,9 @@ const VISUAL_WORDS = ['görüyor\\p{L}*', 'çizim\\p{L}*', 'video\\p{L}*',];const VISUAL_QUESTION = new RegExp(`(?<!\\p{L})(?:${VISUAL_WORDS.join('|')})(?!\\p{L})`, 'iu');-// A question about the current page itself. On a page with almost no text-// only these get a screenshot: "Who was Ada Lovelace?" asked on an image page-// is a general question and keeps its offline references.+// A question about the current page itself. Only these get a screenshot:+// "What does melanoma look like?" or "Who was Ada Lovelace?" is a general+// question even on an image page, and keeps its offline references.const PAGE_WORDS = [// English'this', 'here', 'page', 'site', 'tab', 'screen', 'summari[sz]e', 'describe', 'explain', 'tl;?dr',@@ -77,6 +77,15 @@ const PAGE_WORDS = ['açıkla\\p{L}*', 'anlat\\p{L}*', 'ne var', 'ne yazıyor', 'ne gösteriyor',];const PAGE_QUESTION = new RegExp(`(?<!\\p{L})(?:${PAGE_WORDS.join('|')})(?!\\p{L})`, 'iu');+// A specific visual thing, which can only be on the page: "the photo", "the+// Buy button", "can you see …"; in Turkish a visual stem with a suffix+// ("fotoğraftaki", "grafikte", "logonun rengi"), since there are no articles.+const PAGE_THINGS = 'images?|pictures?|photo(?:graph)?s?|pics?|screenshots?|charts?|graphs?|diagrams?|maps?|logos?|icons?|videos?|thumbnails?|banners?|buttons?|headers?|menus?|backgrounds?|illustrations?|drawings?|fonts?|text';+const PAGE_VISUAL_REFERENCE = new RegExp([+ `(?<!\\p{L})(?:the|this|that|these|those)\\s+(?:\\p{L}+\\s+){0,2}(?:${PAGE_THINGS})(?!\\p{L})`,+ '(?<!\\p{L})(?:do|can) you see(?!\\p{L})',+ '(?<!\\p{L})(?:resim|resm|görsel|görüntü|foto|grafi|grafiğ|şema|diyagram|harita|logo|ikon|simge|renk|reng|çizim|video|ekran)\\p{L}+',+].join('|'), 'iu');// Precedes the screenshot in the user message; the system prompt says the// same, so text inside the image is never taken as an instruction.@@ -90,14 +99,16 @@ export function tinyAskScreenshotNote(page = null) {? TINY_ASK_TEXTLESS_SCREENSHOT_NOTE : TINY_ASK_SCREENSHOT_NOTE;}-// Whether a tiny Ask turn should also show the model the visible page.-export function tinyAskWantsScreenshot({ question = '', page = null } = {}) {+// Whether a tiny Ask turn should also show the model the visible page: a+// visual question about the page, or any question about a page that was read+// but has almost no text (the answer is likely only in the pixels). Never+// with an attached image, which is then what the question is about.+export function tinyAskWantsScreenshot({ question = '', page = null, imageAttachment = false } = {}) {+ if (imageAttachment) return false;const text = String(question || '');+ if (!PAGE_QUESTION.test(text) && !PAGE_VISUAL_REFERENCE.test(text)) return false;if (VISUAL_QUESTION.test(text)) return true;- // A question about a page that was read but has almost no text: the- // answer is likely only in the pixels.- return !!page?.kind && String(page.text || '').trim().length < TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS- && PAGE_QUESTION.test(text);+ return !!page?.kind && String(page.text || '').trim().length < TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS;}const CHUNK_TARGET_CHARS = 700;diff --git a/test/run.js b/test/run.jsindex 789fea56d..18da1b3ff 100644--- a/test/run.js+++ b/test/run.js@@ -41611,18 +41611,21 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visuconst { tinyAskWantsScreenshot, TINY_ASK_SCREENSHOT_NOTE, TINY_ASK_SYSTEM_PROMPT } = TinyAskCh;const page = { kind: 'page', text: 'Plenty of readable page text. '.repeat(20) };for (const question of ['What is in this image?', 'What color is the button?', 'What does the chart show?',- 'Can you see a cat?', 'Bu resimde ne var?', 'Grafikte ne görünüyor?', 'Logonun rengi ne?', 'Ekranda ne yazıyor?']) {+ 'What color is the mug in the photo?', 'Can you see a cat?', 'Bu resimde ne var?', 'Grafikte ne görünüyor?',+ 'Logonun rengi ne?', 'Ekranda ne yazıyor?', 'Fotoğraftaki kupa ne renk?']) {assert.equal(tinyAskWantsScreenshot({ question, page }), true, question);}for (const question of ['How much is the blue plan?', 'Summarize this page', 'figure out the price', 'see the reviews',- 'Görevi özetle', 'Fiyat ne kadar?']) {- assert.equal(tinyAskWantsScreenshot({ question, page }), false, question);+ 'Görevi özetle', 'Fiyat ne kadar?', 'What does melanoma look like?', 'What color is a flamingo?', 'Melanom nasıl görünür?']) {+ assert.equal(tinyAskWantsScreenshot({ question, page }), false, `not a visual question about the page: ${question}`);}+ assert.equal(tinyAskWantsScreenshot({ question: 'What is in this image?', page, imageAttachment: true }), false,+ 'an attached image is the subject, not the page');const textless = { kind: 'page', text: ' ' };for (const question of ['Summarize', 'What is on this page?', 'Describe it', 'Bu sayfada ne var?', 'Burada ne yazıyor?']) {assert.equal(tinyAskWantsScreenshot({ question, page: textless }), true, `a page question on a textless page: ${question}`);}- for (const question of ['Who was Ada Lovelace?', 'How do I treat a burn?', 'Ada Lovelace kimdir?']) {+ for (const question of ['Who was Ada Lovelace?', 'How do I treat a burn?', 'Ada Lovelace kimdir?', 'What does melanoma look like?']) {assert.equal(tinyAskWantsScreenshot({ question, page: textless }), false, `a general question keeps its references: ${question}`);}assert.equal(tinyAskWantsScreenshot({ question: 'Summarize', page: { kind: 'pdf', text: '', unavailable: 'no text' } }), true, 'a scanned PDF');@@ -41667,7 +41670,7 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visucaptures.push(id);return { dataUrl: 'data:image/jpeg;base64,c2NyZWVu', width: 800, height: 600, captureId: 'cap_1' };};- const run = text => agent.processMessage(tabId, text, () => { }, 'ask', [], {});+ const run = (text, attachments = []) => agent.processMessage(tabId, text, () => { }, 'ask', attachments, {});return { agent, tabId, requests, captures, run };};@@ -41705,6 +41708,13 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visuassert.equal(captures.length, 0, 'a general question on a textless page takes no screenshot');assert.equal(requests[0].at(-1).content.filter(block => block.type === 'image_url').length, 0);}+ {+ const { requests, captures, run } = setup();+ await run('What is in this image?', [{ kind: 'image', name: 'cat.png', mimeType: 'image/png', dataUrl: 'data:image/png;base64,Y2F0' }]);+ assert.equal(captures.length, 0, 'an attached image is not joined by a page screenshot');+ const images = requests[0].at(-1).content.filter(block => block.type === 'image_url');+ assert.deepEqual(images.map(block => block.image_url.url), ['data:image/png;base64,Y2F0'], 'only the attached image reaches the model');+ }{const { agent, captures, run } = setup();agent.strictSecretMode = true;