Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Tiny Ask screenshots: page reference and visual intent as two separate checks
Claude184c8dd93e83
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.mdindex b736171dc..6708e12d5 100644--- a/docs/webgpu-models.md+++ b/docs/webgpu-models.md@@ -174,13 +174,16 @@ Use the tools in `scripts/models/bonsai/`:are added only when there is no page text.- When the model reads images (Ternary Bonsai 2 2B Vision), a visualquestion about the page also gets a screenshot of the visible page- (`tinyAskWantsScreenshot`). Examples: "what color is the button?", "what- does the chart show?", "can you see …", "fotoğraftaki kupa ne renk?".+ (`tinyAskWantsScreenshot`). It needs both: the question points at the page+ (this, here, the page, "the photo", "that Buy button", "can you see",+ Turkish bu/burada/sayfa or a word for a thing on it), and asks something+ visual (appearance, colours, images, charts). Examples: "is the Buy+ button red?", "what does the chart show?", "fotoğraftaki kupa ne renk?".- A question about a page with almost no text ("what is on this page?")gets one as well, with a note to answer from the image.- General questions take none, even on an image page: "what does- melanoma look like?", "who was Ada Lovelace?". A screenshot alone never- suppresses offline references.+ melanoma look like?", "explain what melanoma looks like", "who was Ada+ Lovelace?". A screenshot alone never suppresses offline references.- Neither does a turn with an attached image or a linked page: that isthe subject, and the current tab would be the wrong source.- The capture is the same as automatic screenshots: redaction, sizediff --git a/src/chrome/src/agent/tiny-ask.js b/src/chrome/src/agent/tiny-ask.jsindex e99887be9..57d5572ef 100644--- a/src/chrome/src/agent/tiny-ask.js+++ b/src/chrome/src/agent/tiny-ask.js@@ -53,39 +53,47 @@ export const TINY_ASK_READ_MAX_CHARS = 60000;// scanned PDF). Other questions stay text-only: a screenshot adds about a// second of prefill on a desktop GPU and up to ~1k image tokens of context.export const TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS = 200;+// A screenshot needs two things: the question points at the current page, and+// what it asks is visual (or the page has almost no text to answer from).+// "What does melanoma look like?" is visual but general; "Summarize this+// page" points at the page but is answered from its text.++// Visual: appearance words, colours and visual media, in English and Turkish.+const HUES = 'red|green|blue|yellow|orange|purple|violet|pink|black|white|gr[ae]y|brown|beige|gold(?:en)?|silver';const VISUAL_WORDS = [- // English'images?', 'pictures?', 'photo(?:graph)?s?', 'pics?', 'screenshots?', 'screen', 'charts?', 'graphs?','diagrams?', 'maps?', 'logos?', 'icons?', 'colou?r(?:s|ed|ful)?', 'look(?:s|ing)? like', '(?:do|can) you see',- 'visible', 'thumbnails?', 'banners?', 'layout', 'fonts?', 'illustrations?', 'drawings?', 'videos?',- // Turkish (stems take any suffix: resmi, fotoğrafta, grafikte, rengi)+ 'visible', 'thumbnails?', 'banners?', 'layout', 'fonts?', 'illustrations?', 'drawings?', 'videos?', 'appearance',+ `(?:${HUES})(?:dish)?`,'resim\\p{L}*', 'resm\\p{L}*', 'görsel\\p{L}*', 'görüntü\\p{L}*', 'foto\\p{L}*', 'ekran\\p{L}*','grafi\\p{L}*', 'grafiğ\\p{L}*', 'şema\\p{L}*', 'diyagram\\p{L}*', 'harita\\p{L}*', 'logo\\p{L}*','ikon\\p{L}*', 'simge\\p{L}*', 'renk\\p{L}*', 'reng\\p{L}*', 'görünüyor\\p{L}*', 'görünüm\\p{L}*','görüyor\\p{L}*', 'çizim\\p{L}*', 'video\\p{L}*',+ '(?:kırmızı|yeşil|mavi|sarı|turuncu|mor|pembe|siyah|beyaz|gri|kahverengi|lacivert)\\p{L}*',];const VISUAL_QUESTION = new RegExp(`(?<!\\p{L})(?:${VISUAL_WORDS.join('|')})(?!\\p{L})`, 'iu');-// A question about the current page itself. Only these get a screenshot:-// "What does melanoma look like?" or "Who was Ada Lovelace?" is a general-// question even on an image page, and keeps its offline references.-const PAGE_WORDS = [- // English- 'this', 'here', 'page', 'site', 'tab', 'screen', 'summari[sz]e', 'describe', 'explain', 'tl;?dr',- "what(?:'s| is) (?:on|shown|happening)",- // Turkish- 'bu', 'bunu', 'bunda', 'burada', 'buradaki', 'sayfa\\p{L}*', 'site\\p{L}*', 'sekme\\p{L}*', 'özetle\\p{L}*',- 'açıkla\\p{L}*', 'anlat\\p{L}*', 'ne var', 'ne yazıyor', 'ne gösteriyor',-];-const PAGE_QUESTION = new RegExp(`(?<!\\p{L})(?:${PAGE_WORDS.join('|')})(?!\\p{L})`, 'iu');-// A specific visual thing, which can only be on the page: "the photo", "the-// Buy button", "can you see …"; in Turkish a visual stem with a suffix-// ("fotoğraftaki", "grafikte", "logonun rengi"), since there are no articles.-const PAGE_THINGS = 'images?|pictures?|photo(?:graph)?s?|pics?|screenshots?|charts?|graphs?|diagrams?|maps?|logos?|icons?|videos?|thumbnails?|banners?|buttons?|headers?|menus?|backgrounds?|illustrations?|drawings?|fonts?|text';-const PAGE_VISUAL_REFERENCE = new RegExp([- `(?<!\\p{L})(?:the|this|that|these|those)\\s+(?:\\p{L}+\\s+){0,2}(?:${PAGE_THINGS})(?!\\p{L})`,++// Pointing at the page: deictic words, the page itself, or a specific thing+// on it ("the photo", "that Buy button"). Verbs such as "explain" do not+// count on their own: "Explain what melanoma looks like" is general.+const PAGE_THINGS = 'images?|pictures?|photo(?:graph)?s?|pics?|screenshots?|charts?|graphs?|diagrams?|maps?|logos?|icons?|videos?|thumbnails?|banners?|buttons?|links?|headers?|menus?|backgrounds?|illustrations?|drawings?|fonts?|text|headings?|titles?';+const PAGE_REFERENCE = new RegExp([+ '(?<!\\p{L})(?:this|here)(?!\\p{L})',+ '(?<!\\p{L})on (?:the )?screen(?!\\p{L})',+ '(?<!\\p{L})(?:the|current|open) (?:page|site|website|tab|screen|article|document)(?!\\p{L})',+ `(?<!\\p{L})(?:the|that|these|those)\\s+(?:\\p{L}+\\s+){0,2}(?:${PAGE_THINGS})(?!\\p{L})`,'(?<!\\p{L})(?:do|can) you see(?!\\p{L})',- '(?<!\\p{L})(?:resim|resm|görsel|görüntü|foto|grafi|grafiğ|şema|diyagram|harita|logo|ikon|simge|renk|reng|çizim|video|ekran)\\p{L}+',+ "(?<!\\p{L})what(?:'s| is) (?:on|shown|happening)(?!\\p{L})",+ // Turkish has no articles: demonstratives, the page itself, or a word for+ // a thing on it ("fotoğraftaki", "logonun", "buton").+ '(?<!\\p{L})(?:bu|şu|bunu|bunda|burada|buradaki|bunlar\\p{L}*)(?!\\p{L})',+ '(?<!\\p{L})(?:sayfa|sekme|ekran)\\p{L}*',+ '(?<!\\p{L})(?:resim|resm|görsel|görüntü|foto|grafi|grafiğ|şema|diyagram|harita|logo|ikon|simge|çizim|video|buton|düğme|başlık)\\p{L}*',+ '(?<!\\p{L})ne (?:yazıyor|gösteriyor)(?!\\p{L})',].join('|'), 'iu');+// A bare request with nothing else to act on ("Summarize", "Describe it",+// "Özetle", "Bunu açıkla") can only mean the current page.+const BARE_PAGE_REQUEST = /^\s*(?:please\s+)?(?:summari[sz]e|describe|explain|tl;?dr)(?:\s+(?:it|(?:the|this) page))?(?:\s+please)?\s*[.?!]*\s*$|^\s*(?:bunu\s+|bu sayfayı\s+)?(?:özetle|açıkla|anlat)\p{L}*\s*[.?!]*\s*$/iu;// Precedes the screenshot in the user message; the system prompt says the// same, so text inside the image is never taken as an instruction.@@ -99,16 +107,16 @@ export function tinyAskScreenshotNote(page = null) {? TINY_ASK_TEXTLESS_SCREENSHOT_NOTE : TINY_ASK_SCREENSHOT_NOTE;}-// Whether a tiny Ask turn should also show the model the visible page: a-// visual question about the page, or any question about a page that was read-// but has almost no text (the answer is likely only in the pixels). Never+// Whether a tiny Ask turn should also show the model the visible page. Never// with an attached image or a linked page, which is then what the question is// about: a screenshot of the current tab would be the wrong source.export function tinyAskWantsScreenshot({ question = '', page = null, imageAttachment = false, linkedPage = false } = {}) {if (imageAttachment || linkedPage) return false;const text = String(question || '');- if (!PAGE_QUESTION.test(text) && !PAGE_VISUAL_REFERENCE.test(text)) return false;+ if (!PAGE_REFERENCE.test(text) && !BARE_PAGE_REQUEST.test(text)) return false;if (VISUAL_QUESTION.test(text)) return true;+ // A page that was read but has almost no text: the answer is likely only+ // in the pixels.return !!page?.kind && String(page.text || '').trim().length < TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS;}diff --git a/test/run.js b/test/run.jsindex 55d762fb7..c5f27cad4 100644--- a/test/run.js+++ b/test/run.js@@ -41612,11 +41612,13 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visuconst page = { kind: 'page', text: 'Plenty of readable page text. '.repeat(20) };for (const question of ['What is in this image?', 'What color is the button?', 'What does the chart show?','What color is the mug in the photo?', 'Can you see a cat?', 'Bu resimde ne var?', 'Grafikte ne görünüyor?',- 'Logonun rengi ne?', 'Ekranda ne yazıyor?', 'Fotoğraftaki kupa ne renk?']) {+ 'Logonun rengi ne?', 'Ekranda ne yazıyor?', 'Fotoğraftaki kupa ne renk?', 'Is the Buy button red?', 'Is this button blue?',+ 'Buton kırmızı mı?']) {assert.equal(tinyAskWantsScreenshot({ question, page }), true, question);}for (const question of ['How much is the blue plan?', 'Summarize this page', 'figure out the price', 'see the reviews',- 'Görevi özetle', 'Fiyat ne kadar?', 'What does melanoma look like?', 'What color is a flamingo?', 'Melanom nasıl görünür?']) {+ 'Görevi özetle', 'Fiyat ne kadar?', 'What does melanoma look like?', 'What color is a flamingo?', 'Melanom nasıl görünür?',+ 'Explain what melanoma looks like', 'Melanomun rengi ne?', 'Is the sky blue?', 'I heard that melanoma looks dark']) {assert.equal(tinyAskWantsScreenshot({ question, page }), false, `not a visual question about the page: ${question}`);}assert.equal(tinyAskWantsScreenshot({ question: 'What is in this image?', page, imageAttachment: true }), false,@@ -41624,10 +41626,11 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visuassert.equal(tinyAskWantsScreenshot({ question: 'What color is the logo at https://example.com?', page, linkedPage: true }), false,'a linked page is the subject, not the current tab');const textless = { kind: 'page', text: ' ' };- for (const question of ['Summarize', 'What is on this page?', 'Describe it', 'Bu sayfada ne var?', 'Burada ne yazıyor?']) {+ for (const question of ['Summarize', 'What is on this page?', 'Describe it', 'Bu sayfada ne var?', 'Burada ne yazıyor?', 'Özetle', 'Bunu açıkla']) {assert.equal(tinyAskWantsScreenshot({ question, page: textless }), true, `a page question on a textless page: ${question}`);}- for (const question of ['Who was Ada Lovelace?', 'How do I treat a burn?', 'Ada Lovelace kimdir?', 'What does melanoma look like?']) {+ for (const question of ['Who was Ada Lovelace?', 'How do I treat a burn?', 'Ada Lovelace kimdir?', 'What does melanoma look like?',+ 'Explain what melanoma looks like']) {assert.equal(tinyAskWantsScreenshot({ question, page: textless }), false, `a general question keeps its references: ${question}`);}assert.equal(tinyAskWantsScreenshot({ question: 'Summarize', page: { kind: 'pdf', text: '', unavailable: 'no text' } }), true, 'a scanned PDF');