Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Tiny Ask screenshots: keep the page capture for mixed-source questions
Claudef413791aa9f1
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.mdindex 6708e12d5..4f2921b50 100644--- a/docs/webgpu-models.md+++ b/docs/webgpu-models.md@@ -184,8 +184,10 @@ Use the tools in `scripts/models/bonsai/`:- General questions take none, even on an image page: "what doesmelanoma look like?", "explain what melanoma looks like", "who was AdaLovelace?". A screenshot alone never suppresses offline references.- - Neither does a turn with an attached image or a linked page: that is- the subject, and the current tab would be the wrong source.+ - A turn with an attached image or a linked page takes one only when the+ question names the page outright ("compare the attached image with the+ logo on this page"). Otherwise that source is the subject, and the+ current tab would be the wrong one.- The capture is the same as automatic screenshots: redaction, sizebudget and the per-turn limit. It is ephemeral like page text, and itnever goes to a separate vision model.diff --git a/src/chrome/src/agent/tiny-ask.js b/src/chrome/src/agent/tiny-ask.jsindex 57d5572ef..4444c8a86 100644--- a/src/chrome/src/agent/tiny-ask.js+++ b/src/chrome/src/agent/tiny-ask.js@@ -91,6 +91,10 @@ const PAGE_REFERENCE = new RegExp(['(?<!\\p{L})(?:resim|resm|görsel|görüntü|foto|grafi|grafiğ|şema|diyagram|harita|logo|ikon|simge|çizim|video|buton|düğme|başlık)\\p{L}*','(?<!\\p{L})ne (?:yazıyor|gösteriyor)(?!\\p{L})',].join('|'), 'iu');+// The page named outright ("this page", "on screen", "sayfada"). With an+// attached image or a linked page in the turn, "this image" means that+// source, so only this explicit form still points at the current page.+const EXPLICIT_PAGE_REFERENCE = /(?<!\p{L})(?:(?:the|this|current|open) (?:page|site|website|tab|screen)|on (?:the )?screen)(?!\p{L})|(?<!\p{L})(?:sayfa|sekme|ekran)\p{L}*/iu;// A bare request with nothing else to act on ("Summarize", "Describe it",// "Özetle", "Bunu açıkla") can only mean the current page.const BARE_PAGE_REQUEST = /^\s*(?:please\s+)?(?:summari[sz]e|describe|explain|tl;?dr)(?:\s+(?:it|(?:the|this) page))?(?:\s+please)?\s*[.?!]*\s*$|^\s*(?:bunu\s+|bu sayfayı\s+)?(?:özetle|açıkla|anlat)\p{L}*\s*[.?!]*\s*$/iu;@@ -107,12 +111,16 @@ export function tinyAskScreenshotNote(page = null) {? TINY_ASK_TEXTLESS_SCREENSHOT_NOTE : TINY_ASK_SCREENSHOT_NOTE;}-// Whether a tiny Ask turn should also show the model the visible page. Never-// with an attached image or a linked page, which is then what the question is-// about: a screenshot of the current tab would be the wrong source.+// Whether a tiny Ask turn should also show the model the visible page. With+// an attached image or a linked page, that source is the subject unless the+// question names the current page outright ("compare the attached image with+// the logo on this page"); otherwise a current-tab screenshot would be the+// wrong source.export function tinyAskWantsScreenshot({ question = '', page = null, imageAttachment = false, linkedPage = false } = {}) {- if (imageAttachment || linkedPage) return false;const text = String(question || '');+ if (imageAttachment || linkedPage) {+ return EXPLICIT_PAGE_REFERENCE.test(text) && VISUAL_QUESTION.test(text);+ }if (!PAGE_REFERENCE.test(text) && !BARE_PAGE_REQUEST.test(text)) return false;if (VISUAL_QUESTION.test(text)) return true;// A page that was read but has almost no text: the answer is likely onlydiff --git a/test/run.js b/test/run.jsindex c5f27cad4..3646e30a2 100644--- a/test/run.js+++ b/test/run.js@@ -41625,6 +41625,15 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visu'an attached image is the subject, not the page');assert.equal(tinyAskWantsScreenshot({ question: 'What color is the logo at https://example.com?', page, linkedPage: true }), false,'a linked page is the subject, not the current tab');+ for (const [question, options] of [+ ['Compare the attached image with the logo on this page.', { imageAttachment: true }],+ ['Bu resim sayfadaki logoya benziyor mu?', { imageAttachment: true }],+ ['Is the logo at https://example.com the same colour as the one on this page?', { linkedPage: true }],+ ]) {+ assert.equal(tinyAskWantsScreenshot({ question, page, ...options }), true, `a mixed-source question names the page: ${question}`);+ }+ assert.equal(tinyAskWantsScreenshot({ question: 'Describe this image', page, imageAttachment: true }), false,+ 'with an attachment, "this image" is the attachment');const textless = { kind: 'page', text: ' ' };for (const question of ['Summarize', 'What is on this page?', 'Describe it', 'Bu sayfada ne var?', 'Burada ne yazıyor?', 'Özetle', 'Bunu açıkla']) {assert.equal(tinyAskWantsScreenshot({ question, page: textless }), true, `a page question on a textless page: ${question}`);@@ -41720,6 +41729,10 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visuassert.equal(captures.length, 0, 'an attached image is not joined by a page screenshot');const images = requests[0].at(-1).content.filter(block => block.type === 'image_url');assert.deepEqual(images.map(block => block.image_url.url), ['data:image/png;base64,Y2F0'], 'only the attached image reaches the model');+ await run('Compare the attached image with the logo on this page.', [{ kind: 'image', name: 'logo.png', mimeType: 'image/png', dataUrl: 'data:image/png;base64,bG9nbw==' }]);+ assert.equal(captures.length, 1, 'a question that names the page as well takes a page screenshot');+ assert.deepEqual(requests[1].at(-1).content.filter(block => block.type === 'image_url').map(block => block.image_url.url),+ ['data:image/jpeg;base64,c2NyZWVu', 'data:image/png;base64,bG9nbw=='], 'both the page screenshot and the attachment reach the model');}{const { requests, captures, run } = setup();