Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Tiny Ask screenshots: keep the page capture for mixed-source questions

Claudef413791aa9f1
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.md
index 6708e12d5..4f2921b50 100644
--- a/docs/webgpu-models.md
+++ b/docs/webgpu-models.md
@@ -184,8 +184,10 @@ Use the tools in `scripts/models/bonsai/`:
- General questions take none, even on an image page: "what does
melanoma look like?", "explain what melanoma looks like", "who was Ada
Lovelace?". A screenshot alone never suppresses offline references.
- - Neither does a turn with an attached image or a linked page: that is
- the subject, and the current tab would be the wrong source.
+ - A turn with an attached image or a linked page takes one only when the
+ question names the page outright ("compare the attached image with the
+ logo on this page"). Otherwise that source is the subject, and the
+ current tab would be the wrong one.
- The capture is the same as automatic screenshots: redaction, size
budget and the per-turn limit. It is ephemeral like page text, and it
never goes to a separate vision model.
diff --git a/src/chrome/src/agent/tiny-ask.js b/src/chrome/src/agent/tiny-ask.js
index 57d5572ef..4444c8a86 100644
--- a/src/chrome/src/agent/tiny-ask.js
+++ b/src/chrome/src/agent/tiny-ask.js
@@ -91,6 +91,10 @@ const PAGE_REFERENCE = new RegExp([
'(?<!\\p{L})(?:resim|resm|görsel|görüntü|foto|grafi|grafiğ|şema|diyagram|harita|logo|ikon|simge|çizim|video|buton|düğme|başlık)\\p{L}*',
'(?<!\\p{L})ne (?:yazıyor|gösteriyor)(?!\\p{L})',
].join('|'), 'iu');
+// The page named outright ("this page", "on screen", "sayfada"). With an
+// attached image or a linked page in the turn, "this image" means that
+// source, so only this explicit form still points at the current page.
+const EXPLICIT_PAGE_REFERENCE = /(?<!\p{L})(?:(?:the|this|current|open) (?:page|site|website|tab|screen)|on (?:the )?screen)(?!\p{L})|(?<!\p{L})(?:sayfa|sekme|ekran)\p{L}*/iu;
// A bare request with nothing else to act on ("Summarize", "Describe it",
// "Özetle", "Bunu açıkla") can only mean the current page.
const BARE_PAGE_REQUEST = /^\s*(?:please\s+)?(?:summari[sz]e|describe|explain|tl;?dr)(?:\s+(?:it|(?:the|this) page))?(?:\s+please)?\s*[.?!]*\s*$|^\s*(?:bunu\s+|bu sayfayı\s+)?(?:özetle|açıkla|anlat)\p{L}*\s*[.?!]*\s*$/iu;
@@ -107,12 +111,16 @@ export function tinyAskScreenshotNote(page = null) {
? TINY_ASK_TEXTLESS_SCREENSHOT_NOTE : TINY_ASK_SCREENSHOT_NOTE;
}
-// Whether a tiny Ask turn should also show the model the visible page. Never
-// with an attached image or a linked page, which is then what the question is
-// about: a screenshot of the current tab would be the wrong source.
+// Whether a tiny Ask turn should also show the model the visible page. With
+// an attached image or a linked page, that source is the subject unless the
+// question names the current page outright ("compare the attached image with
+// the logo on this page"); otherwise a current-tab screenshot would be the
+// wrong source.
export function tinyAskWantsScreenshot({ question = '', page = null, imageAttachment = false, linkedPage = false } = {}) {
- if (imageAttachment || linkedPage) return false;
const text = String(question || '');
+ if (imageAttachment || linkedPage) {
+ return EXPLICIT_PAGE_REFERENCE.test(text) && VISUAL_QUESTION.test(text);
+ }
if (!PAGE_REFERENCE.test(text) && !BARE_PAGE_REQUEST.test(text)) return false;
if (VISUAL_QUESTION.test(text)) return true;
// A page that was read but has almost no text: the answer is likely only
diff --git a/test/run.js b/test/run.js
index c5f27cad4..3646e30a2 100644
--- a/test/run.js
+++ b/test/run.js
@@ -41625,6 +41625,15 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visu
'an attached image is the subject, not the page');
assert.equal(tinyAskWantsScreenshot({ question: 'What color is the logo at https://example.com?', page, linkedPage: true }), false,
'a linked page is the subject, not the current tab');
+ for (const [question, options] of [
+ ['Compare the attached image with the logo on this page.', { imageAttachment: true }],
+ ['Bu resim sayfadaki logoya benziyor mu?', { imageAttachment: true }],
+ ['Is the logo at https://example.com the same colour as the one on this page?', { linkedPage: true }],
+ ]) {
+ assert.equal(tinyAskWantsScreenshot({ question, page, ...options }), true, `a mixed-source question names the page: ${question}`);
+ }
+ assert.equal(tinyAskWantsScreenshot({ question: 'Describe this image', page, imageAttachment: true }), false,
+ 'with an attachment, "this image" is the attachment');
const textless = { kind: 'page', text: ' ' };
for (const question of ['Summarize', 'What is on this page?', 'Describe it', 'Bu sayfada ne var?', 'Burada ne yazıyor?', 'Özetle', 'Bunu açıkla']) {
assert.equal(tinyAskWantsScreenshot({ question, page: textless }), true, `a page question on a textless page: ${question}`);
@@ -41720,6 +41729,10 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visu
assert.equal(captures.length, 0, 'an attached image is not joined by a page screenshot');
const images = requests[0].at(-1).content.filter(block => block.type === 'image_url');
assert.deepEqual(images.map(block => block.image_url.url), ['data:image/png;base64,Y2F0'], 'only the attached image reaches the model');
+ await run('Compare the attached image with the logo on this page.', [{ kind: 'image', name: 'logo.png', mimeType: 'image/png', dataUrl: 'data:image/png;base64,bG9nbw==' }]);
+ assert.equal(captures.length, 1, 'a question that names the page as well takes a page screenshot');
+ assert.deepEqual(requests[1].at(-1).content.filter(block => block.type === 'image_url').map(block => block.image_url.url),
+ ['data:image/jpeg;base64,c2NyZWVu', 'data:image/png;base64,bG9nbw=='], 'both the page screenshot and the attachment reach the model');
}
{
const { requests, captures, run } = setup();