Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Tiny Ask screenshots: none for a question about a linked page
Claude9fe66678b731
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.mdindex a2a387fcf..b736171dc 100644--- a/docs/webgpu-models.md+++ b/docs/webgpu-models.md@@ -181,7 +181,8 @@ Use the tools in `scripts/models/bonsai/`:- General questions take none, even on an image page: "what doesmelanoma look like?", "who was Ada Lovelace?". A screenshot alone neversuppresses offline references.- - Neither does a turn with an attached image: that image is the subject.+ - Neither does a turn with an attached image or a linked page: that is+ the subject, and the current tab would be the wrong source.- The capture is the same as automatic screenshots: redaction, sizebudget and the per-turn limit. It is ephemeral like page text, and itnever goes to a separate vision model.diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 20c655408..0472ad316 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -12007,7 +12007,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d: { url: linkedUrl, text, incomplete: fetched.hasMore === true };if (linked.unavailable) onUpdate?.('thinking', { step: 0, note: 'Could not read the linked page.' });}- const screenshot = tinyAskWantsScreenshot({ question: questionText, page, imageAttachment })+ const screenshot = tinyAskWantsScreenshot({ question: questionText, page, imageAttachment, linkedPage: !!linkedUrl })? await this._tinyAskScreenshot(tabId, onUpdate) : null;let modelLabel = '';try {diff --git a/src/chrome/src/agent/tiny-ask.js b/src/chrome/src/agent/tiny-ask.jsindex 6caf844a0..e99887be9 100644--- a/src/chrome/src/agent/tiny-ask.js+++ b/src/chrome/src/agent/tiny-ask.js@@ -102,9 +102,10 @@ export function tinyAskScreenshotNote(page = null) {// Whether a tiny Ask turn should also show the model the visible page: a// visual question about the page, or any question about a page that was read// but has almost no text (the answer is likely only in the pixels). Never-// with an attached image, which is then what the question is about.-export function tinyAskWantsScreenshot({ question = '', page = null, imageAttachment = false } = {}) {- if (imageAttachment) return false;+// with an attached image or a linked page, which is then what the question is+// about: a screenshot of the current tab would be the wrong source.+export function tinyAskWantsScreenshot({ question = '', page = null, imageAttachment = false, linkedPage = false } = {}) {+ if (imageAttachment || linkedPage) return false;const text = String(question || '');if (!PAGE_QUESTION.test(text) && !PAGE_VISUAL_REFERENCE.test(text)) return false;if (VISUAL_QUESTION.test(text)) return true;diff --git a/test/run.js b/test/run.jsindex 18da1b3ff..55d762fb7 100644--- a/test/run.js+++ b/test/run.js@@ -41621,6 +41621,8 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visu}assert.equal(tinyAskWantsScreenshot({ question: 'What is in this image?', page, imageAttachment: true }), false,'an attached image is the subject, not the page');+ assert.equal(tinyAskWantsScreenshot({ question: 'What color is the logo at https://example.com?', page, linkedPage: true }), false,+ 'a linked page is the subject, not the current tab');const textless = { kind: 'page', text: ' ' };for (const question of ['Summarize', 'What is on this page?', 'Describe it', 'Bu sayfada ne var?', 'Burada ne yazıyor?']) {assert.equal(tinyAskWantsScreenshot({ question, page: textless }), true, `a page question on a textless page: ${question}`);@@ -41663,6 +41665,7 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visuagent.executeTool = async (_tabId, name) => {if (name === 'read_page') return { text: pageText, hasMore: false, nextOffset: null };if (name === 'get_selection') return { text: '' };+ if (name === 'fetch_url') return { success: true, text: 'Linked page text.', hasMore: false };throw new Error(`unexpected tool ${name}`);};const captures = [];@@ -41715,6 +41718,12 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visuconst images = requests[0].at(-1).content.filter(block => block.type === 'image_url');assert.deepEqual(images.map(block => block.image_url.url), ['data:image/png;base64,Y2F0'], 'only the attached image reaches the model');}+ {+ const { requests, captures, run } = setup();+ await run('What color is the logo at https://brand.example/about?');+ assert.equal(captures.length, 0, 'a question about a linked page takes no current-tab screenshot');+ assert.match(JSON.stringify(requests[0].at(-1).content), /Linked page text\./);+ }{const { agent, captures, run } = setup();agent.strictSecretMode = true;