Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
WIP gate
Claude4de9b7cae207
diff --git a/scripts/check-tiny-ask-screenshot-gate.mjs b/scripts/check-tiny-ask-screenshot-gate.mjsnew file mode 100644index 000000000..9696804ca--- /dev/null+++ b/scripts/check-tiny-ask-screenshot-gate.mjs@@ -0,0 +1,154 @@+// Hardware eval for the tiny Ask screenshot gate: the in-browser model's own+// Yes/No readout on whether a question needs a screenshot of the current page+// (src/chrome/src/agent/tiny-ask.js, tinyAskScreenshotGate). Runs labelled+// questions in several languages through the real extension's decision path+// and reports accuracy, latency and every miss.+//+// node scripts/check-tiny-ask-screenshot-gate.mjs --profile <profile with the model cached> \+// --adapter-luid 0,82094 --output .build/tiny-ask-gate/result.json+//+// --extension DIR unpacked extension (default src/chrome); use a fresh profile per build+// --threshold P Yes threshold to report (default TINY_ASK_SCREENSHOT_GATE_THRESHOLD)+import fs from 'node:fs/promises';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { chromium } from 'playwright';+import { tinyAskScreenshotGate, TINY_ASK_SCREENSHOT_GATE_THRESHOLD } from '../src/chrome/src/agent/tiny-ask.js';++const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');+const args = process.argv.slice(2);+const option = (name, fallback) => { const i = args.indexOf(name); return i < 0 ? fallback : args[i + 1]; };+const extension = path.resolve(option('--extension', path.join(root, 'src/chrome')));+const profile = path.resolve(option('--profile', '.build/tiny-ask-gate/profile'));+const output = path.resolve(option('--output', '.build/tiny-ask-gate/result.json'));+const model = 'webbrain-one/Ternary-Bonsai-2-2B-ONNX-WebGPU-Vision';+const threshold = Number(option('--threshold', TINY_ASK_SCREENSHOT_GATE_THRESHOLD));++// [expected, language, question, context]. Expected Yes: the answer depends on+// what the current page visibly shows. Pages have text unless `pageChars` says+// otherwise.+const TEXT_PAGE = { pageChars: 5000 };+const TEXTLESS = { pageChars: 0 };+const CASES = [+ // Visual questions about the current page.+ [true, 'en', 'What color is the Buy button?', TEXT_PAGE],+ [true, 'en', 'What does the chart on this page show?', TEXT_PAGE],+ [true, 'en', 'Describe the photo at the top.', TEXT_PAGE],+ [true, 'en', 'Is the logo red or blue?', TEXT_PAGE],+ [true, 'en', 'How does this page look on my screen?', TEXT_PAGE],+ [true, 'tr', 'Satın al butonu ne renk?', TEXT_PAGE],+ [true, 'tr', 'Bu sayfadaki grafik ne gösteriyor?', TEXT_PAGE],+ [true, 'tr', 'Fotoğraftaki kupa ne renk?', TEXT_PAGE],+ [true, 'de', 'Welche Farbe hat der Kaufen-Knopf?', TEXT_PAGE],+ [true, 'de', 'Was zeigt das Diagramm auf dieser Seite?', TEXT_PAGE],+ [true, 'es', '¿De qué color es el botón de compra?', TEXT_PAGE],+ [true, 'es', '¿Qué muestra la foto de esta página?', TEXT_PAGE],+ [true, 'fr', 'De quelle couleur est le logo ?', TEXT_PAGE],+ [true, 'fr', 'Que montre le graphique sur cette page ?', TEXT_PAGE],+ [true, 'it', 'Di che colore è il pulsante Acquista?', TEXT_PAGE],+ [true, 'pt', 'O que mostra a imagem desta página?', TEXT_PAGE],+ [true, 'ja', 'このページの画像には何が写っていますか?', TEXT_PAGE],+ [true, 'ar', 'ما لون زر الشراء في هذه الصفحة؟', TEXT_PAGE],+ // Questions about a page with almost no text.+ [true, 'en', 'What is on this page?', TEXTLESS],+ [true, 'tr', 'Bu sayfada ne var?', TEXTLESS],+ [true, 'de', 'Was ist auf dieser Seite zu sehen?', TEXTLESS],+ [true, 'es', 'Resume esta página.', TEXTLESS],+ // Mixed sources that name the current page.+ [true, 'en', 'Compare the attached image with the logo on this page.', { ...TEXT_PAGE, imageAttachment: true }],+ [true, 'tr', 'Eklediğim resim bu sayfadaki logoya benziyor mu?', { ...TEXT_PAGE, imageAttachment: true }],+ // Answered from the page text.+ [false, 'en', 'How much is the Pro plan per month?', TEXT_PAGE],+ [false, 'en', 'Summarize this page in two sentences.', TEXT_PAGE],+ [false, 'en', 'When did the repairs end?', TEXT_PAGE],+ [false, 'en', 'Who wrote this article?', TEXT_PAGE],+ [false, 'tr', 'Pro planı aylık ne kadar?', TEXT_PAGE],+ [false, 'tr', 'Bu sayfayı özetle.', TEXT_PAGE],+ [false, 'de', 'Wie viel kostet der Pro-Tarif im Monat?', TEXT_PAGE],+ [false, 'de', 'Fasse diese Seite zusammen.', TEXT_PAGE],+ [false, 'es', '¿Cuánto cuesta el plan Pro al mes?', TEXT_PAGE],+ [false, 'fr', 'Résume cette page en deux phrases.', TEXT_PAGE],+ [false, 'it', 'Quando è stato pubblicato questo articolo?', TEXT_PAGE],+ [false, 'pt', 'Quanto custa o plano Pro por mês?', TEXT_PAGE],+ [false, 'ja', 'このページを要約してください。', TEXT_PAGE],+ [false, 'ar', 'لخص هذه الصفحة.', TEXT_PAGE],+ // General knowledge, including visual-sounding questions.+ [false, 'en', 'What does melanoma look like?', TEXT_PAGE],+ [false, 'en', 'Explain what melanoma looks like.', TEXTLESS],+ [false, 'en', 'Who was Ada Lovelace?', TEXTLESS],+ [false, 'en', 'What color is a flamingo?', TEXT_PAGE],+ [false, 'en', 'How do I treat a burn?', TEXT_PAGE],+ [false, 'tr', 'Melanom nasıl görünür?', TEXT_PAGE],+ [false, 'tr', 'Ada Lovelace kimdir?', TEXTLESS],+ [false, 'de', 'Wie sieht ein Flamingo aus?', TEXT_PAGE],+ [false, 'es', '¿Qué es la fotosíntesis?', TEXT_PAGE],+ [false, 'fr', 'Quelle est la capitale du Japon ?', TEXTLESS],+ [false, 'ja', '富士山の高さは?', TEXT_PAGE],+ [false, 'ar', 'من هو ابن سينا؟', TEXT_PAGE],+ // Only about an attached image or a linked page.+ [false, 'en', 'What is in this image?', { ...TEXT_PAGE, imageAttachment: true }],+ [false, 'tr', 'Bu resimde ne var?', { ...TEXT_PAGE, imageAttachment: true }],+ [false, 'de', 'Beschreibe dieses Bild.', { ...TEXT_PAGE, imageAttachment: true }],+ [false, 'en', 'What color is the logo at https://brand.example/about?', { ...TEXT_PAGE, linkedPage: true }],+ [false, 'es', 'Resume https://news.example/launch en una frase.', { ...TEXT_PAGE, linkedPage: true }],+];++const result = { model, extension, threshold, startedAt: new Date().toISOString(), cases: [] };+let context;+try {+ context = await chromium.launchPersistentContext(profile, {+ headless: true,+ channel: 'chromium',+ args: [`--disable-extensions-except=${extension}`, `--load-extension=${extension}`,+ '--enable-unsafe-webgpu', '--force-high-performance-gpu',+ ...(option('--adapter-luid', '') ? [`--use-adapter-luid=${option('--adapter-luid')}`] : [])],+ });+ const worker = context.serviceWorkers()[0] || await context.waitForEvent('serviceworker', { timeout: 30000 });+ const id = new URL(worker.url()).host;+ const harness = await context.newPage();+ await harness.goto(`chrome-extension://${id}/src/ui/settings.html`);+ const message = (action, data = {}) => harness.evaluate(({ action, data }) => chrome.runtime.sendMessage({ target: 'background', action, ...data }), { action, data });+ await harness.evaluate(() => chrome.storage.local.set({ onboardingComplete: true, wbLocale: 'en', helpImproveWebBrain: false }));+ result.extensionVersion = await harness.evaluate(() => chrome.runtime.getManifest().version);+ const preset = (await import('../src/chrome/src/providers/webgpu.js')).webgpuModelPreset(model);+ await message('update_provider', { providerId: 'webgpu', config: { model, dtype: preset.dtype } });+ const status = await message('get_webgpu_download_status', { model });+ if (!status?.ready) throw new Error(`${model} is not cached in this profile`);+ for (const [index, [expected, language, question, facts]] of CASES.entries()) {+ const gate = tinyAskScreenshotGate({ question, ...facts });+ const started = Date.now();+ const reply = await message('webgpu_decide_fixture', { fixture: { ...gate, promptVariant: 'readout' } });+ const seconds = (Date.now() - started) / 1000;+ const probability = Number(reply?.result?.answers?.needs_screenshot?.noul);+ const entry = {+ expected, language, question, ...facts, probability: Number.isFinite(probability) ? probability : null,+ predicted: probability >= threshold, seconds, ...(reply?.success ? {} : { error: reply?.error || 'no reply' }),+ };+ entry.correct = entry.error ? false : entry.predicted === expected;+ if (index > 0) result.cases.push(entry); else result.cold = entry;+ console.log(JSON.stringify({ ok: entry.correct, p: entry.probability?.toFixed(3), s: seconds, language, question }));+ }+ // The first call loads the model; score it, but keep it out of the latency.+ const all = [result.cold, ...result.cases];+ const latency = result.cases.map(entry => entry.seconds).sort((a, b) => a - b);+ const share = (rows, test) => `${rows.filter(test).length}/${rows.length}`;+ result.summary = {+ accuracy: share(all, entry => entry.correct),+ yesRecall: share(all.filter(entry => entry.expected), entry => entry.predicted),+ falseYes: share(all.filter(entry => !entry.expected), entry => entry.predicted),+ byLanguage: Object.fromEntries([...new Set(all.map(entry => entry.language))]+ .map(language => [language, share(all.filter(entry => entry.language === language), entry => entry.correct)])),+ latencySeconds: { p50: latency[Math.floor(latency.length / 2)], p95: latency[Math.floor(latency.length * 0.95)] },+ misses: all.filter(entry => !entry.correct).map(entry => ({ expected: entry.expected, p: entry.probability, question: entry.question })),+ };+ console.log(JSON.stringify(result.summary, null, 1));+} catch (error) {+ result.error = String(error?.stack || error);+ console.error(error);+ process.exitCode = 1;+} finally {+ result.finishedAt = new Date().toISOString();+ await fs.mkdir(path.dirname(output), { recursive: true });+ await fs.writeFile(output, JSON.stringify(result, null, 2));+ await context?.close().catch(() => {});+}diff --git a/scripts/check-webgpu-ask.mjs b/scripts/check-webgpu-ask.mjsindex 5f66844cb..292bb046a 100644--- a/scripts/check-webgpu-ask.mjs+++ b/scripts/check-webgpu-ask.mjs@@ -113,11 +113,15 @@ const tasks = [{ name: 'act-request', page: '/pricing.html', text: 'Click the Buy Pro button.', expect: /\S/, reject: /\b(?:I(?:'ve| have)? clicked|clicked the)\b/i },{ name: 'linked-url', page: '/article.html', text: `Summarize ${base}/linked.html in one sentence.`, expect: /May|lettuce|Orbital/i },{ name: 'browser-page', page: 'chrome://version/', text: 'hello', expect: /\S/ },- // A vision model also sees the visible page for visual questions and for- // pages with almost no text; text questions (price above) stay text-only.+ // A vision model also sees the visible page when its own readout says the+ // question needs it; text questions (price above) stay text-only.{ name: 'image-color', page: '/image.html', text: 'What color is the mug in the photo?', expect: /green/i, screenshot: true },{ name: 'image-turkish', page: '/image.html', text: 'Fotoğraftaki kupa ne renk?', expect: /yeşil|green/i, screenshot: true },{ name: 'image-only-page', page: '/poster.html', text: 'What is on this page?', expect: /orange|circle|sun/i, screenshot: true },+ // The decision to look is the model's own readout, so it works in any language.+ { name: 'image-german', page: '/image.html', text: 'Welche Farbe hat die Tasse auf dem Foto?', expect: /grün|green/i, screenshot: true },+ { name: 'image-spanish', page: '/image.html', text: '¿De qué color es la taza de la foto?', expect: /verde|green/i, screenshot: true },+ { name: 'text-german', page: '/pricing.html', text: 'Wie viel kostet der Pro-Tarif im Monat?', expect: /29/, screenshot: false },].filter(task => !taskFilter || taskFilter.has(task.name));await fs.mkdir(path.dirname(output), { recursive: true });diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 0472ad316..7205ce602 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -208,7 +208,7 @@ import { shouldAutoGroupTabs } from '../tab-group-preference.js';import {TINY_ASK_ACT_BLOCKED_MESSAGE, TINY_ASK_COMPACT_LINKED_MAX_CHARS, TINY_ASK_COMPACT_PAGE_MAX_CHARS,TINY_ASK_COMPACT_SELECTION_MAX_CHARS, TINY_ASK_READ_MAX_CHARS, TINY_ASK_READ_WINDOW_CHARS, TINY_ASK_SYSTEM_PROMPT,- tinyAskLinkedUrl, tinyAskScreenshotNote, tinyAskPageContext, tinyAskPageReadable, tinyAskRuntimeLine, tinyAskWantsScreenshot,+ TINY_ASK_SCREENSHOT_GATE_THRESHOLD, tinyAskLinkedUrl, tinyAskScreenshotGate, tinyAskScreenshotNote, tinyAskPageContext, tinyAskPageReadable, tinyAskRuntimeLine,} from './tiny-ask.js';import { webgpuModelDisplayName } from '../providers/webgpu.js';@@ -12007,8 +12007,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d: { url: linkedUrl, text, incomplete: fetched.hasMore === true };if (linked.unavailable) onUpdate?.('thinking', { step: 0, note: 'Could not read the linked page.' });}- const screenshot = tinyAskWantsScreenshot({ question: questionText, page, imageAttachment, linkedPage: !!linkedUrl })- ? await this._tinyAskScreenshot(tabId, onUpdate) : null;+ const screenshot = await this._tinyAskScreenshot(tabId, {+ question: questionText, page, imageAttachment, linkedPage: !!linkedUrl, onUpdate,+ });let modelLabel = '';try {const provider = this._activeProvider(tabId);@@ -12055,23 +12056,45 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn enriched;}- // The visible page for a tiny Ask turn, only when the in-browser model reads- // images itself and Strict Secret Mode is off. Never a separate describing- // model: tiny Ask stays on the user's device. Same capture as the agent's+ // The visible page for a tiny Ask turn, when the in-browser model reads+ // images itself and its own Yes/No readout says the question needs it (see+ // tinyAskScreenshotGate: language-independent, and blind to page content).+ // Never in Strict Secret Mode, never through a separate describing model:+ // tiny Ask stays on the user's device. Same capture as the agent's// automatic screenshots (redaction, size budget, per-turn screenshot limit).- async _tinyAskScreenshot(tabId, onUpdate = null) {+ // Any failure leaves the turn text-only, as before.+ async _tinyAskScreenshot(tabId, { question = '', page = null, imageAttachment = false, linkedPage = false, onUpdate = null } = {}) {// Redaction covers form fields, not text drawn in images or canvases.- if (this.strictSecretMode) return null;+ if (this.strictSecretMode || !page?.kind) return null;let provider = null;let route = null;try {provider = this._activeProvider(tabId);- if (!this._isInBrowserProvider(provider)) return null;+ if (!this._isInBrowserProvider(provider) || typeof provider.decide !== 'function') return null;route = await this._resolveVisionRoute(tabId, provider);} catch {return null;}if (route?.route !== 'active_raw' || route.rawImage !== true || route.provider !== provider) return null;+ const started = Date.now();+ let probability = null;+ try {+ const gate = tinyAskScreenshotGate({+ question, pageChars: String(page.text || '').trim().length, imageAttachment, linkedPage,+ });+ const result = await provider.decide({ ...gate, promptVariant: 'readout', signal: this._runAbortSignal(tabId) });+ probability = Number(result?.answers?.needs_screenshot?.noul);+ } catch (error) {+ if (error?.name === 'AbortError' || this._checkAbort(tabId)) throw error;+ }+ const runId = this.currentRunId.get(tabId);+ if (runId) {+ trace.recordNote(runId, null, 'tiny_ask_screenshot_gate', {+ probability: Number.isFinite(probability) ? Math.round(probability * 1000) / 1000 : null,+ ms: Date.now() - started,+ });+ }+ if (!(probability >= TINY_ASK_SCREENSHOT_GATE_THRESHOLD)) return null;onUpdate?.('thinking', { step: 0, note: 'Looking at the page…' });try {const shot = await this._captureBudgetedAutoScreenshot(tabId);diff --git a/src/chrome/src/agent/tiny-ask.js b/src/chrome/src/agent/tiny-ask.jsindex 4444c8a86..d93e411fd 100644--- a/src/chrome/src/agent/tiny-ask.js+++ b/src/chrome/src/agent/tiny-ask.js@@ -47,57 +47,33 @@ export const TINY_ASK_BACKGROUND_PAGE_MAX_CHARS = 3000;// up to this cap so the excerpt can come from anywhere in the document.export const TINY_ASK_READ_WINDOW_CHARS = 6000;export const TINY_ASK_READ_MAX_CHARS = 60000;-// In-browser models that read images (Ternary Bonsai 2 2B Vision) also get-// the visible part of the page when the question is about something visual,-// or when the page has almost no readable text (an image, a canvas app, a-// scanned PDF). Other questions stay text-only: a screenshot adds about a-// second of prefill on a desktop GPU and up to ~1k image tokens of context.+// In-browser models that read images (Ternary Bonsai 2 2B Vision) can also be+// shown the visible part of the page. Whether a question needs it is decided+// by the model itself: one Yes/No readout through WebGPUProvider.decide+// before the answer (no text generated, about 0.2-0.4 s on a desktop GPU), so+// it works in any language. The readout sees only the user's message and+// facts WebBrain knows (how much page text there is, an attached image, a+// linked page), never page content, so a page cannot steer the decision.export const TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS = 200;-// A screenshot needs two things: the question points at the current page, and-// what it asks is visual (or the page has almost no text to answer from).-// "What does melanoma look like?" is visual but general; "Summarize this-// page" points at the page but is answered from its text.+export const TINY_ASK_SCREENSHOT_GATE_THRESHOLD = 0.5;+export const TINY_ASK_SCREENSHOT_GATE_QUESTION = 'Does answering this message well require looking at a screenshot of '+ + 'the current page? Yes only if the answer depends on what the current page visibly shows that its text does '+ + 'not: images, photos, charts, colours, layout or design, or anything at all when the page has almost no text '+ + 'and the message is about the page. No for questions answered from the page text, for general knowledge '+ + 'questions, and for questions only about an attached image or a linked page.';-// Visual: appearance words, colours and visual media, in English and Turkish.-const HUES = 'red|green|blue|yellow|orange|purple|violet|pink|black|white|gr[ae]y|brown|beige|gold(?:en)?|silver';-const VISUAL_WORDS = [- 'images?', 'pictures?', 'photo(?:graph)?s?', 'pics?', 'screenshots?', 'screen', 'charts?', 'graphs?',- 'diagrams?', 'maps?', 'logos?', 'icons?', 'colou?r(?:s|ed|ful)?', 'look(?:s|ing)? like', '(?:do|can) you see',- 'visible', 'thumbnails?', 'banners?', 'layout', 'fonts?', 'illustrations?', 'drawings?', 'videos?', 'appearance',- `(?:${HUES})(?:dish)?`,- 'resim\\p{L}*', 'resm\\p{L}*', 'görsel\\p{L}*', 'görüntü\\p{L}*', 'foto\\p{L}*', 'ekran\\p{L}*',- 'grafi\\p{L}*', 'grafiğ\\p{L}*', 'şema\\p{L}*', 'diyagram\\p{L}*', 'harita\\p{L}*', 'logo\\p{L}*',- 'ikon\\p{L}*', 'simge\\p{L}*', 'renk\\p{L}*', 'reng\\p{L}*', 'görünüyor\\p{L}*', 'görünüm\\p{L}*',- 'görüyor\\p{L}*', 'çizim\\p{L}*', 'video\\p{L}*',- '(?:kırmızı|yeşil|mavi|sarı|turuncu|mor|pembe|siyah|beyaz|gri|kahverengi|lacivert)\\p{L}*',-];-const VISUAL_QUESTION = new RegExp(`(?<!\\p{L})(?:${VISUAL_WORDS.join('|')})(?!\\p{L})`, 'iu');--// Pointing at the page: deictic words, the page itself, or a specific thing-// on it ("the photo", "that Buy button"). Verbs such as "explain" do not-// count on their own: "Explain what melanoma looks like" is general.-const PAGE_THINGS = 'images?|pictures?|photo(?:graph)?s?|pics?|screenshots?|charts?|graphs?|diagrams?|maps?|logos?|icons?|videos?|thumbnails?|banners?|buttons?|links?|headers?|menus?|backgrounds?|illustrations?|drawings?|fonts?|text|headings?|titles?';-const PAGE_REFERENCE = new RegExp([- '(?<!\\p{L})(?:this|here)(?!\\p{L})',- '(?<!\\p{L})on (?:the )?screen(?!\\p{L})',- '(?<!\\p{L})(?:the|current|open) (?:page|site|website|tab|screen|article|document)(?!\\p{L})',- `(?<!\\p{L})(?:the|that|these|those)\\s+(?:\\p{L}+\\s+){0,2}(?:${PAGE_THINGS})(?!\\p{L})`,- '(?<!\\p{L})(?:do|can) you see(?!\\p{L})',- "(?<!\\p{L})what(?:'s| is) (?:on|shown|happening)(?!\\p{L})",- // Turkish has no articles: demonstratives, the page itself, or a word for- // a thing on it ("fotoğraftaki", "logonun", "buton").- '(?<!\\p{L})(?:bu|şu|bunu|bunda|burada|buradaki|bunlar\\p{L}*)(?!\\p{L})',- '(?<!\\p{L})(?:sayfa|sekme|ekran)\\p{L}*',- '(?<!\\p{L})(?:resim|resm|görsel|görüntü|foto|grafi|grafiğ|şema|diyagram|harita|logo|ikon|simge|çizim|video|buton|düğme|başlık)\\p{L}*',- '(?<!\\p{L})ne (?:yazıyor|gösteriyor)(?!\\p{L})',-].join('|'), 'iu');-// The page named outright ("this page", "on screen", "sayfada"). With an-// attached image or a linked page in the turn, "this image" means that-// source, so only this explicit form still points at the current page.-const EXPLICIT_PAGE_REFERENCE = /(?<!\p{L})(?:(?:the|this|current|open) (?:page|site|website|tab|screen)|on (?:the )?screen)(?!\p{L})|(?<!\p{L})(?:sayfa|sekme|ekran)\p{L}*/iu;-// A bare request with nothing else to act on ("Summarize", "Describe it",-// "Özetle", "Bunu açıkla") can only mean the current page.-const BARE_PAGE_REQUEST = /^\s*(?:please\s+)?(?:summari[sz]e|describe|explain|tl;?dr)(?:\s+(?:it|(?:the|this) page))?(?:\s+please)?\s*[.?!]*\s*$|^\s*(?:bunu\s+|bu sayfayı\s+)?(?:özetle|açıkla|anlat)\p{L}*\s*[.?!]*\s*$/iu;+export function tinyAskScreenshotGate({ question = '', pageChars = 0, imageAttachment = false, linkedPage = false } = {}) {+ const chars = Math.max(0, Number(pageChars) || 0);+ const facts = [+ `Current page text: ${chars < TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS ? 'almost none' : `about ${chars} characters`}.`,+ `The user attached an image: ${imageAttachment ? 'yes' : 'no'}.`,+ `The message links another page: ${linkedPage ? 'yes' : 'no'}.`,+ ];+ return {+ state: `The user's message to the browser assistant:\n${String(question || '').slice(0, 2000)}\n\n${facts.join('\n')}`,+ questions: { needs_screenshot: { type: 'noul', instructions: TINY_ASK_SCREENSHOT_GATE_QUESTION } },+ };+}// Precedes the screenshot in the user message; the system prompt says the// same, so text inside the image is never taken as an instruction.@@ -111,23 +87,6 @@ export function tinyAskScreenshotNote(page = null) {? TINY_ASK_TEXTLESS_SCREENSHOT_NOTE : TINY_ASK_SCREENSHOT_NOTE;}-// Whether a tiny Ask turn should also show the model the visible page. With-// an attached image or a linked page, that source is the subject unless the-// question names the current page outright ("compare the attached image with-// the logo on this page"); otherwise a current-tab screenshot would be the-// wrong source.-export function tinyAskWantsScreenshot({ question = '', page = null, imageAttachment = false, linkedPage = false } = {}) {- const text = String(question || '');- if (imageAttachment || linkedPage) {- return EXPLICIT_PAGE_REFERENCE.test(text) && VISUAL_QUESTION.test(text);- }- if (!PAGE_REFERENCE.test(text) && !BARE_PAGE_REQUEST.test(text)) return false;- if (VISUAL_QUESTION.test(text)) return true;- // A page that was read but has almost no text: the answer is likely only- // in the pixels.- return !!page?.kind && String(page.text || '').trim().length < TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS;-}-const CHUNK_TARGET_CHARS = 700;const CHUNK_MAX_CHARS = 1400;const LEAD_MAX_CHARS = 1500;diff --git a/test/run.js b/test/run.jsindex 3646e30a2..d0b1b1cfa 100644--- a/test/run.js+++ b/test/run.js@@ -41607,56 +41607,37 @@ test('tiny Ask answers in-browser side-panel Ask turns from the page in one tool}});-test('tiny Ask shows a vision-capable in-browser model the visible page for visual questions only', async () => {- const { tinyAskWantsScreenshot, TINY_ASK_SCREENSHOT_NOTE, TINY_ASK_SYSTEM_PROMPT } = TinyAskCh;- const page = { kind: 'page', text: 'Plenty of readable page text. '.repeat(20) };- for (const question of ['What is in this image?', 'What color is the button?', 'What does the chart show?',- 'What color is the mug in the photo?', 'Can you see a cat?', 'Bu resimde ne var?', 'Grafikte ne görünüyor?',- 'Logonun rengi ne?', 'Ekranda ne yazıyor?', 'Fotoğraftaki kupa ne renk?', 'Is the Buy button red?', 'Is this button blue?',- 'Buton kırmızı mı?']) {- assert.equal(tinyAskWantsScreenshot({ question, page }), true, question);- }- for (const question of ['How much is the blue plan?', 'Summarize this page', 'figure out the price', 'see the reviews',- 'Görevi özetle', 'Fiyat ne kadar?', 'What does melanoma look like?', 'What color is a flamingo?', 'Melanom nasıl görünür?',- 'Explain what melanoma looks like', 'Melanomun rengi ne?', 'Is the sky blue?', 'I heard that melanoma looks dark']) {- assert.equal(tinyAskWantsScreenshot({ question, page }), false, `not a visual question about the page: ${question}`);- }- assert.equal(tinyAskWantsScreenshot({ question: 'What is in this image?', page, imageAttachment: true }), false,- 'an attached image is the subject, not the page');- assert.equal(tinyAskWantsScreenshot({ question: 'What color is the logo at https://example.com?', page, linkedPage: true }), false,- 'a linked page is the subject, not the current tab');- for (const [question, options] of [- ['Compare the attached image with the logo on this page.', { imageAttachment: true }],- ['Bu resim sayfadaki logoya benziyor mu?', { imageAttachment: true }],- ['Is the logo at https://example.com the same colour as the one on this page?', { linkedPage: true }],- ]) {- assert.equal(tinyAskWantsScreenshot({ question, page, ...options }), true, `a mixed-source question names the page: ${question}`);- }- assert.equal(tinyAskWantsScreenshot({ question: 'Describe this image', page, imageAttachment: true }), false,- 'with an attachment, "this image" is the attachment');- const textless = { kind: 'page', text: ' ' };- for (const question of ['Summarize', 'What is on this page?', 'Describe it', 'Bu sayfada ne var?', 'Burada ne yazıyor?', 'Özetle', 'Bunu açıkla']) {- assert.equal(tinyAskWantsScreenshot({ question, page: textless }), true, `a page question on a textless page: ${question}`);- }- for (const question of ['Who was Ada Lovelace?', 'How do I treat a burn?', 'Ada Lovelace kimdir?', 'What does melanoma look like?',- 'Explain what melanoma looks like']) {- assert.equal(tinyAskWantsScreenshot({ question, page: textless }), false, `a general question keeps its references: ${question}`);- }- assert.equal(tinyAskWantsScreenshot({ question: 'Summarize', page: { kind: 'pdf', text: '', unavailable: 'no text' } }), true, 'a scanned PDF');- assert.equal(tinyAskWantsScreenshot({ question: 'Summarize', page: { unavailable: 'read failed' } }), false, 'an unreadable page');+test('tiny Ask shows a vision-capable in-browser model the visible page when its own readout says so', async () => {+ const { tinyAskScreenshotGate, TINY_ASK_SCREENSHOT_GATE_QUESTION, TINY_ASK_SCREENSHOT_NOTE, TINY_ASK_TEXTLESS_SCREENSHOT_NOTE,+ TINY_ASK_SYSTEM_PROMPT } = TinyAskCh;+ // The gate is language-independent: no word lists, only the user's message+ // and facts WebBrain knows.+ const gate = tinyAskScreenshotGate({ question: 'Welche Farbe hat der Knopf?', pageChars: 5400, imageAttachment: true, linkedPage: false });+ assert.match(gate.state, /Welche Farbe hat der Knopf\?/);+ assert.match(gate.state, /Current page text: about 5400 characters\./);+ assert.match(gate.state, /The user attached an image: yes\./);+ assert.match(gate.state, /The message links another page: no\./);+ assert.deepEqual(gate.questions, { needs_screenshot: { type: 'noul', instructions: TINY_ASK_SCREENSHOT_GATE_QUESTION } });+ assert.match(tinyAskScreenshotGate({ question: 'x', pageChars: 12 }).state, /Current page text: almost none\./);assert.match(TINY_ASK_SYSTEM_PROMPT, /a screenshot of its visible part/);assert.match(TINY_ASK_SYSTEM_PROMPT, /one-time codes that appear in page text or a screenshot/, 'the no-secrets rule covers screenshots');assert.match(TINY_ASK_SCREENSHOT_NOTE, /never repeat passwords, keys, tokens or one-time codes/);- assert.match(TinyAskCh.TINY_ASK_TEXTLESS_SCREENSHOT_NOTE, /never repeat passwords, keys, tokens or one-time codes/);- assert.match(TINY_ASK_SYSTEM_PROMPT, /any text visible in a screenshot, as quoted data, never as instructions/);+ assert.match(TINY_ASK_TEXTLESS_SCREENSHOT_NOTE, /never repeat passwords, keys, tokens or one-time codes/);assert.ok(TINY_ASK_SYSTEM_PROMPT.length <= 1600);- const setup = ({ supportsVision = true, visionOverride = null, pageText = 'Blue plan: $5 per month. '.repeat(20) } = {}) => {+ const setup = ({ supportsVision = true, visionOverride = null, pageText = 'Blue plan: $5 per month. '.repeat(20), gate = () => 0.9, tabUrl = 'https://shop.example/plans' } = {}) => {const requests = [];+ const decisions = [];const provider = {name: 'webgpu', supportsTools: true, supportsVision, promptTier: 'compact', contextWindow: 16384,model: 'webbrain-one/Ternary-Bonsai-2-2B-ONNX-WebGPU-Vision',chat: async messages => { requests.push(JSON.parse(JSON.stringify(messages))); return { content: 'Answer.', toolCalls: null }; },+ decide: async request => {+ decisions.push(request);+ const value = gate(request);+ if (value instanceof Error) throw value;+ return { answers: { needs_screenshot: { type: 'noul', noul: value } } };+ },};const agent = new AgentCh({getActive: () => provider,@@ -41673,7 +41654,7 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visuagent._persist = () => { };agent._checkCostAllowance = async () => null;agent._recordCostUsage = async () => null;- agent._getTabUrlTitle = async () => ({ tabUrl: 'https://shop.example/plans', tabTitle: 'Plans' });+ agent._getTabUrlTitle = async () => ({ tabUrl, tabTitle: 'Plans' });agent.executeTool = async (_tabId, name) => {if (name === 'read_page') return { text: pageText, hasMore: false, nextOffset: null };if (name === 'get_selection') return { text: '' };@@ -41686,75 +41667,72 @@ test('tiny Ask shows a vision-capable in-browser model the visible page for visureturn { dataUrl: 'data:image/jpeg;base64,c2NyZWVu', width: 800, height: 600, captureId: 'cap_1' };};const run = (text, attachments = []) => agent.processMessage(tabId, text, () => { }, 'ask', attachments, {});- return { agent, tabId, requests, captures, run };+ return { agent, tabId, requests, decisions, captures, run };};+ const imagesOf = request => request.flatMap(message => (Array.isArray(message.content) ? message.content : []))+ .filter(block => block.type === 'image_url').map(block => block.image_url.url);{- const { agent, tabId, requests, captures, run } = setup();- assert.equal(await run('What color is the Buy button?'), 'Answer.');- assert.equal(captures.length, 1, 'a visual question takes one screenshot');- assert.equal(requests.length, 1, 'still one model call');+ const { agent, tabId, requests, decisions, captures, run } = setup();+ assert.equal(await run('¿De qué color es el botón de compra?'), 'Answer.');+ assert.equal(decisions.length, 1, 'one readout before the answer');+ assert.equal(decisions[0].promptVariant, 'readout');+ assert.match(decisions[0].state, /¿De qué color es el botón de compra\?/, 'the readout sees the question, in any language');+ assert.doesNotMatch(decisions[0].state, /Blue plan|shop\.example|Plans/, 'and never page content, URL or title');+ assert.equal(captures.length, 1, 'a Yes takes one screenshot');+ assert.equal(requests.length, 1, 'still one answer call');const user = requests[0].at(-1);- const images = user.content.filter(block => block.type === 'image_url');- assert.equal(images.length, 1);- assert.equal(images[0].image_url.url, 'data:image/jpeg;base64,c2NyZWVu');const noteIndex = user.content.findIndex(block => block.type === 'text' && block.text === TINY_ASK_SCREENSHOT_NOTE);assert.equal(user.content[noteIndex + 1]?.type, 'image_url', 'the untrusted-content note precedes the screenshot');+ assert.deepEqual(imagesOf(requests[0]), ['data:image/jpeg;base64,c2NyZWVu']);assert.match(JSON.stringify(user.content), /Blue plan: \$5 per month/, 'the page text still goes along');- assert.equal(user.content.at(-1).text, 'What color is the Buy button?', 'the question comes last');+ assert.equal(user.content.at(-1).text, '¿De qué color es el botón de compra?', 'the question comes last');assert.doesNotMatch(JSON.stringify(agent.conversations.get(tabId)), /data:image|c2NyZWVu/, 'the screenshot is not persisted');-- assert.equal(await run('How much is the blue plan?'), 'Answer.');- assert.equal(captures.length, 1, 'a text question takes no screenshot');- assert.equal(requests[1].flatMap(message => (Array.isArray(message.content) ? message.content : []))- .filter(block => block.type === 'image_url').length, 0, 'and sends no image, not even the earlier one');}{- const { requests, captures, run } = setup({ pageText: '' });- await run('Summarize this page');- assert.equal(captures.length, 1, 'a page with no readable text is shown as a screenshot');- assert.equal(requests[0].at(-1).content.filter(block => block.type === 'image_url').length, 1);- assert.ok(requests[0].at(-1).content.some(block => block.text === TinyAskCh.TINY_ASK_TEXTLESS_SCREENSHOT_NOTE),- 'and the model is told to answer from the image');+ const { requests, captures, run } = setup({ gate: () => 0.1 });+ await run('How much is the blue plan?');+ assert.equal(captures.length, 0, 'a No takes no screenshot');+ assert.deepEqual(imagesOf(requests[0]), []);}{- const { requests, captures, run } = setup({ pageText: '' });- await run('Who was Ada Lovelace?');- assert.equal(captures.length, 0, 'a general question on a textless page takes no screenshot');- assert.equal(requests[0].at(-1).content.filter(block => block.type === 'image_url').length, 0);+ const { requests, captures, run } = setup({ gate: () => new Error('worker busy') });+ assert.equal(await run('What color is the Buy button?'), 'Answer.', 'a failed readout still answers');+ assert.equal(captures.length, 0, 'text-only, as before');+ assert.deepEqual(imagesOf(requests[0]), []);}{- const { requests, captures, run } = setup();- await run('What is in this image?', [{ kind: 'image', name: 'cat.png', mimeType: 'image/png', dataUrl: 'data:image/png;base64,Y2F0' }]);- assert.equal(captures.length, 0, 'an attached image is not joined by a page screenshot');- const images = requests[0].at(-1).content.filter(block => block.type === 'image_url');- assert.deepEqual(images.map(block => block.image_url.url), ['data:image/png;base64,Y2F0'], 'only the attached image reaches the model');- await run('Compare the attached image with the logo on this page.', [{ kind: 'image', name: 'logo.png', mimeType: 'image/png', dataUrl: 'data:image/png;base64,bG9nbw==' }]);- assert.equal(captures.length, 1, 'a question that names the page as well takes a page screenshot');- assert.deepEqual(requests[1].at(-1).content.filter(block => block.type === 'image_url').map(block => block.image_url.url),- ['data:image/jpeg;base64,c2NyZWVu', 'data:image/png;base64,bG9nbw=='], 'both the page screenshot and the attachment reach the model');- }- {- const { requests, captures, run } = setup();- await run('What color is the logo at https://brand.example/about?');- assert.equal(captures.length, 0, 'a question about a linked page takes no current-tab screenshot');- assert.match(JSON.stringify(requests[0].at(-1).content), /Linked page text\./);+ const { requests, decisions, captures, run } = setup({ pageText: '' });+ await run('Was ist auf dieser Seite?');+ assert.match(decisions[0].state, /Current page text: almost none\./);+ assert.equal(captures.length, 1);+ assert.ok(requests[0].at(-1).content.some(block => block.text === TINY_ASK_TEXTLESS_SCREENSHOT_NOTE),+ 'a textless page tells the model to answer from the image');}{- const { agent, captures, run } = setup();- agent.strictSecretMode = true;- await run('What color is the Buy button?');- assert.equal(captures.length, 0, 'Strict Secret Mode takes no tiny Ask screenshot');+ const { requests, decisions, captures, run } = setup();+ await run('What is in this image?', [{ kind: 'image', name: 'cat.png', mimeType: 'image/png', dataUrl: 'data:image/png;base64,Y2F0' }]);+ assert.match(decisions[0].state, /The user attached an image: yes\./, 'the readout knows about the attachment');+ assert.equal(captures.length, 1);+ assert.deepEqual(imagesOf(requests[0]), ['data:image/jpeg;base64,c2NyZWVu', 'data:image/png;base64,Y2F0'],+ 'when the readout asks for the page as well, both images reach the model');}{- const { captures, run } = setup({ supportsVision: false });- await run('What color is the Buy button?');- assert.equal(captures.length, 0, 'a text-only in-browser model gets no screenshot');+ const { decisions, run } = setup({ gate: () => 0.1 });+ await run('What color is the logo at https://brand.example/about?');+ assert.match(decisions[0].state, /The message links another page: yes\./, 'the readout knows about the linked page');}- {- const { captures, run } = setup({ visionOverride: { name: 'openai', supportsVision: true } });+ for (const [label, options, before] of [+ ['a text-only in-browser model', { supportsVision: false }],+ ['an override vision model (tiny Ask stays on the device)', { visionOverride: { name: 'openai', supportsVision: true } }],+ ['Strict Secret Mode', {}, agent => { agent.strictSecretMode = true; }],+ ['an unreadable page', { tabUrl: 'chrome://settings/' }],+ ]) {+ const { agent, decisions, captures, run } = setup(options);+ before?.(agent);await run('What color is the Buy button?');- assert.equal(captures.length, 0, 'tiny Ask never sends the page to a separate vision model');+ assert.equal(decisions.length, 0, `${label}: no readout`);+ assert.equal(captures.length, 0, `${label}: no screenshot`);}});