Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fix the local vision connection probe image
Emre Sokullud15e94272e24
diff --git a/src/chrome/assets/vision-color-connection-test.png b/src/chrome/assets/vision-color-connection-test.pngnew file mode 100644index 000000000..5329f8609Binary files /dev/null and b/src/chrome/assets/vision-color-connection-test.png differdiff --git a/src/chrome/src/providers/connection-test-assets.js b/src/chrome/src/providers/connection-test-assets.jsindex d0144a81e..fb26e6168 100644--- a/src/chrome/src/providers/connection-test-assets.js+++ b/src/chrome/src/providers/connection-test-assets.js@@ -1,4 +1,5 @@const VISION_CONNECTION_TEST_IMAGE_PATH = 'assets/vision-connection-test.png';+const VISION_COLOR_CONNECTION_TEST_IMAGE_PATH = 'assets/vision-color-connection-test.png';const TRANSCRIPTION_CONNECTION_TEST_AUDIO_PATH = 'assets/transcription-connection-test.wav';async function readPackagedAsset(runtime, relativePath) {@@ -18,8 +19,9 @@ function bytesToBase64(bytes) {return btoa(binary);}-export async function loadVisionConnectionTestImage(runtime) {- const bytes = await readPackagedAsset(runtime, VISION_CONNECTION_TEST_IMAGE_PATH);+export async function loadVisionConnectionTestImage(runtime, { colorPanels = false } = {}) {+ const imagePath = colorPanels ? VISION_COLOR_CONNECTION_TEST_IMAGE_PATH : VISION_CONNECTION_TEST_IMAGE_PATH;+ const bytes = await readPackagedAsset(runtime, imagePath);return `data:image/png;base64,${bytesToBase64(bytes)}`;}diff --git a/src/chrome/src/providers/manager.js b/src/chrome/src/providers/manager.jsindex 12fa7ec22..8b271ceb6 100644--- a/src/chrome/src/providers/manager.js+++ b/src/chrome/src/providers/manager.js@@ -2388,15 +2388,15 @@ export class ProviderManager {const provider = await this.getVisionOverrideProvider()|| await this.getLocalVisionFallbackProvider();if (!provider) return { ok: false, error: 'Vision model not configured' };+ // Both in-browser readers get matching color panels; small local models+ // misread the tiny glyphs in the remote providers' WB7 probe.+ const isLocalWebgpu = provider.name === 'webgpu-vision' || provider.config?.providerName === 'webgpu-vision';let imageDataUrl;try {- imageDataUrl = await loadVisionConnectionTestImage(chrome.runtime);+ imageDataUrl = await loadVisionConnectionTestImage(chrome.runtime, { colorPanels: isLocalWebgpu });} catch (error) {return { ok: false, error: error.message };}- // Both in-browser readers (WebBrain VL 2 450M and Ternary Bonsai 2 2B- // Vision) get the colour probe; small local models misread tiny glyphs.- const isLocalWebgpu = provider.name === 'webgpu-vision' || provider.config?.providerName === 'webgpu-vision';const probePrompt = isLocalWebgpu? 'The image contains three solid vertical color panels. Name their colors from left to right. Reply with only the three color names, in English.': 'Read the three-character black code centered in the attached image. Reply with only that code.';diff --git a/src/firefox/assets/vision-color-connection-test.png b/src/firefox/assets/vision-color-connection-test.pngnew file mode 100644index 000000000..5329f8609Binary files /dev/null and b/src/firefox/assets/vision-color-connection-test.png differdiff --git a/src/firefox/src/providers/connection-test-assets.js b/src/firefox/src/providers/connection-test-assets.jsindex d0144a81e..fb26e6168 100644--- a/src/firefox/src/providers/connection-test-assets.js+++ b/src/firefox/src/providers/connection-test-assets.js@@ -1,4 +1,5 @@const VISION_CONNECTION_TEST_IMAGE_PATH = 'assets/vision-connection-test.png';+const VISION_COLOR_CONNECTION_TEST_IMAGE_PATH = 'assets/vision-color-connection-test.png';const TRANSCRIPTION_CONNECTION_TEST_AUDIO_PATH = 'assets/transcription-connection-test.wav';async function readPackagedAsset(runtime, relativePath) {@@ -18,8 +19,9 @@ function bytesToBase64(bytes) {return btoa(binary);}-export async function loadVisionConnectionTestImage(runtime) {- const bytes = await readPackagedAsset(runtime, VISION_CONNECTION_TEST_IMAGE_PATH);+export async function loadVisionConnectionTestImage(runtime, { colorPanels = false } = {}) {+ const imagePath = colorPanels ? VISION_COLOR_CONNECTION_TEST_IMAGE_PATH : VISION_CONNECTION_TEST_IMAGE_PATH;+ const bytes = await readPackagedAsset(runtime, imagePath);return `data:image/png;base64,${bytesToBase64(bytes)}`;}diff --git a/test/run.js b/test/run.jsindex 9a95a56ed..76b38745d 100644--- a/test/run.js+++ b/test/run.js@@ -23745,6 +23745,7 @@ test('vendored PDF.js and multimodal probes remain store-reviewable in both builtest('multimodal connection-test assets preserve the prior image and silent WAV bytes', () => {const expectedHashes = {'vision-connection-test.png': '514fd28946340ea4e7e7bb7817d876ef1679456c635fbf3afc1b978031e30482',+ 'vision-color-connection-test.png': 'c182ee730b7b5a12fec0e8bed430a7822f09e5cc15bfaae3bd754685efbbe676','transcription-connection-test.wav': '56d4af65701c26df20bd4021eda95b6e830348ce3a746086079fe89285548dc9',};for (const [filename, expectedHash] of Object.entries(expectedHashes)) {@@ -131437,23 +131438,42 @@ test('multimodal connection tests exercise image and audio routes instead of onlassert.equal(visionResult.ok, true, `${label}: real vision probe should pass`);assert.equal(visionCalls.length, 1);assert.match(visionCalls[0].messages[0].content[0].image_url.url, /^data:image\/png;base64,/);+ assert.deepEqual(+ Buffer.from(visionCalls[0].messages[0].content[0].image_url.url.split(',')[1], 'base64'),+ fs.readFileSync(path.join(ROOT, `src/${label}/assets/vision-connection-test.png`)),+ `${label}: the code-reading prompt must receive the WB7 image`,+ );assert.equal(visionCalls[0].options.extraBody.reasoning_tokens, 0);assert.equal(visionCalls[0].options.webbrainVisionProbe, true);if (label === 'chrome') {visionManager.getVisionOverrideProvider = async () => null;- visionManager.getLocalVisionFallbackProvider = async () => ({- name: 'webgpu-vision',- model: WEBGPU_VISION_MODEL_ID,- baseUrl: 'local://webgpu',- chat: async (messages, options) => {- assert.match(messages[0].content[1].text, /three solid vertical color panels/i);- assert.equal(options.webbrainVisionProbe, true);- return { content: 'yellow, blue, red' };- },- });- const localVisionResult = await visionManager.testVisionProvider();- assert.equal(localVisionResult.ok, true, 'chrome: local color-panel vision probe should pass');+ for (const [name, model, config] of [+ ['webgpu-vision', WEBGPU_VISION_MODEL_ID, {}],+ ['webgpu-vision-bonsai', WEBGPU_VISION_BONSAI_MODEL_ID, { providerName: 'webgpu-vision' }],+ ]) {+ let reply = 'yellow, blue, red';+ visionManager.getLocalVisionFallbackProvider = async () => ({+ name, model, config,+ baseUrl: 'local://webgpu',+ chat: async (messages, options) => {+ assert.match(messages[0].content[1].text, /three solid vertical color panels/i);+ assert.equal(options.webbrainVisionProbe, true);+ assert.deepEqual(+ Buffer.from(messages[0].content[0].image_url.url.split(',')[1], 'base64'),+ fs.readFileSync(path.join(ROOT, 'src/chrome/assets/vision-color-connection-test.png')),+ `${name}: the color prompt must receive the yellow, blue, red panels`,+ );+ return { content: reply };+ },+ });+ const localVisionResult = await visionManager.testVisionProvider();+ assert.equal(localVisionResult.ok, true, `${name}: local color-panel vision probe should pass`);+ for (reply of ['Black, White, Black', 'red, blue, yellow', 'WB7']) {+ const wrongColors = await visionManager.testVisionProvider();+ assert.equal(wrongColors.ok, false, `${name}: incorrect colors or order must fail`);+ }+ }}visionManager.getVisionOverrideProvider = async () => ({