Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix the local vision connection probe image

Emre Sokullud15e94272e24
diff --git a/src/chrome/assets/vision-color-connection-test.png b/src/chrome/assets/vision-color-connection-test.png
new file mode 100644
index 000000000..5329f8609
Binary files /dev/null and b/src/chrome/assets/vision-color-connection-test.png differ
diff --git a/src/chrome/src/providers/connection-test-assets.js b/src/chrome/src/providers/connection-test-assets.js
index d0144a81e..fb26e6168 100644
--- a/src/chrome/src/providers/connection-test-assets.js
+++ b/src/chrome/src/providers/connection-test-assets.js
@@ -1,4 +1,5 @@
const VISION_CONNECTION_TEST_IMAGE_PATH = 'assets/vision-connection-test.png';
+const VISION_COLOR_CONNECTION_TEST_IMAGE_PATH = 'assets/vision-color-connection-test.png';
const TRANSCRIPTION_CONNECTION_TEST_AUDIO_PATH = 'assets/transcription-connection-test.wav';
async function readPackagedAsset(runtime, relativePath) {
@@ -18,8 +19,9 @@ function bytesToBase64(bytes) {
return btoa(binary);
}
-export async function loadVisionConnectionTestImage(runtime) {
- const bytes = await readPackagedAsset(runtime, VISION_CONNECTION_TEST_IMAGE_PATH);
+export async function loadVisionConnectionTestImage(runtime, { colorPanels = false } = {}) {
+ const imagePath = colorPanels ? VISION_COLOR_CONNECTION_TEST_IMAGE_PATH : VISION_CONNECTION_TEST_IMAGE_PATH;
+ const bytes = await readPackagedAsset(runtime, imagePath);
return `data:image/png;base64,${bytesToBase64(bytes)}`;
}
diff --git a/src/chrome/src/providers/manager.js b/src/chrome/src/providers/manager.js
index 12fa7ec22..8b271ceb6 100644
--- a/src/chrome/src/providers/manager.js
+++ b/src/chrome/src/providers/manager.js
@@ -2388,15 +2388,15 @@ export class ProviderManager {
const provider = await this.getVisionOverrideProvider()
|| await this.getLocalVisionFallbackProvider();
if (!provider) return { ok: false, error: 'Vision model not configured' };
+ // Both in-browser readers get matching color panels; small local models
+ // misread the tiny glyphs in the remote providers' WB7 probe.
+ const isLocalWebgpu = provider.name === 'webgpu-vision' || provider.config?.providerName === 'webgpu-vision';
let imageDataUrl;
try {
- imageDataUrl = await loadVisionConnectionTestImage(chrome.runtime);
+ imageDataUrl = await loadVisionConnectionTestImage(chrome.runtime, { colorPanels: isLocalWebgpu });
} catch (error) {
return { ok: false, error: error.message };
}
- // Both in-browser readers (WebBrain VL 2 450M and Ternary Bonsai 2 2B
- // Vision) get the colour probe; small local models misread tiny glyphs.
- const isLocalWebgpu = provider.name === 'webgpu-vision' || provider.config?.providerName === 'webgpu-vision';
const probePrompt = isLocalWebgpu
? 'The image contains three solid vertical color panels. Name their colors from left to right. Reply with only the three color names, in English.'
: 'Read the three-character black code centered in the attached image. Reply with only that code.';
diff --git a/src/firefox/assets/vision-color-connection-test.png b/src/firefox/assets/vision-color-connection-test.png
new file mode 100644
index 000000000..5329f8609
Binary files /dev/null and b/src/firefox/assets/vision-color-connection-test.png differ
diff --git a/src/firefox/src/providers/connection-test-assets.js b/src/firefox/src/providers/connection-test-assets.js
index d0144a81e..fb26e6168 100644
--- a/src/firefox/src/providers/connection-test-assets.js
+++ b/src/firefox/src/providers/connection-test-assets.js
@@ -1,4 +1,5 @@
const VISION_CONNECTION_TEST_IMAGE_PATH = 'assets/vision-connection-test.png';
+const VISION_COLOR_CONNECTION_TEST_IMAGE_PATH = 'assets/vision-color-connection-test.png';
const TRANSCRIPTION_CONNECTION_TEST_AUDIO_PATH = 'assets/transcription-connection-test.wav';
async function readPackagedAsset(runtime, relativePath) {
@@ -18,8 +19,9 @@ function bytesToBase64(bytes) {
return btoa(binary);
}
-export async function loadVisionConnectionTestImage(runtime) {
- const bytes = await readPackagedAsset(runtime, VISION_CONNECTION_TEST_IMAGE_PATH);
+export async function loadVisionConnectionTestImage(runtime, { colorPanels = false } = {}) {
+ const imagePath = colorPanels ? VISION_COLOR_CONNECTION_TEST_IMAGE_PATH : VISION_CONNECTION_TEST_IMAGE_PATH;
+ const bytes = await readPackagedAsset(runtime, imagePath);
return `data:image/png;base64,${bytesToBase64(bytes)}`;
}
diff --git a/test/run.js b/test/run.js
index 9a95a56ed..76b38745d 100644
--- a/test/run.js
+++ b/test/run.js
@@ -23745,6 +23745,7 @@ test('vendored PDF.js and multimodal probes remain store-reviewable in both buil
test('multimodal connection-test assets preserve the prior image and silent WAV bytes', () => {
const expectedHashes = {
'vision-connection-test.png': '514fd28946340ea4e7e7bb7817d876ef1679456c635fbf3afc1b978031e30482',
+ 'vision-color-connection-test.png': 'c182ee730b7b5a12fec0e8bed430a7822f09e5cc15bfaae3bd754685efbbe676',
'transcription-connection-test.wav': '56d4af65701c26df20bd4021eda95b6e830348ce3a746086079fe89285548dc9',
};
for (const [filename, expectedHash] of Object.entries(expectedHashes)) {
@@ -131437,23 +131438,42 @@ test('multimodal connection tests exercise image and audio routes instead of onl
assert.equal(visionResult.ok, true, `${label}: real vision probe should pass`);
assert.equal(visionCalls.length, 1);
assert.match(visionCalls[0].messages[0].content[0].image_url.url, /^data:image\/png;base64,/);
+ assert.deepEqual(
+ Buffer.from(visionCalls[0].messages[0].content[0].image_url.url.split(',')[1], 'base64'),
+ fs.readFileSync(path.join(ROOT, `src/${label}/assets/vision-connection-test.png`)),
+ `${label}: the code-reading prompt must receive the WB7 image`,
+ );
assert.equal(visionCalls[0].options.extraBody.reasoning_tokens, 0);
assert.equal(visionCalls[0].options.webbrainVisionProbe, true);
if (label === 'chrome') {
visionManager.getVisionOverrideProvider = async () => null;
- visionManager.getLocalVisionFallbackProvider = async () => ({
- name: 'webgpu-vision',
- model: WEBGPU_VISION_MODEL_ID,
- baseUrl: 'local://webgpu',
- chat: async (messages, options) => {
- assert.match(messages[0].content[1].text, /three solid vertical color panels/i);
- assert.equal(options.webbrainVisionProbe, true);
- return { content: 'yellow, blue, red' };
- },
- });
- const localVisionResult = await visionManager.testVisionProvider();
- assert.equal(localVisionResult.ok, true, 'chrome: local color-panel vision probe should pass');
+ for (const [name, model, config] of [
+ ['webgpu-vision', WEBGPU_VISION_MODEL_ID, {}],
+ ['webgpu-vision-bonsai', WEBGPU_VISION_BONSAI_MODEL_ID, { providerName: 'webgpu-vision' }],
+ ]) {
+ let reply = 'yellow, blue, red';
+ visionManager.getLocalVisionFallbackProvider = async () => ({
+ name, model, config,
+ baseUrl: 'local://webgpu',
+ chat: async (messages, options) => {
+ assert.match(messages[0].content[1].text, /three solid vertical color panels/i);
+ assert.equal(options.webbrainVisionProbe, true);
+ assert.deepEqual(
+ Buffer.from(messages[0].content[0].image_url.url.split(',')[1], 'base64'),
+ fs.readFileSync(path.join(ROOT, 'src/chrome/assets/vision-color-connection-test.png')),
+ `${name}: the color prompt must receive the yellow, blue, red panels`,
+ );
+ return { content: reply };
+ },
+ });
+ const localVisionResult = await visionManager.testVisionProvider();
+ assert.equal(localVisionResult.ok, true, `${name}: local color-panel vision probe should pass`);
+ for (reply of ['Black, White, Black', 'red, blue, yellow', 'WB7']) {
+ const wrongColors = await visionManager.testVisionProvider();
+ assert.equal(wrongColors.ok, false, `${name}: incorrect colors or order must fail`);
+ }
+ }
}
visionManager.getVisionOverrideProvider = async () => ({