Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

54.4 KB
Raw Blame
// Vision mode: browser-free pieces (sizing, diffing, coordinate mapping,
// tool surface, prompt, settings). Real-browser behaviour (captures at
// device pixel ratios 1/2/3, browser zoom, trusted clicks) is covered by
// test/vision-pipeline-browser.mjs and test/quality/vision-mode-mv3.mjs.
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { test } from 'node:test';

const ROOT = fileURLToPath(new URL('../', import.meta.url));
const read = file => fs.readFileSync(path.join(ROOT, file), 'utf8');
const SHARED = ['vision-budget.js', 'vision-diff.js', 'vision-session.js', 'vision-tools.js', 'vision-mode.js', 'vision-encoder.js', 'vision-runtime.js', 'vision-dom-probe.js'];

const area = { get: async () => ({}), set: async () => {}, remove: async () => {} };
const api = {
  storage: { local: area, session: area, onChanged: { addListener() {} } },
  runtime: { getURL: value => `extension://test/${value}`, sendMessage: async () => ({}), getManifest: () => ({ version: '0.0.0' }) },
  tabs: { get: async id => ({ id, url: 'https://example.test/', title: 'Example', status: 'complete' }), sendMessage: async () => ({}) },
  scripting: { executeScript: async () => [{ result: null }] },
};
globalThis.chrome ??= api;
globalThis.browser ??= api;

test('vision modules are shared verbatim by both builds', () => {
  for (const file of SHARED) {
    assert.equal(read(`src/chrome/src/agent/${file}`), read(`src/firefox/src/agent/${file}`), `${file} differs between builds`);
  }
});

const budget = await import('../src/chrome/src/agent/vision-budget.js');
const diff = await import('../src/chrome/src/agent/vision-diff.js');
const sessionModule = await import('../src/chrome/src/agent/vision-session.js');
const visionTools = await import('../src/chrome/src/agent/vision-tools.js');
const visionMode = await import('../src/chrome/src/agent/vision-mode.js');
const { VisionSession } = sessionModule;

test('profiles: model ids pick the provider family before router names', () => {
  const pick = (name, model, extra = {}) => budget.detectVisionProfile({ name, config: { model, ...extra } }).id;
  assert.equal(pick('anthropic', 'claude-sonnet-5'), 'anthropic');
  assert.equal(pick('openrouter', 'anthropic/claude-opus-4.5'), 'anthropic');
  assert.equal(pick('openrouter', 'openai/gpt-5'), 'openai');
  assert.equal(pick('openai', 'gpt-4.1-mini'), 'openai');
  assert.equal(pick('openrouter', 'google/gemini-2.5-pro'), 'gemini');
  assert.equal(pick('llamacpp', 'Qwen2.5-VL-7B-Instruct'), 'qwen');
  assert.equal(pick('openrouter', 'bytedance/ui-tars-1.5-7b'), 'qwen');
  assert.equal(pick('llamacpp', 'Qwen3-VL-8B-Instruct'), 'qwen3');
  assert.equal(pick('vllm', 'Intel/Qwen3.6-27B-int4-AutoRound'), 'qwen3');
  assert.equal(pick('llamacpp', 'gemma-3-12b-it'), 'local');
  assert.equal(pick('ollama', 'llava:13b'), 'local');
  assert.equal(pick('custom', 'pixtral-large', { category: 'cloud' }), 'default');
  // An OpenAI base URL is recognized by its host, never by a substring of the URL.
  assert.equal(pick('custom', 'my-model', { baseUrl: 'https://api.openai.com/v1' }), 'openai');
  assert.equal(pick('custom', 'my-model', { baseUrl: 'api.openai.com/v1' }), 'openai');
  for (const baseUrl of ['https://proxy.example.test/api.openai.com/v1', 'https://api.openai.com.example.test/v1', 'https://example.test/?u=api.openai.com']) {
    assert.equal(pick('custom', 'my-model', { baseUrl, category: 'cloud' }), 'default', baseUrl);
  }
  for (const build of ['chrome', 'firefox']) {
    assert.doesNotMatch(read(`src/${build}/src/agent/vision-budget.js`), /\.includes\(['"]api\.openai\.com/);
  }
  assert.equal(budget.VISION_PROFILES.gemini.coordinates, 'normalized');
  assert.equal(budget.VISION_PROFILES.qwen3.coordinates, 'normalized');
  assert.equal(budget.VISION_PROFILES.anthropic.coordinates, 'pixel');
});

test('sizing: never upscales, respects provider limits, aligns to the patch grid', () => {
  const { planVisionImageSize, VISION_PROFILES } = budget;
  // A 1280x800 viewport fits Anthropic's balanced budget at 1:1.
  const anthropic = planVisionImageSize({ cssWidth: 1280, cssHeight: 800, profile: VISION_PROFILES.anthropic });
  assert.deepEqual([anthropic.width, anthropic.height], [1280, 800]);
  assert.equal(anthropic.scaleX, 1);
  // A small viewport is never upscaled (retina pixels are never sent).
  const small = planVisionImageSize({ cssWidth: 800, cssHeight: 600, profile: VISION_PROFILES.anthropic, quality: 'sharp' });
  assert.deepEqual([small.width, small.height], [800, 600]);
  // A large viewport is reduced below every server-side resize threshold.
  const big = planVisionImageSize({ cssWidth: 2560, cssHeight: 1440, profile: VISION_PROFILES.anthropic, quality: 'sharp' });
  assert.ok(big.width <= 1568 && big.height <= 1568, JSON.stringify(big));
  assert.ok(big.width * big.height <= 1_150_000, JSON.stringify(big));
  assert.ok(Math.abs(big.width / big.height - 16 / 9) < 0.01, 'aspect ratio is preserved');
  // OpenAI keeps the short side at 768 so the server does not rescale.
  const openai = planVisionImageSize({ cssWidth: 1920, cssHeight: 1080, profile: VISION_PROFILES.openai, quality: 'sharp' });
  assert.ok(Math.min(openai.width, openai.height) <= 768, JSON.stringify(openai));
  // Qwen sizes are multiples of 28 so smart_resize is the identity.
  const qwen = planVisionImageSize({ cssWidth: 1366, cssHeight: 768, profile: VISION_PROFILES.qwen });
  assert.equal(qwen.width % 28, 0);
  assert.equal(qwen.height % 28, 0);
  assert.ok(qwen.width * qwen.height <= 1_003_520);
  const qwen3 = planVisionImageSize({ cssWidth: 1366, cssHeight: 768, profile: VISION_PROFILES.qwen3 });
  assert.equal(qwen3.width % 32, 0);
  // Economy is cheaper than balanced, which is no more than sharp.
  const tokens = q => planVisionImageSize({ cssWidth: 1920, cssHeight: 1080, profile: VISION_PROFILES.anthropic, quality: q }).estimatedTokens;
  assert.ok(tokens('economy') < tokens('balanced'));
  assert.ok(tokens('balanced') <= tokens('sharp'));
});

test('token estimates follow each provider formula', () => {
  const { estimateVisionTokens, VISION_PROFILES } = budget;
  assert.equal(estimateVisionTokens(1000, 750, VISION_PROFILES.anthropic), 1000);
  // 1024x640 high detail: short side already <= 768 → 2x2 tiles.
  assert.equal(estimateVisionTokens(1024, 640, VISION_PROFILES.openai), 85 + 170 * 4);
  assert.equal(estimateVisionTokens(300, 300, VISION_PROFILES.gemini), 258);
  assert.equal(estimateVisionTokens(280, 280, VISION_PROFILES.qwen), 100 + 2);
  assert.equal(estimateVisionTokens(320, 320, VISION_PROFILES.qwen3), 100 + 2);
});

test('zoom sizing renders up to 2x CSS pixels within a bounded budget', () => {
  const { planZoomImageSize, VISION_PROFILES } = budget;
  const zoom = planZoomImageSize({ regionWidth: 200, regionHeight: 100, devicePixelRatio: 2, profile: VISION_PROFILES.anthropic });
  assert.deepEqual([zoom.width, zoom.height], [400, 200]);
  const capped = planZoomImageSize({ regionWidth: 1200, regionHeight: 800, devicePixelRatio: 2, profile: VISION_PROFILES.anthropic });
  assert.ok(capped.width <= 1280 && capped.width * capped.height <= 900_000, JSON.stringify(capped));
});

test('encoding: WebP for cloud profiles, JPEG for local servers and when WebP is unavailable', () => {
  const { visionEncodingFor, VISION_PROFILES } = budget;
  assert.equal(visionEncodingFor(VISION_PROFILES.anthropic).mimeType, 'image/webp');
  assert.equal(visionEncodingFor(VISION_PROFILES.local).mimeType, 'image/jpeg');
  assert.equal(visionEncodingFor(VISION_PROFILES.anthropic, { webpSupported: false }).mimeType, 'image/jpeg');
});

function frame(width, height, fill = [255, 255, 255]) {
  const data = new Uint8ClampedArray(width * height * 4);
  for (let i = 0; i < data.length; i += 4) {
    data[i] = fill[0]; data[i + 1] = fill[1]; data[i + 2] = fill[2]; data[i + 3] = 255;
  }
  return data;
}

function paint(data, width, { x, y, w, h }, color) {
  for (let yy = y; yy < y + h; yy++) {
    for (let xx = x; xx < x + w; xx++) {
      const i = (yy * width + xx) * 4;
      data[i] = color[0]; data[i + 1] = color[1]; data[i + 2] = color[2];
    }
  }
  return data;
}

test('diff: identical frames and encoder-level noise produce no change', () => {
  const a = frame(320, 200);
  const b = frame(320, 200);
  assert.equal(diff.diffFrames(a, b, 320, 200).regions.length, 0);
  // +-10 per channel on scattered pixels (JPEG/WebP noise) stays below the threshold.
  for (let i = 0; i < b.length; i += 4 * 7) { b[i] = 245; b[i + 1] = 247; }
  const noisy = diff.diffFrames(a, b, 320, 200);
  assert.equal(noisy.regions.length, 0, JSON.stringify(noisy));
  // A blinking caret (1x14 px) flips a handful of pixels in one cell: ignored.
  const caret = paint(frame(320, 200), 320, { x: 100, y: 50, w: 1, h
: 5 }, [0, 0, 0]);
  assert.equal(diff.diffFrames(a, caret, 320, 200).regions.length, 0);
});

test('diff: a local change yields one padded region; distant changes stay separate', () => {
  const a = frame(640, 400);
  const b = paint(frame(640, 400), 640, { x: 100, y: 120, w: 80, h: 30 }, [20, 40, 200]);
  const one = diff.diffFrames(a, b, 640, 400);
  assert.equal(one.regions.length, 1, JSON.stringify(one.regions));
  const [r] = one.regions;
  assert.ok(r.x <= 100 && r.y <= 120 && r.x + r.width >= 180 && r.y + r.height >= 150, JSON.stringify(r));
  assert.ok(r.width < 160 && r.height < 90, `region stays tight: ${JSON.stringify(r)}`);
  paint(b, 640, { x: 520, y: 320, w: 60, h: 40 }, [200, 0, 0]);
  const two = diff.diffFrames(a, b, 640, 400);
  assert.equal(two.regions.length, 2, JSON.stringify(two.regions));
  assert.ok(two.regionAreaRatio < 0.1);
});

test('diff: nearby changes merge and scattered changes are reduced to maxRegions', () => {
  const a = frame(640, 400);
  const b = frame(640, 400);
  paint(b, 640, { x: 100, y: 100, w: 20, h: 20 }, [0, 0, 0]);
  paint(b, 640, { x: 130, y: 100, w: 20, h: 20 }, [0, 0, 0]);
  assert.equal(diff.diffFrames(a, b, 640, 400).regions.length, 1, 'changes 10px apart share a region');
  const c = frame(640, 400);
  for (let i = 0; i < 8; i++) paint(c, 640, { x: 20 + i * 76, y: 20 + (i % 2) * 300, w: 12, h: 12 }, [0, 0, 0]);
  const reduced = diff.diffFrames(a, c, 640, 400, { maxRegions: 3 });
  assert.ok(reduced.regions.length <= 3, JSON.stringify(reduced.regions));
  for (let i = 0; i < 8; i++) {
    const x = 20 + i * 76 + 6;
    const y = 20 + (i % 2) * 300 + 6;
    assert.ok(diff.pointTouchesRegions({ x, y }, reduced.regions), `change ${i} is still covered`);
  }
});

test('frame update planning: none, patch, keyframe', () => {
  const opts = { width: 1280, height: 800, estimateTokens: (w, h) => Math.ceil((w * h) / 750) };
  assert.equal(diff.planFrameUpdate({ comparable: true, regions: [], regionAreaRatio: 0 }, opts).kind, 'none');
  const small = { comparable: true, regions: [{ x: 100, y: 100, width: 200, height: 60 }], regionAreaRatio: 0.012 };
  const patch = diff.planFrameUpdate(small, opts);
  assert.equal(patch.kind, 'patch');
  assert.ok(patch.patchTokens < patch.frameTokens / 5);
  assert.equal(diff.planFrameUpdate({ ...small, regionAreaRatio: 0.6 }, opts).kind, 'keyframe');
  assert.equal(diff.planFrameUpdate(small, { ...opts, patchChainLength: 6 }).kind, 'keyframe');
  assert.equal(diff.planFrameUpdate(small, { ...opts, forceKeyframe: true }).kind, 'keyframe');
  assert.equal(diff.planFrameUpdate({ comparable: false }, opts).kind, 'keyframe');
  // A 2px sliver grows to a readable minimum size.
  const sliver = diff.planFrameUpdate({ comparable: true, regions: [{ x: 600, y: 10, width: 2, height: 30 }], regionAreaRatio: 0.0001 }, opts);
  assert.ok(sliver.regions[0].width >= 48 && sliver.regions[0].height >= 48, JSON.stringify(sliver.regions));
});

test('frame update planning: patches follow the provider\'s image grid', () => {
  // A Qwen2.5-VL frame for a 1440x900 viewport is 1260x784 (multiples of 28).
  const width = 1260, height = 784;
  const opts = { width, height, align: 28, estimateTokens: (w, h) => Math.ceil(w / 28) * Math.ceil(h / 28) + 2 };
  const changes = [{ x: 517, y: 358, width: 64, height: 48 }, { x: 1200, y: 740, width: 60, height: 44 }, { x: 0, y: 0, width: 50, height: 30 }];
  for (const change of changes) {
    const plan = diff.planFrameUpdate({ comparable: true, regions: [change], regionAreaRatio: 0.004 }, opts);
    assert.equal(plan.kind, 'patch');
    for (const region of plan.regions) {
      assert.equal(region.width % 28, 0, JSON.stringify(region));
      assert.equal(region.height % 28, 0, JSON.stringify(region));
      assert.ok(region.width >= 56 && region.height >= 56, JSON.stringify(region));
      assert.ok(region.x >= 0 && region.y >= 0 && region.x + region.width <= width && region.y + region.height <= height, JSON.stringify(region));
      assert.ok(region.x <= change.x && region.y <= change.y && region.x + region.width >= change.x + change.width
        && region.y + region.height >= change.y + change.height, `the patch covers the change: ${JSON.stringify({ region, change })}`);
    }
  }
  // Two changes whose aligned patches would overlap become one aligned patch.
  const close = diff.planFrameUpdate({ comparable: true, regions: [{ x: 100, y: 100, width: 50, height: 40 }, { x: 160, y: 110, width: 40, height: 40 }], regionAreaRatio: 0.004 }, opts);
  assert.equal(close.regions.length, 1, JSON.stringify(close.regions));
  assert.equal(close.regions[0].width % 28, 0);
  assert.equal(close.regions[0].height % 28, 0);
  // Profiles without a grid keep the exact padded region.
  const plain = diff.planFrameUpdate({ comparable: true, regions: [{ x: 517, y: 358, width: 64, height: 48 }], regionAreaRatio: 0.004 }, { ...opts, align: 1 });
  assert.deepEqual(plain.regions.map(({ x, y, width: w, height: h }) => ({ x, y, w, h })), [{ x: 517, y: 358, w: 64, h: 48 }]);
});

test('copyRegions updates only the patched area of the seen buffer', () => {
  const seen = frame(64, 64, [0, 0, 0]);
  const next = frame(64, 64, [255, 255, 255]);
  diff.copyRegions(seen, next, 64, [{ x: 8, y: 8, width: 16, height: 16 }]);
  const at = (x, y) => seen[(y * 64 + x) * 4];
  assert.equal(at(10, 10), 255);
  assert.equal(at(30, 30), 0);
});

test('session: keyframe points map to CSS pixels on downscaled and retina captures', () => {
  // A 1440x900 CSS viewport on a DPR 2 display, sent as 1280x800.
  const s = new VisionSession();
  const f = s.registerKeyframe({ imageWidth: 1280, imageHeight: 800, cssWidth: 1440, cssHeight: 900, devicePixelRatio: 2, url: 'https://a.test/' });
  assert.equal(f.id, 'f1');
  const p = s.resolvePoint({ x: 640, y: 400 });
  assert.equal(p.ok, true);
  assert.deepEqual([p.cssX, p.cssY], [720, 450]);
  // Out of bounds and invalid points are refused with an actionable error.
  assert.equal(s.resolvePoint({ x: 1300, y: 10 }).code, 'out_of_bounds');
  assert.equal(s.resolvePoint({ x: 'a', y: 10 }).code, 'invalid_point');
  assert.equal(s.resolvePoint({ x: 10, y: 10, image: 'p9' }).code, 'unknown_image');
  // Without any frame the model is told to take one.
  assert.equal(new VisionSession().resolvePoint({ x: 1, y: 1 }).code, 'no_frame');
});

test('session: patch-local points, stale areas and retired frames', () => {
  const s = new VisionSession();
  s.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 1000, cssHeight: 500 });
  const [patch] = s.registerPatches([{ x: 200, y: 100, width: 300, height: 100 }]);
  assert.equal(patch.id, 'p2');
  // (10, 20) inside p2 is (210, 120) on the screen.
  const local = s.resolvePoint({ x: 10, y: 20, image: 'p2' });
  assert.deepEqual([local.cssX, local.cssY], [210, 120]);
  // The same point read from f1 is stale: that area changed in p2.
  const stale = s.resolvePoint({ x: 210, y: 120, image: 'f1' });
  assert.equal(stale.code, 'stale_area');
  assert.match(stale.error, /p2/);
  // An unchanged area of f1 is still valid, and the current screen always is.
  assert.equal(s.resolvePoint({ x: 800, y: 400, image: 'f1' }).ok, true);
  assert.equal(s.resolvePoint({ x: 210, y: 120 }).ok, true);
  // A new keyframe retires every earlier image.
  s.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 1000, cssHeight: 500 });
  const retired = s.resolvePoint({ x: 1, y: 1, image: 'p2' });
  assert.equal(retired.code, 'stale_image');
  assert.match(retired.error, /f3/);
});

test('session: normalized coordinates and zoom crops', () => {
  const s = new VisionSession({ coordinates: 'normalized' });
  s.registerKeyframe({ imageWidth: 1024, imageHeight: 640, cssWidth: 1280, cssHeight: 800 });
  const centre = s.resolvePoint({ x: 500, y: 500 });
  assert.deepEqual([centre.cssX, centre.cssY], [640, 400]);
  assert.equal(s.resolvePoint({ x: 1200, y: 10 }).code, 'out_of_bounds');

  const px = new VisionSession();
  px.registerKeyframe({ imageWidth: 1024, imageHeight: 640, cssWidth: 1280, cssHeight: 800 });
  const region = px.resolveRegion({ x: 100, y: 100, width: 200, height: 100 });
  assert.equal(region.ok, true);
  assert.deepEqual(region.keyframeRegion, { x: 100, y: 100, width: 200, height: 100 });
  assert.deepEqual(region.cssRegion, { x: 125, y: 125, width: 250, height: 125 });
  const zoom = px.registerZoom(region.keyframeRegion, 500, 250);
  // The centre of the zoom image is the centre of the region.
  const z = px.resolvePoint({ x: 250, y: 125, image: zoom.id });
  assert.deepEqual([z.cssX, z.cssY], [250, 187.5]);
});

test('session: a layout, URL or DPR change requires a new keyframe', () => {
  const s = new VisionSession();
  assert.equal(s.needsKeyframe({}), true);
  s.registerKeyframe({ imageWidth: 100, imageHeight: 50, cssWidth: 100, cssHeight: 50, devicePixelRatio: 1, url: 'https://a.test/' }, frame(100, 50));
  assert.equal(s.needsKeyframe({ cssWidth: 100, cssHeight: 50, devicePixelRatio: 1, url: 'https://a.test/' }), false);
  assert.equal(s.needsKeyframe({ cssWidth: 120, cssHeight: 50, devicePixelRatio: 1, url: 'https://a.test/' }), true);
  assert.equal(s.needsKeyframe({ cssWidth: 100, cssHeight: 50, devicePixelRatio: 2, url: 'https://a.test/' }), true);
  assert.equal(s.needsKeyframe({ cssWidth: 100, cssHeight: 50, devicePixelRatio: 1, url: 'https://a.test/b' }), true);
  s.forceKeyframe = true;
  assert.equal(s.needsKeyframe({ cssWidth: 100, cssHeight: 50, devicePixelRatio: 1, url: 'https://a.test/' }), true);
});

test('click_at translates into a canonical CSS coordinate click', () => {
  const s = new VisionSession();
  s.registerKeyframe({ imageWidth: 1024, imageHeight: 640, cssWidth: 1280, cssHeight: 800 });
  const t = visionTools.translateVisionClick({ x: 512, y: 320, clicks: 2, button: 'left' }, s);
  assert.equal(t.ok, true);
  assert.equal(t.name, 'click');
  assert.deepEqual(t.args, { x: 640, y: 400, coordinate_space: 'css' });
  assert.equal(t.vision.clickCount, 2);
  assert.equal(t.vision.tool, 'click_at');
  const bad = visionTools.translateVisionClick({ x: 5000, y: 1 }, s);
  assert.equal(bad.ok, false);
  assert.equal(bad.result.noDispatch, true);
  assert.equal(bad.result.visionCoordinateError, 'out_of_bounds');
});

test('settings: Auto by default, off disables, only Act with an image-capable model qualifies', () => {
  const { normalizeVisionModeSetting, resolveVisionMode } = visionMode;
  assert.equal(normalizeVisionModeSetting(undefined), 'auto');
  assert.equal(normalizeVisionModeSetting('OFF'), 'off');
  assert.equal(normalizeVisionModeSetting(false), 'off');
  assert.equal(normalizeVisionModeSetting('maybe'), 'auto');
  // Auto was called On: a stored 'on' or true reads as Auto, and still applies.
  assert.equal(normalizeVisionModeSetting('on'), 'auto');
  assert.equal(normalizeVisionModeSetting(' ON '), 'auto');
  assert.equal(normalizeVisionModeSetting(true), 'auto');
  assert.equal(normalizeVisionModeSetting('auto'), 'auto');
  assert.equal(normalizeVisionModeSetting('maybe', 'off'), 'off');
  const seeing = { name: 'anthropic', supportsVision: true, config: { model: 'claude-sonnet-5' } };
  assert.deepEqual(
    { ...resolveVisionMode('auto', { mode: 'act', provider: seeing }) },
    { active: true, setting: 'auto', reason: 'vision_capable_model', profileId: 'anthropic', coordinates: 'pixel', quality: 'balanced' },
  );
  assert.deepEqual(
    { ...resolveVisionMode('on', { mode: 'act', provider: seeing }) },
    { active: true, setting: 'auto', reason: 'vision_capable_model', profileId: 'anthropic', coordinates: 'pixel', quality: 'balanced' },
  );
  assert.equal(resolveVisionMode('off', { mode: 'act', provider: seeing }).reason, 'setting_off');
  assert.equal(resolveVisionMode('auto', { mode: 'ask', provider: seeing }).reason, 'not_act_mode');
  assert.equal(resolveVisionMode('auto', { mode: 'dev', provider: seeing }).reason, 'not_act_mode');
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: { ...seeing, supportsVision: false } }).reason, 'model_without_images');
  // A describe-only vision sub-model cannot aim clicks.
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: seeing, rawImage: false }).reason, 'model_without_images');
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: seeing, standalone: true }).reason, 'standalone_chat');
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: { name: 'openrouter', supportsVision: true, config: { model: 'google/gemini-2.5-flash' } } }).coordinates, 'normalized');
  // A site workflow proved against the element inventory keeps the tree path.
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: seeing, ledgerWorkflow: true }).reason, 'workflow_needs_element_inventory');
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: seeing, ledgerWorkflow: false }).active, true);
});

test('settings: an upload handed to the element tools keeps those steps on the tree path', () => {
  const { resolveVisionMode } = visionMode;
  const seeing = { name: 'anthropic', supportsVision: true, config: { model: 'claude-sonnet-5' } };
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: seeing, uploadHandoff: true }).reason, 'file_upload_needs_element_inventory');
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: seeing, uploadHandoff: true }).active, false);
  assert.equal(resolveVisionMode('auto', { mode: 'act', provider: seeing, uploadHandoff: false }).active, true);
  // The setting and the mode still come first.
  assert.equal(resolveVisionMode('off', { mode: 'act', provider: seeing, uploadHandoff: true }).reason, 'setting_off');
  assert.equal(resolveVisionMode('auto', { mode: 'ask', provider: seeing, uploadHandoff: true }).reason, 'not_act_mode');
});

for (const build of ['chrome', 'firefox']) {
  const tools = await import(`../src/${build}/src/agent/tools.js`);
  const { validateToolArguments } = await import(`../src/${build}/src/agent/tool-arguments.js`);
  const gate = await import(`../src/${build}/src/agent/permission-gate.js`);
  const names = list => list.map(tool => tool.function.name);

  test(`${build}: vision tool surface replaces tree, ref and selector tools for every tier`, () => {
    for (const tier of ['compact', 'mid', 'full']) {
      const legacy = names(tools.getToolsForMode('act', { tier }));
      const vision = tools.getToolsForMode('act', { tier, vision: { coordinates: 'pixel' } });
      const visionNames = names(vision);
      for (const removed of visionTools.VISION_REMOVED_TOOLS) {
        assert.ok(!visionNames.includes(removed), `${build}/${tier}: ${removed} must not be offered in vision mode`);
      }
      for (const added of ['take_screenshot', 'click_at', 'scroll_at', 'hover_at', 'drag_at', 'zoom', 'wait', 'type_text', 'done']) {
        assert.ok(visionNames.includes(added), `${build}/${tier}: ${added} is offered`);
      }
      // Everything that does not target DOM nodes is kept.
      for (const name of legacy) {
        if (visionTools.VISION_REMOVED_TOOLS.has(name)) continue;
        assert.ok(visionNames.includes(name), `${build}/${tier}: ${name} is kept`);
      }
      assert.equal(new Set(visionNames).size, visionNames.length, 'no duplicate tool names');
      const typeText = vision.find(tool => tool.function.name === 'type_text');
      assert.deepEqual(Object.keys(typeText.function.parameters.properties).sort(), ['clear', 'text']);
      // Ask and Dev never get the vision surface.
      assert.ok(!names(tools.getToolsForMode('ask', { tier, vision: { coordinates: 'pixel' } })).includes('click_at'));
      assert.ok(!names(tools.getToolsForMode('dev', { tier, vision: { coordinates: 'pixel' } })).includes('click_at'));
    }
  });

  test(`${build}: vision schemas are closed and validate model calls`, () => {
    const vision = tools.getToolsForMode('act', { tier: 'full', vision: { coordinates: 'pixel' } });
    const schema = name => vision.find(tool => tool.function.name === name).function.parameters;
    assert.equal(validateToolArguments('click_at', { x: 10, y: 20 }, schema('click_at')).ok, true);
    assert.equal(validateToolArguments('click_at', { x: 10, y: 20, image: 'p3', clicks: 2, button: 'right' }, schema('click_at')).ok, true);
    assert.notEqual(validateToolArguments('click_at', { x: 10 }, schema('click_at')).ok, true);
    assert.notEqual(validateToolArguments('click_at', { x: 1, y: 2, ref_id: 'ref_1' }, schema('click_at')).ok, true);
    assert.notEqual(validateToolArguments('click_at', { x: 1, y: 2, clicks: 4 }, schema('click_at')).ok, true);
    assert.equal(validateToolArguments('scroll_at', { direction: 'down' }, schema('scroll_at')).ok, true);
    assert.equal(validateToolArguments('drag_at', { start_x: 1, start_y: 2, end_x: 3, end_y: 4 }, schema('drag_at')).ok, true);
    assert.equal(validateToolArguments('zoom', { x: 1, y: 2, width: 30, height: 40 }, schema('zoom')).ok, true);
    assert.equal(validateToolArguments('wait', { seconds: 1 }, schema('wait')).ok, true);
    assert.notEqual(validateToolArguments('wait', { seconds: 60 }, schema('wait')).ok, true);
    // Normalized profiles describe 0..1000 coordinates.
    const normalized = tools.getToolsForMode('act', { tier: 'full', vision: { coordinates: 'normalized' } });
    assert.match(normalized.find(tool => tool.function.name === 'click_at').function.description, /0\.\.1000/);
  });

  test(`${build}: every vision tool is classified for the permission gate and untrusted results`, () => {
    for (const name of visionTools.VISION_TOOL_NAMES) {
      const gated = gate.capabilityFor(name, {}) !== null;
      const untrusted = gate.UNTRUSTED_CONTENT_TOOLS.has(name);
      assert.ok(gated || untrusted || name === 'wait', `${build}: ${name} is unclassified`);
    }
    assert.equal(gate.capabilityFor('click_at', {}), gate.Capability.CLICK);
    assert.equal(gate.capabilityFor('drag_at', {}), gate.Capability.CLICK);
    assert.equal(gate.capabilityFor('scroll_at', {}), null);
    for (const name of visionTools.VISION_TOOL_NAMES) {
      assert.ok(tools.RESERVED_AGENT_TOOL_NAMES.has(name), `${build}: skills cannot shadow ${name}`);
    }
  });

  test(`${build}: the vision Act prompt teaches coordinates and never mentions tree tools`, () => {
    for (const tier of ['compact', 'mid', 'full']) {
      const prompt = tools.getVisionActPrompt({ coordinates: 'pixel', tier });
      assert.match(prompt, /VISION CONTROL/);
      assert.match(prompt, /click_at/);
      assert.match(prompt, /UNTRUSTED/);
      assert.match(prompt, /retina/);
      assert.doesNotMatch(prompt, /get_accessibility_tree|click_ax|set_field|ref_id/);
    }
    assert.match(tools.getVisionActPrompt({ coordinates: 'normalized' }), /0\.\.1000/);
    assert.match(tools.getVisionActPrompt({ tier: 'compact' }), /cannot schedule/);
  });

  test(`${build}: the vision Act prompt says how a file gets attached without the element tools`, () => {
    // upload_file is not a vision tool; clicking the upload control hands it over.
    for (const tier of ['compact', 'mid', 'full']) {
      const prompt = tools.getVisionActPrompt({ coordinates: 'pixel', tier });
      assert.match(prompt, /File uploads: to attach [^\n]*click the page's own attach or upload control once with click_at\. WebBrain blocks the OS file dialog[^\n]*gives you upload_file for the next steps/);
      assert.match(prompt, /Only if WebBrain then says the upload is unavailable, finish the rest and tell the user\./);
    }
    // Compact upload_file attaches only the user's own attachments.
    assert.match(tools.getVisionActPrompt({ tier: 'compact' }), /to attach a file the user attached to their message,/);
    assert.match(tools.getVisionActPrompt({ tier: 'full' }), /to attach a file the user attached, or one downloaded in this conversation,/);
  });
}

for (const build of ['chrome', 'firefox']) {
  test(`${build}: settings, background, export, locales and traces carry the vision settings`, async () => {
    const html = read(`src/${build}/src/ui/settings.html`);
    const settings = read(`src/${build}/src/ui/settings.js`);
    const background = read(`src/${build}/src/background.js`);
    const runtime = build === 'chrome' ? 'chrome' : 'browser';
    assert.match(html, /<select id="select-vision-mode" data-i18n-aria-label="st\.display\.vision_mode\.label">\s*<option value="auto" data-i18n="st\.display\.vision_mode\.auto"><\/option>\s*<option value="off" data-i18n="st\.display\.vision_mode\.off"><\/option>/);
    assert.match(html, /<select id="select-vision-quality"[\s\S]*?value="economy"[\s\S]*?value="balanced"[\s\S]*?value="sharp"/);
    assert.ok(html.indexOf('select-vision-mode') > html.indexOf('select-lean-mode'), 'next to Lean mode in Advanced');
    assert.match(settings, /visionModeSelect\.value = normalizeVisionModeSetting\(stored\.visionMode\)/);
    assert.match(settings, new RegExp(`await ${runtime}\\.storage\\.local\\.set\\(\\{ visionMode: normalizeVisionModeSetting\\(visionModeSelect\\.value\\) \\}\\)`));
    assert.match(settings, new RegExp(`await ${runtime}\\.storage\\.local\\.set\\(\\{ visionQuality: normalizeVisionQuality\\(visionQualitySelect\\.value\\) \\}\\)`));
    // Background applies the stored setting (default Auto) before the first chat.
    assert.match(background, /agent\.visionMode = normalizeVisionModeSetting\(stored\?\.visionMode\);/);
    assert.match(background, /if \(changes\.visionMode\) \{\s*agent\.visionMode = normalizeVisionModeSetting\(changes\.visionMode\.newValue\);/);
    assert.equal((background.match(/await strictSecretModeReady;/g) || []).length, (background.match(/await visionModeReady;/g) || []).length,
      'every hydration wait also waits for the vision setting');
    const config = await import(`../src/${build}/src/config-transfer.js`);
    assert.equal(config.DEFAULT_CONFIG_SETTINGS.visionMode, 'auto');
    assert.equal(config.DEFAULT_CONFIG_SETTINGS.visionQuality, 'balanced');
    assert.ok(config.CONFIG_STORAGE_KEYS.includes('visionMode'));
    assert.ok(config.CONFIG_STORAGE_KEYS.includes('visionQuality'));
    const exported = config.createConfigExport({});
    assert.equal(config.parseConfigImport(JSON.stringify({ ...exported, settings: { ...exported.settings, visionMode: 'off', visionQuality: 'sharp' } })).settings.visionMode, 'off');
    assert.equal(config.parseConfigImport(JSON.stringify({ ...exported, settings: { ...exported.settings, visionMode: 'auto' } })).settings.visionMode, 'auto');
    // An export made while Auto was called On imports as Auto, and a stored 'on' exports as Auto.
    assert.equal(config.parseConfigImport(JSON.stringify({ ...exported, settings: { ...exported.settings, visionMode: 'on' } })).settings.visionMode, 'auto');
    assert.equal(config.createConfigExport({ visionMode: 'on' }).settings.visionMode, 'auto');
    assert.throws(() => config.parseConfigImport(JSON.stringify({ ...exported, settings: { ...exported.settings, visionMode: 'maybe' } })), /visionMode/);
    assert.throws(() => config.parseConfigImport(JSON.stringify({ ...exported, settings: { ...exported.settings, visionQuality: 'huge' } })), /visionQuality/);
    // Every locale has real, translated copy.
    const dir = path.join(ROOT, `src/${build}/src/ui/locales`);
    const english = (await import(`../src/${build}/src/ui/locales/en.js`)).default;
    const locales = fs.readdirSync(dir).filter(name => /^[a-z]{2}\.js$/.test(name));
    assert.equal(locales.length, 23);
    const keys = ['vision_mode.label', 'vision_mode.desc', 'vision_mode.auto', 'vision_mode.off', 'vision_quality.label', 'vision_quality.desc',
      'vision_quality.economy', 'vision_quality.balanced', 'vision_quality.sharp'];
    for (const file of locales) {
      const dict = (await import(`../src/${build}/src/ui/locales/${file}`)).default;
      for (const key of keys) assert.ok(String(dict[`st.display.${key}`] || '').trim(), `${file}: ${key}`);
      if (file !== 'en.js') assert.notEqual(dict['st.display.vision_mode.desc'], english['st.display.vision_mode.desc'], `${file}: translated description`);
      // Auto reads as Lean mode's Auto beside it, and On is gone.
      assert.equal(dict['st.display.vision_mode.auto'], dict['st.display.lean_mode.auto'], `${file}: Auto label`);
      assert.equal(Object.hasOwn(dict, 'st.display.vision_mode.on'), false, `${file}: no On label`);
    }
    // Traces record the setting and the run's resolved state, nothing else.
    const runtimeConfig = await import(`../src/${build}/src/trace/runtime-config.js`);
    assert.deepEqual(runtimeConfig.normalizeRuntimeTraceConfig({ vision_mode: true, vision_mode_setting: 'auto', token: 'x' }),
      { schema_version: 1, vision_mode: true, vision_mode_setting: 'auto' });
  });
}

const probeModule = await import('../src/chrome/src/agent/vision-dom-probe.js');
const runtimeModule = await import('../src/chrome/src/agent/vision-runtime.js');

test('pruning: after the current frame only the newest zooms keep their images, and oldest zooms go first past the size budget', () => {
  const image = (chars = 40) => ({ type: 'image_url', image_url: { url: `data:image/webp;base64,${'A'.repeat(chars)}` } });
  const frame = (kind, ids, images) => ({ role: 'user', content: [{ type: 'text', text: ids.join(',') }, ...images], webbrainVisionFrame: { epoch: 1, kind, ids } });
  const messages = [
    frame('key', ['f1'], [image()]),
    ...['z2', 'z3', 'z4', 'z5', 'z6', 'z7'].map(id => frame('zoom', [id], [image()])),
    frame('patch', ['p8', 'z9'], [image(), image()]),
  ];
  const kept = runtimeModule.pruneVisionImages(messages).messages;
  const shown = kept.flatMap(message => message.content.filter(block => block.type === 'image_url').length ? message.webbrainVisionFrame.ids : []);
  const omitted = kept.flatMap(message => message.content.filter(block => block.type === 'text' && /^\[zoom z\d+ omitted/.test(block.text)).map(block => /z\d+/.exec(block.text)[0]));
  assert.deepEqual(omitted, ['z2', 'z3', 'z4']);
  assert.deepEqual(shown, ['f1', 'z5', 'z6', 'z7', 'p8', 'z9']);
  // A huge zoom pushes the kept images past the budget: the oldest zoom goes, never the frame or a patch.
  const big = runtimeModule.pruneVisionImages([
    frame('key', ['f1'], [image()]), frame('zoom', ['z2'], [image(8_100_000)]), frame('zoom', ['z3'], [image()]), frame('patch', ['p4'], [image()]),
  ]).messages;
  assert.equal(big[1].content.some(block => block.type === 'image_url'), false);
  for (const index of [0, 2, 3]) assert.equal(big[index].content.some(block => block.type === 'image_url'), true, String(index));
});

test('session: numbering continues across sessions and a retired screen refuses points', () => {
  const session = new VisionSession({ startEpoch: 3, startCounter: 7 });
  const frame = session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 1000, cssHeight: 500 });
  assert.equal(frame.id, 'f8');
  assert.equal(frame.epoch, 4);
  assert.ok(session.resolvePoint({ x: 10, y: 10 }).ok);
  session.retireKeyframe();
  assert.equal(session.resolvePoint({ x: 10, y: 10 }).code, 'no_frame');
  assert.equal(session.forceKeyframe, true);
  const next = session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 1000, cssHeight: 500 });
  assert.equal(next.id, 'f9');
  assert.equal(session.resolvePoint({ x: 10, y: 10, image: 'f8' }).code, 'stale_image');
});

test('session: a failed update stays recorded until the model is shown the screen again', () => {
  const session = new VisionSession();
  assert.equal(session.observationFailed, false);
  session.registerKeyframe({ imageWidth: 100, imageHeight: 50, cssWidth: 100, cssHeight: 50 }, new Uint8ClampedArray(100 * 50 * 4));
  for (const shown of [
    () => session.registerNoChange(),
    () => session.registerPatches([{ x: 0, y: 0, width: 10, height: 10 }], new Uint8ClampedArray(100 * 50 * 4)),
    () => session.registerKeyframe({ imageWidth: 100, imageHeight: 50, cssWidth: 100, cssHeight: 50 }, new Uint8ClampedArray(100 * 50 * 4)),
  ]) {
    session.observationFailed = true;
    session.registerZoom({ x: 0, y: 0, width: 10, height: 10 }, 20, 20);
    assert.equal(session.observationFailed, true, 'a zoom is not the screen');
    shown();
    assert.equal(session.observationFailed, false);
  }
});

test('session: only the newest four zooms stay usable', () => {
  const session = new VisionSession();
  session.registerKeyframe({ imageWidth: 100, imageHeight: 50, cssWidth: 100, cssHeight: 50 });
  const zooms = [1, 2, 3, 4, 5].map(() => session.registerZoom({ x: 0, y: 0, width: 10, height: 10 }, 20, 20));
  assert.equal(sessionModule.VISION_MAX_LIVE_ZOOMS, 4);
  const oldest = session.resolvePoint({ x: 5, y: 5, image: zooms[0].id });
  assert.equal(oldest.code, 'stale_image');
  assert.match(oldest.error, /no longer shown: only the newest 4 zooms are kept/);
  for (const zoom of zooms.slice(1)) assert.equal(session.resolvePoint({ x: 5, y: 5, image: zoom.id }).ok, true, zoom.id);
});

test('traces: metadata traces keep vision notes as vision notes, without error texts', async () => {
  const { projectTraceEventData } = await import('../src/chrome/src/trace/privacy.js');
  assert.equal(read('src/chrome/src/trace/privacy.js'), read('src/firefox/src/trace/privacy.js'));
  const failed = projectTraceEventData('note', { step: 3, note: 'vision_frame_error', extra: { stage: 'observe', error: 'Cannot read https://bank.example/account?id=42' } });
  assert.deepEqual(failed, { step: 3, note: 'vision_frame_error', extra: { stage: 'observe' } });
  const frame = projectTraceEventData('note', { step: 1, note: 'vision_frame', extra: {
    kind: 'keyframe', id: 'f4', reason: 'vision resumed', width: 1356, height: 847, devicePixelRatio: 2, bytes: 81234,
    mimeType: 'image/webp', profile: 'anthropic', regions: [{ x: 1, y: 2, width: 3, height: 4 }], url: 'https://bank.example/',
  } });
  assert.deepEqual(frame.extra, { kind: 'keyframe', reason: 'vision resumed', width: 1356, height: 847, devicePixelRatio: 2, bytes: 81234, mimeType: 'image/webp', id: 'f4', profile: 'anthropic' });
  const skipped = projectTraceEventData('note', { step: 2, note: 'vision_redaction_skipped', extra: { code: 'scan_unavailable', context: 'zoom' } });
  assert.deepEqual(skipped.extra, { code: 'scan_unavailable', context: 'zoom' });
  // Values outside the closed lists are dropped, not passed through.
  assert.equal(projectTraceEventData('note', { step: 2, note: 'vision_frame', extra: { kind: 'secret text', reason: 'typed: hunter2' } }).extra, undefined);
  // The opt-in content tier keeps the note as recorded.
  const raw = { step: 3, note: 'vision_frame_error', extra: { stage: 'observe', error: 'details' } };
  assert.deepEqual(projectTraceEventData('note', raw, { includeContent: true }), raw);
});

test('session: an image value that is not an id is refused, not read as the current screen', () => {
  const session = new VisionSession();
  session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 2000, cssHeight: 1000 });
  const zoom = session.registerZoom({ x: 500, y: 250, width: 250, height: 125 }, 500, 250);
  assert.equal(zoom.id, 'z2');
  for (const value of ['zoom z2', 'z2.png', 'Z-2', 'image z2']) {
    const point = session.resolvePoint({ x: 200, y: 100, image: value });
    assert.equal(point.code, 'unknown_image', value);
    assert.match(point.error, /Name an image exactly as labeled/);
  }
  // Case and whitespace are forgiven; the zoom maps through its own scale.
  const exact = session.resolvePoint({ x: 200, y: 100, image: ' Z2 ' });
  assert.ok(exact.ok);
  assert.deepEqual({ x: exact.cssX, y: exact.cssY }, { x: 1200, y: 600 });
  for (const value of [undefined, null, '']) assert.equal(session.resolvePoint({ x: 10, y: 10, image: value }).imageId, 'f1');
});

test('session: a normalized zoom whose far corner changed keeps its requested size', () => {
  const session = new VisionSession({ coordinates: 'normalized' });
  session.registerKeyframe({ imageWidth: 1280, imageHeight: 720, cssWidth: 1280, cssHeight: 720 });
  // The bottom-right area changed after f1: the far corner is stale.
  session.registerPatches([{ x: 600, y: 300, width: 400, height: 300 }]);
  const region = session.resolveRegion({ x: 250, y: 250, width: 250, height: 250, image: 'f1' });
  assert.ok(region.ok, JSON.stringify(region));
  // 250/1000 of 1280x720, not 250 keyframe px.
  assert.deepEqual({ width: region.keyframeRegion.width, height: region.keyframeRegion.height }, { width: 320, height: 180 });
});

test('pruning: the last frame in the history is current, whatever its epoch number', () => {
  const img = id => ({ type: 'image_url', image_url: { url: `data:image/webp;base64,${id}` } });
  const frame = (epoch, kind, ids) => ({ role: 'user', content: [{ type: 'text', text: ids.join(',') }, ...ids.map(img)], webbrainVisionFrame: { epoch, kind, ids } });
  // Session A (epochs 1..3), then session B restarted at epoch 1.
  const messages = [frame(1, 'key', ['f1']), frame(1, 'patch', ['p2']), frame(3, 'key', ['f3']), frame(3, 'zoom', ['z4']), frame(1, 'key', ['f1']), frame(1, 'patch', ['p2'])];
  const kept = list => list.flatMap(message => message.content).filter(block => block.type === 'image_url').length;
  const { messages: pruned, latestEpoch } = runtimeModule.pruneVisionImages(messages);
  assert.equal(latestEpoch, 1);
  assert.equal(kept(pruned.slice(0, 4)), 0, 'session A images are omitted');
  assert.equal(kept(pruned.slice(4)), 2, 'session B frame and patch are kept');
  // A failed first capture retires the last frame for the model too.
  const retired = runtimeModule.pruneVisionImages([...messages, { role: 'user', content: 'no frame', webbrainVisionFrame: { epoch: 0, kind: 'retire' } }]).messages;
  assert.equal(kept(retired), 0);
  assert.deepEqual(runtimeModule.visionHistoryCounters(messages), { epoch: 3, counter: 4 });
});

test('dispatch checks: repeated clicks, drags that click, wheels over number fields', () => {
  const at = extra => ({ points: [{ found: true, tag: 'span', clickable: false, opaqueFrame: false, ...extra }] });
  const { visionMultiClickRefusal, visionDragRefusal, visionWheelRefusal } = probeModule;
  assert.equal(visionMultiClickRefusal(at({ clickable: true, control: 'button' }), { clickCount: 1 }), null);
  assert.match(visionMultiClickRefusal(at({ clickable: true, control: 'button' }), { clickCount: 2 }).error, /double click was refused because the point is on a button/);
  assert.equal(visionMultiClickRefusal(at({}), { clickCount: 3 }), null);
  assert.ok(visionMultiClickRefusal(at({}), { clickCount: 2, recipientGuarded: true }));
  assert.ok(visionMultiClickRefusal(null, { clickCount: 2 }));
  const pair = (pairInfo, extra = {}) => ({ points: [at(extra).points[0], at(extra).points[0]], pair: pairInfo });
  const from = { x: 10, y: 10 };
  assert.equal(visionDragRefusal(pair({ known: true, clickEquivalent: false }), from, { x: 12, y: 13 }).visionInputRefused, 'drag_too_short');
  assert.equal(visionDragRefusal(pair({ known: true, clickEquivalent: false }), from, { x: 200, y: 10 }), null);
  assert.equal(visionDragRefusal(pair({ known: true, clickEquivalent: true, control: 'a' }), from, { x: 200, y: 10 }).visionInputRefused, 'drag_is_click');
  assert.equal(visionDragRefusal(pair({ known: true, clickEquivalent: false }, { opaqueFrame: true }), from, { x: 200, y: 10 }).visionInputRefused, 'drag_opaque_frame');
  assert.equal(visionDragRefusal(null, from, { x: 200, y: 10 }).visionInputRefused, 'drag_unchecked');
  assert.equal(visionWheelRefusal(at({ focusedNumberInput: true })).visionInputRefused, 'wheel_number_input');
  assert.equal(visionWheelRefusal(at({ focusedNumberInput: false })), null);
  assert.equal(visionWheelRefusal(null), null, 'an unreadable target does not block scrolling');
  // The probe travels as text: it must stay self-contained.
  const expression = probeModule.visionDomProbeExpression({ points: [{ x: 1, y: 2 }], pair: false });
  assert.match(expression, /^\(function visionDomProbe\(request\)/);
  assert.match(expression, /\(\{"points":\[\{"x":1,"y":2\}\],"pair":false\}\)$/);
});

test('profiles: GLM-4.xV and InternVL point in relative 0..1000', () => {
  for (const model of ['z-ai/glm-4.5v', 'zai-org/GLM-4.6V', 'THUDM/GLM-4.1V-9B-Thinking', 'OpenGVLab/InternVL3-78B']) {
    const profile = budget.detectVisionProfile({ name: 'openrouter', config: { model } });
    assert.equal(profile.id, 'relative1000', model);
    assert.equal(profile.coordinates, 'normalized');
    assert.equal(profile.format, 'jpeg');
  }
  assert.equal(budget.detectVisionProfile({ name: 'openrouter', config: { model: 'z-ai/glm-4.6' } }).id, 'default', 'text GLM is unaffected');
});

test('click_at at the top-left CSS pixel is nudged off the normalized-fraction range', () => {
  const session = new VisionSession();
  session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 1000, cssHeight: 500 });
  const result = visionTools.translateVisionClick({ x: 0, y: 1 }, session);
  assert.deepEqual(result.args, { x: 1.5, y: 1.5, coordinate_space: 'css' });
  assert.deepEqual(visionTools.translateVisionClick({ x: 0, y: 40 }, session).args, { x: 0, y: 40, coordinate_space: 'css' });
});

test('page feedback: vision points survive page churn, keyboard follow-ups ride on their click', async () => {
  const policyModule = await import('../src/chrome/src/agent/page-feedback-policy.js');
  const { pageFeedbackCallPolicy, canRetainPageFeedbackCalls } = policyModule;
  for (const build of ['chrome', 'firefox']) {
    assert.equal(read(`src/${build}/src/agent/page-feedback-policy.js`), read('src/chrome/src/agent/page-feedback-policy.js'));
  }
  for (const name of ['click_at', 'hover_at', 'drag_at']) assert.equal(pageFeedbackCallPolicy(name, { x: 1, y: 2 }).kind, 'vision_point', name);
  assert.equal(pageFeedbackCallPolicy('scroll_at', { direction: 'down' }).kind, 'scroll');
  for (const name of ['take_screenshot', 'zoom', 'wait']) assert.equal(pageFeedbackCallPolicy(name, {}).kind, 'observe', name);
  const call = (name, args = {}) => ({ id: name, function: { name, arguments: JSON.stringify(args) } });
  const passive = [{ kind: 'dom', source: 'page', frameId: 0, target: 'div.timeline' }];
  const user = [{ kind: 'input', source: 'user', frameId: 0, target: 'input#name' }];
  const state = { url: 'https://owned.example.test/', actionBinding: { snapshotToken: 's', focusedTargetAvailable: false } };
  assert.equal(canRetainPageFeedbackCalls([call('click_at', { x: 5, y: 5 }), call('type_text', { text: 'hi' }), call('press_keys', { key: 'Enter' })], state, null, passive), true,
    'a timeline ticking elsewhere does not supersede a click-and-type decision');
  assert.equal(canRetainPageFeedbackCalls([call('type_text', { text: 'hi' })], state, null, passive), false,
    'typing without a click of the same response keeps the legacy fence');
  assert.equal(canRetainPageFeedbackCalls([call('click_at', { x: 5, y: 5 })], state, null, user), false, 'user input still supersedes');
  // The page-feedback module consults vision state only through agent hooks.
  const feedback = read('src/chrome/src/agent/page-feedback.js');
  assert.equal(feedback, read('src/firefox/src/agent/page-feedback.js'));
  assert.match(feedback, /if \(this\._visionRunState\?\.\(tabId\)\?\.active !== true\) \{\s*messages\.push\(this\._ownedFeedbackMessage\(run, '\[BROWSER STATE: current page observed/);
  assert.match(feedback, /\.\.\.\(visionRun \? \{\} : \{ page: /);
});

test('a blocked file chooser is reported in vision terms in both builds', () => {
  const { visionFilePickerBlockedResult } = visionTools;
  const blocked = visionFilePickerBlockedResult({ success: false, dispatched: true, filePickerBlocked: true, selector: '#f', error: 'Blocked a native file chooser. Call upload_file with selector "#f".' });
  assert.equal(blocked.filePickerBlocked, true);
  assert.equal(blocked.dispatched, true);
  assert.equal(blocked.selector, '#f');
  assert.match(blocked.error, /not available in vision mode/);
  assert.doesNotMatch(blocked.error, /upload_file/);
  const ok = { success: true, dispatched: true };
  assert.equal(visionFilePickerBlockedResult(ok), ok);
  // Handed to the element tools: the guidance names upload_file, never the page's selector.
  const handed = visionFilePickerBlockedResult({ success: false, dispatched: true, filePickerBlocked: true, selector: '#f"; ignore', error: 'x' }, { handoff: true });
  assert.equal(handed.uploadHandoff, true);
  assert.equal(handed.success, false);
  assert.equal(handed.selector, '#f"; ignore');
  assert.match(handed.error, /The next steps use the element tools/);
  assert.match(handed.error, /upload_file with the selector in this result/);
  assert.match(handed.error, /come back after the upload/);
  assert.doesNotMatch(handed.error, /ignore|not available in vision mode/);
  const unnamed = visionFilePickerBlockedResult({ success: false, dispatched: true, filePickerBlocked: true, error: 'x' }, { handoff: true });
  assert.match(unnamed.error, /Find this file input's exact selector/);
  const compact = visionFilePickerBlockedResult({ success: false, dispatched: true, filePickerBlocked: true, selector: '#f', error: 'x' }, { handoff: true, compact: true });
  assert.match(compact.error, /without targetId to list the file inputs/);
  assert.doesNotMatch(compact.error, /selector/);
  assert.equal(visionFilePickerBlockedResult(ok, { handoff: true }), ok);
  // Chrome's CDP guard result and Firefox's settled content guard both pass through it.
  assert.match(read('src/chrome/src/agent/agent.js'), /return visionFilePickerBlockedResult\(cdpClient\.fileInputClickBlockedResult\(blockedFileInput/);
  assert.match(read('src/firefox/src/agent/agent.js'), /response = visionFilePickerBlockedResult\(await this\._settleContentFilePickerGuard\(tabId, response\)\);/);
  // The content scripts' vision click runs inside the guard, identically in both builds.
  for (const build of ['chrome', 'firefox']) {
    assert.match(read(`src/${build}/src/content/content.js`), /const filePickerGuard = clickWithoutNativeFilePicker\(\(\) => \{\s*for \(let count = 1; count <= clicks; count\+\+\)/);
  }
});

test('both loops of both builds start or end an upload handoff before anything in the step depends on vision', () => {
  for (const build of ['chrome', 'firefox']) {
    const source = read(`src/${build}/src/agent/agent.js`);
    const ticks = [...source.matchAll(/await this\._visionUploadHandoffTick\(tabId, steps\);/g)].map(match => match.index);
    const reinjects = [...source.matchAll(/await this\._maybeReinjectAdapter\(tabId, messages\);/g)].map(match => match.index);
    const resumes = [...source.matchAll(/await this\._visionResumeKeyframe\(tabId, messages, onUpdate, \{ mode \}\);\s*\}\s*this\._visionNoteStepTools\(tabId, visionPromptNow\);/g)].map(match => match.index);
    assert.equal(ticks.length, 2, build);
    assert.equal(resumes.length, 2, build);
    for (const [index, tick] of ticks.entries()) {
      assert.ok(tick < reinjects[index] && reinjects[index] - tick < 400, `${build}: tick before adapter notes`);
      assert.ok(tick < resumes[index], `${build}: tick before the tools`);
    }
  }
  // Screenshot redaction's pixelation is the same module in both builds.
  assert.equal(read('src/chrome/src/agent/screenshot-redaction.js'), read('src/firefox/src/agent/screenshot-redaction.js'));
  // Chrome's protocol guard marks a chooser only it saw.
  assert.match(read('src/chrome/src/cdp/cdp-client.js'), /selector: null,\s*protocolOnly: true,/);
});

test('content scripts: the vision input path is the same in both builds', () => {
  const visionBlock = file => {
    const source = read(file);
    const start = source.indexOf('  function _visionTargetAt(');
    const end = source.indexOf('\n  }\n', source.indexOf('  function visionInput(')) + 4;
    assert.ok(start > 0 && end > start, file);
    return source.slice(start, end);
  };
  assert.equal(visionBlock('src/firefox/src/content/content.js'), visionBlock('src/chrome/src/content/content.js'));
});

test('micro screen changes: a caret, a clock or a spinner settle; a menu or a toast does not', () => {
  const { isMicroScreenChange } = runtimeModule;
  const width = 1296, height = 810;
  const base = frame(width, height);
  const changed = rect => diff.diffFrames(base, paint(frame(width, height), width, rect, [0, 0, 0]), width, height);
  assert.equal(isMicroScreenChange(diff.diffFrames(base, frame(width, height), width, height)), true, 'no change at all');
  assert.equal(isMicroScreenChange(changed({ x: 400, y: 300, w: 2, h: 18 })), true, 'a 2 px caret');
  assert.equal(isMicroScreenChange(changed({ x: 1100, y: 760, w: 60, h: 14 })), true, 'a clock or ticker line');
  assert.equal(isMicroScreenChange(changed({ x: 600, y: 380, w: 36, h: 36 })), true, 'a spinner');
  assert.equal(isMicroScreenChange(changed({ x: 500, y: 100, w: 220, h: 300 })), false, 'a menu opening');
  assert.equal(isMicroScreenChange(changed({ x: 900, y: 40, w: 320, h: 64 })), false, 'a toast sliding in');
  assert.equal(isMicroScreenChange({ comparable: false, changedCells: 0, totalCells: 100 }), false, 'incomparable frames never count');
  assert.equal(isMicroScreenChange(null), false);
});

test('page monitor: vision points, focused follow-ups and wheels rebase over churn, never over a person', () => {
  const monitor = read('src/chrome/src/content/page-monitor.js');
  assert.equal(monitor, read('src/firefox/src/content/page-monitor.js'));
  // One predicate for the three vision operations, with intervention-only
  // authority (as for certified targets and scrolls).
  assert.match(monitor, /function visionOperation\(op\) \{\s*return op\?\.visionPoint === true \|\| op\?\.visionFocusFollowUp === true \|\| op\?\.visionWheel === true;/);
  assert.match(monitor, /return op\.scrollIntent \|\| op\.preparedBinding\?\.model === true \|\| visionOperation\(op\)\s*\? op\.preparedInterventionRevision !== interventionRevision/);
  assert.match(monitor, /function rebasePassivePreparation\(op, params = \{\}\) \{\s*if \(visionOperation\(op\)\) return rebaseVisionPreparation\(op, params\);/);
  assert.match(monitor, /\|\| authorityChanged\(op\) \|\| \(params\.documentToken && params\.documentToken !== documentToken\)\) return false;/);
  // No tree binding is prepared for them (a body binding tracks only churn).
  assert.match(monitor, /: !scrollIntent && !visionOperation\(params\) && allowPassiveRebase && operationTarget/);
  const feedback = read('src/chrome/src/agent/page-feedback.js');
  assert.match(feedback, /\.\.\.\(name === 'scroll_at' \? \{ visionWheel: true \} : \{\}\)/);
});