Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

15.5 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { fakeBrowser } from 'wxt/testing';
import { OVERLAY_ATTRIBUTE } from '@/lib/overlay';
import { AUTO_SEND_MS, HANDS_FREE_CHANNEL, type OrbCommand, type OrbView } from './events';
import { exposeHandsFree } from './host';

type Frame = { op: string } & Record<string, unknown>;
type Unsent<T> = T extends unknown ? Omit<T, 'channel'> : never;
type Trigger = (message: unknown, sender: unknown, respond: (value: unknown) => void) => Promise<unknown>;

const view = (patch: Partial<OrbView> = {}): OrbView => ({
  theme: 'ember',
  since: 0,
  position: null,
  link: 'live',
  run: 'idle',
  voice: 'listening',
  pushToTalk: false,
  cues: true,
  ...patch,
});

/* A person pressing a key; a key event the page makes up is untrusted, and the orb ignores it. */
function key(type: 'keydown' | 'keyup', code: string, init: KeyboardEventInit = {}, trusted = true): void {
  const event = new KeyboardEvent(type, { code, bubbles: true, ...init });
  if (trusted) Object.defineProperty(event, 'isTrusted', { value: true });
  window.dispatchEvent(event);
}

function trustedClick(target: Element): void {
  const event = new MouseEvent('click', { bubbles: true });
  Object.defineProperty(event, 'isTrusted', { value: true });
  target.dispatchEvent(event);
}

let root: ShadowRoot | null;
let sent: Frame[];
const answers = new Map<string, unknown>();

/* The orb answers through sendResponse, which the fake's typing leaves out. */
async function deliver(command: Unsent<OrbCommand>): Promise<unknown> {
  let reply: unknown;
  await (fakeBrowser.runtime.onMessage.trigger as unknown as Trigger)({ channel: HANDS_FREE_CHANNEL, ...command }, {}, (value) => {
    reply = value;
  });
  return reply;
}

const show = (patch: Partial<OrbView> = {}) => deliver({ op: 'show', view: view(patch) });
const heard = (text: string, final: boolean) => deliver({ op: 'heard', text, final });
const $ = <T extends Element = HTMLElement>(selector: string) => root!.querySelector(selector) as T;
const requests = (op: string) => sent.filter((frame) => frame.op === op);
const shown = () => [...root!.querySelectorAll('.tok')].map((token) => token.textContent).join(' ');
const press = () => $('.orb').dispatchEvent(new MouseEvent('click', { bubbles: true }));

beforeEach(() => {
  vi.useFakeTimers();
  fakeBrowser.reset();
  sent = [];
  answers.clear();
  root = null;
  vi.spyOn(fakeBrowser.runtime, 'sendMessage').mockImplementation(async (message: unknown) => {
    const frame = message as Frame;
    sent.push(frame);
    return answers.get(frame.op) ?? { ok: true };
  });
  const attach = Element.prototype.attachShadow;
  vi.spyOn(Element.prototype, 'attachShadow').mockImplementation(function (this: Element, init: ShadowRootInit) {
    root = attach.call(this, { ...init, mode: 'open' });
    return root;
  });
  if (!HTMLElement.prototype.setPointerCapture) HTMLElement.prototype.setPointerCapture = () => undefined;
  exposeHandsFree();
});

afterEach(() => {
  vi.useRealTimers();
  vi.restoreAllMocks();
  document.getElementById('browsentic-hands-free')?.remove();
});

describe('the hands-free orb', () => {
  it('asks the background what to show as soon as the page loads', () => {
    expect(requests('sync')).toHaveLength(1);
  });

  it('mounts as a marked overlay A-Eye will skip, and says so to the background', async () => {
    expect(await show()).toEqual({ ok: true });
    expect(document.getElementById('browsentic-hands-free')?.hasAttribute(OVERLAY_ATTRIBUTE)).toBe(true);
    expect($('.wrap').dataset.tone).toBe('listen');
  });

  it('streams what is said, counts down after a pause, then sends it', async () => {
    await show();
    await heard('open the', false);
    await heard('open the second result', false);
    expect(shown()).toBe('open the second result');
    expect($('.wrap').classList.contains('counting')).toBe(false);

    await heard('open the second result', true);
    expect($('.wrap').classList.contains('counting')).toBe(true);
    expect(requests('submit')).toHaveLength(0);

    vi.advanceTimersByTime(AUTO_SEND_MS);
    expect(requests('submit')).toEqual([
      expect.objectContaining({ text: 'open the second result', liveTools: false }),
    ]);
    expect($('.caption').dataset.kind).toBe('sent');
  });

  it('keeps words that did not change, and only streams in the new ones', async () => {
    await show();
    await heard('scroll down', false);
    const first = root!.querySelector('.tok');
    await heard('scroll down a bit', false);
    expect(root!.querySelector('.tok')).toBe(first);
    expect(shown()).toBe('scroll down a bit');
  });

  it('throws the sentence away when pressed during the countdown', async () => {
    await show();
    await heard('delete my account', true);
    press();
    expect(shown()).toBe('Cleared — say it again.');
    vi.advanceTimersByTime(AUTO_SEND_MS * 2);
    expect(requests('submit')).toHaveLength(0);
  });

  it('waits for the next pause when speech resumes during the countdown', async () => {
    await show();
    await heard('open settings', true);
    vi.advanceTimersByTime(AUTO_SEND_MS - 200);
    await heard('and then', false);
    vi.advanceTimersByTime(AUTO_SEND_MS);
    expect(requests('submit')).toHaveLength(0);
    await heard('and then privacy', true);
    vi.advanceTimersByTime(AUTO_SEND_MS);
    expect(requests('submit')[0]).toMatchObject({ text: 'open settings and then privacy' });
  });

  it('ignores speech while the agent works, and a press stops the run', async () => {
    await show({ run: 'working', voice: 'paused' });
    await heard('click buy', true);
    vi.advanceTimersByTime(AUTO_SEND_MS);
    expect(requests('submit')).toHaveLength(0);
    expect($('.wrap').dataset.face).toBe('dots');
    press();
    expect(requests('cancel')).toHaveLength(1);
  });

  it('mutes, listens, asks for the microphone, or opens the panel, by what it is showing', async () => {
    await show();
    press();
    expect(requests('listen').at(-1)).toMatchObject({ on: false });

    await show({ voice: 'muted' });
    press();
    expect(requests('listen').at(-1)).toMatchObject({ on: true });

    await show({ voice: 'needs-mic' });
    press();
    expect(requests('grantMic')).toHaveLength(1);

    await show({ voice: 'paused', link: 'off' });
    press();
    expect(requests('openPanel')).toHaveLength(1);
  });

  it('says so when recognition stops, and a press tries again', async () => {
    await show();
    await show({ voice: 'failed' });
    expect($('.wrap').dataset.tone).toBe('trouble');
    expect(shown()).toBe('Speech recognition stopped — tap the mic to try again.');
    press();
    expect(requests('listen').at(-1)).toMatchObject({ on: true });
  });

  it('turns ember for an approval and opens the request on a press, which a forged click cannot answer', async () => {
    await show();
    await show({
      run: 'approval',
      voice: 'paused',
      approval: { toolId: 't1', action: 'clickElement', site: 'shop.test', detail: 'selector: #buy' },
    });
    expect($('.wrap').dataset.tone).toBe('ask');
    expect($('.caption').dataset.kind).toBe('ask');

    press();
    expect($('.pop').hidden).toBe(false);
    expect($('.pop-title').textContent).toBe('Allow click element?');
    expect($('.pop-detail').textContent).toBe('selector: #buy');
    expect(root!.querySelector('[data-decide="always"]')?.textContent).toBe('Always on shop.test');

    $('[data-decide="allow"]').dispatchEvent(new MouseEvent('click', { bubbles: true }));
    expect(requests('decide')).toHaveLength(0);
    trustedClick($('[data-decide="always"]'));
    expect(requests('decide')).toEqual([expect.objectContaining({ toolId: 't1', allow: true, remember: true })]);

    await show({ run: 'working', voice: 'paused' });
    expect($('.pop').hidden).toBe(true);
  });

  it('shows the code the agent wrote, and never offers “always” for it', async () => {
    await show({
      run: 'approval',
      voice: 'paused',
      approval: { toolId: 't2', action: 'injectCode', site: 'shop.test', purpose: 'Total the cart', code: 'return 1 + 1;' },
    });
    press();
    expect($('.pop-code').textContent).toBe('return 1 + 1;');
    expect($('.pop-purpose').textContent).toBe('Total the cart');
    expect(root!.querySelector('[data-decide="always"]')).toBeNull();
  });

  it('opens the arc menu after a second of hover, and not before', async () => {
    await show();
    $('.wrap').dispatchEvent(new PointerEvent('pointerenter', { pointerType: 'mouse' }));
    vi.advanceTimersByTime(900);
    expect($('.wrap').classList.contains('open')).toBe(false);
    vi.advanceTimersByTime(100);
    expect($('.wrap').classList.contains('open')).toBe(true);
    const slots = [...root!.querySelectorAll<HTMLElement>('.item')].map((item) => Number.parseFloat(item.style.getPropertyValue('--x')));
    expect(slots).toEqual([...slots].sort((a, b) => a - b));
  });

  it('switches action cues from the menu and shows which way they are set', async () => {
    await show();
    const item = $('[data-item="cues"]');
    expect(item.classList.contains('on')).toBe(true);
    item.dispatchEvent(new MouseEvent('click', { bubbles: true }));
    expect(requests('cues')).toEqual([expect.objectContaining({ on: false })]);
    await show({ cues: false });
    expect(item.classList.contains('on')).toBe(false);
    expect(item.querySelector('.tip')?.textContent).toBe('Show what the agent does');
  });

  it('rides live code and an A-Eye pick along with the next instruction, then lets them go', async () => {
    await show();
    $('[data-item="code"]').dispatchEvent(new MouseEvent('click', { bubbles: true }));
    answers.set('pick', { focus: { tag: 'button', label: 'Buy now', selector: '#buy' } });
    $('[data-item="focus"]').dispatchEvent(new MouseEvent('click', { bubbles: true }));
    await vi.waitFor(() => expect(root!.querySelectorAll('.chip')).toHaveLength(2));
    expect(root!.querySelector('.chips')?.textContent).toContain('A-Eye · Buy now');

    await heard('buy it', true);
    vi.advanceTimersByTime(AUTO_SEND_MS);
    expect(requests('submit')[0]).toMatchObject({ text: 'buy it', liveTools: true, focus: { label: 'Buy now' } });
    expect(root!.querySelectorAll('.chip')).toHaveLength(1);
  });

  it('holds a reply for a tab nobody is looking at, and reads it once they are', async () => {
    await show();
    vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('hidden');
    await deliver({ op: 'say', text: 'Done — **three** results.', tone: 'reply' });
    expect(shown()).not.toContain('results.');

    vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('visible');
    document.dispatchEvent(new Event('visibilitychange'));
    expect($('.caption').dataset.kind).toBe('reply');
    expect(shown()).toBe('Done — **three** results.');
  });

  it('drags on a long press and remembers where it was dropped', async () => {
    await show();
    const orb = $('.orb');
    const at = (type: string, x: number, y: number) =>
      orb.dispatchEvent(new PointerEvent(type, { pointerId: 1, isPrimary: true, button: 0, clientX: x, clientY: y, bubbles: true }));
    const start = { x: Number.parseFloat($('.wrap').style.left), y: Number.parseFloat($('.wrap').style.top) };
    at('pointerdown', start.x, start.y);
    vi.advanceTimersByTime(400);
    expect($('.wrap').classList.contains('lifted')).toBe(true);
    at('pointermove', start.x - 200, start.y - 150);
    at('pointerup', start.x - 200, start.y - 150);
    expect(requests('move')).toHaveLength(1);
    press();
    expect(requests('listen')).toHaveLength(0);
  });

  describe('hold to talk', () => {
    const held = { pushToTalk: true, voice: 'held' as const };

    it('toggles from the menu, and wears the key on the orb', async () => {
      await show();
      $('[data-item="talk"]').dispatchEvent(new MouseEvent('click', { bubbles: true }));
      expect(requests('pushToTalk')).toEqual([expect.objectContaining({ on: true })]);
      await show(held);
      expect($('.wrap').classList.contains('push-to-talk')).toBe(true);
      expect($('[data-item="talk"]').classList.contains('on')).toBe(true);
    });

    it('talks while left Control is held, and sends the moment the words are in', async () => {
      await show(held);
      key('keydown', 'ControlLeft');
      vi.advanceTimersByTime(250);
      expect(requests('talk')).toEqual([expect.objectContaining({ on: true })]);
      expect($('.wrap').classList.contains('holding')).toBe(true);

      await show({ ...held, voice: 'listening' });
      await heard('scroll to the', false);
      await heard('scroll to the reviews', true);
      vi.advanceTimersByTime(AUTO_SEND_MS * 2);
      expect(requests('submit')).toHaveLength(0);

      key('keyup', 'ControlLeft');
      expect(requests('talk').at(-1)).toMatchObject({ on: false });
      await show(held);
      expect(requests('submit')).toEqual([expect.objectContaining({ text: 'scroll to the reviews' })]);
    });

    it('sends what was still being heard when the key came up', async () => {
      await show(held);
      key('keydown', 'ControlLeft');
      vi.advanceTimersByTime(250);
      await show({ ...held, voice: 'listening' });
      await heard('go back', false);
      key('keyup', 'ControlLeft');
      vi.advanceTimersByTime(2500);
      expect(requests('submit')[0]).toMatchObject({ text: 'go back' });
    });

    it('never opens the mic for a tap, a shortcut, a click or a scroll', async () => {
      await show(held);
      key('keydown', 'ControlLeft');
      key('keyup', 'ControlLeft');
      vi.advanceTimersByTime(500);

      key('keydown', 'ControlLeft');
      key('keydown', 'KeyC', { ctrlKey: true });
      vi.advanceTimersByTime(500);

      key('keydown', 'ControlLeft', { shiftKey: true });
      vi.advanceTimersByTime(500);

      key('keydown', 'ControlLeft');
      const click = new PointerEvent('pointerdown', { bubbles: true });
      Object.defineProperty(click, 'isTrusted', { value: true });
      window.dispatchEvent(click);
      vi.advanceTimersByTime(500);

      expect(requests('talk')).toHaveLength(0);
    });

    it('throws the words away when a shortcut lands mid-hold', async () => {
      await show(held);
      key('keydown', 'ControlLeft');
      vi.advanceTimersByTime(250);
      await show({ ...held, voice: 'listening' });
      await heard('select all', false);
      key('keydown', 'KeyA', { ctrlKey: true });
      expect(requests('talk').at(-1)).toMatchObject({ on: false });
      await heard('select all', true);
      await show(held);
      vi.advanceTimersByTime(3000);
      expect(requests('submit')).toHaveLength(0);
    });

    it('ignores a Control press the page made up', async () => {
      await show(held);
      key('keydown', 'ControlLeft', {}, false);
      vi.advanceTimersByTime(500);
      expect(requests('talk')).toHaveLength(0);
    });

    it('lets go of the mic when the page takes the orb away mid-hold', async () => {
      await show(held);
      key('keydown', 'ControlLeft');
      vi.advanceTimersByTime(250);
      document.getElementById('browsentic-hands-free')?.remove();
      key('keydown', 'ControlLeft', { repeat: true });
      expect(requests('talk').at(-1)).toMatchObject({ on: false });
    });

    it('stays quiet where it is not the tab being listened for', async () => {
      await show({ pushToTalk: true, voice: 'paused' });
      key('keydown', 'ControlLeft');
      vi.advanceTimersByTime(500);
      expect(requests('talk')).toHaveLength(0);
    });
  });

  it('leaves when hands-free ends', async () => {
    await show();
    await deliver({ op: 'hide' });
    vi.advanceTimersByTime(400);
    expect(document.getElementById('browsentic-hands-free')).toBeNull();
  });
});