Skip to content

Self-improving AI Employees for everyone

21.5 KB
Raw Blame
import { createRef } from 'react';
import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { SurfaceControl } from '../../workspace/surface';

const { actions } = vi.hoisted(() => ({ actions: { isReady: true, sendRequest: vi.fn() } }));
vi.mock('@/contexts/WebSocketContext', () => ({ useWebSocketActions: () => actions }));
import BrowserWorkspace from '../BrowserWorkspace';

class MockSocket {
  static OPEN = 1;
  static instances: MockSocket[] = [];
  readyState = 0;
  binaryType = '';
  onopen: (() => void) | null = null;
  onmessage: ((event: { data: unknown }) => void) | null = null;
  onclose: ((event: { code: number; reason: string }) => void) | null = null;
  onerror: (() => void) | null = null;
  sent: Record<string, unknown>[] = [];
  close = vi.fn(() => { this.readyState = 3; });
  constructor(public url: string) { MockSocket.instances.push(this); }
  send(message: string) { this.sent.push(JSON.parse(message)); }
  open() { this.readyState = 1; this.onopen?.(); }
  message(data: Record<string, unknown>) { this.onmessage?.({ data: JSON.stringify(data) }); }
}
const nodes = [{ node_id: 'browser-1', label: 'Research' }];

function mockFrameRenderer(deferred = false) {
  const loads: (() => void)[] = [];
  const draw = vi.fn();
  vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ drawImage: draw } as unknown as CanvasRenderingContext2D);
  const revoke = vi.fn(); const create = vi.fn(() => 'blob:browser-frame');
  const NativeURL = URL;
  vi.stubGlobal('URL', class extends NativeURL { static createObjectURL = create; static revokeObjectURL = revoke; });
  vi.stubGlobal('Image', class {
    naturalWidth = 640; naturalHeight = 360;
    onload: (() => void) | null = null; onerror: (() => void) | null = null;
    set src(value: string) { if (!value) return; const load = () => this.onload?.(); if (deferred) loads.push(load); else queueMicrotask(load); }
  });
  const header = new TextEncoder().encode(JSON.stringify({ seq: 1, device_width: 1280, device_height: 720 }));
  const frame = new Uint8Array(4 + header.length + 1); frame.set([1, 1]);
  new DataView(frame.buffer).setUint16(2, header.length, false); frame.set(header, 4);
  return { draw, create, revoke, frame, loads };
}

beforeEach(() => {
  MockSocket.instances = []; actions.isReady = true; actions.sendRequest.mockReset();
  actions.sendRequest.mockResolvedValue({ success: true });
  vi.stubGlobal('WebSocket', MockSocket);
  vi.stubGlobal('ResizeObserver', class { observe() {} disconnect() {} });
});
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); });

describe('BrowserWorkspace session lifecycle', () => {
  it('does not connect without a saved workflow or a browser node', () => {
    const { rerender } = render(<BrowserWorkspace nodes={nodes} />);
    expect(screen.getByText('Save this workflow to open its browser.')).toBeInTheDocument();
    rerender(<BrowserWorkspace workflowId="wf" nodes={[]} />);
    expect(MockSocket.instances).toHaveLength(0);
  });
  it('attaches immediately on open, watches idle sessions, and launches only on Start', async () => {
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    expect(socket.url).toMatch(/\/ws\/browser$/);
    act(() => { socket.open(); socket.message({ type: 'attached', running: false }); });
    expect(socket.sent[0]).toMatchObject({ type: 'attach', target: { kind: 'node', workflow_id: 'wf', node_id: 'browser-1' }, visible: true });
    expect(actions.sendRequest.mock.calls.some(([type]) => type === 'browser_session_open')).toBe(false);
    fireEvent.click(screen.getByRole('button', { name: 'Start browser' }));
    await waitFor(() => expect(actions.sendRequest).toHaveBeenCalledWith('browser_session_open', { workflow_id: 'wf', node_id: 'browser-1' }, 60_000));
  });
  it('sends hidden visibility and closes the old socket when the selected node changes', () => {
    const { rerender, unmount } = render(<BrowserWorkspace workflowId="wf" nodes={[...nodes, { node_id: 'browser-2', label: 'Orders' }]} />);
    const old = MockSocket.instances[0]; act(() => old.open());
    rerender(<BrowserWorkspace workflowId="wf" nodes={[...nodes, { node_id: 'browser-2', label: 'Orders' }]} visible={false} />);
    expect(old.sent).toContainEqual({ type: 'visibility', visible: false });
    fireEvent.change(screen.getByRole('combobox', { name: 'Browser node' }), { target: { value: 'browser-2' } });
    expect(old.close).toHaveBeenCalledOnce();
    const current = MockSocket.instances[1]; act(() => current.open());
    expect(current.sent[0]).toMatchObject({ target: { node_id: 'browser-2' }, visible: false });
    unmount(); expect(current.close).toHaveBeenCalledOnce();
  });
  it('gates navigation and keyboard input on server-confirmed control and hands back on unmount', () => {
    const { unmount } = render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'state', state: 'agent', controller: null }); });
    fireEvent.click(screen.getByRole('button', { name: 'Browser controls' }));
    expect(screen.getByRole('textbox', { name: 'Browser address' })).toBeDisabled();
    fireEvent.click(screen.getByRole('button', { name: 'Take control' }));
    expect(socket.sent.at(-1)).toEqual({ type: 'control_request' });
    act(() => socket.message({ type: 'state', state: 'user', controller: 'you' }));
    expect(screen.getByRole('textbox', { name: 'Browser address' })).toBeEnabled();
    fireEvent.keyDown(screen.getByRole('group', { name: 'Live browser page' }), { key: 'Enter', code: 'Enter', keyCode: 13 });
    expect(socket.sent.at(-1)).toMatchObject({ type: 'key', action: 'down', key: 'Enter', key_code: 13 });
    unmount(); expect(socket.sent.at(-1)).toEqual({ type: 'control_release', outcome: 'control_lost' });
  });
  it('offers the page the browser shows in a new tab, for web pages only', () => {
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'state', state: 'agent', controller: null }); });
    fireEvent.click(screen.getByRole('button', { name: 'Browser controls' }));
    expect(screen.queryByRole('link', { name: 'Open in new tab' })).not.toBeInTheDocument();
    act(() => socket.message({ type: 'page', url: 'https://example.com/book' }));
    const link = screen.getByRole('link', { name: 'Open in new tab' });
    expect(link).toHaveAttribute('href', 'https://example.com/book');
    expect(link).toHaveAttribute('target', '_blank');
    expect(link).toHaveAttribute('rel', 'noopener noreferrer');
    for (const url of ['chrome://settings', 'javascript:alert(document.cookie)', 'data:text/html,<script>alert(1)</script>']) {
      act(() => socket.message({ type: 'page', url }));
      expect(screen.queryByRole('link', { name: 'Open in new tab' })).not.toBeInTheDocument();
    }
  });
  it('acknowledges malformed binary frames so a bad frame cannot stall the stream', async () => {
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.onmessage?.({ data: new ArrayBuffer(2) }); });
    await waitFor(() => expect(socket.sent.at(-1)).toEqual({ type: 'ack' }));
    expect(screen.getByRole('alert')).toHaveTextContent('Incomplete browser frame');
  });
  it('paints and acknowledges frames, revokes JPEG URLs, and skips decoding hidden frames', async () => {
    const { draw, create, revoke, frame } = mockFrameRenderer();
    const { rerender } = render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(draw).toHaveBeenCalledOnce());
    expect(socket.sent.at(-1)).toEqual({ type: 'ack', seq: 1 });
    expect(revoke).toHaveBeenCalledWith('blob:browser-frame');
    rerender(<BrowserWorkspace workflowId="wf" nodes={nodes} visible={false} />);
    act(() => socket.onmessage?.({ data: frame.buffer }));
    await waitFor(() => expect(socket.sent.filter((message) => message.type === 'ack')).toHaveLength(2));
    expect(create).toHaveBeenCalledOnce();
  });
  it('clears the picture and offers Reconnect after terminal screencast failure', async () => {
    const { draw, frame } = mockFrameRenderer();
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(draw).toHaveBeenCalledOnce());
    act(() => {
      socket.message({ type: 'error', code: 'screencast', retrying: false, message: 'Reconnect the live view.' });
      // A controller broadcast does not repair a failed picture.
      socket.message({ type: 'state', state: 'idle', controller: null });
    });
    expect(screen.getByLabelText('Browser screenshot')).not.toBeVisible();
    expect(screen.getByRole('alert')).toHaveTextContent('Reconnect the live view.');
    fireEvent.click(screen.getByRole('button', { name: 'Reconnect' }));
    expect(socket.close).toHaveBeenCalledOnce();
    expect(MockSocket.instances).toHaveLength(2);
    expect(screen.queryByRole('alert')).not.toBeInTheDocument();
  });
  it('recovers a retrying screencast error on a fresh frame without hiding navigation errors', async () => {
    const { draw, frame } = mockFrameRenderer();
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => {
      socket.open(); socket.message({ type: 'state', state: 'agent' });
      socket.message({ type: 'error', code: 'screencast', retrying: true, message: 'Retrying live view…' });
    });
    expect(screen.queryByRole('button', { name: 'Reconnect' })).not.toBeInTheDocument();
    expect(screen.getByRole('status')).toHaveTextContent('Employee is browsing');
    act(() => socket.onmessage?.({ data: frame.buffer }));
    await waitFor(() => expect(draw).toHaveBeenCalledOnce());
    expect(screen.queryByRole('alert')).not.toBeInTheDocument();
    act(() => {
      socket.message({ type: 'error', code: 'screencast', retrying: true, message: 'Retrying live view…' });
      socket.message({ type: 'error', code: 'navigation', message: 'This address is blocked.' });
      socket.onmessage?.({ data: frame.buffer });
    });
    await waitFor(() => expect(draw).toHaveBeenCalledTimes(2));
    expect(screen.getByRole('alert')).toHaveTextContent('This address is blocked.');
  });
});


describe('browser frame and input lifecycle boundaries', () => {
  it('distinguishes an explicit challenge resume from losing browser focus', () => {
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'state', state: 'awaiting_user', challenge_required: true, request: { reason: 'captcha', message: 'Finish the site verification.' } }); });
    expect(screen.getByRole('status')).toHaveTextContent('Paused for site verification');
    expect(screen.getByText('Finish the site verification.')).toBeInTheDocument();
    act(() => socket.message({ type: 'state', state: 'user', controller: 'you', challenge_required: true }));
    fireEvent.click(screen.getByRole('button', { name: 'Hand back and resume' }));
    expect(socket.sent.at(-1)).toEqual({ type: 'control_release' });
    fireEvent(window, new Event('blur'));
    expect(socket.sent.at(-1)).toEqual({ type: 'control_release', outcome: 'control_lost' });
  });
  it('does not reset an unchanged canvas backing store', async () => {
    const { frame, draw } = mockFrameRenderer();
    const width = vi.spyOn(HTMLCanvasElement.prototype, 'width', 'set');
    const height = vi.spyOn(HTMLCanvasElement.prototype, 'height', 'set');
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(draw).toHaveBeenCalledOnce());
    act(() => socket.onmessage?.({ data: frame.buffer }));
    await waitFor(() => expect(draw).toHaveBeenCalledTimes(2));
    expect(width).toHaveBeenCalledTimes(1); expect(height).toHaveBeenCalledTimes(1);
  });
  it('does not paint after hiding during decode, but still acknowledges the frame', async () => {
    const { frame, loads, draw, revoke } = mockFrameRenderer(true);
    const { rerender } = render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(loads).toHaveLength(1));
    rerender(<BrowserWorkspace workflowId="wf" nodes={nodes} visible={false} />);
    await act(async () => loads[0]());
    expect(draw).not.toHaveBeenCalled(); expect(revoke).toHaveBeenCalled();
    expect(socket.sent).toContainEqual({ type: 'ack', seq: 1 });
  });
  it('does not restore live state when an old decode finishes after disconnect', async () => {
    const { frame, loads, draw, revoke } = mockFrameRenderer(true);
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(loads).toHaveLength(1));
    act(() => { socket.readyState = 3; socket.onclose?.({ code: 4004, reason: 'Session closed' }); });
    await act(async () => loads[0]());
    expect(draw).not.toHaveBeenCalled(); expect(revoke).toHaveBeenCalled();
    expect(screen.getByRole('status')).toHaveTextContent('Disconnected');
    expect(screen.getByLabelText('Browser screenshot')).not.toBeVisible();
    expect(screen.getByRole('button', { name: 'Reconnect' })).toBeInTheDocument();
  });
  it('cleans up an in-progress decode on disposal without painting', async () => {
    const { frame, loads, draw, revoke } = mockFrameRenderer(true);
    const { unmount } = render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(loads).toHaveLength(1));
    unmount(); await act(async () => loads[0]());
    expect(draw).not.toHaveBeenCalled(); expect(revoke).toHaveBeenCalled();
    expect(socket.close).toHaveBeenCalledOnce();
  });
  it.each(['blur', 'pointercancel'])('releases control on %s and stops further keys', (kind) => {
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'state', state: 'user', controller: 'you' }); });
    const surface = screen.getByRole('group', { name: 'Live browser page' });
    fireEvent.keyDown(surface, { key: 'Shift', code: 'ShiftLeft' });
    if (kind === 'blur') fireEvent(window, new Event('blur'));
    else fireEvent.pointerCancel(surface);
    expect(socket.sent.at(-1)).toEqual({ type: 'control_release', outcome: 'control_lost' });
    const count = socket.sent.length;
    fireEvent.keyDown(surface, { key: 'a', code: 'KeyA' });
    expect(socket.sent).toHaveLength(count);
  });
  it('answers the Workspace’s Take over no while the browser is stopped', async () => {
    const control = createRef<SurfaceControl>();
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} ref={control} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'attached', running: false }); socket.message({ type: 'idle', reason: 'not running' }); });
    let claim!: Promise<(() => void) | null>;
    act(() => { claim = control.current!.claim(); });
    expect(socket.sent.at(-1)).toEqual({ type: 'control_request', force: true });
    // A stopped browser answers every request with `idle`.
    act(() => socket.message({ type: 'idle', reason: 'not running' }));
    await expect(claim).resolves.toBeNull();
  });
  it('gives back the control the Workspace’s Take over was granted', async () => {
    const control = createRef<SurfaceControl>();
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} ref={control} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'state', state: 'agent' }); });
    let claim!: Promise<(() => void) | null>;
    act(() => { claim = control.current!.claim(); });
    act(() => { socket.message({ type: 'control', granted: true }); socket.message({ type: 'state', state: 'user', controller: 'you' }); });
    const giveBack = await claim;
    expect(giveBack).toBeTypeOf('function');
    act(() => giveBack!());
    expect(socket.sent.at(-1)).toEqual({ type: 'control_release' });
  });
  it('cancels a pending takeover when the document becomes hidden', () => {
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'state', state: 'agent' }); });
    fireEvent.click(screen.getByRole('button', { name: 'Take control' }));
    expect(socket.sent.at(-1)).toEqual({ type: 'control_request' });
    vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('hidden');
    fireEvent(document, new Event('visibilitychange'));
    expect(socket.sent.slice(-2)).toEqual([{ type: 'visibility', visible: false }, { type: 'control_release', outcome: 'control_lost' }]);
    expect(screen.getByRole('button', { name: 'Take control' })).toBeEnabled();
  });
  it('bounds decoding when a sender exceeds its two-frame credit', async () => {
    const { frame, loads } = mockFrameRenderer(true);
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); for (let i = 0; i < 10; i++) socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(loads).toHaveLength(1));
    expect(socket.sent.filter((m) => m.type === 'ack')).toHaveLength(8);
    await act(async () => loads[0]());
    expect(loads).toHaveLength(2);
    await act(async () => loads[1]());
    expect(socket.sent.filter((m) => m.type === 'ack')).toHaveLength(10);
  });
  it('discards a pending picture during protected login and exposes authorized close for manual recovery', async () => {
    const { frame, loads, draw } = mockFrameRenderer(true);
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'state', state: 'agent' }); socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(loads).toHaveLength(1));
    act(() => socket.message({ type: 'sensitive', enabled: true }));
    await act(async () => loads[0]());
    expect(draw).not.toHaveBeenCalled();
    expect(screen.queryByRole('button', { name: 'Take control' })).not.toBeInTheDocument();
    fireEvent.click(screen.getByRole('button', { name: 'Close browser for manual login' }));
    await waitFor(() => expect(actions.sendRequest).toHaveBeenCalledWith('browser_session_stop', { workflow_id: 'wf', node_id: 'browser-1' }, 60_000));
    await waitFor(() => expect(screen.getByRole('button', { name: 'Start browser' })).toBeInTheDocument());
  });
});

describe('The agent’s cursor', () => {
  /** A live picture of a 1280x720 page, while the employee browses. */
  async function livePicture() {
    const { draw, frame } = mockFrameRenderer();
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} agentName="Maya" />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'state', state: 'agent', controller: null }); socket.onmessage?.({ data: frame.buffer }); });
    await waitFor(() => expect(draw).toHaveBeenCalledOnce());
    return socket;
  }

  it('shows where the employee acted, with their name, on the picture', async () => {
    const socket = await livePicture();
    expect(screen.queryByText('Maya')).toBeNull();
    act(() => socket.message({ type: 'agent_action', action: 'click', x: 640, y: 360 }));
    // The middle of the page is the middle of the picture.
    const cursor = screen.getByText('Maya').parentElement!;
    expect([cursor.style.left, cursor.style.top]).toEqual(['50%', '50%']);
  });

  it('rings the field the employee typed into', async () => {
    const socket = await livePicture();
    act(() => socket.message({ type: 'agent_action', action: 'type', box: [320, 180, 640, 180] }));
    const ring = screen.getByText('Maya').closest('[aria-hidden="true"]')!.querySelector<HTMLElement>('.rounded-sm')!;
    expect([ring.style.left, ring.style.top, ring.style.width, ring.style.height]).toEqual(['25%', '25%', '50%', '25%']);
  });

  it('hides while the owner has control, and goes with the picture', async () => {
    const socket = await livePicture();
    act(() => socket.message({ type: 'agent_action', action: 'hover', x: 100, y: 100 }));
    const layer = screen.getByText('Maya').closest('[aria-hidden="true"]')!;
    expect(layer).not.toHaveClass('invisible');
    act(() => socket.message({ type: 'state', state: 'user', controller: 'you' }));
    expect(layer).toHaveClass('invisible');
    act(() => socket.message({ type: 'sensitive', enabled: true }));
    expect(screen.queryByText('Maya')).toBeNull();
  });

  it('draws nothing before the first picture', () => {
    mockFrameRenderer();
    render(<BrowserWorkspace workflowId="wf" nodes={nodes} agentName="Maya" />);
    const socket = MockSocket.instances[0];
    act(() => { socket.open(); socket.message({ type: 'agent_action', action: 'click', x: 1, y: 1 }); });
    expect(screen.queryByText('Maya')).toBeNull();
  });
});