Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

6.3 KB
Raw Blame
import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";
import { GlobalRegistrator } from "@happy-dom/global-registrator";
import { cleanup, fireEvent, render, waitFor } from "@testing-library/react";
import { LiveScreen } from "@/components/computer/live-screen";

/**
 * What a person's click turns into while they hold the wheel.
 *
 * The canvas sends nothing until it knows how large the frames are, because a click has to be mapped
 * from where it landed on screen to where it landed on the page. So each test feeds one frame first.
 * The bitmap decode inside that handler is allowed to fail here — there is no `createImageBitmap`
 * under happy-dom, and the component already treats a frame it cannot draw as one to skip — but the
 * size is recorded before the decode is attempted, which is the part these need.
 */

class SocketDouble {
  static readonly OPEN = 1;
  static readonly CLOSED = 3;
  static latest: SocketDouble | undefined;

  readyState = SocketDouble.OPEN;
  onopen: (() => void) | null = null;
  onmessage: ((event: { data: string }) => void) | null = null;
  onerror: (() => void) | null = null;
  onclose: (() => void) | null = null;
  readonly sent: Record<string, unknown>[] = [];

  constructor(_url: string) {
    SocketDouble.latest = this;
    queueMicrotask(() => this.onopen?.());
  }

  send(payload: string) {
    this.sent.push(JSON.parse(payload) as Record<string, unknown>);
  }

  close() {
    this.readyState = SocketDouble.CLOSED;
    this.onclose?.();
  }
}

let originalWebSocket: typeof WebSocket;

beforeAll(() => {
  GlobalRegistrator.register();
  originalWebSocket = globalThis.WebSocket;
  globalThis.WebSocket = SocketDouble as unknown as typeof WebSocket;
});

afterEach(() => {
  cleanup();
  SocketDouble.latest = undefined;
});

afterAll(() => {
  globalThis.WebSocket = originalWebSocket;
  GlobalRegistrator.unregister();
});

/** The frames Chrome is casting, and the canvas they are drawn on, at the same size. */
const FRAME = { width: 800, height: 600 };

async function liveCanvas(): Promise<{
  canvas: HTMLCanvasElement;
  socket: SocketDouble;
}> {
  const { container } = render(<LiveScreen computerId="mouse-test" driving />);
  await waitFor(() => expect(SocketDouble.latest).toBeDefined());
  const socket = SocketDouble.latest as SocketDouble;

  socket.onmessage?.({
    data: JSON.stringify({ type: "frame", data: "AAAA", ...FRAME }),
  });
  await waitFor(() => expect(socket.sent).toEqual([]));

  const canvas = container.querySelector("canvas") as HTMLCanvasElement;
  canvas.getBoundingClientRect = () =>
    ({
      left: 0,
      top: 0,
      width: FRAME.width,
      height: FRAME.height,
    }) as DOMRect;
  return { canvas, socket };
}

/** One press and release where the person's browser says this is the nth click there. */
function clickAt(canvas: HTMLCanvasElement, detail: number) {
  const at = { clientX: 40, clientY: 30, button: 0, detail };
  fireEvent.mouseDown(canvas, at);
  fireEvent.mouseUp(canvas, at);
}

test("a double click is sent as a double click", async () => {
  // Chrome fires `dblclick` on the page only when the second press says it is the second, and
  // nothing else does: two presses that both claim to be the first are two separate clicks, so
  // opening a row, expanding a node and selecting a word were all impossible while driving.
  const { canvas, socket } = await liveCanvas();

  clickAt(canvas, 1);
  clickAt(canvas, 2);

  expect(socket.sent.map((message) => message.clickCount)).toEqual([
    1, 1, 2, 2,
  ]);
});

test("an ordinary click is still one click", async () => {
  const { canvas, socket } = await liveCanvas();

  clickAt(canvas, 1);

  expect(socket.sent).toEqual([
    {
      type: "mouse",
      event: "pressed",
      x: 40,
      y: 30,
      button: "left",
      clickCount: 1,
      modifiers: 0,
    },
    {
      type: "mouse",
      event: "released",
      x: 40,
      y: 30,
      button: "left",
      clickCount: 1,
      modifiers: 0,
    },
  ]);
});

test("a press that claims no click at all is still a click", async () => {
  // `detail` is 0 on an event a script dispatched rather than a person, and the computer refuses a
  // press of zero: Chrome then sees a move that happens to have a button set and no click fires.
  const { canvas, socket } = await liveCanvas();

  clickAt(canvas, 0);

  expect(socket.sent.map((message) => message.clickCount)).toEqual([1, 1]);
});

test("moving the mouse is not a click", async () => {
  const { canvas, socket } = await liveCanvas();

  fireEvent.mouseMove(canvas, {
    clientX: 40,
    clientY: 30,
    button: 0,
    detail: 0,
  });

  expect(socket.sent).toEqual([
    {
      type: "mouse",
      event: "moved",
      x: 40,
      y: 30,
      button: "left",
      clickCount: 0,
      modifiers: 0,
    },
  ]);
});

test("a turn of the wheel is stopped here as well as sent there", async () => {
  // In a browser React attaches `onWheel` to its root as a passive listener, so a `preventDefault`
  // there is ignored and the wheel scrolled this page as well as the Bot's. Under happy-dom React
  // does not detect passive support and binds it actively, which hides exactly that, so this pins
  // the listener the canvas holds itself, and that it is not passive.
  const wheelListeners: unknown[] = [];
  const addEventListener = HTMLCanvasElement.prototype.addEventListener;
  HTMLCanvasElement.prototype.addEventListener = function (
    this: HTMLCanvasElement,
    ...args: Parameters<typeof addEventListener>
  ) {
    if (args[0] === "wheel") wheelListeners.push(args[2]);
    return addEventListener.apply(this, args);
  };
  try {
    const { canvas, socket } = await liveCanvas();
    expect(wheelListeners).toEqual([{ passive: false }]);

    // happy-dom's WheelEvent carries no coordinates, so a mouse event of that type stands in.
    const wheel = new MouseEvent("wheel", {
      clientX: 40,
      clientY: 30,
      bubbles: true,
      cancelable: true,
    });
    Object.defineProperties(wheel, {
      deltaX: { value: 0 },
      deltaY: { value: 120 },
    });
    canvas.dispatchEvent(wheel);

    expect(wheel.defaultPrevented).toBe(true);
    expect(socket.sent).toEqual([
      { type: "wheel", x: 40, y: 30, deltaX: 0, deltaY: 120, modifiers: 0 },
    ]);
  } finally {
    HTMLCanvasElement.prototype.addEventListener = addEventListener;
  }
});