Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

10.2 KB
Raw Blame
/**
 * Live screen and live input over one WebSocket, using Chrome's own screencast.
 *
 * Chrome pushes frames as the page changes, which supports human takeover better than polling a PNG
 * once a second.
 *
 * This follows permissively-licensed references: `steel-dev/steel-browser`'s `casting.handler.ts`
 * (Apache-2.0) for the server loop and Chrome DevTools' `InputModel.ts` (BSD-3) for key event
 * translation.
 *
 * noVNC is not used because it requires Xvfb, x11vnc and websockify, while this container runs
 * headless. This implementation streams the page and forwards input through the Chrome DevTools
 * Protocol.
 */
import type { CDPSession, Page } from "playwright";

/** What the surface sends us. */
export type InputMessage =
  | {
      type: "mouse";
      event: "pressed" | "released" | "moved";
      x: number;
      y: number;
      button?: "left" | "right" | "middle";
      clickCount?: number;
      modifiers?: number;
    }
  | {
      type: "wheel";
      x: number;
      y: number;
      deltaX: number;
      deltaY: number;
      modifiers?: number;
    }
  | {
      type: "key";
      event: "down" | "up";
      key: string;
      code: string;
      text?: string;
      /** Legacy browser keyCode, which CDP uses as its Windows virtual key code. */
      windowsVirtualKeyCode?: number;
      modifiers?: number;
    }
  | { type: "text"; text: string };

/** What we send back. */
export type FrameMessage = {
  type: "frame";
  /** Base64 JPEG. Chrome's own encoding; we do not re-encode. */
  data: string;
  width: number;
  height: number;
};

/**
 * Chrome's virtual key codes, for the keys that need one.
 *
 * `Input.dispatchKeyEvent` is not satisfied by `key` alone. A form field will ignore a bare Backspace
 * or Enter unless `windowsVirtualKeyCode` is set, which is the common reason a hand-written
 * screencast works for letters but not editing keys. Lifted from the mapping DevTools uses for the
 * same purpose.
 */
const VIRTUAL_KEY_CODES: Record<string, number> = {
  Backspace: 8,
  Tab: 9,
  Enter: 13,
  Shift: 16,
  Control: 17,
  Alt: 18,
  Escape: 27,
  " ": 32,
  PageUp: 33,
  PageDown: 34,
  End: 35,
  Home: 36,
  ArrowLeft: 37,
  ArrowUp: 38,
  ArrowRight: 39,
  ArrowDown: 40,
  Delete: 46,
  ";": 186,
  "=": 187,
  ",": 188,
  "-": 189,
  ".": 190,
  "/": 191,
  "`": 192,
  "[": 219,
  "\\": 220,
  "]": 221,
  "'": 222,
};

/**
 * The character a key stands for, where Chrome will not act on the key without one.
 *
 * `Input.dispatchKeyEvent` has two kinds of key-down. `rawKeyDown` is a key that produces no
 * character: Chrome delivers it to the page and stops there, so a listener fires and nothing else
 * happens. `keyDown` carries text, and that is what makes Chrome perform the key's own default
 * action.
 *
 * Sending text only for a printable character therefore left Enter as a key a page could hear and
 * not act on. Measured against Chromium 151 through this file: with a person holding the wheel,
 * Enter did not submit the form they had just filled in, did not start a new line in a textarea and
 * did not press the button they had tabbed to, while the page's own `keydown` listener saw every
 * one of them. Which is the sign-in at the end of almost every takeover.
 *
 * One key rather than a list, measured the same way: Backspace, Delete, Tab, Home, End and the four
 * arrows all do what they mean as a `rawKeyDown`, because their default action is not the insertion
 * of a character.
 *
 * A carriage return, which is what a keyboard's Enter carries. Chrome turns it into whatever the
 * field it lands in needs, so a single-line input is left holding no extra character.
 */
const TEXT_FOR_KEY: Record<string, string | undefined> = { Enter: "\r" };

function virtualKeyCode(key: string): number {
  if (VIRTUAL_KEY_CODES[key] !== undefined) return VIRTUAL_KEY_CODES[key];
  // A single printable character carries its own code point, upper-cased, which is what Chrome expects
  // for a key event as opposed to the text it produces.
  if (key.length === 1) return key.toUpperCase().charCodeAt(0);
  return 0;
}

export type Screencast = {
  /** Stop the cast and detach. Safe to call twice. */
  stop: () => Promise<void>;
  /** Apply one thing the person did. */
  send: (message: InputMessage) => Promise<void>;
};

/**
 * Start casting `page` to `onFrame`, and return a handle that accepts input.
 *
 * `maxWidth`/`maxHeight` cap what Chrome encodes; it scales to fit and tells us the real dimensions in
 * the metadata, which the surface needs in order to map a click back. Capping matters because the cost
 * of a frame is mostly encoding, and oversized casts waste bandwidth.
 */
export async function startScreencast(
  page: Page,
  onFrame: (frame: FrameMessage) => void,
  options: { maxWidth?: number; maxHeight?: number; quality?: number } = {},
): Promise<Screencast> {
  const client: CDPSession = await page.context().newCDPSession(page);
  let stopped = false;

  type ScreencastFrame = {
    data: string;
    sessionId: number;
    metadata: { deviceWidth: number; deviceHeight: number };
  };

  client.on("Page.screencastFrame", (event: ScreencastFrame) => {
    const { data, sessionId, metadata } = event;
    // Acknowledge every frame. Chrome will not send the next one until the current is acked, which is
    // the backpressure that stops a slow client drowning in frames. Forgetting this is why a naive
    // implementation delivers one frame and then appears to hang.
    void client
      .send("Page.screencastFrameAck", { sessionId })
      .catch(() => undefined);
    if (stopped) return;
    onFrame({
      type: "frame",
      data,
      width: metadata.deviceWidth,
      height: metadata.deviceHeight,
    });
  });

  await client.send("Page.startScreencast", {
    format: "jpeg",
    quality: options.quality ?? 70,
    maxWidth: options.maxWidth ?? 1280,
    maxHeight: options.maxHeight ?? 800,
    // One frame per change, not per interval. Chrome decides when something moved.
    everyNthFrame: 1,
  });

  return {
    async stop() {
      if (stopped) return;
      stopped = true;
      await client.send("Page.stopScreencast").catch(() => undefined);
      await client.detach().catch(() => undefined);
    },

    async send(message: InputMessage) {
      if (stopped) return;
      if (message.type === "mouse") {
        await client.send("Input.dispatchMouseEvent", {
          type:
            message.event === "pressed"
              ? "mousePressed"
              : message.event === "released"
                ? "mouseReleased"
                : "mouseMoved",
          x: message.x,
          y: message.y,
          button: message.button ?? "left",
          // Chrome needs a non-zero clickCount on press/release or the page sees a move that happens
          // to have a button set, and no click ever fires.
          clickCount: message.event === "moved" ? 0 : (message.clickCount ?? 1),
          modifiers: message.modifiers ?? 0,
        });
        return;
      }

      if (message.type === "wheel") {
        await client.send("Input.dispatchMouseEvent", {
          type: "mouseWheel",
          x: message.x,
          y: message.y,
          deltaX: message.deltaX,
          deltaY: message.deltaY,
          modifiers: message.modifiers ?? 0,
        });
        return;
      }

      if (message.type === "key") {
        const offeredCode = message.windowsVirtualKeyCode;
        const code =
          Number.isInteger(offeredCode) &&
          (offeredCode ?? 0) > 0 &&
          (offeredCode ?? 0) <= 255
            ? (offeredCode as number)
            : virtualKeyCode(message.key);
        /*
         * The text this key carries: the surface supplies it for a printable character, and
         * `TEXT_FOR_KEY` fills in the keys whose default action Chrome will not perform without one.
         *
         * Only on the way down, because that is where a default action happens. A key going up
         * carries whatever it was given and nothing more.
         */
        const text =
          message.event === "up"
            ? message.text
            : (message.text ?? TEXT_FOR_KEY[message.key]);
        /*
         * NO `nativeVirtualKeyCode`, so the key reaches the page and never the browser itself.
         *
         * Chrome hands a key the page did not handle on to the browser's own shortcuts only when the
         * event names a native key code (content/browser/devtools/protocol/input_handler.cc: without
         * one the event is `skip_if_unhandled`, which also skips the shortcuts Chrome takes before the
         * page, such as closing the tab). And on macOS that code is read as the Mac's own key number,
         * not the Windows one this used to send: 37, the left arrow's, is the Mac's L, and 36, Home's,
         * is its Return (native_input_event_builder_mac.mm builds the browser's event from it).
         * Measured on Chrome for Testing 151 headless on macOS, a person's ArrowLeft reached the page
         * as an unnamed key 76, and with Home after it the browser stopped answering DevTools at all;
         * in the end-to-end run it opened chrome://settings/help and then spun a core until its
         * computer was restarted. Without the native code every key arrived as itself and moved the
         * caret, deleted and tabbed as it should. Playwright's and Puppeteer's own keyboards never send
         * one either.
         */
        await client.send("Input.dispatchKeyEvent", {
          // `keyDown` only when there is text to insert; otherwise `rawKeyDown`, which is what Chrome
          // expects for keys that do not produce a character. Sending keyDown with no text makes
          // editing keys arrive as nothing.
          type:
            message.event === "up" ? "keyUp" : text ? "keyDown" : "rawKeyDown",
          key: message.key,
          code: message.code,
          ...(text ? { text } : {}),
          windowsVirtualKeyCode: code,
          modifiers: message.modifiers ?? 0,
        });
        return;
      }

      // A block of text at once: a paste, or a one-time code the person did not type character by
      // character. `Input.insertText` bypasses key events entirely, which is correct here, it is not
      // pretending to be a keyboard.
      await client.send("Input.insertText", { text: message.text });
    },
  };
}