Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

20.5 KB
Raw Blame
import {
  type Ref,
  useCallback,
  useEffect,
  useImperativeHandle,
  useRef,
  useState,
} from "react";
import { readControl } from "@/lib/computers/control";
import {
  connectScreen,
  type ScreenSocket,
} from "@/lib/computers/screen-connection";
import { keyOf } from "@/lib/hotkeys/hotkeys";
import { socketUrl } from "@/lib/socket-url";
import { currentPageVisible } from "./preview-visibility";
import { pageCoordinates } from "./take-the-wheel";

/**
 * Low-latency screencast used while a human is driving the Bot's browser.
 *
 * The inline card keeps using cheap polling for passive watching. This view uses Chrome's
 * screencast socket so input and visual feedback stay synchronized during takeover.
 *
 * Follows Chrome DevTools' own `InputModel.ts` (BSD-3) for the event translation and
 * `steel-dev/steel-browser`'s casting handler (Apache-2.0) for the frame loop, because no maintained
 * library publishes this and every real implementation is one app-internal file.
 */

/**
 * CDP's modifier bitmask. Alt 1, Control 2, Meta 4, Shift 8.
 *
 * Needed or a capital letter typed with Shift arrives lower-case, and Ctrl+A selects nothing.
 * Command is sent as Control: the Bot's computer is Linux, where Meta+A selects nothing, and a Mac
 * user's hands reach for Command+A, Command+C and Command+Z.
 */
function modifierBits(event: {
  altKey: boolean;
  ctrlKey: boolean;
  metaKey: boolean;
  shiftKey: boolean;
}): number {
  return (
    (event.altKey ? 1 : 0) |
    (event.ctrlKey || event.metaKey ? 2 : 0) |
    (event.shiftKey ? 8 : 0)
  );
}

/** The Command key itself, pressed or released, as the Control key the Linux browser expects. */
function remoteKey(event: KeyboardEvent) {
  return event.key === "Meta"
    ? {
        key: "Control",
        code: event.code === "MetaRight" ? "ControlRight" : "ControlLeft",
        windowsVirtualKeyCode: 17,
      }
    : {
        key: event.key,
        code: event.code,
        windowsVirtualKeyCode: event.keyCode,
      };
}

/**
 * Let the local browser create a paste event, whose clipboard text is forwarded separately.
 *
 * The V is read the way a shortcut is (`keyOf`), so a layout that writes another script still has
 * one: Ctrl and the V key report "м" on Russian and "ω" on Greek.
 */
function isPasteShortcut(event: KeyboardEvent): boolean {
  return (event.ctrlKey || event.metaKey) && keyOf(event) === "v";
}

/** What a page can ask of its live screen: to take the keyboard, as a click on the screen does. */
export type LiveScreenHandle = { focus: () => void };

type Props = {
  /**
   * Computer identity is part of the stream URL so input and frames stay scoped to the active Bot.
   */
  computerId: string;
  /** Whether the user currently holds the wheel. Input is only sent when true. */
  driving: boolean;
  /** Called with a human-readable reason when the stream cannot be established. */
  onProblem?: (problem: string | null) => void;
  /** Called while a dropped stream is being retried on its own, and again when it is back or gone. */
  onReconnecting?: (reconnecting: boolean) => void;
  retryKey?: number;
  /** What a screen reader calls the screen while driving. A Bot's screen unless this says whose it is. */
  label?: string;
  /**
   * Take the keyboard as soon as there is input to take, without a click: for a view opened to drive the screen, where
   * the screen is what the person came to type into (the full-size view of a Bot's screen).
   */
  autoFocus?: boolean;
  ref?: Ref<LiveScreenHandle>;
};

type FrameMessage = {
  data: string;
  width: number;
  height: number;
};

export function LiveScreen({
  computerId,
  driving,
  onProblem,
  onReconnecting,
  retryKey = 0,
  label = "The Bot's screen. You have control. Click and type here.",
  autoFocus = false,
  ref,
}: Props) {
  const canvasRef = useRef<HTMLCanvasElement | null>(null);
  const socketRef = useRef<WebSocket | null>(null);
  /** Keys sent down to the remote browser, by code, whose release goes there too, wherever the focus is by then. */
  const keysDown = useRef(new Map<string, ReturnType<typeof remoteKey>>());
  /**
   * Every key still down in the remote browser, released there: when the keyboard stops being forwarded (control handed
   * back, the screen closed, its stream ended) with a key held, its release would never be sent, and a Shift or a Control
   * left down there changes every key and click after it. Straight to the socket, as the person's own release would
   * have gone; with modifiers none, since every one is being let go.
   */
  const releaseKeys = useCallback(() => {
    const socket = socketRef.current;
    for (const key of keysDown.current.values())
      if (socket?.readyState === WebSocket.OPEN)
        socket.send(
          JSON.stringify({ type: "key", event: "up", ...key, modifiers: 0 }),
        );
    keysDown.current.clear();
  }, []);

  /** The size of the frames Chrome is sending, which is what input coordinates are relative to. */
  const frameSize = useRef<{ width: number; height: number } | null>(null);
  /** Latest validated encoded frame. Hidden tabs keep only this, never decoded bitmaps. */
  const latestFrame = useRef<FrameMessage | null>(null);
  /** Monotonic guard so a slow older decode cannot replace a newer frame. */
  const latestFrameId = useRef(0);
  const [connected, setConnected] = useState(false);
  const inputReady = useRef(false);
  const previouslyDriving = useRef(driving);

  // biome-ignore lint/correctness/useExhaustiveDependencies: explicit Retry restarts this connection.
  useEffect(() => {
    let closed = false;
    setConnected(false);
    inputReady.current = false;
    frameSize.current = null;
    latestFrame.current = null;
    latestFrameId.current++;

    const drawFrame = async (frame: FrameMessage, frameId: number) => {
      /**
       * Decoded off the main thread and drawn as a bitmap.
       *
       * `createImageBitmap` rather than assigning a data URI to an `<img>`: the image path decodes
       * synchronously on the main thread for every frame, which at screencast rates is the difference
       * between a smooth page and one that stutters while you are trying to click something on it.
       */
      try {
        const binary = Uint8Array.from(atob(frame.data), (c) =>
          c.charCodeAt(0),
        );
        const bitmap = await createImageBitmap(
          new Blob([binary], { type: "image/jpeg" }),
        );
        if (
          closed ||
          frameId !== latestFrameId.current ||
          !currentPageVisible()
        ) {
          bitmap.close();
          return;
        }
        const canvas = canvasRef.current;
        if (!canvas) {
          bitmap.close();
          return;
        }
        canvas.width = bitmap.width;
        canvas.height = bitmap.height;
        canvas.getContext("2d")?.drawImage(bitmap, 0, 0);
        bitmap.close();
      } catch {
        // Ignore a single corrupt frame; the next frame replaces it.
      }
    };

    const drawLatestFrame = () => {
      if (!currentPageVisible()) return;
      const frame = latestFrame.current;
      if (!frame) return;
      void drawFrame(frame, latestFrameId.current);
    };

    const onMessage = (event: { data: unknown }) => {
      let parsed: unknown;
      try {
        parsed = JSON.parse(String(event.data));
      } catch {
        return;
      }
      // A frame that is not an object (`null`, a number, a string, an array) has
      // no `type` to read: reaching for it throws a `TypeError` inside this
      // handler and stops the live view from drawing further frames. Drop it.
      if (
        typeof parsed !== "object" ||
        parsed === null ||
        Array.isArray(parsed)
      ) {
        return;
      }
      const message = parsed as {
        type: string;
        data?: string;
        width?: number;
        height?: number;
        error?: string;
      };
      if (typeof message.type !== "string") return;
      if (message.type === "error") {
        onProblem?.(message.error ?? "Couldn't show the screen.");
        return;
      }
      if (message.type !== "frame" || !message.data) return;
      // Live-run JSON only checks `typeof type === "string"` upstream. Non-finite or negative
      // dimensions would poison frameSize and every coordinate scaled from it; a huge payload
      // would hit `atob` before any bound. Both are dropped as corrupt frames.
      const width = message.width ?? 1280;
      const height = message.height ?? 800;
      if (
        typeof width !== "number" ||
        typeof height !== "number" ||
        !Number.isFinite(width) ||
        !Number.isFinite(height) ||
        width <= 0 ||
        height <= 0 ||
        width > 8192 ||
        height > 8192
      ) {
        return;
      }
      if (
        typeof message.data !== "string" ||
        message.data.length > 20_000_000
      ) {
        return;
      }

      const canvas = canvasRef.current;
      if (!canvas || closed) return;

      frameSize.current = {
        width: message.width ?? 1280,
        height: message.height ?? 800,
      };
      const frame = { data: message.data, width, height };
      latestFrame.current = frame;
      const frameId = ++latestFrameId.current;

      if (!currentPageVisible()) return;
      void drawFrame(frame, frameId);
    };

    const connection = connectScreen({
      open: () => {
        const socket = new WebSocket(
          socketUrl(`/api/computers/${encodeURIComponent(computerId)}/stream`),
        );
        socketRef.current = socket;
        // Event handlers use only data, so adapt the native event surface without changing the protocol.
        return {
          set onopen(callback: ScreenSocket["onopen"]) {
            socket.onopen = callback
              ? () => {
                  void callback();
                }
              : null;
          },
          set onclose(callback: ScreenSocket["onclose"]) {
            socket.onclose = callback;
          },
          set onerror(callback: ScreenSocket["onerror"]) {
            socket.onerror = callback;
          },
          set onmessage(callback: ScreenSocket["onmessage"]) {
            socket.onmessage = callback;
          },
          close: () => socket.close(),
        };
      },
      verifyOwnership: async () => {
        const state = await readControl(computerId);
        return state?.holder === "human" && !state.transitioning;
      },
      onReady: (ready) => {
        inputReady.current = ready;
        setConnected(ready);
      },
      onProblem: (problem) => onProblem?.(problem),
      onReconnecting: (reconnecting) => onReconnecting?.(reconnecting),
      onDisconnect: () => {
        frameSize.current = null;
        latestFrame.current = null;
        latestFrameId.current++;
        const canvas = canvasRef.current;
        if (canvas)
          canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height);
      },
      onMessage,
    });
    document.addEventListener("visibilitychange", drawLatestFrame);
    return () => {
      closed = true;
      // Before the socket goes: a key still down over there would stay held in the remote browser.
      releaseKeys();
      connection.stop();
      document.removeEventListener("visibilitychange", drawLatestFrame);
      socketRef.current = null;
    };
  }, [computerId, onProblem, onReconnecting, retryKey]);

  useEffect(() => {
    const wasDriving = previouslyDriving.current;
    previouslyDriving.current = driving;
    if (
      wasDriving ||
      !driving ||
      inputReady.current ||
      socketRef.current?.readyState !== WebSocket.OPEN
    )
      return;
    let live = true;
    void readControl(computerId)
      .then((state) => {
        if (!live || socketRef.current?.readyState !== WebSocket.OPEN) return;
        inputReady.current = state?.holder === "human" && !state.transitioning;
        setConnected(inputReady.current);
      })
      .catch(() => {
        inputReady.current = false;
      });
    return () => {
      live = false;
    };
  }, [computerId, driving]);

  const send = useCallback(
    (message: Record<string, unknown>) => {
      const socket = socketRef.current;
      if (
        !driving ||
        !inputReady.current ||
        socket?.readyState !== WebSocket.OPEN
      )
        return;
      socket.send(JSON.stringify(message));
    },
    [driving],
  );

  /**
   * Convert from displayed canvas coordinates to page coordinates with the shared, tested helper.
   * A screencast frame is the viewport, so its frame size stands in for natural image size.
   */
  const at = useCallback((event: { clientX: number; clientY: number }) => {
    const canvas = canvasRef.current;
    const size = frameSize.current;
    if (!canvas || !size) return null;
    return pageCoordinates(
      { naturalWidth: size.width, naturalHeight: size.height },
      canvas.getBoundingClientRect(),
      event,
    );
  }, []);

  const onMouse = useCallback(
    (kind: "pressed" | "released" | "moved") =>
      (event: React.MouseEvent<HTMLCanvasElement>) => {
        // A press on the screen gives it the keyboard, as a press on any focusable element does, so typing into the
        // remote page after clicking a field on it needs no second click.
        if (kind === "pressed")
          event.currentTarget.focus({ preventScroll: true });
        const point = at(event);
        if (!point) return;
        send({
          type: "mouse",
          event: kind,
          ...point,
          button:
            event.button === 2
              ? "right"
              : event.button === 1
                ? "middle"
                : "left",
          /*
           * The browser's own count, not a fixed one.
           *
           * `MouseEvent.detail` is how many times in a row this button has been pressed in the same
           * place, worked out by the browser to its own timing and distance rules. Chrome fires
           * `dblclick` on the far page only when the second press says it is the second, so sending 1
           * every time meant a double click arrived as two separate clicks: no `dblclick` ever
           * reached the page, `event.detail` was always 1, and opening a row, expanding a node and
           * selecting a word were all things a person holding the wheel could not do.
           *
           * At least one on a press, because the computer refuses a press of zero for the reason its
           * own comment gives -- Chrome would see a move that happens to have a button set, and no
           * click at all. `detail` is zero on an event a script dispatched rather than a person.
           */
          clickCount: kind === "moved" ? 0 : Math.max(1, event.detail),
          modifiers: modifierBits(event),
        });
      },
    [at, send],
  );

  useImperativeHandle(
    ref,
    () => ({ focus: () => canvasRef.current?.focus({ preventScroll: true }) }),
    [],
  );

  /*
   * A view opened to drive the screen puts the keyboard on it, as a dialog puts focus inside itself: taking control
   * and typing needs no click on the page first, as it needed none before the screen took keys only with focus.
   */
  useEffect(() => {
    if (autoFocus && driving && connected)
      canvasRef.current?.focus({ preventScroll: true });
  }, [autoFocus, driving, connected]);

  /**
   * Keystrokes, forwarded while driving, and only those the screen itself has: it takes the keyboard the way any focused
   * element does, by a click on it (or Tab to it), and gives it back on Escape or a click anywhere else.
   *
   * It used to take every keystroke on the window that was not aimed at a text field, because the canvas could not hold
   * focus. So nothing else on the page could have a key: the Browser page's tabs never heard their arrows, Home, End or
   * Delete, no button took Enter or Space, and Tab never left anything. And every one of those keys went to the remote
   * browser instead, where a stray arrow is not harmless. Now the canvas holds focus, as noVNC's does (its keyboard
   * listens on the canvas it focuses on a click), and a key aimed anywhere else is that element's.
   *
   * Still listened for on the window, in the capture phase, and stopped as well as prevented, because a keystroke sent to
   * the Bot's browser is not also this page's: the app's own shortcuts listen on this window too, and they were bound
   * first, when the signed-in app mounted, so they would see it first (a capital N typed into the remote page started a
   * new chat, and Ctrl+B there toggled the sidebar here). The paste shortcut is not stopped, because the paste event it
   * makes is how the clipboard's text arrives. Nor is Escape, which closes a full-size view; on its way it also takes the
   * keyboard off the screen, which is how a person who reached it with Tab leaves it again (WCAG's "no keyboard trap").
   */
  useEffect(() => {
    const canvas = canvasRef.current;
    if (!driving || !connected || !canvas) return;
    const held = keysDown.current;
    const onKeyDown = (event: KeyboardEvent) => {
      if (event.target !== canvas) return;
      if (event.key === "Escape") {
        canvas.blur();
        return;
      }
      if (isPasteShortcut(event)) return;
      event.preventDefault();
      event.stopPropagation();
      held.set(event.code, remoteKey(event));
      send({
        type: "key",
        event: "down",
        ...remoteKey(event),
        // Only a printable character carries text. Sending text for Backspace makes Chrome insert a
        // character instead of deleting one.
        ...(event.key.length === 1 ? { text: event.key } : {}),
        modifiers: modifierBits(event),
      });
    };
    /**
     * The release of a key that went down to the remote browser, wherever the focus is now: a key released after a
     * click took the focus elsewhere is still held over there otherwise. Only those: the Tab that brought the focus to
     * the screen, say, was this page's, and its release is too.
     */
    const onKeyUp = (event: KeyboardEvent) => {
      if (!held.delete(event.code)) return;
      event.preventDefault();
      send({
        type: "key",
        event: "up",
        ...remoteKey(event),
        modifiers: modifierBits(event),
      });
    };
    /** Paste arrives as one block; CDP inserts it as text rather than key events. */
    const onPaste = (event: ClipboardEvent) => {
      if (event.target !== canvas) return;
      const text = event.clipboardData?.getData("text");
      if (!text) return;
      event.preventDefault();
      send({ type: "text", text });
    };

    window.addEventListener("keydown", onKeyDown, true);
    window.addEventListener("keyup", onKeyUp);
    window.addEventListener("paste", onPaste);
    return () => {
      window.removeEventListener("keydown", onKeyDown, true);
      window.removeEventListener("keyup", onKeyUp);
      window.removeEventListener("paste", onPaste);
      releaseKeys();
    };
  }, [driving, connected, send, releaseKeys]);

  /**
   * The wheel, forwarded while driving, from a listener that is allowed to stop it here.
   *
   * Not React's `onWheel`: React attaches that to its root as a passive listener, so the
   * `preventDefault` in it was ignored ("Unable to preventDefault inside passive event listener
   * invocation."). The wheel reached the Bot's page and also scrolled whatever on this page was
   * under it, the frame that holds this screen included, and Ctrl and the wheel zoomed this page.
   */
  useEffect(() => {
    const canvas = canvasRef.current;
    if (!driving || !connected || !canvas) return;
    const onWheel = (event: WheelEvent) => {
      const point = at(event);
      if (!point) return;
      event.preventDefault();
      send({
        type: "wheel",
        ...point,
        deltaX: event.deltaX,
        deltaY: event.deltaY,
        modifiers: modifierBits(event),
      });
    };
    canvas.addEventListener("wheel", onWheel, { passive: false });
    return () => canvas.removeEventListener("wheel", onWheel);
  }, [driving, connected, at, send]);

  return (
    <canvas
      ref={canvasRef}
      className={`block h-auto w-full ${driving ? "cursor-crosshair focus-visible:outline-2 focus-visible:-outline-offset-2 focus-visible:outline-ring" : ""}`}
      // Only forward input during takeover. Then the screen is a stop in the Tab order and takes every key while it has
      // the focus, which is what a screen reader is told by `application`.
      {...(driving
        ? {
            onMouseDown: onMouse("pressed"),
            onMouseUp: onMouse("released"),
            onMouseMove: onMouse("moved"),
            onContextMenu: (event: React.MouseEvent) => event.preventDefault(),
            role: "application",
            tabIndex: 0,
          }
        : {})}
      aria-label={driving ? label : "The Bot's screen, live"}
      data-connected={connected}
    />
  );
}