Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import {
type Ref,
useCallback,
useEffect,
useImperativeHandle,
useRef,
useState,
} from "react";
import { readControl } from "@/lib/computers/control";
import {
connectScreen,
type ScreenSocket,
} from "@/lib/computers/screen-connection";
import { keyOf } from "@/lib/hotkeys/hotkeys";
import { socketUrl } from "@/lib/socket-url";
import { currentPageVisible } from "./preview-visibility";
import { pageCoordinates } from "./take-the-wheel";
/**
* Low-latency screencast used while a human is driving the Bot's browser.
*
* The inline card keeps using cheap polling for passive watching. This view uses Chrome's
* screencast socket so input and visual feedback stay synchronized during takeover.
*
* Follows Chrome DevTools' own `InputModel.ts` (BSD-3) for the event translation and
* `steel-dev/steel-browser`'s casting handler (Apache-2.0) for the frame loop, because no maintained
* library publishes this and every real implementation is one app-internal file.
*/
/**
* CDP's modifier bitmask. Alt 1, Control 2, Meta 4, Shift 8.
*
* Needed or a capital letter typed with Shift arrives lower-case, and Ctrl+A selects nothing.
* Command is sent as Control: the Bot's computer is Linux, where Meta+A selects nothing, and a Mac
* user's hands reach for Command+A, Command+C and Command+Z.
*/
function modifierBits(event: {
altKey: boolean;
ctrlKey: boolean;
metaKey: boolean;
shiftKey: boolean;
}): number {
return (
(event.altKey ? 1 : 0) |
(event.ctrlKey || event.metaKey ? 2 : 0) |
(event.shiftKey ? 8 : 0)
);
}
/** The Command key itself, pressed or released, as the Control key the Linux browser expects. */
function remoteKey(event: KeyboardEvent) {
return event.key === "Meta"
? {
key: "Control",
code: event.code === "MetaRight" ? "ControlRight" : "ControlLeft",
windowsVirtualKeyCode: 17,
}
: {
key: event.key,
code: event.code,
windowsVirtualKeyCode: event.keyCode,
};
}
/**
* Let the local browser create a paste event, whose clipboard text is forwarded separately.
*
* The V is read the way a shortcut is (`keyOf`), so a layout that writes another script still has
* one: Ctrl and the V key report "м" on Russian and "ω" on Greek.
*/
function isPasteShortcut(event: KeyboardEvent): boolean {
return (event.ctrlKey || event.metaKey) && keyOf(event) === "v";
}
/** What a page can ask of its live screen: to take the keyboard, as a click on the screen does. */
export type LiveScreenHandle = { focus: () => void };
type Props = {
/**
* Computer identity is part of the stream URL so input and frames stay scoped to the active Bot.
*/
computerId: string;
/** Whether the user currently holds the wheel. Input is only sent when true. */
driving: boolean;
/** Called with a human-readable reason when the stream cannot be established. */
onProblem?: (problem: string | null) => void;
/** Called while a dropped stream is being retried on its own, and again when it is back or gone. */
onReconnecting?: (reconnecting: boolean) => void;
retryKey?: number;
/** What a screen reader calls the screen while driving. A Bot's screen unless this says whose it is. */
label?: string;
/**
* Take the keyboard as soon as there is input to take, without a click: for a view opened to drive the screen, where
* the screen is what the person came to type into (the full-size view of a Bot's screen).
*/
autoFocus?: boolean;
ref?: Ref<LiveScreenHandle>;
};
type FrameMessage = {
data: string;
width: number;
height: number;
};
export function LiveScreen({
computerId,
driving,
onProblem,
onReconnecting,
retryKey = 0,
label = "The Bot's screen. You have control. Click and type here.",
autoFocus = false,
ref,
}: Props) {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const socketRef = useRef<WebSocket | null>(null);
/** Keys sent down to the remote browser, by code, whose release goes there too, wherever the focus is by then. */
const keysDown = useRef(new Map<string, ReturnType<typeof remoteKey>>());
/**
* Every key still down in the remote browser, released there: when the keyboard stops being forwarded (control handed
* back, the screen closed, its stream ended) with a key held, its release would never be sent, and a Shift or a Control
* left down there changes every key and click after it. Straight to the socket, as the person's own release would
* have gone; with modifiers none, since every one is being let go.
*/
const releaseKeys = useCallback(() => {
const socket = socketRef.current;
for (const key of keysDown.current.values())
if (socket?.readyState === WebSocket.OPEN)
socket.send(
JSON.stringify({ type: "key", event: "up", ...key, modifiers: 0 }),
);
keysDown.current.clear();
}, []);
/** The size of the frames Chrome is sending, which is what input coordinates are relative to. */
const frameSize = useRef<{ width: number; height: number } | null>(null);
/** Latest validated encoded frame. Hidden tabs keep only this, never decoded bitmaps. */
const latestFrame = useRef<FrameMessage | null>(null);
/** Monotonic guard so a slow older decode cannot replace a newer frame. */
const latestFrameId = useRef(0);
const [connected, setConnected] = useState(false);
const inputReady = useRef(false);
const previouslyDriving = useRef(driving);
// biome-ignore lint/correctness/useExhaustiveDependencies: explicit Retry restarts this connection.
useEffect(() => {
let closed = false;
setConnected(false);
inputReady.current = false;
frameSize.current = null;
latestFrame.current = null;
latestFrameId.current++;
const drawFrame = async (frame: FrameMessage, frameId: number) => {
/**
* Decoded off the main thread and drawn as a bitmap.
*
* `createImageBitmap` rather than assigning a data URI to an `<img>`: the image path decodes
* synchronously on the main thread for every frame, which at screencast rates is the difference
* between a smooth page and one that stutters while you are trying to click something on it.
*/
try {
const binary = Uint8Array.from(atob(frame.data), (c) =>
c.charCodeAt(0),
);
const bitmap = await createImageBitmap(
new Blob([binary], { type: "image/jpeg" }),
);
if (
closed ||
frameId !== latestFrameId.current ||
!currentPageVisible()
) {
bitmap.close();
return;
}
const canvas = canvasRef.current;
if (!canvas) {
bitmap.close();
return;
}
canvas.width = bitmap.width;
canvas.height = bitmap.height;
canvas.getContext("2d")?.drawImage(bitmap, 0, 0);
bitmap.close();
} catch {
// Ignore a single corrupt frame; the next frame replaces it.
}
};
const drawLatestFrame = () => {
if (!currentPageVisible()) return;
const frame = latestFrame.current;
if (!frame) return;
void drawFrame(frame, latestFrameId.current);
};
const onMessage = (event: { data: unknown }) => {
let parsed: unknown;
try {
parsed = JSON.parse(String(event.data));
} catch {
return;
}
// A frame that is not an object (`null`, a number, a string, an array) has
// no `type` to read: reaching for it throws a `TypeError` inside this
// handler and stops the live view from drawing further frames. Drop it.
if (
typeof parsed !== "object" ||
parsed === null ||
Array.isArray(parsed)
) {
return;
}
const message = parsed as {
type: string;
data?: string;
width?: number;
height?: number;
error?: string;
};
if (typeof message.type !== "string") return;
if (message.type === "error") {
onProblem?.(message.error ?? "Couldn't show the screen.");
return;
}
if (message.type !== "frame" || !message.data) return;
// Live-run JSON only checks `typeof type === "string"` upstream. Non-finite or negative
// dimensions would poison frameSize and every coordinate scaled from it; a huge payload
// would hit `atob` before any bound. Both are dropped as corrupt frames.
const width = message.width ?? 1280;
const height = message.height ?? 800;
if (
typeof width !== "number" ||
typeof height !== "number" ||
!Number.isFinite(width) ||
!Number.isFinite(height) ||
width <= 0 ||
height <= 0 ||
width > 8192 ||
height > 8192
) {
return;
}
if (
typeof message.data !== "string" ||
message.data.length > 20_000_000
) {
return;
}
const canvas = canvasRef.current;
if (!canvas || closed) return;
frameSize.current = {
width: message.width ?? 1280,
height: message.height ?? 800,
};
const frame = { data: message.data, width, height };
latestFrame.current = frame;
const frameId = ++latestFrameId.current;
if (!currentPageVisible()) return;
void drawFrame(frame, frameId);
};
const connection = connectScreen({
open: () => {
const socket = new WebSocket(
socketUrl(`/api/computers/${encodeURIComponent(computerId)}/stream`),
);
socketRef.current = socket;
// Event handlers use only data, so adapt the native event surface without changing the protocol.
return {
set onopen(callback: ScreenSocket["onopen"]) {
socket.onopen = callback
? () => {
void callback();
}
: null;
},
set onclose(callback: ScreenSocket["onclose"]) {
socket.onclose = callback;
},
set onerror(callback: ScreenSocket["onerror"]) {
socket.onerror = callback;
},
set onmessage(callback: ScreenSocket["onmessage"]) {
socket.onmessage = callback;
},
close: () => socket.close(),
};
},
verifyOwnership: async () => {
const state = await readControl(computerId);
return state?.holder === "human" && !state.transitioning;
},
onReady: (ready) => {
inputReady.current = ready;
setConnected(ready);
},
onProblem: (problem) => onProblem?.(problem),
onReconnecting: (reconnecting) => onReconnecting?.(reconnecting),
onDisconnect: () => {
frameSize.current = null;
latestFrame.current = null;
latestFrameId.current++;
const canvas = canvasRef.current;
if (canvas)
canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height);
},
onMessage,
});
document.addEventListener("visibilitychange", drawLatestFrame);
return () => {
closed = true;
// Before the socket goes: a key still down over there would stay held in the remote browser.
releaseKeys();
connection.stop();
document.removeEventListener("visibilitychange", drawLatestFrame);
socketRef.current = null;
};
}, [computerId, onProblem, onReconnecting, retryKey]);
useEffect(() => {
const wasDriving = previouslyDriving.current;
previouslyDriving.current = driving;
if (
wasDriving ||
!driving ||
inputReady.current ||
socketRef.current?.readyState !== WebSocket.OPEN
)
return;
let live = true;
void readControl(computerId)
.then((state) => {
if (!live || socketRef.current?.readyState !== WebSocket.OPEN) return;
inputReady.current = state?.holder === "human" && !state.transitioning;
setConnected(inputReady.current);
})
.catch(() => {
inputReady.current = false;
});
return () => {
live = false;
};
}, [computerId, driving]);
const send = useCallback(
(message: Record<string, unknown>) => {
const socket = socketRef.current;
if (
!driving ||
!inputReady.current ||
socket?.readyState !== WebSocket.OPEN
)
return;
socket.send(JSON.stringify(message));
},
[driving],
);
/**
* Convert from displayed canvas coordinates to page coordinates with the shared, tested helper.
* A screencast frame is the viewport, so its frame size stands in for natural image size.
*/
const at = useCallback((event: { clientX: number; clientY: number }) => {
const canvas = canvasRef.current;
const size = frameSize.current;
if (!canvas || !size) return null;
return pageCoordinates(
{ naturalWidth: size.width, naturalHeight: size.height },
canvas.getBoundingClientRect(),
event,
);
}, []);
const onMouse = useCallback(
(kind: "pressed" | "released" | "moved") =>
(event: React.MouseEvent<HTMLCanvasElement>) => {
// A press on the screen gives it the keyboard, as a press on any focusable element does, so typing into the
// remote page after clicking a field on it needs no second click.
if (kind === "pressed")
event.currentTarget.focus({ preventScroll: true });
const point = at(event);
if (!point) return;
send({
type: "mouse",
event: kind,
...point,
button:
event.button === 2
? "right"
: event.button === 1
? "middle"
: "left",
/*
* The browser's own count, not a fixed one.
*
* `MouseEvent.detail` is how many times in a row this button has been pressed in the same
* place, worked out by the browser to its own timing and distance rules. Chrome fires
* `dblclick` on the far page only when the second press says it is the second, so sending 1
* every time meant a double click arrived as two separate clicks: no `dblclick` ever
* reached the page, `event.detail` was always 1, and opening a row, expanding a node and
* selecting a word were all things a person holding the wheel could not do.
*
* At least one on a press, because the computer refuses a press of zero for the reason its
* own comment gives -- Chrome would see a move that happens to have a button set, and no
* click at all. `detail` is zero on an event a script dispatched rather than a person.
*/
clickCount: kind === "moved" ? 0 : Math.max(1, event.detail),
modifiers: modifierBits(event),
});
},
[at, send],
);
useImperativeHandle(
ref,
() => ({ focus: () => canvasRef.current?.focus({ preventScroll: true }) }),
[],
);
/*
* A view opened to drive the screen puts the keyboard on it, as a dialog puts focus inside itself: taking control
* and typing needs no click on the page first, as it needed none before the screen took keys only with focus.
*/
useEffect(() => {
if (autoFocus && driving && connected)
canvasRef.current?.focus({ preventScroll: true });
}, [autoFocus, driving, connected]);
/**
* Keystrokes, forwarded while driving, and only those the screen itself has: it takes the keyboard the way any focused
* element does, by a click on it (or Tab to it), and gives it back on Escape or a click anywhere else.
*
* It used to take every keystroke on the window that was not aimed at a text field, because the canvas could not hold
* focus. So nothing else on the page could have a key: the Browser page's tabs never heard their arrows, Home, End or
* Delete, no button took Enter or Space, and Tab never left anything. And every one of those keys went to the remote
* browser instead, where a stray arrow is not harmless. Now the canvas holds focus, as noVNC's does (its keyboard
* listens on the canvas it focuses on a click), and a key aimed anywhere else is that element's.
*
* Still listened for on the window, in the capture phase, and stopped as well as prevented, because a keystroke sent to
* the Bot's browser is not also this page's: the app's own shortcuts listen on this window too, and they were bound
* first, when the signed-in app mounted, so they would see it first (a capital N typed into the remote page started a
* new chat, and Ctrl+B there toggled the sidebar here). The paste shortcut is not stopped, because the paste event it
* makes is how the clipboard's text arrives. Nor is Escape, which closes a full-size view; on its way it also takes the
* keyboard off the screen, which is how a person who reached it with Tab leaves it again (WCAG's "no keyboard trap").
*/
useEffect(() => {
const canvas = canvasRef.current;
if (!driving || !connected || !canvas) return;
const held = keysDown.current;
const onKeyDown = (event: KeyboardEvent) => {
if (event.target !== canvas) return;
if (event.key === "Escape") {
canvas.blur();
return;
}
if (isPasteShortcut(event)) return;
event.preventDefault();
event.stopPropagation();
held.set(event.code, remoteKey(event));
send({
type: "key",
event: "down",
...remoteKey(event),
// Only a printable character carries text. Sending text for Backspace makes Chrome insert a
// character instead of deleting one.
...(event.key.length === 1 ? { text: event.key } : {}),
modifiers: modifierBits(event),
});
};
/**
* The release of a key that went down to the remote browser, wherever the focus is now: a key released after a
* click took the focus elsewhere is still held over there otherwise. Only those: the Tab that brought the focus to
* the screen, say, was this page's, and its release is too.
*/
const onKeyUp = (event: KeyboardEvent) => {
if (!held.delete(event.code)) return;
event.preventDefault();
send({
type: "key",
event: "up",
...remoteKey(event),
modifiers: modifierBits(event),
});
};
/** Paste arrives as one block; CDP inserts it as text rather than key events. */
const onPaste = (event: ClipboardEvent) => {
if (event.target !== canvas) return;
const text = event.clipboardData?.getData("text");
if (!text) return;
event.preventDefault();
send({ type: "text", text });
};
window.addEventListener("keydown", onKeyDown, true);
window.addEventListener("keyup", onKeyUp);
window.addEventListener("paste", onPaste);
return () => {
window.removeEventListener("keydown", onKeyDown, true);
window.removeEventListener("keyup", onKeyUp);
window.removeEventListener("paste", onPaste);
releaseKeys();
};
}, [driving, connected, send, releaseKeys]);
/**
* The wheel, forwarded while driving, from a listener that is allowed to stop it here.
*
* Not React's `onWheel`: React attaches that to its root as a passive listener, so the
* `preventDefault` in it was ignored ("Unable to preventDefault inside passive event listener
* invocation."). The wheel reached the Bot's page and also scrolled whatever on this page was
* under it, the frame that holds this screen included, and Ctrl and the wheel zoomed this page.
*/
useEffect(() => {
const canvas = canvasRef.current;
if (!driving || !connected || !canvas) return;
const onWheel = (event: WheelEvent) => {
const point = at(event);
if (!point) return;
event.preventDefault();
send({
type: "wheel",
...point,
deltaX: event.deltaX,
deltaY: event.deltaY,
modifiers: modifierBits(event),
});
};
canvas.addEventListener("wheel", onWheel, { passive: false });
return () => canvas.removeEventListener("wheel", onWheel);
}, [driving, connected, at, send]);
return (
<canvas
ref={canvasRef}
className={`block h-auto w-full ${driving ? "cursor-crosshair focus-visible:outline-2 focus-visible:-outline-offset-2 focus-visible:outline-ring" : ""}`}
// Only forward input during takeover. Then the screen is a stop in the Tab order and takes every key while it has
// the focus, which is what a screen reader is told by `application`.
{...(driving
? {
onMouseDown: onMouse("pressed"),
onMouseUp: onMouse("released"),
onMouseMove: onMouse("moved"),
onContextMenu: (event: React.MouseEvent) => event.preventDefault(),
role: "application",
tabIndex: 0,
}
: {})}
aria-label={driving ? label : "The Bot's screen, live"}
data-connected={connected}
/>
);
}