Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Stop the wheel on a Bot's screen from also scrolling the page around it (#603)
kevin3a1ec6787647
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c8507f3a..b031ba16 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.## Unreleased+### Scrolling a Bot's browser no longer scrolls or zooms the page around it++While somebody drives a Bot's browser, a turn of the mouse wheel over its screen is sent to it, and+the screen was meant to keep the wheel from also acting on the app. React attaches its wheel handler+as a passive listener, which a browser does not allow to do that, so the wheel scrolled the frame+holding the Bot's screen along with the Bot's page, and Ctrl with the wheel zoomed the app. The+wheel is now handled by a listener that can hold it, so it reaches only the Bot's browser.### Typing into a Bot's browser no longer triggers the app's own shortcutsWhile somebody drives a Bot's browser, every keystroke is sent to it. The app's shortcuts listen fordiff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsxindex 70ad2214..5d5393fe 100644--- a/app/src/components/computer/live-screen.tsx+++ b/app/src/components/computer/live-screen.tsx@@ -227,7 +227,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {* Convert from displayed canvas coordinates to page coordinates with the shared, tested helper.* A screencast frame is the viewport, so its frame size stands in for natural image size.*/- const at = useCallback((event: React.MouseEvent) => {+ const at = useCallback((event: { clientX: number; clientY: number }) => {const canvas = canvasRef.current;const size = frameSize.current;if (!canvas || !size) return null;@@ -343,6 +343,33 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {};}, [driving, send]);+ /**+ * The wheel, forwarded while driving, from a listener that is allowed to stop it here.+ *+ * Not React's `onWheel`: React attaches that to its root as a passive listener, so the+ * `preventDefault` in it was ignored ("Unable to preventDefault inside passive event listener+ * invocation."). The wheel reached the Bot's page and also scrolled whatever on this page was+ * under it, the frame that holds this screen included, and Ctrl and the wheel zoomed this page.+ */+ useEffect(() => {+ const canvas = canvasRef.current;+ if (!driving || !canvas) return;+ const onWheel = (event: WheelEvent) => {+ const point = at(event);+ if (!point) return;+ event.preventDefault();+ send({+ type: "wheel",+ ...point,+ deltaX: event.deltaX,+ deltaY: event.deltaY,+ modifiers: modifierBits(event),+ });+ };+ canvas.addEventListener("wheel", onWheel, { passive: false });+ return () => canvas.removeEventListener("wheel", onWheel);+ }, [driving, at, send]);+return (<canvasref={canvasRef}@@ -354,18 +381,6 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {onMouseUp: onMouse("released"),onMouseMove: onMouse("moved"),onContextMenu: (event: React.MouseEvent) => event.preventDefault(),- onWheel: (event: React.WheelEvent<HTMLCanvasElement>) => {- const point = at(event);- if (!point) return;- event.preventDefault();- send({- type: "wheel",- ...point,- deltaX: event.deltaX,- deltaY: event.deltaY,- modifiers: modifierBits(event),- });- },}: {})}aria-label={diff --git a/app/tests/live-screen-mouse.test.tsx b/app/tests/live-screen-mouse.test.tsxindex daacda29..46dc2aee 100644--- a/app/tests/live-screen-mouse.test.tsx+++ b/app/tests/live-screen-mouse.test.tsx@@ -165,3 +165,43 @@ test("moving the mouse is not a click", async () => {},]);});++test("a turn of the wheel is stopped here as well as sent there", async () => {+ // In a browser React attaches `onWheel` to its root as a passive listener, so a `preventDefault`+ // there is ignored and the wheel scrolled this page as well as the Bot's. Under happy-dom React+ // does not detect passive support and binds it actively, which hides exactly that, so this pins+ // the listener the canvas holds itself, and that it is not passive.+ const wheelListeners: unknown[] = [];+ const addEventListener = HTMLCanvasElement.prototype.addEventListener;+ HTMLCanvasElement.prototype.addEventListener = function (+ this: HTMLCanvasElement,+ ...args: Parameters<typeof addEventListener>+ ) {+ if (args[0] === "wheel") wheelListeners.push(args[2]);+ return addEventListener.apply(this, args);+ };+ try {+ const { canvas, socket } = await liveCanvas();+ expect(wheelListeners).toEqual([{ passive: false }]);++ // happy-dom's WheelEvent carries no coordinates, so a mouse event of that type stands in.+ const wheel = new MouseEvent("wheel", {+ clientX: 40,+ clientY: 30,+ bubbles: true,+ cancelable: true,+ });+ Object.defineProperties(wheel, {+ deltaX: { value: 0 },+ deltaY: { value: 120 },+ });+ canvas.dispatchEvent(wheel);++ expect(wheel.defaultPrevented).toBe(true);+ expect(socket.sent).toEqual([+ { type: "wheel", x: 40, y: 30, deltaX: 0, deltaY: 120, modifiers: 0 },+ ]);+ } finally {+ HTMLCanvasElement.prototype.addEventListener = addEventListener;+ }+});