Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Stop the wheel on a Bot's screen from also scrolling the page around it (#603)

kevin3a1ec6787647
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c8507f3a..b031ba16 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
## Unreleased
+### Scrolling a Bot's browser no longer scrolls or zooms the page around it
+
+While somebody drives a Bot's browser, a turn of the mouse wheel over its screen is sent to it, and
+the screen was meant to keep the wheel from also acting on the app. React attaches its wheel handler
+as a passive listener, which a browser does not allow to do that, so the wheel scrolled the frame
+holding the Bot's screen along with the Bot's page, and Ctrl with the wheel zoomed the app. The
+wheel is now handled by a listener that can hold it, so it reaches only the Bot's browser.
### Typing into a Bot's browser no longer triggers the app's own shortcuts
While somebody drives a Bot's browser, every keystroke is sent to it. The app's shortcuts listen for
diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx
index 70ad2214..5d5393fe 100644
--- a/app/src/components/computer/live-screen.tsx
+++ b/app/src/components/computer/live-screen.tsx
@@ -227,7 +227,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
* Convert from displayed canvas coordinates to page coordinates with the shared, tested helper.
* A screencast frame is the viewport, so its frame size stands in for natural image size.
*/
- const at = useCallback((event: React.MouseEvent) => {
+ const at = useCallback((event: { clientX: number; clientY: number }) => {
const canvas = canvasRef.current;
const size = frameSize.current;
if (!canvas || !size) return null;
@@ -343,6 +343,33 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
};
}, [driving, send]);
+ /**
+ * The wheel, forwarded while driving, from a listener that is allowed to stop it here.
+ *
+ * Not React's `onWheel`: React attaches that to its root as a passive listener, so the
+ * `preventDefault` in it was ignored ("Unable to preventDefault inside passive event listener
+ * invocation."). The wheel reached the Bot's page and also scrolled whatever on this page was
+ * under it, the frame that holds this screen included, and Ctrl and the wheel zoomed this page.
+ */
+ useEffect(() => {
+ const canvas = canvasRef.current;
+ if (!driving || !canvas) return;
+ const onWheel = (event: WheelEvent) => {
+ const point = at(event);
+ if (!point) return;
+ event.preventDefault();
+ send({
+ type: "wheel",
+ ...point,
+ deltaX: event.deltaX,
+ deltaY: event.deltaY,
+ modifiers: modifierBits(event),
+ });
+ };
+ canvas.addEventListener("wheel", onWheel, { passive: false });
+ return () => canvas.removeEventListener("wheel", onWheel);
+ }, [driving, at, send]);
+
return (
<canvas
ref={canvasRef}
@@ -354,18 +381,6 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
onMouseUp: onMouse("released"),
onMouseMove: onMouse("moved"),
onContextMenu: (event: React.MouseEvent) => event.preventDefault(),
- onWheel: (event: React.WheelEvent<HTMLCanvasElement>) => {
- const point = at(event);
- if (!point) return;
- event.preventDefault();
- send({
- type: "wheel",
- ...point,
- deltaX: event.deltaX,
- deltaY: event.deltaY,
- modifiers: modifierBits(event),
- });
- },
}
: {})}
aria-label={
diff --git a/app/tests/live-screen-mouse.test.tsx b/app/tests/live-screen-mouse.test.tsx
index daacda29..46dc2aee 100644
--- a/app/tests/live-screen-mouse.test.tsx
+++ b/app/tests/live-screen-mouse.test.tsx
@@ -165,3 +165,43 @@ test("moving the mouse is not a click", async () => {
},
]);
});
+
+test("a turn of the wheel is stopped here as well as sent there", async () => {
+ // In a browser React attaches `onWheel` to its root as a passive listener, so a `preventDefault`
+ // there is ignored and the wheel scrolled this page as well as the Bot's. Under happy-dom React
+ // does not detect passive support and binds it actively, which hides exactly that, so this pins
+ // the listener the canvas holds itself, and that it is not passive.
+ const wheelListeners: unknown[] = [];
+ const addEventListener = HTMLCanvasElement.prototype.addEventListener;
+ HTMLCanvasElement.prototype.addEventListener = function (
+ this: HTMLCanvasElement,
+ ...args: Parameters<typeof addEventListener>
+ ) {
+ if (args[0] === "wheel") wheelListeners.push(args[2]);
+ return addEventListener.apply(this, args);
+ };
+ try {
+ const { canvas, socket } = await liveCanvas();
+ expect(wheelListeners).toEqual([{ passive: false }]);
+
+ // happy-dom's WheelEvent carries no coordinates, so a mouse event of that type stands in.
+ const wheel = new MouseEvent("wheel", {
+ clientX: 40,
+ clientY: 30,
+ bubbles: true,
+ cancelable: true,
+ });
+ Object.defineProperties(wheel, {
+ deltaX: { value: 0 },
+ deltaY: { value: 120 },
+ });
+ canvas.dispatchEvent(wheel);
+
+ expect(wheel.defaultPrevented).toBe(true);
+ expect(socket.sent).toEqual([
+ { type: "wheel", x: 40, y: 30, deltaX: 0, deltaY: 120, modifiers: 0 },
+ ]);
+ } finally {
+ HTMLCanvasElement.prototype.addEventListener = addEventListener;
+ }
+});