Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Send a double click as a double click while driving (#478)

kevin779edbd3e9b2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9236e397..24a3ce1e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,16 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
## Unreleased
+### Double-clicking works while a person is driving a Bot's browser
+
+A double click was sent to the Bot's browser as two separate first clicks, because every press said
+it was the first one. Chrome fires `dblclick` on the page only when the second press says it is the
+second, so the page never saw one at all and `event.detail` was always 1. Opening a row in a table,
+expanding a node in a tree and double-clicking a word to select it were all things a person holding
+the wheel simply could not do, with nothing on screen to say why — the clicks landed, they just each
+counted as the first. The count the person's own browser worked out is now the one that is sent, so a
+double click is a double click and a single one is unchanged.
+
### Pressing Enter works while a person is driving a Bot's browser
Taking the wheel of a Bot's browser is mostly for the sign-in it cannot do itself, and Enter is how a
diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx
index b9486f25..cd9e1784 100644
--- a/app/src/components/computer/live-screen.tsx
+++ b/app/src/components/computer/live-screen.tsx
@@ -1,6 +1,6 @@
import { useCallback, useEffect, useRef, useState } from "react";
-import { pageCoordinates } from "./take-the-wheel";
import { socketUrl } from "@/lib/socket-url";
+import { pageCoordinates } from "./take-the-wheel";
/**
* Low-latency screencast used while a human is driving the Bot's browser.
@@ -187,7 +187,21 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
: event.button === 1
? "middle"
: "left",
- clickCount: kind === "moved" ? 0 : 1,
+ /*
+ * The browser's own count, not a fixed one.
+ *
+ * `MouseEvent.detail` is how many times in a row this button has been pressed in the same
+ * place, worked out by the browser to its own timing and distance rules. Chrome fires
+ * `dblclick` on the far page only when the second press says it is the second, so sending 1
+ * every time meant a double click arrived as two separate clicks: no `dblclick` ever
+ * reached the page, `event.detail` was always 1, and opening a row, expanding a node and
+ * selecting a word were all things a person holding the wheel could not do.
+ *
+ * At least one on a press, because the computer refuses a press of zero for the reason its
+ * own comment gives -- Chrome would see a move that happens to have a button set, and no
+ * click at all. `detail` is zero on an event a script dispatched rather than a person.
+ */
+ clickCount: kind === "moved" ? 0 : Math.max(1, event.detail),
modifiers: modifierBits(event),
});
},
diff --git a/app/tests/live-screen-mouse.test.tsx b/app/tests/live-screen-mouse.test.tsx
new file mode 100644
index 00000000..daacda29
--- /dev/null
+++ b/app/tests/live-screen-mouse.test.tsx
@@ -0,0 +1,167 @@
+import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";
+import { GlobalRegistrator } from "@happy-dom/global-registrator";
+import { cleanup, fireEvent, render, waitFor } from "@testing-library/react";
+import { LiveScreen } from "@/components/computer/live-screen";
+
+/**
+ * What a person's click turns into while they hold the wheel.
+ *
+ * The canvas sends nothing until it knows how large the frames are, because a click has to be mapped
+ * from where it landed on screen to where it landed on the page. So each test feeds one frame first.
+ * The bitmap decode inside that handler is allowed to fail here — there is no `createImageBitmap`
+ * under happy-dom, and the component already treats a frame it cannot draw as one to skip — but the
+ * size is recorded before the decode is attempted, which is the part these need.
+ */
+
+class SocketDouble {
+ static readonly OPEN = 1;
+ static readonly CLOSED = 3;
+ static latest: SocketDouble | undefined;
+
+ readyState = SocketDouble.OPEN;
+ onopen: (() => void) | null = null;
+ onmessage: ((event: { data: string }) => void) | null = null;
+ onerror: (() => void) | null = null;
+ onclose: (() => void) | null = null;
+ readonly sent: Record<string, unknown>[] = [];
+
+ constructor(_url: string) {
+ SocketDouble.latest = this;
+ queueMicrotask(() => this.onopen?.());
+ }
+
+ send(payload: string) {
+ this.sent.push(JSON.parse(payload) as Record<string, unknown>);
+ }
+
+ close() {
+ this.readyState = SocketDouble.CLOSED;
+ this.onclose?.();
+ }
+}
+
+let originalWebSocket: typeof WebSocket;
+
+beforeAll(() => {
+ GlobalRegistrator.register();
+ originalWebSocket = globalThis.WebSocket;
+ globalThis.WebSocket = SocketDouble as unknown as typeof WebSocket;
+});
+
+afterEach(() => {
+ cleanup();
+ SocketDouble.latest = undefined;
+});
+
+afterAll(() => {
+ globalThis.WebSocket = originalWebSocket;
+ GlobalRegistrator.unregister();
+});
+
+/** The frames Chrome is casting, and the canvas they are drawn on, at the same size. */
+const FRAME = { width: 800, height: 600 };
+
+async function liveCanvas(): Promise<{
+ canvas: HTMLCanvasElement;
+ socket: SocketDouble;
+}> {
+ const { container } = render(<LiveScreen computerId="mouse-test" driving />);
+ await waitFor(() => expect(SocketDouble.latest).toBeDefined());
+ const socket = SocketDouble.latest as SocketDouble;
+
+ socket.onmessage?.({
+ data: JSON.stringify({ type: "frame", data: "AAAA", ...FRAME }),
+ });
+ await waitFor(() => expect(socket.sent).toEqual([]));
+
+ const canvas = container.querySelector("canvas") as HTMLCanvasElement;
+ canvas.getBoundingClientRect = () =>
+ ({
+ left: 0,
+ top: 0,
+ width: FRAME.width,
+ height: FRAME.height,
+ }) as DOMRect;
+ return { canvas, socket };
+}
+
+/** One press and release where the person's browser says this is the nth click there. */
+function clickAt(canvas: HTMLCanvasElement, detail: number) {
+ const at = { clientX: 40, clientY: 30, button: 0, detail };
+ fireEvent.mouseDown(canvas, at);
+ fireEvent.mouseUp(canvas, at);
+}
+
+test("a double click is sent as a double click", async () => {
+ // Chrome fires `dblclick` on the page only when the second press says it is the second, and
+ // nothing else does: two presses that both claim to be the first are two separate clicks, so
+ // opening a row, expanding a node and selecting a word were all impossible while driving.
+ const { canvas, socket } = await liveCanvas();
+
+ clickAt(canvas, 1);
+ clickAt(canvas, 2);
+
+ expect(socket.sent.map((message) => message.clickCount)).toEqual([
+ 1, 1, 2, 2,
+ ]);
+});
+
+test("an ordinary click is still one click", async () => {
+ const { canvas, socket } = await liveCanvas();
+
+ clickAt(canvas, 1);
+
+ expect(socket.sent).toEqual([
+ {
+ type: "mouse",
+ event: "pressed",
+ x: 40,
+ y: 30,
+ button: "left",
+ clickCount: 1,
+ modifiers: 0,
+ },
+ {
+ type: "mouse",
+ event: "released",
+ x: 40,
+ y: 30,
+ button: "left",
+ clickCount: 1,
+ modifiers: 0,
+ },
+ ]);
+});
+
+test("a press that claims no click at all is still a click", async () => {
+ // `detail` is 0 on an event a script dispatched rather than a person, and the computer refuses a
+ // press of zero: Chrome then sees a move that happens to have a button set and no click fires.
+ const { canvas, socket } = await liveCanvas();
+
+ clickAt(canvas, 0);
+
+ expect(socket.sent.map((message) => message.clickCount)).toEqual([1, 1]);
+});
+
+test("moving the mouse is not a click", async () => {
+ const { canvas, socket } = await liveCanvas();
+
+ fireEvent.mouseMove(canvas, {
+ clientX: 40,
+ clientY: 30,
+ button: 0,
+ detail: 0,
+ });
+
+ expect(socket.sent).toEqual([
+ {
+ type: "mouse",
+ event: "moved",
+ x: 40,
+ y: 30,
+ button: "left",
+ clickCount: 0,
+ modifiers: 0,
+ },
+ ]);
+});