Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: pace X11 wheel events and release interrupted scroll input

Task Builder4ce348b6ffa1
diff --git a/apps/desktop/electron/computer-use-native-worker.mjs b/apps/desktop/electron/computer-use-native-worker.mjs
index 07f54036..afd3a15d 100644
--- a/apps/desktop/electron/computer-use-native-worker.mjs
+++ b/apps/desktop/electron/computer-use-native-worker.mjs
@@ -152,7 +152,7 @@ function windows() {
for (const p of points) {
const position = pointValue(p); checked(position); move(position.x, position.y); checked(position);
if (action.type === "drag" && !pointerDown) { pointerDown = true; mouseEvent(2); }
- await new Promise((resolve) => setTimeout(resolve, action.type === "drag" ? 12 : 2));
+ await new Promise((resolve) => setTimeout(resolve, action.type === "scroll" ? 32 : action.type === "drag" ? 12 : 2));
}
const position = pointValue(points.at(-1)); checked(position);
if (action.type === "scroll") mouseEvent(action.axis === "horizontal" ? 0x1000 : 0x800, action.delta * (action.axis === "horizontal" ? 120 : -120));
@@ -343,8 +343,8 @@ function x11() {
const fakeMotion = (...args) => inject(rawMotion, args);
const keysym = x.func("ulong XStringToKeysym(str name)"), keycode = x.func("uint8_t XKeysymToKeycode(void *display, ulong symbol)");
const downKeys = new Set();
- let pointerDown = false;
- const release = () => { for (const key of downKeys) inject(rawKey, [display, key, 0, 0], false); downKeys.clear(); if (pointerDown) inject(rawButton, [display, 1, 0, 0], false); pointerDown = false; sync(display, 0); };
+ let pointerDown = false, wheelDown = 0;
+ const release = () => { for (const key of downKeys) inject(rawKey, [display, key, 0, 0], false); downKeys.clear(); if (pointerDown) inject(rawButton, [display, 1, 0, 0], false); pointerDown = false; if (wheelDown) inject(rawButton, [display, wheelDown, 0, 0], false); wheelDown = 0; sync(display, 0); };
const symbolAt = x.func("ulong XKeycodeToKeysym(void *display, uint8_t code, int index)");
const key = (name, down) => { const symbol = typeof name === "number" ? name : keysym(name); const code = keycode(display, symbol); if (!code) fail("unsupported_action", "This character is unavailable in the active keyboard layout. Use an app connector for arbitrary Unicode text."); if (down) downKeys.add(code); else downKeys.delete(code); if (!fakeKey(display, code, down ? 1 : 0, 0)) fail("input_unavailable", "X11 rejected keyboard input."); sync(display, 0); };
return {
@@ -382,12 +382,19 @@ function x11() {
for (const p of points) {
checked(); fakeMotion(display, -1, expected.bounds.x + Math.round(p.x), expected.bounds.y + Math.round(p.y), 0); sync(display, 0); checked(p);
if (action.type === "drag" && !pointerDown) { pointerDown = true; fakeButton(display, 1, 1, 0); sync(display, 0); }
- await new Promise((resolve) => setTimeout(resolve, action.type === "drag" ? 12 : 2));
+ await new Promise((resolve) => setTimeout(resolve, action.type === "scroll" ? 32 : action.type === "drag" ? 12 : 2));
}
checked(points.at(-1));
if (action.type === "scroll") {
const button = action.axis === "horizontal" ? action.delta > 0 ? 7 : 6 : action.delta > 0 ? 5 : 4;
- for (let i = 0; i < Math.abs(action.delta); i++) { checked(points.at(-1)); fakeButton(display, button, 1, 0); fakeButton(display, button, 0, 0); sync(display, 0); }
+ // Let the app process pointer entry and distinct wheel ticks.
+ // Keep releases tracked so person input or Stop cannot leave a button held.
+ for (let i = 0; i < Math.abs(action.delta); i++) {
+ checked(points.at(-1)); wheelDown = button; fakeButton(display, button, 1, 0); sync(display, 0);
+ await new Promise((resolve) => setTimeout(resolve, 8));
+ fakeButton(display, button, 0, 0); wheelDown = 0; sync(display, 0);
+ await new Promise((resolve) => setTimeout(resolve, 12));
+ }
} else if (action.type !== "drag") {
for (let i = 0; i < (action.type === "double_click" ? 2 : 1); i++) { checked(points.at(-1)); pointerDown = true; fakeButton(display, 1, 1, 0); fakeButton(display, 1, 0, 0); pointerDown = false; sync(display, 0); if (i === 0 && action.type === "double_click") await new Promise((resolve) => setTimeout(resolve, 60)); }
}
diff --git a/evals/packages/labs/fixtures/portable-computer-use-app.mjs b/evals/packages/labs/fixtures/portable-computer-use-app.mjs
index b9a19e43..e86499f4 100644
--- a/evals/packages/labs/fixtures/portable-computer-use-app.mjs
+++ b/evals/packages/labs/fixtures/portable-computer-use-app.mjs
@@ -11,7 +11,7 @@ app.setPath("userData", process.argv.at(-1));
const ready = app.whenReady().then(async () => {
for (const [index, title] of ["Workspace window", "Other window"].entries()) {
const window = new BrowserWindow({ width: 620, height: 450, x: 50 + index * 700, y: 120, title, webPreferences: { nodeIntegration: false, contextIsolation: true } });
- const html = "<title>" + title + "</title>" + "<style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}#doubles{left:420px;width:150px}#scroll{position:absolute;left:420px;top:160px;width:150px;height:230px;overflow:auto;background:#ddddff}#drag{position:absolute;left:70px;top:290px;width:120px;height:55px;background:#008080;touch-action:none;user-select:none}</style><button id=\"increment\">Increment</button><input id=\"draft\" aria-label=\"Fixture draft\" value=\"Initial draft\"><p id=\"count\">0</p><button id=\"doubles\">Double click</button><div id=\"scroll\"><div style=\"height:1000px\">Scroll me</div></div><div id=\"drag\">Drag me</div><script>window.counter=0;window.pointer={doubles:0,held:false};document.getElementById(\"increment\").onclick=()=>{document.getElementById(\"count\").textContent=String(++window.counter)};document.getElementById(\"doubles\").ondblclick=()=>{document.getElementById(\"doubles\").textContent=String(++window.pointer.doubles)};const handle=document.getElementById(\"drag\");let start;handle.onpointerdown=e=>{window.pointer.held=true;start={x:e.clientX,y:e.clientY,left:handle.offsetLeft,top:handle.offsetTop};handle.setPointerCapture(e.pointerId)};handle.onpointermove=e=>{if(window.pointer.held&&e.buttons===1){handle.style.left=(start.left+e.clientX-start.x)+\"px\";handle.style.top=(start.top+e.clientY-start.y)+\"px\"}};handle.onpointerup=()=>{window.pointer.held=false};</script>";
+ const html = "<title>" + title + "</title>" + "<style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}#doubles{left:420px;width:150px}#scroll{position:absolute;left:420px;top:160px;width:150px;height:230px;overflow:auto;background:#ddddff}#drag{position:absolute;left:70px;top:290px;width:120px;height:55px;background:#008080;touch-action:none;user-select:none}</style><button id=\"increment\">Increment</button><input id=\"draft\" aria-label=\"Fixture draft\" value=\"Initial draft\"><p id=\"count\">0</p><button id=\"doubles\">Double click</button><div id=\"scroll\"><div style=\"height:1000px\">Scroll me</div></div><div id=\"drag\">Drag me</div><script>window.counter=0;window.pointer={doubles:0,held:false};window.wheelEvents=[];document.addEventListener(\"wheel\",e=>{window.wheelEvents.push({x:e.clientX,y:e.clientY,deltaY:e.deltaY,ctrl:e.ctrlKey,shift:e.shiftKey,alt:e.altKey,target:e.target.id,parent:e.target.parentElement?.id});if(window.wheelEvents.length>20)window.wheelEvents.shift()},{passive:true});document.getElementById(\"increment\").onclick=()=>{document.getElementById(\"count\").textContent=String(++window.counter)};document.getElementById(\"doubles\").ondblclick=()=>{document.getElementById(\"doubles\").textContent=String(++window.pointer.doubles)};const handle=document.getElementById(\"drag\");let start;handle.onpointerdown=e=>{window.pointer.held=true;start={x:e.clientX,y:e.clientY,left:handle.offsetLeft,top:handle.offsetTop};handle.setPointerCapture(e.pointerId)};handle.onpointermove=e=>{if(window.pointer.held&&e.buttons===1){handle.style.left=(start.left+e.clientX-start.x)+\"px\";handle.style.top=(start.top+e.clientY-start.y)+\"px\"}};handle.onpointerup=()=>{window.pointer.held=false};</script>";
await window.loadURL("data:text/html," + encodeURIComponent(html)); windows.push(window);
}
});
@@ -20,6 +20,7 @@ createInterface({ input: channel }).on("line", async (line) => {
try {
await ready; let result;
if (request.method === "state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ count: window.counter, draft: document.getElementById("draft").value })')));
+ else if (request.method === "input_events") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript("window.wheelEvents")));
else if (request.method === "pointer_state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ doubles: window.pointer.doubles, scrolled: document.getElementById("scroll").scrollTop > 0, dragged: document.getElementById("drag").offsetLeft > 170, released: !window.pointer.held })')));
else if (request.method === "bounds") result = windows.map((w) => ({ bounds: w.getBounds(), contentBounds: w.getContentBounds(), scale: screen.getDisplayMatching(w.getBounds()).scaleFactor }));
else if (request.method === "focus_other") { windows[1].focus(); result = {}; }
diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.ts
index 7b75d709..67194b62 100644
--- a/evals/specs/computer-use-portable-window.e2e.test.ts
+++ b/evals/specs/computer-use-portable-window.e2e.test.ts
@@ -69,7 +69,7 @@ test("Windows and Linux X11 Computer Use keeps native input within the approved
await act("double-click", async (observed) => ({ type: "double_click", ...await world.point(observed, 480, 95) }));
await act("scroll", async (observed) => ({ type: "scroll", axis: "vertical", delta: 3, ...await world.point(observed, 480, 240) }));
await act("drag", async (observed) => ({ type: "drag", path: await Promise.all([{ x: 120, y: 310 }, { x: 140, y: 315 }, { x: 180, y: 325 }, { x: 240, y: 330 }].map(({ x, y }) => world.point(observed, x, y))) }));
- await expect.poll(() => world.pointerState()).toEqual([
+ await expect.poll(() => world.pointerState(), { message: JSON.stringify(await world.inputEvents()) }).toEqual([
{ doubles: 1, scrolled: true, dragged: true, released: true },
{ doubles: 0, scrolled: false, dragged: false, released: true },
]);
diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.ts
index 1effb90a..4ecace06 100644
--- a/evals/worlds/computer-use.ts
+++ b/evals/worlds/computer-use.ts
@@ -361,6 +361,7 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:
cover: () => fixture.request("cover"),
outsideInput: () => fixture.request("outside_input"),
pointerState: () => fixture.request("pointer_state"),
+ inputEvents: () => fixture.request("input_events"),
imagePixel(reply: unknown) {
if (!record(reply) || !Array.isArray(reply.content)) throw new Error("Missing image content.");
const image = reply.content.find((item: unknown) => record(item) && item.type === "image");