A desktop coding agent with free access to frontier models.
TypeScript
fix: pace X11 wheel events and release interrupted scroll input
Task Builder4ce348b6ffa1
diff --git a/apps/desktop/electron/computer-use-native-worker.mjs b/apps/desktop/electron/computer-use-native-worker.mjsindex 07f54036..afd3a15d 100644--- a/apps/desktop/electron/computer-use-native-worker.mjs+++ b/apps/desktop/electron/computer-use-native-worker.mjs@@ -152,7 +152,7 @@ function windows() {for (const p of points) {const position = pointValue(p); checked(position); move(position.x, position.y); checked(position);if (action.type === "drag" && !pointerDown) { pointerDown = true; mouseEvent(2); }- await new Promise((resolve) => setTimeout(resolve, action.type === "drag" ? 12 : 2));+ await new Promise((resolve) => setTimeout(resolve, action.type === "scroll" ? 32 : action.type === "drag" ? 12 : 2));}const position = pointValue(points.at(-1)); checked(position);if (action.type === "scroll") mouseEvent(action.axis === "horizontal" ? 0x1000 : 0x800, action.delta * (action.axis === "horizontal" ? 120 : -120));@@ -343,8 +343,8 @@ function x11() {const fakeMotion = (...args) => inject(rawMotion, args);const keysym = x.func("ulong XStringToKeysym(str name)"), keycode = x.func("uint8_t XKeysymToKeycode(void *display, ulong symbol)");const downKeys = new Set();- let pointerDown = false;- const release = () => { for (const key of downKeys) inject(rawKey, [display, key, 0, 0], false); downKeys.clear(); if (pointerDown) inject(rawButton, [display, 1, 0, 0], false); pointerDown = false; sync(display, 0); };+ let pointerDown = false, wheelDown = 0;+ const release = () => { for (const key of downKeys) inject(rawKey, [display, key, 0, 0], false); downKeys.clear(); if (pointerDown) inject(rawButton, [display, 1, 0, 0], false); pointerDown = false; if (wheelDown) inject(rawButton, [display, wheelDown, 0, 0], false); wheelDown = 0; sync(display, 0); };const symbolAt = x.func("ulong XKeycodeToKeysym(void *display, uint8_t code, int index)");const key = (name, down) => { const symbol = typeof name === "number" ? name : keysym(name); const code = keycode(display, symbol); if (!code) fail("unsupported_action", "This character is unavailable in the active keyboard layout. Use an app connector for arbitrary Unicode text."); if (down) downKeys.add(code); else downKeys.delete(code); if (!fakeKey(display, code, down ? 1 : 0, 0)) fail("input_unavailable", "X11 rejected keyboard input."); sync(display, 0); };return {@@ -382,12 +382,19 @@ function x11() {for (const p of points) {checked(); fakeMotion(display, -1, expected.bounds.x + Math.round(p.x), expected.bounds.y + Math.round(p.y), 0); sync(display, 0); checked(p);if (action.type === "drag" && !pointerDown) { pointerDown = true; fakeButton(display, 1, 1, 0); sync(display, 0); }- await new Promise((resolve) => setTimeout(resolve, action.type === "drag" ? 12 : 2));+ await new Promise((resolve) => setTimeout(resolve, action.type === "scroll" ? 32 : action.type === "drag" ? 12 : 2));}checked(points.at(-1));if (action.type === "scroll") {const button = action.axis === "horizontal" ? action.delta > 0 ? 7 : 6 : action.delta > 0 ? 5 : 4;- for (let i = 0; i < Math.abs(action.delta); i++) { checked(points.at(-1)); fakeButton(display, button, 1, 0); fakeButton(display, button, 0, 0); sync(display, 0); }+ // Let the app process pointer entry and distinct wheel ticks.+ // Keep releases tracked so person input or Stop cannot leave a button held.+ for (let i = 0; i < Math.abs(action.delta); i++) {+ checked(points.at(-1)); wheelDown = button; fakeButton(display, button, 1, 0); sync(display, 0);+ await new Promise((resolve) => setTimeout(resolve, 8));+ fakeButton(display, button, 0, 0); wheelDown = 0; sync(display, 0);+ await new Promise((resolve) => setTimeout(resolve, 12));+ }} else if (action.type !== "drag") {for (let i = 0; i < (action.type === "double_click" ? 2 : 1); i++) { checked(points.at(-1)); pointerDown = true; fakeButton(display, 1, 1, 0); fakeButton(display, 1, 0, 0); pointerDown = false; sync(display, 0); if (i === 0 && action.type === "double_click") await new Promise((resolve) => setTimeout(resolve, 60)); }}diff --git a/evals/packages/labs/fixtures/portable-computer-use-app.mjs b/evals/packages/labs/fixtures/portable-computer-use-app.mjsindex b9a19e43..e86499f4 100644--- a/evals/packages/labs/fixtures/portable-computer-use-app.mjs+++ b/evals/packages/labs/fixtures/portable-computer-use-app.mjs@@ -11,7 +11,7 @@ app.setPath("userData", process.argv.at(-1));const ready = app.whenReady().then(async () => {for (const [index, title] of ["Workspace window", "Other window"].entries()) {const window = new BrowserWindow({ width: 620, height: 450, x: 50 + index * 700, y: 120, title, webPreferences: { nodeIntegration: false, contextIsolation: true } });- const html = "<title>" + title + "</title>" + "<style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}#doubles{left:420px;width:150px}#scroll{position:absolute;left:420px;top:160px;width:150px;height:230px;overflow:auto;background:#ddddff}#drag{position:absolute;left:70px;top:290px;width:120px;height:55px;background:#008080;touch-action:none;user-select:none}</style><button id=\"increment\">Increment</button><input id=\"draft\" aria-label=\"Fixture draft\" value=\"Initial draft\"><p id=\"count\">0</p><button id=\"doubles\">Double click</button><div id=\"scroll\"><div style=\"height:1000px\">Scroll me</div></div><div id=\"drag\">Drag me</div><script>window.counter=0;window.pointer={doubles:0,held:false};document.getElementById(\"increment\").onclick=()=>{document.getElementById(\"count\").textContent=String(++window.counter)};document.getElementById(\"doubles\").ondblclick=()=>{document.getElementById(\"doubles\").textContent=String(++window.pointer.doubles)};const handle=document.getElementById(\"drag\");let start;handle.onpointerdown=e=>{window.pointer.held=true;start={x:e.clientX,y:e.clientY,left:handle.offsetLeft,top:handle.offsetTop};handle.setPointerCapture(e.pointerId)};handle.onpointermove=e=>{if(window.pointer.held&&e.buttons===1){handle.style.left=(start.left+e.clientX-start.x)+\"px\";handle.style.top=(start.top+e.clientY-start.y)+\"px\"}};handle.onpointerup=()=>{window.pointer.held=false};</script>";+ const html = "<title>" + title + "</title>" + "<style>body{margin:0;background:white;color:black;font:16px sans-serif}button{position:absolute;left:70px;top:70px;width:180px;height:55px}input{position:absolute;left:70px;top:160px;width:300px;height:40px;caret-color:transparent}#count{position:absolute;left:70px;top:230px}#doubles{left:420px;width:150px}#scroll{position:absolute;left:420px;top:160px;width:150px;height:230px;overflow:auto;background:#ddddff}#drag{position:absolute;left:70px;top:290px;width:120px;height:55px;background:#008080;touch-action:none;user-select:none}</style><button id=\"increment\">Increment</button><input id=\"draft\" aria-label=\"Fixture draft\" value=\"Initial draft\"><p id=\"count\">0</p><button id=\"doubles\">Double click</button><div id=\"scroll\"><div style=\"height:1000px\">Scroll me</div></div><div id=\"drag\">Drag me</div><script>window.counter=0;window.pointer={doubles:0,held:false};window.wheelEvents=[];document.addEventListener(\"wheel\",e=>{window.wheelEvents.push({x:e.clientX,y:e.clientY,deltaY:e.deltaY,ctrl:e.ctrlKey,shift:e.shiftKey,alt:e.altKey,target:e.target.id,parent:e.target.parentElement?.id});if(window.wheelEvents.length>20)window.wheelEvents.shift()},{passive:true});document.getElementById(\"increment\").onclick=()=>{document.getElementById(\"count\").textContent=String(++window.counter)};document.getElementById(\"doubles\").ondblclick=()=>{document.getElementById(\"doubles\").textContent=String(++window.pointer.doubles)};const handle=document.getElementById(\"drag\");let start;handle.onpointerdown=e=>{window.pointer.held=true;start={x:e.clientX,y:e.clientY,left:handle.offsetLeft,top:handle.offsetTop};handle.setPointerCapture(e.pointerId)};handle.onpointermove=e=>{if(window.pointer.held&&e.buttons===1){handle.style.left=(start.left+e.clientX-start.x)+\"px\";handle.style.top=(start.top+e.clientY-start.y)+\"px\"}};handle.onpointerup=()=>{window.pointer.held=false};</script>";await window.loadURL("data:text/html," + encodeURIComponent(html)); windows.push(window);}});@@ -20,6 +20,7 @@ createInterface({ input: channel }).on("line", async (line) => {try {await ready; let result;if (request.method === "state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ count: window.counter, draft: document.getElementById("draft").value })')));+ else if (request.method === "input_events") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript("window.wheelEvents")));else if (request.method === "pointer_state") result = await Promise.all(windows.map((w) => w.webContents.executeJavaScript('({ doubles: window.pointer.doubles, scrolled: document.getElementById("scroll").scrollTop > 0, dragged: document.getElementById("drag").offsetLeft > 170, released: !window.pointer.held })')));else if (request.method === "bounds") result = windows.map((w) => ({ bounds: w.getBounds(), contentBounds: w.getContentBounds(), scale: screen.getDisplayMatching(w.getBounds()).scaleFactor }));else if (request.method === "focus_other") { windows[1].focus(); result = {}; }diff --git a/evals/specs/computer-use-portable-window.e2e.test.ts b/evals/specs/computer-use-portable-window.e2e.test.tsindex 7b75d709..67194b62 100644--- a/evals/specs/computer-use-portable-window.e2e.test.ts+++ b/evals/specs/computer-use-portable-window.e2e.test.ts@@ -69,7 +69,7 @@ test("Windows and Linux X11 Computer Use keeps native input within the approvedawait act("double-click", async (observed) => ({ type: "double_click", ...await world.point(observed, 480, 95) }));await act("scroll", async (observed) => ({ type: "scroll", axis: "vertical", delta: 3, ...await world.point(observed, 480, 240) }));await act("drag", async (observed) => ({ type: "drag", path: await Promise.all([{ x: 120, y: 310 }, { x: 140, y: 315 }, { x: 180, y: 325 }, { x: 240, y: 330 }].map(({ x, y }) => world.point(observed, x, y))) }));- await expect.poll(() => world.pointerState()).toEqual([+ await expect.poll(() => world.pointerState(), { message: JSON.stringify(await world.inputEvents()) }).toEqual([{ doubles: 1, scrolled: true, dragged: true, released: true },{ doubles: 0, scrolled: false, dragged: false, released: true },]);diff --git a/evals/worlds/computer-use.ts b/evals/worlds/computer-use.tsindex 1effb90a..4ecace06 100644--- a/evals/worlds/computer-use.ts+++ b/evals/worlds/computer-use.ts@@ -361,6 +361,7 @@ export async function portableComputerUseWorld(_seed: Seed, { place }: { place:cover: () => fixture.request("cover"),outsideInput: () => fixture.request("outside_input"),pointerState: () => fixture.request("pointer_state"),+ inputEvents: () => fixture.request("input_events"),imagePixel(reply: unknown) {if (!record(reply) || !Array.isArray(reply.content)) throw new Error("Missing image content.");const image = reply.content.find((item: unknown) => record(item) && item.type === "image");