Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Pin runtime c920b66b: click and scroll take uia_only; live checks of the input routes (LPT-942)

alexshpunt-clanker[bot]0a86636ac46f
diff --git a/src/driver.ts b/src/driver.ts
index d25e0e9..8fc2b9e 100644
--- a/src/driver.ts
+++ b/src/driver.ts
@@ -86,8 +86,8 @@ export const WINDOW_RESTORE_RUNTIME = {
} as const;
/** Window management plus every earlier native capability; `windows-desktop` selects these bytes. Browser pages use Playwright, not this runtime. */
export const WINDOW_MANAGEMENT_RUNTIME = {
- source: "f754b1046ac1d6807f63365a8ebad10cb10e0de4", version: "0.37.0",
- sha256: "41143483eed9f16d8ea7a1c4f1b5ae13ad38918b490b4dea4c2bd89fa94e00e4",
+ source: "c920b66bd60d064aa0d9f5662e9b30f1648222fd", version: "0.37.0",
+ sha256: "7a633781120a6648dca16a332cc302f42e0137db5f874e1d1dd7d4d17d727120",
} as const;
const forkRuntimes = {
"windows-desktop": WINDOW_MANAGEMENT_RUNTIME,
diff --git a/src/runtime-pins.json b/src/runtime-pins.json
index 3d4b9b5..8fe69ab 100644
--- a/src/runtime-pins.json
+++ b/src/runtime-pins.json
@@ -1,12 +1,12 @@
{
"format": 2,
- "source": "f754b1046ac1d6807f63365a8ebad10cb10e0de4",
+ "source": "c920b66bd60d064aa0d9f5662e9b30f1648222fd",
"version": "0.37.0",
- "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-f754b104/",
+ "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-c920b66b/",
"files": {
- "cua-driver.exe": "41143483eed9f16d8ea7a1c4f1b5ae13ad38918b490b4dea4c2bd89fa94e00e4",
- "cua-driver-uia.exe": "85e8b6982fcd103306f48c21ef5ac0dae93fbf4fb13cbb91314b9e88ca98ac79",
- "cua-cursor-theme.exe": "eb8afb30ddae9a4d08855952c1fddb708ab7aba50f4734c67f463725c783d108",
+ "cua-driver.exe": "7a633781120a6648dca16a332cc302f42e0137db5f874e1d1dd7d4d17d727120",
+ "cua-driver-uia.exe": "7789da185c8d4c62da8adf245825311e48f914880580c996d7c30552917159d6",
+ "cua-cursor-theme.exe": "9ca25534dd2d2c95088521e0547863482f05e37591ff699531796dd5f3a433b0",
"LICENSE.md": "d0a32419a44fa38d5023d1431dfcbedd1457eb5e7b2e6c7c87ca064facb41b41",
"THIRD_PARTY_NOTICES.md": "88a24f59293f03b3e03655e8144bcfadc9861bef1182cfeea91639e11f573fd1"
}
diff --git a/test/live/quick-fixes.live.ts b/test/live/quick-fixes.live.ts
index cd87ad4..e3962f8 100644
--- a/test/live/quick-fixes.live.ts
+++ b/test/live/quick-fixes.live.ts
@@ -45,8 +45,11 @@ test("a point click after a refused region Read captures the window again", asyn
try {
const view = await live.read(window, ["image"]);
await live.read(`computer:observation/${view.details.observation_id}`, ["region:5000,5000,10,10"]).catch(() => undefined);
- const clicked = await live.resources.input(requireComputerAddress(window), live.cwd, clickPlan({ window, x: 60, y: 20 }));
- expect(clicked.details.status, text(clicked)).toBe("delivered");
+ const clicked = await live.resources.input(requireComputerAddress(window), live.cwd, clickPlan({ window, x: 200, y: 250 }));
+ // The click is real input; right after an app starts, Windows can still hand the foreground back to another window,
+ // and the driver then answers unknown. What this checks is that the spent observation no longer refuses the click.
+ expect(clicked.details.status, text(clicked)).not.toBe("refused");
+ expect(text(clicked)).not.toMatch(/expired/);
} finally {
await live.resources.delete(`${window}?force=true`, live.cwd);
}
diff --git a/test/live/typed-input.live.ts b/test/live/typed-input.live.ts
index dc8f1f2..7c37220 100644
--- a/test/live/typed-input.live.ts
+++ b/test/live/typed-input.live.ts
@@ -50,10 +50,11 @@ test("one action vocabulary drives a native window by locator, ref and point", a
expect(await holds(window, [{ target: field, value: "hello" }])).toBe("satisfied");
expect((await act(window, typePlan({ window, target: field, text: "world", replace: true, submit: true }))).result.details.status).toBe("delivered");
expect(await holds(window, [{ target: field, value: "world" }])).toBe("satisfied");
- expect((await act(window, pressPlan({ window, key: "ctrl+a" }))).result.details.status).toBe("delivered");
- expect((await act(window, pressPlan({ window, key: "Backspace" }))).result.details.status).toBe("delivered");
+ // Keys are real key presses, so the app behaves as for a person: a WinForms TextBox ignores ctrl+a. End clears the
+ // selection replace left, and shift+Home selects the text again only if the chord arrives as a chord: a bare Home
+ // would leave the caret at the start and Backspace would delete nothing.
+ for (const key of ["End", "shift+Home", "Backspace"]) expect((await act(window, pressPlan({ window, key }))).result.details.status).toBe("delivered");
// An empty field's value falls back to its name in conditions, so its aria line shows the emptiness: no ": value".
- // A chord must reach the app as a chord: a bare "a" would leave "worlda" behind instead of an empty field.
const field_line = /textbox "Field".*$/m.exec(presentComputerResult(await live.read(window, ["aria"])))![0];
expect(field_line).toMatch(/^textbox "Field" \[ref=[0-9a-z]+:[a-z]+\]$/);
expect((await act(window, selectPlan({ window, target: "getByRole('list', { name: 'Pick' })", values: ["beta"] }))).result.details.status).toBe("delivered");
@@ -69,6 +70,29 @@ test("one action vocabulary drives a native window by locator, ref and point", a
} finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
});
+// LPT-942: what the agent sees structurally is pressed structurally; without a pattern the same call uses the real pointer.
+test("a ref is pressed through UI Automation without taking the foreground, and a ref without a pattern gets the real pointer", async () => {
+ const title = `pi-route-${Date.now()}`;
+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, nativeForm(title), title);
+ try {
+ const foreground = async () => ((await live.read(window, ["state"])).details.state?.state as { foreground_at_target: boolean }).foreground_at_target;
+ const refOf = async (pattern: RegExp) => pattern.exec(presentComputerResult(await live.read(window, ["ui"])))![1];
+ const button = await refOf(/^([0-9a-z]+:[a-z]+) \| Button \| Count 0/m);
+ const structural = await act(window, clickPlan({ window, target: button }));
+ expect(structural.result.details.status, structural.body).toBe("delivered");
+ expect(structural.body).not.toContain("real pointer");
+ expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 1' })", is: "exists" }])).toBe("satisfied");
+ // UI Automation pressed the button in the background: the window never came to the front.
+ expect(await foreground()).toBe(false);
+ const field = await refOf(/^([0-9a-z]+:[a-z]+) \| Edit \| Field/m);
+ const real = await act(window, clickPlan({ window, target: field }));
+ expect(real.result.details.status, real.body).not.toBe("refused");
+ expect(real.body).toContain("No UI Automation pattern of the target carried this click, so this call used the real pointer.");
+ } finally {
+ await deleteComputer(live.resources, `${window}?force=true`, { path: `${window}?force=true` }, live.cwd);
+ }
+});
+
test("points of a native canvas draw strokes that the receipt shows as changed pixels", async () => {
const opened = await live.insert(live.machine, { action: "open", app: "Paint" });
expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");