Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Follow-ups keep their session and stop after a cancel (LPT-915)

alexshpunt-clanker[bot]3ec69ac34d32
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index bc0240c..0877330 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -56,7 +56,9 @@ const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "do
* by default; `explicit` means the agent asked for it (`successor: true` or `expect`), which also overrides the
* defaults that skip it. `frame` holds the window's pixels before the action, when the input observation had them.
*/
-interface FollowUp { expect?: WaitSpec; successor: boolean; explicit: boolean; preexisting?: boolean; frame?: WindowFrame }
+interface FollowUp { expect?: WaitSpec; successor: boolean; explicit: boolean; preexisting?: boolean; frame?: WindowFrame;
+ /** The window's session, resolved before the action: a failed action can make the session forget the window. */
+ session?: ComputerSession }
/** How long a native successor keeps looking for a change before reporting that nothing changed. */
const SETTLE_MS = 300;
/**
@@ -844,8 +846,9 @@ export class ComputerResources {
/** Keep the input observation's own pixels as the baseline of the pixel comparison; they must be taken before the action. */
private async withFrame(follow: FollowUp, window: string, observation: string | undefined, cwd: string): Promise<FollowUp> {
if (!follow.successor) return follow;
- const frame = (await this.sessionFor(cwd, { window_ref: window })).windowFrame(window);
- return frame && frame.observation_id === observation ? { ...follow, frame } : follow;
+ const session = await this.sessionFor(cwd, { window_ref: window });
+ const frame = session.windowFrame(window);
+ return { ...follow, session, ...(frame && frame.observation_id === observation ? { frame } : {}) };
}
/** Validate an action's optional `expect` postcondition and `successor` request before anything is dispatched. */
@@ -871,7 +874,9 @@ export class ComputerResources {
// A default successor stays quiet when there is nothing to compare: a refusal sent nothing, and a browser
// action that returned no page snapshot would otherwise be compared with a native walk of the browser window.
if (!follow.explicit && (status === "refused" || (before.result.details.dom && !result.details.dom))) return result;
- const session = await this.sessionFor(cwd, { window_ref: window });
+ // A cancelled call returns its receipt without capturing again.
+ if (signal?.aborted) return result;
+ const session = follow.session ?? await this.sessionFor(cwd, { window_ref: window });
const preexisting = follow.preexisting === true;
let successor: ComputerResult | undefined, settled: StateDiff | undefined, crops: ComputerResult["content"] = [];
let postcondition: ComputerResult["details"]["postcondition"];
diff --git a/test/live/locators.live.ts b/test/live/locators.live.ts
index 4e3ab84..d2f96d7 100644
--- a/test/live/locators.live.ts
+++ b/test/live/locators.live.ts
@@ -46,7 +46,7 @@ test("a WinForms window is driven by Playwright locators, one call per step", as
// Auto-wait: Later is disabled for 1.5 s; the click waits for it instead of failing.
// Later has no click handler, so its change list says nothing changed (LPT-914).
- await expectStatus({ action: "click", target: `getByRole("button", { name: "Later" })`, timeout_ms: 5000 }, "delivered", /Nothing changed in the UI within \d+ ms after the action/);
+ await expectStatus({ action: "click", target: `getByRole("button", { name: "Later" })`, timeout_ms: 5000 }, "delivered", /Nothing changed in the UI or the window's pixels within \d+ ms after the action/);
// Every action reports what it changed: the renamed button and the status text, without a Read.
await expectStatus({ action: "click", target: `getByRole("button", { name: "Count 0" })` }, "delivered",
/~ \S+ Button "Count 0" → "Count 1"[\s\S]*~ \S+ Text "idle" → "clicked 1"|~ \S+ Text "idle" → "clicked 1"[\s\S]*~ \S+ Button "Count 0" → "Count 1"/);
diff --git a/test/live/ui-changes.live.ts b/test/live/ui-changes.live.ts
index 816dfbb..362e9eb 100644
--- a/test/live/ui-changes.live.ts
+++ b/test/live/ui-changes.live.ts
@@ -46,5 +46,5 @@ test("a changes re-read lists only the changed rows and their hints ground input
// A changes Read after an action compares with that action's successor, so a repeated click shows no change.
const after = await live.read(window, ["changes"]);
expect(after.details.changes!.since).toBe(again.details.successor!.observation_id);
- expect(presentComputerResult(after)).toMatch(/Nothing changed in the UI since that observation\./);
+ expect(presentComputerResult(after)).toMatch(/Nothing changed in the UI or the window's pixels since that observation\./);
});