Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

test: integration tests expect real input for points, keys and drags (LPT-942)

alexshpunt-clanker[bot]b19975b48ff3
diff --git a/test/complete-input.integration.test.ts b/test/complete-input.integration.test.ts
index cfb66be..aef9ed4 100644
--- a/test/complete-input.integration.test.ts
+++ b/test/complete-input.integration.test.ts
@@ -5,7 +5,7 @@ import { assistantMessage, getToolExecution, getToolExecutions, getToolResultTex
import { expect, test } from "vitest";
import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
-test("real Pi sends keys, chords and drags in the background and keeps partial and refused outcomes apart", async () => {
+test("real Pi sends keys, chords and drags as real input and keeps partial and refused outcomes apart", async () => {
await mkdir(".tmp/cua-gestures", { recursive: true });
const cwd = await mkdtemp(path.resolve(".tmp/cua-gestures/pi-contract-")); const log = path.join(cwd, "calls.jsonl");
await mkdir(path.join(cwd, ".pi"));
@@ -23,8 +23,8 @@ if (drag.outcome !== "delivered") throw Error("Drag refused: " + drag.text);
const partial = await press("escape");
if (partial.outcome !== "unknown" || partial.effect !== "unknown") throw Error("Partial key reported as " + partial.outcome);
text(partial.text);
-const refused = await press("delete", {foreground:false});
-if (refused.outcome !== "refused" || refused.effect !== "not-applied") throw Error("Background refusal reported as " + refused.outcome);
+const refused = await press("delete");
+if (refused.outcome !== "refused" || refused.effect !== "not-applied") throw Error("Native refusal reported as " + refused.outcome);
text(refused.text);
text("Complete sequence guards passed");`;
try {
@@ -53,7 +53,8 @@ text("Complete sequence guards passed");`;
expect(input.map(c => c.name)).toEqual(["press_key", "hotkey", "drag", "press_key", "press_key"]);
expect(input.map(c => c.arguments.key ?? c.arguments.keys)).toEqual(["a", ["shift", "b"], undefined, "escape", "delete"]);
expect(input.map(c => c.arguments.target)).toEqual(Array(5).fill({ kind: "window", pid: 42, window_id: 91 }));
- expect(input.every(c => c.arguments.delivery_mode === "background")).toBe(true);
+ // Keys and drags are real input: every one goes in the foreground, and a refused key is not retried.
+ expect(input.every(c => c.arguments.delivery_mode === "foreground")).toBe(true);
expect(calls.some(c => c.name === "bring_to_front")).toBe(false);
} finally { await rm(cwd, { recursive: true, force: true }); }
});
diff --git a/test/coordinate-scroll.integration.test.ts b/test/coordinate-scroll.integration.test.ts
index 5761780..3f57628 100644
--- a/test/coordinate-scroll.integration.test.ts
+++ b/test/coordinate-scroll.integration.test.ts
@@ -8,7 +8,7 @@ const root = path.resolve(import.meta.dirname, "..");
const fixture = (name: string) => path.join(root, "test/fixtures", name);
// Pattern: real-Pi extensions example and the ordinary IDE computer-provider scenarios.
-// Contract: computer_scroll preserves capture authority and structured uncertain prefixes; IDE Insert refuses.
+// Contract: computer_scroll at a point is real input that preserves capture authority and structured uncertain prefixes; IDE Insert refuses.
async function scenario(testName: string, scenario: "receipts" | "authority", providerFirst: boolean) {
const consumer = IDE_EXTENSION, provider = fixture("coordinate-scroll-extension.ts");
await mkdir(path.join(root, ".tmp/lpt-730"), { recursive: true });
@@ -45,12 +45,12 @@ const native = (x: number, y: number, delta_x: number, delta_y: number, delivery
test.each([false, true])("ordinary IDE scroll preserves capture-bound deltas and uncertain prefixes (providerFirst=%s)", async providerFirst => {
const actions = inputCalls(await scenario("coordinate-scroll-receipts", "receipts", providerFirst));
- expect(actions.map(call => call.name)).toEqual(Array(10).fill("scroll_at"));
+ // Every scroll at a point is real input in the foreground.
+ expect(actions.map(call => call.name)).toEqual(Array(7).fill("scroll_at"));
expect(actions.map(call => call.arguments)).toEqual([
- native(100, 80, 0, 2, "background"), native(100, 80, -2, 3, "background"), native(100, 80, -2, 3), native(120, 90, 1, 0),
- ...Array.from({ length: 5 }, () => native(130, 100, 1, 1)), native(130, 100, 1, 1, "background"),
+ native(100, 80, -2, 3), native(120, 90, 1, 0), ...Array.from({ length: 5 }, () => native(130, 100, 1, 1)),
]);
- // The refusal spent its capture, so the foreground retry captures the window again: every scroll has its own capture.
+ // Every scroll binds to its own capture.
expect(new Set(actions.map(call => call.arguments.capture_id)).size).toBe(actions.length);
}, 120_000);
diff --git a/test/fixtures/coordinate-scroll-extension.ts b/test/fixtures/coordinate-scroll-extension.ts
index 8e387d4..7b319b4 100644
--- a/test/fixtures/coordinate-scroll-extension.ts
+++ b/test/fixtures/coordinate-scroll-extension.ts
@@ -111,24 +111,17 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
const image = view.content.find(block => block.type === "image");
assert(image?.mimeType === "image/png" && image.data, "Native overview image did not reach actual Pi");
if (scenario === "receipts") {
- // foreground: false keeps a background refusal as it is.
- const refused = await rejected(first, { foreground: false }), refusal = semantic(refused), refusalState = record(refusal.state);
- assert(refusal.status === "refused" && refusal.effect === "not-applied" && refusalState.code === "background_unavailable" && refusalState.pointer_attempted === false,
- "Structured background refusal lost its no-input evidence");
- // A refusal spends the observation; by default the same refusal sends the scroll again in the foreground, and the
- // window keeps foreground delivery.
- await observe(first);
+ // A scroll at a point is real input: the wheel goes in the foreground at once.
const delivered = await scroll(first, { delta_x: -2, delta_y: 3 }), receipt = semantic(delivered), state = record(receipt.state);
assert(!delivered.isError && receipt.status === "delivered" && receipt.effect === "applied" && state.unit === "wheel_detents" &&
JSON.stringify(state.delivered_axes) === '["y","x"]' && state.pointer_changed === null && record(receipt.action_capability).actual_delivery === "foreground",
`Truthful two-axis structured receipt lost: ${textOf(delivered)}`);
- assert(textOf(delivered).includes("refused background input"), "The foreground retry was not reported");
await observe(first, "image");
const imageOnly = await scroll(first, { x: 120, y: 90, delta_x: 1, delta_y: 0 });
assert(!imageOnly.isError && semantic(imageOnly).status === "delivered", "Image-only scroll refused");
- for (const operation of ["partial", "last_check", "wrong_unit", "wrong_delta", "wrong_axes", "background_success"]) {
+ for (const operation of ["partial", "last_check", "wrong_unit", "wrong_delta", "wrong_axes"]) {
await control(operation); view = await observe(first);
- const result = await rejected(first, { x: 130, y: 100, delta_x: 1, delta_y: 1, ...(operation === "background_success" ? { foreground: false } : {}) });
+ const result = await rejected(first, { x: 130, y: 100, delta_x: 1, delta_y: 1 });
const action = semantic(result), native = record(action.state);
assert(action.status === "unknown" && action.effect === "unknown", `${operation} lost structured uncertainty`);
if (operation === "partial") assert(native.pointer_delivered === true && JSON.stringify(native.attempted_axes) === '["y","x"]' &&
@@ -147,18 +140,18 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
assert(cropped.content.some(block => block.type === "image" && block.mimeType === "image/jpeg") && field(cropped.details, "input") === "none" &&
field(cropped.details, "observation_id") === undefined, "Detail granted input authority");
await control("capability_off"); await observe(first);
- await rejected(first, { foreground: true });
+ await rejected(first);
await control("capability_on");
const external = reference(await ok("search", { path: "computer:machine/local/windows", query: "Unowned fixture" }), "window");
await observe(external); await control("safe");
- await rejected(external, { foreground: true });
+ await rejected(external);
await observe(first, "image");
- const owned = await scroll(first, { foreground: true });
+ const owned = await scroll(first);
assert(!owned.isError && semantic(owned).status === "delivered", "Safe denied owned image scroll");
await observe(first); await control("close_first");
- await rejected(first, { foreground: true });
+ await rejected(first);
await observe(second); await control("recycle_second");
- await rejected(second, { foreground: true });
+ await rejected(second);
// Insert never acts on a window; it names the tool that does.
const insert = await denied("insert", { path: first, text: JSON.stringify({ action: "scroll", x: 100, y: 80, delta_y: 2, unit: "wheel_detents" }) });
assert(textOf(insert).includes("computer_scroll"), "Insert refusal named no tool");
diff --git a/test/fixtures/ide-provider-extension.ts b/test/fixtures/ide-provider-extension.ts
index d51c6aa..e2aed0a 100644
--- a/test/fixtures/ide-provider-extension.ts
+++ b/test/fixtures/ide-provider-extension.ts
@@ -121,7 +121,7 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
const hint = hintOf(view), literal = ' {"looks":"like JSON"}\n"quoted text"\n ';
await refused("write", { path: reference(view, "observation"), content: literal }, "computer_type");
const nativeBeforeType = (await journal()).length;
- await ok("computer_type", { window: first, target: hint, text: literal, delay_ms: 0, foreground: true });
+ await ok("computer_type", { window: first, target: hint, text: literal, delay_ms: 0 });
assert(!await capturedBeforeInput(nativeBeforeType), "computer_type refreshed native capture");
const written = (await inputs()).at(-1);
assert(written?.name === "type_text" && written.arguments.text === literal && written.arguments.delay_ms === 0, "Literal typing normalized text, added a command or lost delay");
diff --git a/test/held-path.integration.test.ts b/test/held-path.integration.test.ts
index f46bee2..c6ddad2 100644
--- a/test/held-path.integration.test.ts
+++ b/test/held-path.integration.test.ts
@@ -14,7 +14,7 @@ test("real Pi draws a held path through its points and preserves a partial strok
}));
const code = `${SCRIPT_HELPERS}
const opened = await openApp("Notepad");
-const stopped = await tools.computer_drag({window:opened.window,foreground:true,from:{x:20,y:30},via:[{x:40,y:80},{x:90,y:70}],to:{x:100,y:30},duration_ms:800});
+const stopped = await tools.computer_drag({window:opened.window,from:{x:20,y:30},via:[{x:40,y:80},{x:90,y:70}],to:{x:100,y:30},duration_ms:800});
if(stopped.outcome !== "unknown" || stopped.effect !== "unknown") throw Error("Lost partial path: " + stopped.text);
text(stopped.text);
text("Held path guards passed");`;
diff --git a/test/key-hold.integration.test.ts b/test/key-hold.integration.test.ts
index bae8c18..551260e 100644
--- a/test/key-hold.integration.test.ts
+++ b/test/key-hold.integration.test.ts
@@ -14,7 +14,7 @@ test("real Pi holds keys for a bounded time and preserves release uncertainty",
}));
const code = `${SCRIPT_HELPERS}
const opened = await openApp("Notepad");
-const press = command => tools.computer_press({window:opened.window,foreground:true,hold_ms:500,...command});
+const press = command => tools.computer_press({window:opened.window,hold_ms:500,...command});
const complete = await press({key:"shift+w"});
if(complete.outcome !== "delivered" || !complete.text.includes("keys_released=true")) throw Error("Complete hold receipt lost: " + complete.text);
const partial = await press({key:"escape"});