Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

test: coordinate scroll integration test uses computer_scroll (LPT-934)

alexshpunt-clanker[bot]d431014e4997
diff --git a/test/coordinate-scroll.integration.test.ts b/test/coordinate-scroll.integration.test.ts
index d54da4a..5761780 100644
--- a/test/coordinate-scroll.integration.test.ts
+++ b/test/coordinate-scroll.integration.test.ts
@@ -8,7 +8,7 @@ const root = path.resolve(import.meta.dirname, "..");
const fixture = (name: string) => path.join(root, "test/fixtures", name);
// Pattern: real-Pi extensions example and the ordinary IDE computer-provider scenarios.
-// Contract: actual IDE Insert preserves capture authority and structured uncertain prefixes.
+// Contract: computer_scroll preserves capture authority and structured uncertain prefixes; IDE Insert refuses.
async function scenario(testName: string, scenario: "receipts" | "authority", providerFirst: boolean) {
const consumer = IDE_EXTENSION, provider = fixture("coordinate-scroll-extension.ts");
await mkdir(path.join(root, ".tmp/lpt-730"), { recursive: true });
@@ -21,7 +21,7 @@ async function scenario(testName: string, scenario: "receipts" | "authority", pr
artifactsDir: testArtifactsDir(import.meta.filename), cwd, rawMode: false,
extensions: providerFirst ? [provider, consumer] : [consumer, provider],
isolateUserResources: true, piCommand: PI_COMMAND, timeoutMs: 90_000,
- tools: ["read", "search", "insert", "verify_coordinate_scroll"],
+ tools: ["read", "search", "insert", "delete", "verify_coordinate_scroll"],
conversation: [assistantMessage([toolCall({ id: "scroll", name: "verify_coordinate_scroll", arguments: { scenario } })], { stopReason: "toolUse" }),
assistantMessage([text("Finished the synthetic coordinate-scroll fixture.")])],
}).run("Exercise ordinary IDE coordinate scroll against synthetic MCP windows only. Never operate a real desktop or replay unknown input.");
@@ -45,11 +45,12 @@ const native = (x: number, y: number, delta_x: number, delta_y: number, delivery
test.each([false, true])("ordinary IDE scroll preserves capture-bound deltas and uncertain prefixes (providerFirst=%s)", async providerFirst => {
const actions = inputCalls(await scenario("coordinate-scroll-receipts", "receipts", providerFirst));
- expect(actions.map(call => call.name)).toEqual(Array(9).fill("scroll_at"));
+ expect(actions.map(call => call.name)).toEqual(Array(10).fill("scroll_at"));
expect(actions.map(call => call.arguments)).toEqual([
- native(100, 80, 0, 2, "background"), native(100, 80, -2, 3), native(120, 90, 1, 0),
+ native(100, 80, 0, 2, "background"), native(100, 80, -2, 3, "background"), native(100, 80, -2, 3), native(120, 90, 1, 0),
...Array.from({ length: 5 }, () => native(130, 100, 1, 1)), native(130, 100, 1, 1, "background"),
]);
+ // The refusal spent its capture, so the foreground retry captures the window again: every scroll has its own capture.
expect(new Set(actions.map(call => call.arguments.capture_id)).size).toBe(actions.length);
}, 120_000);
diff --git a/test/fixtures/coordinate-scroll-extension.ts b/test/fixtures/coordinate-scroll-extension.ts
index fc8b02b..8e387d4 100644
--- a/test/fixtures/coordinate-scroll-extension.ts
+++ b/test/fixtures/coordinate-scroll-extension.ts
@@ -5,6 +5,7 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { loadComputerConfig } from "../../src/config.ts";
import { McpDriver } from "../../src/driver.ts";
import { registerComputerProvider } from "../../src/provider.ts";
+import { registerComputerTools } from "../../src/computer-tools.ts";
import { ComputerSession, type ComputerMode, type ComputerResult, type WindowTarget } from "../../src/session.ts";
interface Result { isError?: boolean; content: { type: string; text?: string; data?: string; mimeType?: string }[]; details?: unknown }
@@ -50,7 +51,8 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
const close = async (): Promise<ComputerResult> => session ? session.close() : {
content: [{ type: "text", text: "No synthetic runtime is running." }], details: { status: "closed", mode },
};
- await registerComputerProvider(pi, getSession, close);
+ const provider = await registerComputerProvider(pi, getSession, close);
+ registerComputerTools(pi, provider.resources);
pi.registerTool({
name: "verify_coordinate_scroll", label: "Verify coordinate scroll fixture",
description: "Exercise ordinary IDE resource tools against synthetic MCP windows only; never operate a desktop.",
@@ -86,99 +88,82 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
else if (operation === "capability_on") capability = true;
else await driver!.call("fixture_control", { operation }, signal);
};
- const open = async () => reference(await ok("insert", { path: "computer:machine/local", text: JSON.stringify({ action: "open", app: "Notepad" }) }), "window");
+ const open = async () => reference(await ok("computer_open", { at: "computer:machine/local", app: "Notepad" }), "window");
const observe = (window: string, granularity = "full") => ok("read", { path: window,
views: [granularity, ...(granularity === "ui" ? [] : ['capture:{"hints":false}'])] });
- const command = (extra: Record<string, unknown> = {}) => ({ action: "scroll", x: 100, y: 80, delta_y: 2, unit: "wheel_detents", ...extra });
- const act = async (observation: string, command: Record<string, unknown>) => {
+ // A point scroll of computer_scroll: wheel notches at a point of the window's latest image.
+ const scroll = async (window: string, extra: Record<string, unknown> = {}) => {
const before = (await journal()).length;
- const result = await call("insert", { path: observation, text: JSON.stringify(command) });
- assert(!await capturedBeforeInput(before), "Insert resolution silently refreshed native capture authority");
+ const result = await call("computer_scroll", { window, x: 100, y: 80, delta_y: 2, ...extra });
+ assert(!await capturedBeforeInput(before), "computer_scroll silently refreshed native capture authority");
return result;
};
- const rejected = async (observation: string, command: Record<string, unknown>) => {
- const result = await act(observation, command);
- assert(result.isError, `Insert admitted a forbidden or consumed observation: ${textOf(result)}`);
+ const rejected = async (window: string, extra: Record<string, unknown> = {}) => {
+ const result = await scroll(window, extra);
+ assert(result.isError, `computer_scroll admitted a forbidden or uncertain request: ${textOf(result)}`);
return result;
};
- assert(!pi.getAllTools().some(tool => /^computer_/.test(tool.name)), "Provider exposed duplicate ordinary computer_* tools");
+ assert(!pi.getAllTools().some(tool => /^computer_(read|search|delete)$/.test(tool.name)), "Provider exposed duplicate ordinary computer tools");
const first = await open(), second = await open();
const listed = await ok("search", { path: "computer:machine/local/windows", query: "Owned fixture" });
assert(textOf(listed).includes(first) && textOf(listed).includes(second), "Search lost exact opened resources");
- let view = await observe(first), observation = reference(view, "observation");
+ let view = await observe(first);
const image = view.content.find(block => block.type === "image");
assert(image?.mimeType === "image/png" && image.data, "Native overview image did not reach actual Pi");
if (scenario === "receipts") {
- const background = command();
- const refused = await rejected(observation, background), refusal = semantic(refused), refusalState = record(refusal.state);
+ // foreground: false keeps a background refusal as it is.
+ const refused = await rejected(first, { foreground: false }), refusal = semantic(refused), refusalState = record(refusal.state);
assert(refusal.status === "refused" && refusal.effect === "not-applied" && refusalState.code === "background_unavailable" && refusalState.pointer_attempted === false,
"Structured background refusal lost its no-input evidence");
- await rejected(observation, { ...background, delivery_mode: "foreground" });
- view = await observe(first); observation = reference(view, "observation");
- const full = command({ delta_x: -2, delta_y: 3, delivery_mode: "foreground" });
- const delivered = await act(observation, full), receipt = semantic(delivered), state = record(receipt.state);
+ // A refusal spends the observation; by default the same refusal sends the scroll again in the foreground, and the
+ // window keeps foreground delivery.
+ await observe(first);
+ const delivered = await scroll(first, { delta_x: -2, delta_y: 3 }), receipt = semantic(delivered), state = record(receipt.state);
assert(!delivered.isError && receipt.status === "delivered" && receipt.effect === "applied" && state.unit === "wheel_detents" &&
JSON.stringify(state.delivered_axes) === '["y","x"]' && state.pointer_changed === null && record(receipt.action_capability).actual_delivery === "foreground",
- "Truthful two-axis structured receipt lost");
- await rejected(observation, full);
- view = await observe(first, "image"); observation = reference(view, "observation");
- const imageOnly = await act(observation, command({ x: 120, y: 90, delta_x: 1, delta_y: 0, delivery_mode: "foreground" }));
+ `Truthful two-axis structured receipt lost: ${textOf(delivered)}`);
+ assert(textOf(delivered).includes("refused background input"), "The foreground retry was not reported");
+ await observe(first, "image");
+ const imageOnly = await scroll(first, { x: 120, y: 90, delta_x: 1, delta_y: 0 });
assert(!imageOnly.isError && semantic(imageOnly).status === "delivered", "Image-only scroll refused");
for (const operation of ["partial", "last_check", "wrong_unit", "wrong_delta", "wrong_axes", "background_success"]) {
- await control(operation); view = await observe(first); observation = reference(view, "observation");
- const request = command({ x: 130, y: 100, delta_x: 1, delta_y: 1, delivery_mode: operation === "background_success" ? "background" : "foreground" });
- const result = await rejected(observation, request), action = semantic(result), native = record(action.state);
+ await control(operation); view = await observe(first);
+ const result = await rejected(first, { x: 130, y: 100, delta_x: 1, delta_y: 1, ...(operation === "background_success" ? { foreground: false } : {}) });
+ const action = semantic(result), native = record(action.state);
assert(action.status === "unknown" && action.effect === "unknown", `${operation} lost structured uncertainty`);
if (operation === "partial") assert(native.pointer_delivered === true && JSON.stringify(native.attempted_axes) === '["y","x"]' &&
JSON.stringify(native.delivered_axes) === '["y"]' && native.code === "cancelled", "Interrupted native prefix lost");
if (operation === "last_check") assert(JSON.stringify(native.delivered_axes) === '["y","x"]' && native.code === "foreground_lost",
"Complete native prefix lost after final check failed");
- await rejected(observation, request);
}
await control("success");
- for (const extra of [
- { delta_y: 51 }, { delta_y: 1.5 }, { delta_x: -1.5 }, { delta_x: 0, delta_y: 0 }, { unit: "pixels" }, { unit: undefined }, { x: 500 }, { x: -1 }, { y: undefined },
- { direction: "down" }, { by: "line" }, { amount: 1 }, { button: "left" }, { modifiers: ["ctrl"] }, { animation: "reactive" },
- ]) {
- view = await observe(first); observation = reference(view, "observation");
- await rejected(observation, command({ delivery_mode: "foreground", ...extra }));
- }
- view = await observe(first); observation = reference(view, "observation");
- await rejected(observation, { action: "sequence", sequence: [command({ delivery_mode: "foreground" })] });
+ const sent = (await journal()).filter(row => row.name === "scroll_at").length;
+ for (const extra of [{ delta_y: 51 }, { delta_x: 0, delta_y: 0 }, { x: -1 }, { y: undefined }, { unit: "pixels" }, { direction: "down" },
+ { amount: 1 }, { button: "left" }, { modifiers: ["ctrl"] }]) await rejected(first, extra);
+ assert((await journal()).filter(row => row.name === "scroll_at").length === sent, "An invalid scroll reached native input");
} else {
- await rejected(observation, command({ window_ref: second, delivery_mode: "foreground" }));
- view = await observe(first, "ui"); observation = reference(view, "observation");
- assert(!view.content.some(block => block.type === "image") && field(view.details, "ui"), "UI-only fixture observation lost its projection");
- await rejected(observation, command({ delivery_mode: "foreground" }));
- view = await observe(first); observation = reference(view, "observation");
- const hint = field(view.details, "hint");
- assert(typeof hint === "string", "Native fixture failed to supply a current hint");
- await rejected(observation, command({ hint, delivery_mode: "foreground" }));
- view = await observe(first); observation = reference(view, "observation");
- const token = field(view.details, "element_token");
- assert(typeof token === "string", "Native fixture failed to supply a current token");
- await rejected(observation, command({ element_token: token, delivery_mode: "foreground" }));
- view = await observe(first); observation = reference(view, "observation");
- const cropped = await ok("read", { path: observation, views: ["region:50,50,100,100"] });
+ // A detail crop is readonly: it grants no input and issues no observation.
+ const cropped = await ok("read", { path: reference(view, "observation"), views: ["region:50,50,100,100"] });
assert(cropped.content.some(block => block.type === "image" && block.mimeType === "image/jpeg") && field(cropped.details, "input") === "none" &&
field(cropped.details, "observation_id") === undefined, "Detail granted input authority");
- await rejected(observation, command({ delivery_mode: "foreground" }));
- await control("capability_off"); view = await observe(first); observation = reference(view, "observation");
- await rejected(observation, command({ delivery_mode: "foreground" }));
+ await control("capability_off"); await observe(first);
+ await rejected(first, { foreground: true });
await control("capability_on");
const external = reference(await ok("search", { path: "computer:machine/local/windows", query: "Unowned fixture" }), "window");
- view = await observe(external); observation = reference(view, "observation"); await control("safe");
- await rejected(observation, command({ delivery_mode: "foreground" }));
- view = await observe(first, "image"); observation = reference(view, "observation");
- const owned = await act(observation, command({ delivery_mode: "foreground" }));
+ await observe(external); await control("safe");
+ await rejected(external, { foreground: true });
+ await observe(first, "image");
+ const owned = await scroll(first, { foreground: true });
assert(!owned.isError && semantic(owned).status === "delivered", "Safe denied owned image scroll");
- view = await observe(first); observation = reference(view, "observation"); await control("close_first");
- await rejected(observation, command({ delivery_mode: "foreground" }));
- view = await observe(second); observation = reference(view, "observation"); await control("recycle_second");
- await rejected(observation, command({ delivery_mode: "foreground" }));
- await denied("insert", { path: first, text: JSON.stringify(command({ delivery_mode: "foreground" })) });
+ await observe(first); await control("close_first");
+ await rejected(first, { foreground: true });
+ await observe(second); await control("recycle_second");
+ await rejected(second, { foreground: true });
+ // Insert never acts on a window; it names the tool that does.
+ const insert = await denied("insert", { path: first, text: JSON.stringify({ action: "scroll", x: 100, y: 80, delta_y: 2, unit: "wheel_detents" }) });
+ assert(textOf(insert).includes("computer_scroll"), "Insert refusal named no tool");
}
- await ok("insert", { path: "computer:session", text: JSON.stringify({ action: "close" }) });
+ await ok("delete", { path: "computer:session" });
return { content: [{ type: "text" as const, text: `Scroll ${scenario} contract passed` },
...(image ? [{ type: "image" as const, data: image.data!, mimeType: image.mimeType! }] : [])], details: { scenario } };
},