Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Show details in action previews (#167)
alexshpunt-clanker[bot]1f46fd7dd217
diff --git a/docs/native-observability.md b/docs/native-observability.mdindex b8218f2..7b80fde 100644--- a/docs/native-observability.md+++ b/docs/native-observability.md@@ -18,7 +18,7 @@ A JSON patch controls individual categories and rendering preferences:/computer-visuals {"observations":true,"previews":false,"pointer":true,"scroll_drag":true,"keyboard":true,"typed_text":true,"opacity":0.8,"quality":"balanced","reduced_motion":false}```-`typed_text` shows the text an action types or sets on its card, while `keyboard` is on. Text going into a field that UI Automation reports as a password is never sent to the renderer. Control characters show as spaces, and only the last 200 characters are sent.+Action cards show available key combinations and pointer details. Printable keys are shown only when a native observation identifies a non-password focused element; otherwise they are omitted unless part of a non-text shortcut. `keyboard` and `typed_text` must both be on for keyboard details and typed text. Text going into a field that UI Automation reports as a password is never sent to the renderer. Pointer and scroll details follow their respective visual settings. Control characters show as spaces, and only the last 200 characters are sent.`opacity` is 0..1. `quality` is `performance`, `balanced` or `high`. Preferences are saved on the Pi session branch; navigating to another branch restores its preferences. These commands belong to the human. They do not change Safe/YOLO, capture sizing, input delivery or permission checks.diff --git a/src/native-visuals.ts b/src/native-visuals.tsindex 140ad9f..7f5eb1e 100644--- a/src/native-visuals.ts+++ b/src/native-visuals.ts@@ -133,7 +133,7 @@ export class NativeVisuals {const args: Record<string, unknown> = { generation, kind: mark?.kind ?? "overview", source,...(mark?.region ? { region: { ...mark.region } } : {}), ...(mark?.point ? { point: { ...mark.point } } : {}),...(frame.action ? { action: frame.action } : {}), ...(frame.outcome ? { outcome: frame.outcome } : {}),- ...(mark?.kind === "action" && frame.typedText && options.keyboard && options.typed_text ? { typed_text: frame.typedText } : {}) };+ ...(mark?.kind === "action" && frame.typedText && (["scroll", "scroll_at", "drag"].includes(frame.action ?? "") ? options.scroll_drag : ["click", "right_click", "double_click"].includes(frame.action ?? "") ? options.pointer : options.keyboard && options.typed_text) ? { typed_text: frame.typedText } : {}) };// Bound transient transport memory. Oversized images still get a source frame, without a preview.if (options.previews && image && ["image/png", "image/jpeg"].includes(image.mimeType) && image.data.length <= 4 * 1024 * 1024)args.image = { data: image.data, mime_type: image.mimeType };diff --git a/src/session.ts b/src/session.tsindex 0b8d14a..d071cf4 100644--- a/src/session.ts+++ b/src/session.ts@@ -102,6 +102,38 @@ function presentationAction(command: InputCommand | WebAction): string {if (command.action === "timeline") return command.events?.every(event => event.type === "key_down" || event.type === "key_up") ? "hotkey" : "input_timeline";return command.action;}+/** Display only bounded input metadata; printable keys can be text and are hidden unless the focus is known safe. */+function presentationDetail(command: InputCommand | WebAction, safeText = false): string | undefined {+ const chord = (keys: string[]) => {+ const last = keys.at(-1);+ if (!last || !/^(?:[a-z0-9\-=[\];',./\\`]|enter|return|tab|escape|esc|space|backspace|delete|insert|up|down|left|right|arrowup|arrowdown|arrowleft|arrowright|home|end|pageup|pagedown|f(?:[1-9]|1[0-9]|2[0-4])|ctrl|control|shift|alt|win|meta)$/i.test(last)) return undefined;+ if (keys.slice(0, -1).some(key => !/^(?:ctrl|control|shift|alt|win|meta)$/i.test(key))) return undefined;+ const modifiers = keys.slice(0, -1);+ // Ctrl+Alt can be AltGr, which enters printable text on many keyboard layouts.+ const nonTextModifier = modifiers.some(key => /^(?:ctrl|control|win|meta)$/i.test(key)) && !modifiers.some(key => /^alt$/i.test(key));+ return safeText || last.length > 1 || nonTextModifier ? keys.join("+") : undefined;+ };+ if (command.action === "press") return command.key ? shownTypedText(chord(command.key.split("+"))) : undefined;+ if (command.action === "key") return command.key ? shownTypedText(chord([...(command.modifiers ?? []), command.key])) : undefined;+ if (command.action === "hotkey" || command.action === "hold") {+ const keys = command.keys?.length ? chord(command.keys) : undefined;+ const actions = command.action === "hold" ? command.actions?.map(action => action.action).join(" → ") : undefined;+ return shownTypedText([keys, actions].filter(Boolean).join(" + "));+ }+ if (command.action === "select_all") return "Ctrl+A";+ if (command.action === "timeline" && command.events?.length) {+ const details = command.events.slice(0, 20).map(event => event.key ? (chord([event.key]) ? `${event.type.replaceAll("_", " ")} ${chord([event.key])}` : undefined) : event.type.replaceAll("_", " "));+ return shownTypedText(details.filter(Boolean).join(" → "));+ }+ if (command.action === "scroll" && "direction" in command) return command.direction ? shownTypedText(command.direction) : undefined;+ if (command.action === "click") {+ const count = "count" in command ? command.count : "click_count" in command ? command.click_count : undefined;+ return shownTypedText(`${command.button ?? "left"} click${count && count > 1 ? ` ×${count}` : ""}`);+ }+ if (command.action === "double_click" || command.action === "right_click") return command.action.replaceAll("_", " ");+ if (command.action === "drag") return shownTypedText(`${command.button ?? "left"} drag`);+ return undefined;+}/** A stored Playwright snapshot of the active tab, with the tab list, any held dialog and events since the previous snapshot. */export interface WebDom extends WebPage { truncated: boolean; matches?: number }/** Delivery is not a task postcondition. Unknown input is never automatically replayed. */@@ -426,9 +458,9 @@ export class ComputerSession {/** Source-associated cue only: no UI strings, guessed coordinates, capture or input. */private activity(entry: WindowEntry, mark: HighlightMark, signal?: AbortSignal, image?: Content,- outcome?: "delivered" | "refused" | "unknown" | "verified", action?: string): Promise<HighlightResult | undefined> {+ outcome?: "delivered" | "refused" | "unknown" | "verified", action?: string, typedText?: string): Promise<HighlightResult | undefined> {if (!this.allowed(entry) || signal?.aborted) return Promise.resolve(undefined);- return notifyWindowObserver(this.observer, entry.target, entry.identity, entry.visualBounds ?? entry.observation?.bounds, signal, mark, image, action, outcome);+ return notifyWindowObserver(this.observer, entry.target, entry.identity, entry.visualBounds ?? entry.observation?.bounds, signal, mark, image, action, outcome, typedText);}private async current(ref: string, signal?: AbortSignal): Promise<WindowEntry | undefined> {const entry = this.entries.get(ref);@@ -1428,10 +1460,10 @@ export class ComputerSession {return entry;}- private async webFailure(entry: WindowEntry, error: unknown, acting = false, signal?: AbortSignal, action?: string): Promise<ComputerResult> {+ private async webFailure(entry: WindowEntry, error: unknown, acting = false, signal?: AbortSignal, action?: string, typedText?: string): Promise<ComputerResult> {const reason = error instanceof Error ? error.message : String(error);const uncertain = acting && error instanceof WebError && (error.timeout || error.transport);- if (acting) await this.activity(entry, { kind: "action" }, signal, undefined, uncertain ? "unknown" : "refused", action);+ if (acting) await this.activity(entry, { kind: "action" }, signal, undefined, uncertain ? "unknown" : "refused", action, typedText);return this.result(uncertain ? "unknown" : "refused", uncertain? `The action may have reached the page. ${reason} Read the window with dom before deciding; never replay it blindly.`: `Playwright refused: ${reason}`, undefined, this.addressed(entry));@@ -1522,11 +1554,11 @@ export class ComputerSession {entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;try {const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));- await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action));+ await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action), presentationDetail(action));if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");return this.result("delivered", `Playwright performed ${action.action}${action.target && action.action !== "evaluate" ? " after its actionability checks" : ""}. Check the fresh snapshot below for the application's response.`,undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });- } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action)); }+ } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action), presentationDetail(action)); }});}@@ -1543,11 +1575,11 @@ export class ComputerSession {entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;try {const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));- await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action));+ await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action), presentationDetail(action));if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");return this.result("delivered", `Playwright performed ${action.action}. Check the fresh snapshot below for the page's response.`,undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });- } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action)); }+ } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action), presentationDetail(action)); }});}@@ -2142,9 +2174,11 @@ export class ComputerSession {const mark = { kind: "action" as const, region: step.element_token ? observed.frames.get(step.element_token) : undefined };const activityAction = presentationAction(step);// A password field's text is never shown.- const typedElement = ["type_text", "set_value"].includes(step.action)- ? observed.ui?.elements.find(element => step.element_token ? element.element_token === step.element_token : element.focused) : undefined;- const typedText = ["type_text", "set_value"].includes(step.action) && !typedElement?.password ? shownTypedText(step.text) : undefined;+ const typedElement = observed.ui?.elements.find(element => step.element_token ? element.element_token === step.element_token : element.focused);+ const focusedElement = observed.ui?.elements.find(element => element.focused);+ const typedText = ["type_text", "set_value"].includes(step.action)+ ? !typedElement?.password ? shownTypedText(step.text) : undefined+ : presentationDetail(step, !batch && step.action !== "timeline" && !!focusedElement && !focusedElement.password);let highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, observed.bounds, signal, mark, undefined, activityAction, undefined, typedText);if (signal?.aborted) return finish(this.result("refused", "Cancelled during action mark; input was not dispatched."), "Cancelled during action mark.");if (!this.inputAllowed(entry)) return finish(this.denied(entry.ref), "Permission revoked during action mark.");diff --git a/test/native-visuals.test.ts b/test/native-visuals.test.tsindex 3ace281..c4bfd1d 100644--- a/test/native-visuals.test.ts+++ b/test/native-visuals.test.ts@@ -100,7 +100,7 @@ async function actionFixture(outcome: "delivered" | "refused" | "unknown" = "delif (name === "get_window_state") return { content: [png], structuredContent: {...frame.target, window_bounds: frame.bounds, capture_id: "11111111-1111-4111-8111-111111111111",screenshot_width: frame.bounds.width, screenshot_height: frame.bounds.height,- elements: [{ element_token: "field", role: "TextBox", frame: { x: 0, y: 60, w: 80, h: 40 }, ...(options.password ? { password: true } : {}) }],+ elements: [{ element_token: "field", role: "TextBox", focused: true, frame: { x: 0, y: 60, w: 80, h: 40 }, ...(options.password ? { password: true } : {}) }],} };if (outcome !== "delivered") return { content: [], structuredContent: { effect: outcome } };return { content: [], structuredContent: name === "semantic_action"@@ -135,6 +135,58 @@ const nativeFamilies: { label: string; command: InputCommand; family: string; na] }, family: "input_timeline", native: "input_timeline" },];+test("keyboard previews include the requested shortcut", async () => {+ const f = await actionFixture();+ try {+ const listed = await f.session.windows();+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;+ const read = await f.session.observe(ref); await tick(); f.send.mockClear();+ await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "hotkey", keys: ["ctrl", "shift", "f12"] });+ await tick();+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity").map(([, args]) => args);+ expect(activities).toHaveLength(2);+ for (const args of activities) {+ expect(String(args.typed_text).toLowerCase()).toContain("ctrl");+ expect(String(args.typed_text).toLowerCase()).toContain("shift");+ expect(String(args.typed_text).toLowerCase()).toContain("f12");+ }+ } finally { await f.close(); }+});+test.each([+ { options: { password: true } },+ { options: { settings: (() => { const s = new VisualSettings(); s.update({ typed_text: false }); return s; })() } },+])("printable keys are hidden in a protected field or when typed text is disabled", async ({ options }) => {+ const f = await actionFixture("delivered", options);+ try {+ const listed = await f.session.windows();+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;+ const read = await f.session.observe(ref); await tick(); f.send.mockClear();+ await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "key", key: "a" });+ await tick();+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");+ expect(activities).toHaveLength(2);+ for (const [, args] of activities) expect(args).not.toHaveProperty("typed_text");+ } finally { await f.close(); }+});+test("timeline previews include non-text keys and pointer event types", async () => {+ const f = await actionFixture();+ try {+ const listed = await f.session.windows();+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;+ const read = await f.session.observe(ref); await tick(); f.send.mockClear();+ await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "timeline", events: [+ { at_ms: 0, type: "key_down", key: "f12" }, { at_ms: 1, type: "key_up", key: "f12" },+ { at_ms: 2, type: "wheel", dy: 1 },+ ] });+ await tick();+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");+ expect(activities).toHaveLength(2);+ for (const [, args] of activities) {+ expect(String(args.typed_text)).toContain("f12");+ expect(String(args.typed_text)).toContain("wheel");+ }+ } finally { await f.close(); }+});test.each(nativeFamilies)("native $label retains its presentation family before and after one dispatch", async ({ command, family, native }) => {const f = await actionFixture();try {@@ -154,7 +206,7 @@ test.each(nativeFamilies)("native $label retains its presentation family beforeexpect(Object.keys(args).every(key => ["generation", "kind", "source", "region", "action", "outcome", "typed_text"].includes(key))).toBe(true);if (!command.element_token) expect(args).not.toHaveProperty("region");// The human sees what the agent types, before and after the dispatch.- expect(args.typed_text).toBe("text" in command ? command.text : undefined);+ if ("text" in command) expect(args.typed_text).toBe(command.text);}} finally { await f.close(); }});@@ -199,7 +251,18 @@ test.each(["refused", "unknown"] as const)("native keyboard %s keeps its family} finally { await f.close(); }});-test.each(["dom", "page"] as const)("browser %s key outcomes use keyboard presentation without forwarding key or target", async route => {+test.each(["a", "Shift+a", "Control+Alt+a"]) ("browser printable key %s is not exposed when focus is unknown", async key => {+ const f = await actionFixture();+ try {+ const opened = await f.session.openBrowser(); const ref = opened.details.window_ref!;+ await f.session.observeDom(ref); await tick(); f.send.mockClear();+ await f.session.actPage(ref, { action: "press", key }); await tick();+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");+ expect(activities).toHaveLength(1);+ expect(activities[0][1]).not.toHaveProperty("typed_text");+ } finally { await f.close(); }+});+test.each(["dom", "page"] as const)("browser %s key outcomes include the shortcut without forwarding the target", async route => {for (const outcome of ["delivered", "refused", "unknown"] as const) {const f = await actionFixture();try {@@ -215,7 +278,7 @@ test.each(["dom", "page"] as const)("browser %s key outcomes use keyboard presenexpect(f.browser.page).toHaveBeenCalledExactlyOnceWith("act", action, undefined);expect(f.send.mock.calls).toEqual([["present_agent_activity", {generation: expect.any(Number), kind: "action", source: { ...frame.target, identity: frame.identity, bounds: frame.bounds },- action: "press_key", outcome,+ action: "press_key", outcome, typed_text: action.key,}]]);} finally { await f.close(); }}