Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Show details in action previews (#167)

alexshpunt-clanker[bot]1f46fd7dd217
diff --git a/docs/native-observability.md b/docs/native-observability.md
index b8218f2..7b80fde 100644
--- a/docs/native-observability.md
+++ b/docs/native-observability.md
@@ -18,7 +18,7 @@ A JSON patch controls individual categories and rendering preferences:
/computer-visuals {"observations":true,"previews":false,"pointer":true,"scroll_drag":true,"keyboard":true,"typed_text":true,"opacity":0.8,"quality":"balanced","reduced_motion":false}
```
-`typed_text` shows the text an action types or sets on its card, while `keyboard` is on. Text going into a field that UI Automation reports as a password is never sent to the renderer. Control characters show as spaces, and only the last 200 characters are sent.
+Action cards show available key combinations and pointer details. Printable keys are shown only when a native observation identifies a non-password focused element; otherwise they are omitted unless part of a non-text shortcut. `keyboard` and `typed_text` must both be on for keyboard details and typed text. Text going into a field that UI Automation reports as a password is never sent to the renderer. Pointer and scroll details follow their respective visual settings. Control characters show as spaces, and only the last 200 characters are sent.
`opacity` is 0..1. `quality` is `performance`, `balanced` or `high`. Preferences are saved on the Pi session branch; navigating to another branch restores its preferences. These commands belong to the human. They do not change Safe/YOLO, capture sizing, input delivery or permission checks.
diff --git a/src/native-visuals.ts b/src/native-visuals.ts
index 140ad9f..7f5eb1e 100644
--- a/src/native-visuals.ts
+++ b/src/native-visuals.ts
@@ -133,7 +133,7 @@ export class NativeVisuals {
const args: Record<string, unknown> = { generation, kind: mark?.kind ?? "overview", source,
...(mark?.region ? { region: { ...mark.region } } : {}), ...(mark?.point ? { point: { ...mark.point } } : {}),
...(frame.action ? { action: frame.action } : {}), ...(frame.outcome ? { outcome: frame.outcome } : {}),
- ...(mark?.kind === "action" && frame.typedText && options.keyboard && options.typed_text ? { typed_text: frame.typedText } : {}) };
+ ...(mark?.kind === "action" && frame.typedText && (["scroll", "scroll_at", "drag"].includes(frame.action ?? "") ? options.scroll_drag : ["click", "right_click", "double_click"].includes(frame.action ?? "") ? options.pointer : options.keyboard && options.typed_text) ? { typed_text: frame.typedText } : {}) };
// Bound transient transport memory. Oversized images still get a source frame, without a preview.
if (options.previews && image && ["image/png", "image/jpeg"].includes(image.mimeType) && image.data.length <= 4 * 1024 * 1024)
args.image = { data: image.data, mime_type: image.mimeType };
diff --git a/src/session.ts b/src/session.ts
index 0b8d14a..d071cf4 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -102,6 +102,38 @@ function presentationAction(command: InputCommand | WebAction): string {
if (command.action === "timeline") return command.events?.every(event => event.type === "key_down" || event.type === "key_up") ? "hotkey" : "input_timeline";
return command.action;
}
+/** Display only bounded input metadata; printable keys can be text and are hidden unless the focus is known safe. */
+function presentationDetail(command: InputCommand | WebAction, safeText = false): string | undefined {
+ const chord = (keys: string[]) => {
+ const last = keys.at(-1);
+ if (!last || !/^(?:[a-z0-9\-=[\];',./\\`]|enter|return|tab|escape|esc|space|backspace|delete|insert|up|down|left|right|arrowup|arrowdown|arrowleft|arrowright|home|end|pageup|pagedown|f(?:[1-9]|1[0-9]|2[0-4])|ctrl|control|shift|alt|win|meta)$/i.test(last)) return undefined;
+ if (keys.slice(0, -1).some(key => !/^(?:ctrl|control|shift|alt|win|meta)$/i.test(key))) return undefined;
+ const modifiers = keys.slice(0, -1);
+ // Ctrl+Alt can be AltGr, which enters printable text on many keyboard layouts.
+ const nonTextModifier = modifiers.some(key => /^(?:ctrl|control|win|meta)$/i.test(key)) && !modifiers.some(key => /^alt$/i.test(key));
+ return safeText || last.length > 1 || nonTextModifier ? keys.join("+") : undefined;
+ };
+ if (command.action === "press") return command.key ? shownTypedText(chord(command.key.split("+"))) : undefined;
+ if (command.action === "key") return command.key ? shownTypedText(chord([...(command.modifiers ?? []), command.key])) : undefined;
+ if (command.action === "hotkey" || command.action === "hold") {
+ const keys = command.keys?.length ? chord(command.keys) : undefined;
+ const actions = command.action === "hold" ? command.actions?.map(action => action.action).join(" → ") : undefined;
+ return shownTypedText([keys, actions].filter(Boolean).join(" + "));
+ }
+ if (command.action === "select_all") return "Ctrl+A";
+ if (command.action === "timeline" && command.events?.length) {
+ const details = command.events.slice(0, 20).map(event => event.key ? (chord([event.key]) ? `${event.type.replaceAll("_", " ")} ${chord([event.key])}` : undefined) : event.type.replaceAll("_", " "));
+ return shownTypedText(details.filter(Boolean).join(" → "));
+ }
+ if (command.action === "scroll" && "direction" in command) return command.direction ? shownTypedText(command.direction) : undefined;
+ if (command.action === "click") {
+ const count = "count" in command ? command.count : "click_count" in command ? command.click_count : undefined;
+ return shownTypedText(`${command.button ?? "left"} click${count && count > 1 ? ` ×${count}` : ""}`);
+ }
+ if (command.action === "double_click" || command.action === "right_click") return command.action.replaceAll("_", " ");
+ if (command.action === "drag") return shownTypedText(`${command.button ?? "left"} drag`);
+ return undefined;
+}
/** A stored Playwright snapshot of the active tab, with the tab list, any held dialog and events since the previous snapshot. */
export interface WebDom extends WebPage { truncated: boolean; matches?: number }
/** Delivery is not a task postcondition. Unknown input is never automatically replayed. */
@@ -426,9 +458,9 @@ export class ComputerSession {
/** Source-associated cue only: no UI strings, guessed coordinates, capture or input. */
private activity(entry: WindowEntry, mark: HighlightMark, signal?: AbortSignal, image?: Content,
- outcome?: "delivered" | "refused" | "unknown" | "verified", action?: string): Promise<HighlightResult | undefined> {
+ outcome?: "delivered" | "refused" | "unknown" | "verified", action?: string, typedText?: string): Promise<HighlightResult | undefined> {
if (!this.allowed(entry) || signal?.aborted) return Promise.resolve(undefined);
- return notifyWindowObserver(this.observer, entry.target, entry.identity, entry.visualBounds ?? entry.observation?.bounds, signal, mark, image, action, outcome);
+ return notifyWindowObserver(this.observer, entry.target, entry.identity, entry.visualBounds ?? entry.observation?.bounds, signal, mark, image, action, outcome, typedText);
}
private async current(ref: string, signal?: AbortSignal): Promise<WindowEntry | undefined> {
const entry = this.entries.get(ref);
@@ -1428,10 +1460,10 @@ export class ComputerSession {
return entry;
}
- private async webFailure(entry: WindowEntry, error: unknown, acting = false, signal?: AbortSignal, action?: string): Promise<ComputerResult> {
+ private async webFailure(entry: WindowEntry, error: unknown, acting = false, signal?: AbortSignal, action?: string, typedText?: string): Promise<ComputerResult> {
const reason = error instanceof Error ? error.message : String(error);
const uncertain = acting && error instanceof WebError && (error.timeout || error.transport);
- if (acting) await this.activity(entry, { kind: "action" }, signal, undefined, uncertain ? "unknown" : "refused", action);
+ if (acting) await this.activity(entry, { kind: "action" }, signal, undefined, uncertain ? "unknown" : "refused", action, typedText);
return this.result(uncertain ? "unknown" : "refused", uncertain
? `The action may have reached the page. ${reason} Read the window with dom before deciding; never replay it blindly.`
: `Playwright refused: ${reason}`, undefined, this.addressed(entry));
@@ -1522,11 +1554,11 @@ export class ComputerSession {
entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
try {
const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));
- await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action));
+ await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action), presentationDetail(action));
if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");
return this.result("delivered", `Playwright performed ${action.action}${action.target && action.action !== "evaluate" ? " after its actionability checks" : ""}. Check the fresh snapshot below for the application's response.`,
undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });
- } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action)); }
+ } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action), presentationDetail(action)); }
});
}
@@ -1543,11 +1575,11 @@ export class ComputerSession {
entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
try {
const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));
- await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action));
+ await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action), presentationDetail(action));
if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");
return this.result("delivered", `Playwright performed ${action.action}. Check the fresh snapshot below for the page's response.`,
undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });
- } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action)); }
+ } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action), presentationDetail(action)); }
});
}
@@ -2142,9 +2174,11 @@ export class ComputerSession {
const mark = { kind: "action" as const, region: step.element_token ? observed.frames.get(step.element_token) : undefined };
const activityAction = presentationAction(step);
// A password field's text is never shown.
- const typedElement = ["type_text", "set_value"].includes(step.action)
- ? observed.ui?.elements.find(element => step.element_token ? element.element_token === step.element_token : element.focused) : undefined;
- const typedText = ["type_text", "set_value"].includes(step.action) && !typedElement?.password ? shownTypedText(step.text) : undefined;
+ const typedElement = observed.ui?.elements.find(element => step.element_token ? element.element_token === step.element_token : element.focused);
+ const focusedElement = observed.ui?.elements.find(element => element.focused);
+ const typedText = ["type_text", "set_value"].includes(step.action)
+ ? !typedElement?.password ? shownTypedText(step.text) : undefined
+ : presentationDetail(step, !batch && step.action !== "timeline" && !!focusedElement && !focusedElement.password);
let highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, observed.bounds, signal, mark, undefined, activityAction, undefined, typedText);
if (signal?.aborted) return finish(this.result("refused", "Cancelled during action mark; input was not dispatched."), "Cancelled during action mark.");
if (!this.inputAllowed(entry)) return finish(this.denied(entry.ref), "Permission revoked during action mark.");
diff --git a/test/native-visuals.test.ts b/test/native-visuals.test.ts
index 3ace281..c4bfd1d 100644
--- a/test/native-visuals.test.ts
+++ b/test/native-visuals.test.ts
@@ -100,7 +100,7 @@ async function actionFixture(outcome: "delivered" | "refused" | "unknown" = "del
if (name === "get_window_state") return { content: [png], structuredContent: {
...frame.target, window_bounds: frame.bounds, capture_id: "11111111-1111-4111-8111-111111111111",
screenshot_width: frame.bounds.width, screenshot_height: frame.bounds.height,
- elements: [{ element_token: "field", role: "TextBox", frame: { x: 0, y: 60, w: 80, h: 40 }, ...(options.password ? { password: true } : {}) }],
+ elements: [{ element_token: "field", role: "TextBox", focused: true, frame: { x: 0, y: 60, w: 80, h: 40 }, ...(options.password ? { password: true } : {}) }],
} };
if (outcome !== "delivered") return { content: [], structuredContent: { effect: outcome } };
return { content: [], structuredContent: name === "semantic_action"
@@ -135,6 +135,58 @@ const nativeFamilies: { label: string; command: InputCommand; family: string; na
] }, family: "input_timeline", native: "input_timeline" },
];
+test("keyboard previews include the requested shortcut", async () => {
+ const f = await actionFixture();
+ try {
+ const listed = await f.session.windows();
+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
+ const read = await f.session.observe(ref); await tick(); f.send.mockClear();
+ await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "hotkey", keys: ["ctrl", "shift", "f12"] });
+ await tick();
+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity").map(([, args]) => args);
+ expect(activities).toHaveLength(2);
+ for (const args of activities) {
+ expect(String(args.typed_text).toLowerCase()).toContain("ctrl");
+ expect(String(args.typed_text).toLowerCase()).toContain("shift");
+ expect(String(args.typed_text).toLowerCase()).toContain("f12");
+ }
+ } finally { await f.close(); }
+});
+test.each([
+ { options: { password: true } },
+ { options: { settings: (() => { const s = new VisualSettings(); s.update({ typed_text: false }); return s; })() } },
+])("printable keys are hidden in a protected field or when typed text is disabled", async ({ options }) => {
+ const f = await actionFixture("delivered", options);
+ try {
+ const listed = await f.session.windows();
+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
+ const read = await f.session.observe(ref); await tick(); f.send.mockClear();
+ await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "key", key: "a" });
+ await tick();
+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");
+ expect(activities).toHaveLength(2);
+ for (const [, args] of activities) expect(args).not.toHaveProperty("typed_text");
+ } finally { await f.close(); }
+});
+test("timeline previews include non-text keys and pointer event types", async () => {
+ const f = await actionFixture();
+ try {
+ const listed = await f.session.windows();
+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
+ const read = await f.session.observe(ref); await tick(); f.send.mockClear();
+ await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "timeline", events: [
+ { at_ms: 0, type: "key_down", key: "f12" }, { at_ms: 1, type: "key_up", key: "f12" },
+ { at_ms: 2, type: "wheel", dy: 1 },
+ ] });
+ await tick();
+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");
+ expect(activities).toHaveLength(2);
+ for (const [, args] of activities) {
+ expect(String(args.typed_text)).toContain("f12");
+ expect(String(args.typed_text)).toContain("wheel");
+ }
+ } finally { await f.close(); }
+});
test.each(nativeFamilies)("native $label retains its presentation family before and after one dispatch", async ({ command, family, native }) => {
const f = await actionFixture();
try {
@@ -154,7 +206,7 @@ test.each(nativeFamilies)("native $label retains its presentation family before
expect(Object.keys(args).every(key => ["generation", "kind", "source", "region", "action", "outcome", "typed_text"].includes(key))).toBe(true);
if (!command.element_token) expect(args).not.toHaveProperty("region");
// The human sees what the agent types, before and after the dispatch.
- expect(args.typed_text).toBe("text" in command ? command.text : undefined);
+ if ("text" in command) expect(args.typed_text).toBe(command.text);
}
} finally { await f.close(); }
});
@@ -199,7 +251,18 @@ test.each(["refused", "unknown"] as const)("native keyboard %s keeps its family
} finally { await f.close(); }
});
-test.each(["dom", "page"] as const)("browser %s key outcomes use keyboard presentation without forwarding key or target", async route => {
+test.each(["a", "Shift+a", "Control+Alt+a"]) ("browser printable key %s is not exposed when focus is unknown", async key => {
+ const f = await actionFixture();
+ try {
+ const opened = await f.session.openBrowser(); const ref = opened.details.window_ref!;
+ await f.session.observeDom(ref); await tick(); f.send.mockClear();
+ await f.session.actPage(ref, { action: "press", key }); await tick();
+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");
+ expect(activities).toHaveLength(1);
+ expect(activities[0][1]).not.toHaveProperty("typed_text");
+ } finally { await f.close(); }
+});
+test.each(["dom", "page"] as const)("browser %s key outcomes include the shortcut without forwarding the target", async route => {
for (const outcome of ["delivered", "refused", "unknown"] as const) {
const f = await actionFixture();
try {
@@ -215,7 +278,7 @@ test.each(["dom", "page"] as const)("browser %s key outcomes use keyboard presen
expect(f.browser.page).toHaveBeenCalledExactlyOnceWith("act", action, undefined);
expect(f.send.mock.calls).toEqual([["present_agent_activity", {
generation: expect.any(Number), kind: "action", source: { ...frame.target, identity: frame.identity, bounds: frame.bounds },
- action: "press_key", outcome,
+ action: "press_key", outcome, typed_text: action.key,
}]]);
} finally { await f.close(); }
}