Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
fix: an Insert hold names computer_press with hold_ms, a pointer move computer_hover (LPT-934)
alexshpunt-clanker[bot]db929a8a514f
diff --git a/src/provider.ts b/src/provider.tsindex 0946a6a..a0bafed 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -26,10 +26,10 @@ export const COMPUTER_DELETE_HELP = "A computer window address closes that windo/** Each Insert command the typed tools replaced, by the tool that does it now. */const REPLACED: [RegExp, string][] = [[/^(click|double_click|right_click|invoke|check|uncheck)$/, "computer_click"], [/^(type_text|type|fill|set_value)$/, "computer_type"],- [/^(key|hotkey|press)$/, "computer_press"], [/^(scroll|wheel)$/, "computer_scroll"], [/^(drag|hold)$/, "computer_drag"], [/^(hover|move_pointer)$/, "computer_hover"],+ [/^(key|hotkey|press)$/, "computer_press"], [/^hold$/, "computer_press with hold_ms"], [/^(scroll|wheel)$/, "computer_scroll"], [/^drag$/, "computer_drag"], [/^(hover|move_pointer)$/, "computer_hover"],[/^(select|select_option|menu)$/, "computer_select"], [/^(timeline|sequence)$/, "computer_timeline"],[/^(open|open_browser|open_sandbox|open_session|attach_browser|webview2_debugging)$/, "computer_open"],- [/^(move|resize|minimize|maximize|focus|restore|name)$/, "computer_window (resize of a display: computer_display)"], [/^(wait_for|verify)$/, "computer_wait"],+ [/^(move|resize|minimize|maximize|focus|restore|name)$/, "computer_window (a pointer move: computer_hover; resize of a display: computer_display)"], [/^(wait_for|verify)$/, "computer_wait"],[/^(navigate|back|forward|reload|wait_load|new_tab|switch_tab|close_tab|dialog|upload|wait_download|cancel_download|evaluate)$/, "computer_browser"],[/^(put|get|files)$/, "computer_files"], [/^clipboard_/, "computer_clipboard"], [/^(record_start|record_stop|audio_record)$/, "computer_record"],[/^(close|kill)$/, "Delete (kill: Delete with ?force=true)"], [/^setup$/, "/computer-setup, which only the user runs (ask them)"], [/^batch$/, "Codemode: a script that calls the computer_* tools in order"],diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.tsindex d6f0093..3354540 100644--- a/test/computer-tools.test.ts+++ b/test/computer-tools.test.ts@@ -291,7 +291,7 @@ test("Write and Insert on a computer address refuse and name the tool that doesfor (const [action, tool] of [["click", "computer_click"], ["fill", "computer_type"], ["hotkey", "computer_press"], ["menu", "computer_select"],["open_browser", "computer_open"], ["restore", "computer_window"], ["verify", "computer_wait"], ["navigate", "computer_browser"],["put", "computer_files"], ["clipboard_read", "computer_clipboard"], ["record_start", "computer_record"], ["kill", "Delete"],- ["setup", "/computer-setup"], ["batch", "Codemode"]])+ ["setup", "/computer-setup"], ["batch", "Codemode"], ["hold", "computer_press with hold_ms"], ["drag", "computer_drag"]])expect(computerEditRefusal("insert", { path: "computer:machine/local", text: JSON.stringify({ action }) })).toContain(`Use ${tool}`);// Unknown or unreadable commands still learn where the tools are.expect(computerEditRefusal("insert", { path: "computer:session", text: "not json" })).toMatch(/^Insert does not act on computer resources; nothing was sent\. Each operation has its own computer_\* tool: tool_search/);